chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:22:34 +08:00
commit 4b22cfda96
9037 changed files with 2363717 additions and 0 deletions
+102
View File
@@ -0,0 +1,102 @@
{
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:@typescript-eslint/recommended-requiring-type-checking"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 2022,
"sourceType": "module",
"project": "./tsconfig.json"
},
"plugins": ["@typescript-eslint"],
"ignorePatterns": [
"dist/",
"build/",
"bundle/",
"node_modules/",
"*.d.ts",
"jest.config.js",
"jest.config.cjs"
],
"rules": {
// TypeScript-specific rules
"@typescript-eslint/no-unused-vars": [
"error",
{
"argsIgnorePattern": "^_",
"varsIgnorePattern": "^_"
}
],
"@typescript-eslint/no-non-null-assertion": "warn",
"@typescript-eslint/prefer-optional-chain": "error",
"@typescript-eslint/no-unnecessary-type-assertion": "error",
"@typescript-eslint/no-loss-of-precision": "error",
// We cannot type everything like user inputs, outputs, json, etc.
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-unsafe-assignment": "off",
// Namespaces are useful for organizing API specifications
"@typescript-eslint/no-namespace": "off",
"@typescript-eslint/ban-types": "off",
// General JavaScript/TypeScript rules
"no-console": ["warn", { "allow": ["debug", "warn", "error"] }],
"no-debugger": "error",
"no-alert": "error",
"prefer-const": "error",
"no-var": "error",
"eqeqeq": ["error", "always", { "null": "never" }],
"curly": ["error", "all"],
"no-trailing-spaces": "error",
"no-multiple-empty-lines": ["error", { "max": 2 }],
// Import/Export rules
"no-duplicate-imports": "error",
// Promise/Async rules
"no-async-promise-executor": "error",
"require-await": "error"
},
"overrides": [
{
"files": ["*.test.ts", "*.spec.ts", "examples/**/*.ts", "jest.*.ts"],
"rules": {
"no-console": "off",
"@typescript-eslint/explicit-function-return-type": "off",
"@typescript-eslint/no-unsafe-member-access": "off",
"@typescript-eslint/no-unsafe-argument": "off",
"@typescript-eslint/no-non-null-assertion": "off"
}
},
{
"files": ["scripts/**/*.ts"],
"rules": {
"no-console": "off"
}
},
{
"files": ["integrations/vercel/tests/*.test.ts"],
"rules": {
// AI SDK does not expose the type of createOpenAI return value
"@typescript-eslint/no-unsafe-call": "off"
}
},
{
"files": ["integrations/opencode/tests/*.test.ts"],
"rules": {
// OpenCode plugin types are mocked and not fully typed
"@typescript-eslint/no-unsafe-call": "off",
"@typescript-eslint/unbound-method": "off"
}
},
{
"files": ["integrations/claude-code/tests/*.test.ts"],
"rules": {
// Claude Code transcript types are mocked and not fully typed
"@typescript-eslint/no-unsafe-call": "off",
"@typescript-eslint/unbound-method": "off"
}
}
]
}
+2
View File
@@ -0,0 +1,2 @@
min-release-age=7
omit-lockfile-registry-resolved=true
+3
View File
@@ -0,0 +1,3 @@
**/dist/
**/build/
**/bundle/
+11
View File
@@ -0,0 +1,11 @@
{
"semi": true,
"singleQuote": true,
"printWidth": 100,
"tabWidth": 2,
"useTabs": false,
"bracketSpacing": true,
"bracketSameLine": false,
"arrowParens": "always",
"endOfLine": "lf"
}
+339
View File
@@ -0,0 +1,339 @@
<h1 align="center" style="border-bottom: none">
<div>
<a href="https://mlflow.org/"><picture>
<img alt="MLflow Logo" src="https://raw.githubusercontent.com/mlflow/mlflow/refs/heads/master/assets/logo.svg" width="200" />
</picture></a>
<br>
MLflow TypeScript SDK
</div>
</h1>
<h2 align="center" style="border-bottom: none"></h2>
<p align="center">
<a href="https://github.com/mlflow/mlflow"><img src="https://img.shields.io/github/stars/mlflow/mlflow?style=social" alt="stars"></a>
<a href="https://www.npmjs.com/package/@mlflow/core"><img src="https://img.shields.io/npm/v/%40mlflow%2Fcore.svg" alt="version"></a>
<a href="https://www.npmjs.com/package/@mlflow/core"><img src="https://img.shields.io/npm/dt/%40mlflow%2Fcore.svg" alt="downloads"></a>
<a href="https://github.com/mlflow/mlflow/blob/master/LICENSE.txt"><img src="https://img.shields.io/github/license/mlflow/mlflow" alt="license"></a>
</p>
MLflow Typescript SDK is a variant of the [MLflow Python SDK](https://github.com/mlflow/mlflow) that provides a TypeScript API for MLflow.
> [!IMPORTANT]
> MLflow Typescript SDK is catching up with the Python SDK. Currently only support [Tracing]() and [Feedback Collection]() features. Please raise an issue in Github if you need a feature that is not supported.
## Packages
| Package | NPM | Description |
| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
| [@mlflow/core](./core) | [![npm package](https://img.shields.io/npm/v/%40mlflow%2Fcore?style=flat-square)](https://www.npmjs.com/package/@mlflow/core) | The core tracing functionality and manual instrumentation. |
| [@mlflow/openai](./integrations/openai) | [![npm package](https://img.shields.io/npm/v/%40mlflow%2Fopenai?style=flat-square)](https://www.npmjs.com/package/@mlflow/openai) | Auto-instrumentation integration for OpenAI. |
## Installation
```bash
npm install @mlflow/core
```
> [!NOTE]
> MLflow Typescript SDK requires Node.js 20 or higher.
## Quickstart
Start MLflow Tracking Server if you don't have one already:
```bash
pip install mlflow
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
```
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
Instantiate MLflow SDK in your application:
```typescript
import * as mlflow from '@mlflow/core';
mlflow.init({
trackingUri: 'http://localhost:5000',
experimentId: '<experiment-id>',
});
```
### Configure with environment variables
The SDK can also read configuration from environment variables so you can avoid
hard-coding connection details. If `MLFLOW_TRACKING_URI` and
`MLFLOW_EXPERIMENT_ID` are set, you can initialize the client without passing
any arguments:
```bash
export MLFLOW_TRACKING_URI=http://localhost:5000
export MLFLOW_EXPERIMENT_ID=123456789
```
```typescript
import * as mlflow from '@mlflow/core';
mlflow.init(); // Uses the values from the environment
```
### Authentication
For MLflow tracking servers that require authentication, the SDK supports:
1. **Basic Auth** (username/password):
```typescript
mlflow.init({
trackingUri: 'http://localhost:5000',
experimentId: '123456789',
trackingServerUsername: 'user',
trackingServerPassword: 'pass',
});
```
Or via environment variables:
```bash
export MLFLOW_TRACKING_USERNAME=user
export MLFLOW_TRACKING_PASSWORD=pass
```
2. **Bearer Token**:
```typescript
mlflow.init({
trackingUri: 'http://localhost:5000',
experimentId: '123456789',
trackingServerToken: 'my-token',
});
```
Or via environment variable:
```bash
export MLFLOW_TRACKING_TOKEN=my-token
```
3. **No authentication** (default for local development)
Create a trace:
```typescript
// Wrap a function with mlflow.trace to generate a span when the function is called.
// MLflow will automatically record the function name, arguments, return value,
// latency, and exception information to the span.
const getWeather = mlflow.trace(
(city: string) => {
return `The weather in ${city} is sunny`;
},
// Pass options to set span name. See https://mlflow.org/docs/latest/genai/tracing/quickstart
// for the full list of options.
{ name: 'get-weather' },
);
getWeather('San Francisco');
// Alternatively, start and end span manually
const span = mlflow.startSpan({ name: 'my-span' });
span.end();
```
Tag spans with a severity level so users (or you) can filter by **Minimum log level** in the trace UI:
```typescript
import { SpanLogLevel } from '@mlflow/core';
const tracedAnswer = mlflow.trace((query: string) => llm.generate(query), {
name: 'answer',
spanType: mlflow.SpanType.CHAT_MODEL,
logLevel: SpanLogLevel.INFO,
});
// The string form works too:
mlflow.startSpan({ name: 'plumbing', logLevel: 'DEBUG' });
```
When you use one of the autolog integrations (`@mlflow/openai`, `@mlflow/anthropic`, `@mlflow/gemini`, etc.), MLflow stamps a sensible default level on every span based on its type — you don't need to annotate manually.
View traces in MLflow UI:
![MLflow Tracing UI](https://github.com/mlflow/mlflow/blob/891fed9a746477f808dd2b82d3abb2382293c564/docs/static/images/llms/tracing/quickstart/openai-tool-calling-trace-detail.png?raw=true)
## Publishing
1. Run `yarn bump-version --version <new_version>` from this directory to bump the package versions appropriately
2. `cd` into `core` and run `npm publish`, and repeat for `integrations/openai`
## Adding New Integrations
The TypeScript SDK supports pluggable auto-instrumentation packages under [`integrations/`](./integrations). To add a new integration:
1. Create a new workspace package (for example, `integrations/<provider>`), modeled after the [OpenAI integration](./integrations/openai).
2. Implement the instrumentation entry points in `src/`, exporting a `register()` helper that configures tracing for the target client library.
3. Add package metadata (`package.json`, `tsconfig.json`, and optional `README.md`) so the integration can be built and published.
4. Add unit and/or integration tests under `tests/` that exercise the new instrumentation.
5. Update the root [`package.json`](./package.json) `build:integrations` and `test:integrations` scripts if your package requires additional build or test commands.
Once your integration package is ready, run the local workflow outlined in [Running the SDK after changes](#running-the-sdk-after-changes) and open a pull request that describes the new provider support.
## Contributing
We welcome contributions of new features, bug fixes, and documentation improvements. To contribute:
1. Review the project-wide [contribution guidelines](../../CONTRIBUTING.md) and follow the MLflow [Code of Conduct](../../CODE_OF_CONDUCT.rst).
2. Discuss larger proposals in a GitHub issue or the MLflow community channels before investing significant effort.
3. Fork the repository (or use a feature branch) and make your changes with clear, well-structured commits.
4. Ensure your code includes tests and documentation updates where appropriate.
5. Submit a pull request that summarizes the motivation, implementation details, and validation steps. The MLflow team will review and provide feedback.
## Running the SDK after Changes
The TypeScript workspace uses npm workspaces. After modifying the core SDK or any integration:
```bash
npm install # Install or update workspace dependencies
npm run build # Build the core package and all integrations
npm run test # Execute the test suites for the core SDK and integrations
```
You can run package-specific scripts from their respective directories (for example, `cd core && npm run test`) when iterating on a particular feature. Remember to rebuild before consuming the SDK from another project so that the latest TypeScript output is emitted to `dist/`.
## Trace Usage
MLflow Tracing empowers you throughout the end-to-end lifecycle of your application. Here's how it helps you at each step of the workflow, click on each section to learn more:
<details>
<summary><strong>🔍 Build & Debug</strong></summary>
<table>
<tr>
<td width="60%">
#### Smooth Debugging Experience
MLflow's tracing capabilities provide deep insights into what happens beneath the abstractions of your application, helping you precisely identify where issues occur.
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/observe-with-traces/ui)
</td>
<td width="40%">
![Trace Debug](https://raw.githubusercontent.com/mlflow/mlflow/master/docs/static/images/llms/tracing/genai-trace-debug.png)
</td>
</tr>
</table>
</details>
<details>
<summary><strong>💬 Human Feedback</strong></summary>
<table>
<tr>
<td width="60%">
#### Track Annotation and User Feedback Attached to Traces
Collecting and managing feedback is essential for improving your application. MLflow Tracing allows you to attach user feedback and annotations directly to traces, creating a rich dataset for analysis.
This feedback data helps you understand user satisfaction, identify areas for improvement, and build better evaluation datasets based on real user interactions.
[Learn more →](https://mlflow.org/docs/latest/genai/assessments/feedback)
</td>
<td width="40%">
![Human Feedback](https://raw.githubusercontent.com/mlflow/mlflow/master/docs/static/images/llms/tracing/genai-human-feedback.png)
</td>
</tr>
</table>
</details>
<details>
<summary><strong>📊 Evaluation</strong></summary>
<table>
<tr>
<td width="60%">
#### Systematic Quality Assessment Throughout Your Application
Evaluating the performance of your application is crucial, but creating a reliable evaluation process can be challenging. Traces serve as a rich data source, helping you assess quality with precise metrics for all components.
When combined with MLflow's evaluation capabilities, you get a seamless experience for assessing and improving your application's performance.
[Learn more →](https://mlflow.org/docs/latest/genai/eval-monitor)
</td>
<td width="40%">
![Evaluation](https://raw.githubusercontent.com/mlflow/mlflow/master/docs/static/images/llms/tracing/genai-trace-evaluation.png)
</td>
</tr>
</table>
</details>
<details>
<summary><strong>🚀 Production Monitoring</strong></summary>
<table>
<tr>
<td width="60%">
#### Monitor Applications with Your Favorite Observability Stack
Machine learning projects don't end with the first launch. Continuous monitoring and incremental improvement are critical to long-term success.
Integrated with various observability platforms such as Databricks, Datadog, Grafana, and Prometheus, MLflow Tracing provides a comprehensive solution for monitoring your applications in production.
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/prod-tracing)
</td>
<td width="40%">
![Monitoring](https://raw.githubusercontent.com/mlflow/mlflow/master/docs/static/images/llms/tracing/genai-monitoring.png)
</td>
</tr>
</table>
</details>
<details>
<summary><strong>📦 Dataset Collection</strong></summary>
<table>
<tr>
<td width="60%">
#### Create High-Quality Evaluation Datasets from Production Traces
Traces from production are invaluable for building comprehensive evaluation datasets. By capturing real user interactions and their outcomes, you can create test cases that truly represent your application's usage patterns.
This comprehensive data capture enables you to create realistic test scenarios, validate model performance on actual usage patterns, and continuously improve your evaluation datasets.
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/search-traces#creating-evaluation-datasets)
</td>
<td width="40%">
![Dataset Collection](https://raw.githubusercontent.com/mlflow/mlflow/master/docs/static/images/llms/tracing/genai-trace-dataset.png)
</td>
</tr>
</table>
</details>
## Documentation 📘
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
## License
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
+64
View File
@@ -0,0 +1,64 @@
# MLflow Typescript SDK - Core
This is the core package of the [MLflow Typescript SDK](https://github.com/mlflow/mlflow/tree/main/libs/typescript). It is a skinny package that includes the core tracing functionality and manual instrumentation.
| Package | NPM | Description |
| ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
| [@mlflow/core](./) | [![npm package](https://img.shields.io/npm/v/%40mlflow%2Fcore?style=flat-square)](https://www.npmjs.com/package/@mlflow/core) | The core tracing functionality and manual instrumentation. |
## Installation
```bash
npm install @mlflow/core
```
## Quickstart
Start MLflow Tracking Server. If you have a local Python environment, you can run the following command:
```bash
pip install mlflow
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
```
If you don't have Python environment locally, MLflow also supports Docker deployment or managed services. See [Self-Hosting Guide](https://mlflow.org/docs/latest/self-hosting/index.html) for getting started.
Instantiate MLflow SDK in your application:
```typescript
import * as mlflow from '@mlflow/core';
mlflow.init({
trackingUri: 'http://localhost:5000',
experimentId: '<experiment-id>',
});
```
Create a trace:
```typescript
// Wrap a function with mlflow.trace to generate a span when the function is called.
// MLflow will automatically record the function name, arguments, return value,
// latency, and exception information to the span.
const getWeather = mlflow.trace(
(city: string) => {
return `The weather in ${city} is sunny`;
},
// Pass options to set span name. See https://mlflow.org/docs/latest/genai/tracing/quickstart
// for the full list of options.
{ name: 'get-weather' },
);
getWeather('San Francisco');
// Alternatively, start and end span manually
const span = mlflow.startSpan({ name: 'my-span' });
span.end();
```
## Documentation 📘
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
## License
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
+30
View File
@@ -0,0 +1,30 @@
/**
* Bundles the WAL uploader daemon into a single, executable CJS file.
*
* Why a bundle:
* - The daemon is spawned as a child process by the supervisor (see
* `src/exporters/wal/supervisor.ts`), which resolves it at runtime
* via `@mlflow/core/package.json` + `bundle/daemon.cjs`. Shipping a
* single self-contained file means we don't have to worry about
* transitive `dist/` files or `node_modules` layout in the target
* environment.
* - The `#!/usr/bin/env node` banner + `chmod 0o755` makes the bundle
* directly executable, which is how the npm `bin` field invokes it.
*/
import { build } from 'esbuild';
import { chmodSync } from 'node:fs';
const outfile = 'bundle/daemon.cjs';
await build({
entryPoints: ['dist/exporters/wal/daemon.js'],
bundle: true,
platform: 'node',
format: 'cjs',
outfile,
external: ['node:*'],
banner: { js: '#!/usr/bin/env node' },
});
chmodSync(outfile, 0o755);
+16
View File
@@ -0,0 +1,16 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
roots: ['<rootDir>/tests', '<rootDir>/src'],
testMatch: ['**/*.test.ts'],
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
transform: {
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
},
globalSetup: '<rootDir>/../jest.global-server-setup.ts',
globalTeardown: '<rootDir>/../jest.global-server-teardown.ts',
testTimeout: 30000,
forceExit: true,
detectOpenHandles: true,
};
+74
View File
@@ -0,0 +1,74 @@
{
"name": "@mlflow/core",
"version": "0.3.0",
"description": "TypeScript implementation of MLflow Tracing SDK for LLM observability",
"repository": {
"type": "git",
"url": "https://github.com/mlflow/mlflow.git"
},
"homepage": "https://mlflow.org/",
"author": {
"name": "MLflow",
"url": "https://mlflow.org/"
},
"bugs": {
"url": "https://github.com/mlflow/mlflow/issues"
},
"license": "Apache-2.0",
"keywords": [
"mlflow",
"tracing",
"observability",
"opentelemetry",
"llm",
"javascript",
"typescript"
],
"main": "dist/index.js",
"types": "dist/index.d.ts",
"bin": {
"mlflow-trace-daemon": "./bundle/daemon.cjs"
},
"scripts": {
"build": "tsc && node esbuild.config.mjs",
"test": "jest",
"lint": "eslint . --ext .ts --max-warnings 0",
"lint:fix": "eslint . --ext .ts --fix",
"format": "prettier --write .",
"format:check": "prettier --check ."
},
"dependencies": {
"@databricks/sdk-experimental": "0.15.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/exporter-trace-otlp-proto": "^0.205.0",
"@opentelemetry/otlp-transformer": "^0.205.0",
"@opentelemetry/sdk-node": "^0.205.0",
"@opentelemetry/sdk-trace-base": "^2.1.0",
"fast-safe-stringify": "^2.1.1",
"bignumber.js": "^9.0.0",
"ini": "^5.0.0"
},
"devDependencies": {
"@types/ini": "^4.1.1",
"@types/jest": "^29.5.3",
"@types/node": "^20.4.5",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"esbuild": "^0.25.0",
"eslint": "^8.57.1",
"jest": "^29.6.2",
"msw": "^2.10.3",
"prettier": "^3.5.3",
"ts-jest": "^29.1.1",
"tsx": "^4.7.0",
"typescript": "^5.8.3",
"whatwg-fetch": "^3.6.20"
},
"engines": {
"node": ">=18"
},
"files": [
"dist/",
"bundle/"
]
}
+299
View File
@@ -0,0 +1,299 @@
/**
* MLflow TypeScript SDK Authentication Module
*
* This module provides authentication for MLflow tracking servers:
*
* ## Databricks-hosted MLflow
*
* For `databricks://` URIs, authentication is delegated entirely to the
* Databricks SDK which supports multiple authentication methods:
* - Personal Access Tokens (DATABRICKS_TOKEN)
* - OAuth M2M / Service Principals (DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET)
* - Azure CLI, Azure MSI, Azure Client Secret
* - Google Cloud credentials
* - Config file profiles (~/.databrickscfg)
*
* Host resolution order:
* 1. Explicit `host` option
* 2. DATABRICKS_HOST environment variable
* 3. Host from ~/.databrickscfg (for the specified profile)
*
* ## Self-hosted MLflow (OSS)
*
* For HTTP(S) URIs, authentication is resolved in order:
* 1. Basic Auth (MLFLOW_TRACKING_USERNAME + MLFLOW_TRACKING_PASSWORD)
* 2. Bearer Token (MLFLOW_TRACKING_TOKEN)
* 3. No authentication
*
* Workspace support (servers with workspaces enabled):
* - MLFLOW_WORKSPACE: sets the `X-MLFLOW-WORKSPACE` header for workspace-scoped requests
*
* @module auth
*/
import fs from 'fs';
import os from 'os';
import path from 'path';
import { Config } from '@databricks/sdk-experimental';
/**
* Function that provides HTTP headers for authenticated requests.
* Called before each request to support token refresh for OAuth.
*/
export type HeadersProvider = () => Promise<Record<string, string>>;
/**
* Authentication provider interface used by MLflow clients.
* Provides methods to get the host URL and authentication headers.
*/
export interface AuthProvider {
/** Get the host URL for API requests */
getHost(): string;
/** Get the function that provides authentication headers */
getHeadersProvider(): HeadersProvider;
/**
* Get the Databricks token if available (for backwards compatibility).
* Returns undefined for non-Databricks or OAuth-only authentication.
*/
getDatabricksToken(): string | undefined;
}
/**
* Configuration options for authentication resolution.
*/
export interface AuthOptions {
/** Tracking URI: "databricks", "databricks://profile", or HTTP(S) URL */
trackingUri: string;
/** Explicit host override (takes precedence over environment) */
host?: string;
/** Explicit Databricks token override */
databricksToken?: string;
/** Path to Databricks config file (default: ~/.databrickscfg) */
databricksConfigPath?: string;
/** Basic auth username for OSS MLflow */
trackingServerUsername?: string;
/** Basic auth password for OSS MLflow */
trackingServerPassword?: string;
/** Bearer token for OSS MLflow */
trackingServerToken?: string;
/** Workspace name for MLflow servers with workspaces enabled (sets X-MLFLOW-WORKSPACE header) */
workspace?: string;
}
/**
* Check if a tracking URI targets Databricks.
*/
export function isDatabricksUri(trackingUri: string): boolean {
return trackingUri === 'databricks' || trackingUri.startsWith('databricks://');
}
/**
* Parse profile name from a Databricks tracking URI.
*
* @example
* parseDatabricksProfile('databricks') // returns undefined (use DEFAULT)
* parseDatabricksProfile('databricks://my-profile') // returns 'my-profile'
*/
export function parseDatabricksProfile(trackingUri: string): string | undefined {
if (trackingUri === 'databricks') {
return undefined;
}
const match = trackingUri.match(/^databricks:\/\/(.+)$/);
return match?.[1] || undefined;
}
/**
* Create an authentication provider for the given configuration.
*
* This is the main entry point for authentication. It returns an AuthProvider
* with methods to get the host URL and authentication headers.
*
* @param options - Authentication configuration options
* @returns AuthProvider for making authenticated requests
* @throws Error if required configuration is missing
*/
export function createAuthProvider(options: AuthOptions): AuthProvider {
if (isDatabricksUri(options.trackingUri)) {
return createDatabricksAuth(options);
}
return createOssAuth(options);
}
/**
* Read the host value from a Databricks config file for a specific profile.
*
* The config file uses INI format:
* ```
* [DEFAULT]
* host = https://my-workspace.databricks.com
* token = dapi123...
*
* [profile-name]
* host = https://other-workspace.databricks.com
* ```
*
* @param configPath - Path to the config file
* @param profile - Profile name (undefined for DEFAULT)
* @returns The host value or undefined if not found
*/
function readHostFromConfigFile(configPath: string, profile?: string): string | undefined {
try {
if (!fs.existsSync(configPath)) {
return undefined;
}
const content = fs.readFileSync(configPath, 'utf-8');
const targetSection = profile || 'DEFAULT';
// Simple INI parser - find the section and extract host
const lines = content.split('\n');
let inTargetSection = false;
for (const line of lines) {
const trimmed = line.trim();
// Check for section header
const sectionMatch = trimmed.match(/^\[(.+)\]$/);
if (sectionMatch) {
inTargetSection = sectionMatch[1] === targetSection;
continue;
}
// Look for host in current section
if (inTargetSection) {
const hostMatch = trimmed.match(/^host\s*=\s*(.+)$/);
if (hostMatch) {
return hostMatch[1].trim();
}
}
}
return undefined;
} catch {
return undefined;
}
}
/**
* Create authentication for Databricks-hosted MLflow.
*
* Host resolution order:
* 1. Explicit `host` option
* 2. DATABRICKS_HOST environment variable
* 3. Host from config file (~/.databrickscfg)
*
* All credential resolution is delegated to the Databricks SDK.
* The SDK handles env vars, config files, OAuth, and cloud-specific auth.
*/
function createDatabricksAuth(options: AuthOptions): AuthProvider {
const profile = parseDatabricksProfile(options.trackingUri);
const configPath = options.databricksConfigPath ?? path.join(os.homedir(), '.databrickscfg');
// Resolve host - must be available synchronously
// Priority: explicit option > env var > config file
const host =
options.host || process.env.DATABRICKS_HOST || readHostFromConfigFile(configPath, profile);
if (!host) {
throw new Error(
'Databricks host not found. Please either:\n' +
' 1. Set the DATABRICKS_HOST environment variable, or\n' +
` 2. Add a host to your config file at ${configPath}`,
);
}
// Create Databricks SDK Config - it handles all credential resolution
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const databricksConfig = new Config({
host: options.host,
token: options.databricksToken,
profile,
configFile: options.databricksConfigPath,
});
// Headers provider delegates entirely to the SDK
const headersProvider: HeadersProvider = async () => {
// SDK resolves credentials from env vars, config file, OAuth, etc.
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
await databricksConfig.ensureResolved();
const headers = new Headers();
headers.set('Content-Type', 'application/json');
// SDK adds appropriate auth headers (Bearer token, OAuth, etc.)
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
await databricksConfig.authenticate(headers);
// Convert Headers to plain object
const result: Record<string, string> = {};
headers.forEach((value, key) => {
result[key] = value;
});
return result;
};
// Token for backwards compatibility (may be undefined for OAuth)
const databricksToken = options.databricksToken || process.env.DATABRICKS_TOKEN;
return {
getHost: () => host,
getHeadersProvider: () => headersProvider,
getDatabricksToken: () => databricksToken,
};
}
/**
* Create authentication for self-hosted (OSS) MLflow.
*
* Resolution order:
* 1. Explicit options (username/password or token)
* 2. Environment variables (MLFLOW_TRACKING_*)
* 3. No authentication
*/
function createOssAuth(options: AuthOptions): AuthProvider {
const host = options.trackingUri;
// Resolve credentials from options or environment
const username = options.trackingServerUsername || process.env.MLFLOW_TRACKING_USERNAME;
const password = options.trackingServerPassword || process.env.MLFLOW_TRACKING_PASSWORD;
const token = options.trackingServerToken || process.env.MLFLOW_TRACKING_TOKEN;
// Pre-compute auth header since credentials don't change
let authHeader: string | undefined;
if (username && password) {
const encoded = Buffer.from(`${username}:${password}`).toString('base64');
authHeader = `Basic ${encoded}`;
} else if (token) {
authHeader = `Bearer ${token}`;
}
const workspace = process.env.MLFLOW_WORKSPACE ?? options.workspace;
// Headers provider for OSS MLflow
// eslint-disable-next-line require-await, @typescript-eslint/require-await
const headersProvider: HeadersProvider = async () => {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (authHeader) {
headers['Authorization'] = authHeader;
}
if (workspace) {
headers['X-MLFLOW-WORKSPACE'] = workspace;
}
return headers;
};
return {
getHost: () => host,
getHeadersProvider: () => headersProvider,
getDatabricksToken: () => undefined,
};
}
@@ -0,0 +1,7 @@
import { TraceData } from '../../core/entities/trace_data';
import { TraceInfo } from '../../core/entities/trace_info';
export interface ArtifactsClient {
uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void>;
downloadTraceData(traceInfo: TraceInfo): Promise<TraceData>;
}
@@ -0,0 +1,335 @@
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
import { TraceInfo } from '../../core/entities/trace_info';
import { JSONBig } from '../../core/utils/json';
import { GetCredentialsForTraceDataDownload, GetCredentialsForTraceDataUpload } from '../spec';
import { makeRequest } from '../utils';
import { ArtifactsClient } from './base';
import { AuthProvider, HeadersProvider } from '../../auth';
/**
* Options for creating a DatabricksArtifactsClient.
*/
export interface DatabricksArtifactsClientOptions {
/**
* The Databricks workspace host URL
*/
host: string;
/**
* Authentication provider
*/
authProvider: AuthProvider;
}
export class DatabricksArtifactsClient implements ArtifactsClient {
private host: string;
private headersProvider: HeadersProvider;
constructor(options: DatabricksArtifactsClientOptions) {
this.host = options.host;
this.headersProvider = options.authProvider.getHeadersProvider();
}
/**
* Upload trace data (spans) to the backend using artifact repository pattern.
*
* 1. Get credentials for upload
* 2. Serialize trace data to JSON
* 3. Upload to cloud storage using the credentials
*/
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
try {
const credentials = await this.getCredentialsForTraceDataUpload(traceInfo.traceId);
const traceDataJson = JSONBig.stringify(traceData.toJson());
await this.uploadToCloudStorage(credentials, traceDataJson);
} catch (error) {
console.error(`Trace data upload failed for ${traceInfo.traceId}:`, error);
throw error;
}
}
/**
* Download trace data (spans) from cloud storage
* Uses artifact repository pattern with signed URLs
*/
async downloadTraceData(traceInfo: TraceInfo): Promise<TraceData> {
try {
const credentials = await this.getCredentialsForTraceDataDownload(traceInfo.traceId);
const traceDataJson = await this.downloadFromSignedUrl(credentials);
return TraceData.fromJson(traceDataJson);
} catch (error) {
console.error(`Failed to download trace data for ${traceInfo.traceId}:`, error);
throw error;
}
}
/**
* Get credentials for uploading trace data
* Endpoint: GET /api/2.0/mlflow/traces/{request_id}/credentials-for-data-upload
*/
private async getCredentialsForTraceDataUpload(traceId: string): Promise<ArtifactCredentialInfo> {
const url = GetCredentialsForTraceDataUpload.getEndpoint(this.host, traceId);
const response = await makeRequest<GetCredentialsForTraceDataUpload.Response>(
'GET',
url,
this.headersProvider,
);
return response.credential_info;
}
/**
* Get credentials for downloading trace data
* Endpoint: GET /mlflow/traces/{trace_id}/credentials-for-data-download
*/
private async getCredentialsForTraceDataDownload(
traceId: string,
): Promise<ArtifactCredentialInfo> {
const url = GetCredentialsForTraceDataDownload.getEndpoint(this.host, traceId);
const response = await makeRequest<GetCredentialsForTraceDataDownload.Response>(
'GET',
url,
this.headersProvider,
);
if (response.credential_info) {
return response.credential_info;
} else {
throw new Error('Invalid response format: missing credential_info');
}
}
/**
* Upload data to cloud storage using the provided credentials
*/
private async uploadToCloudStorage(
credentials: ArtifactCredentialInfo,
data: string,
): Promise<void> {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
// Add headers from credentials (if they exist)
if (credentials.headers && Array.isArray(credentials.headers)) {
credentials.headers.forEach((header) => {
headers[header.name] = header.value;
});
}
switch (credentials.type) {
case 'AWS_PRESIGNED_URL':
case 'GCP_SIGNED_URL':
await this.uploadToSignedUrl(credentials.signed_uri, data, headers, credentials.type);
break;
case 'AZURE_SAS_URI':
await this.uploadToAzureBlob(credentials.signed_uri, data, headers);
break;
case 'AZURE_ADLS_GEN2_SAS_URI':
await this.uploadToAzureAdlsGen2(credentials.signed_uri, data, headers);
break;
default:
throw new Error(`Unsupported credential type: ${credentials.type as string}`);
}
}
/**
* Upload data to cloud storage using signed URL (AWS S3 or GCP Storage)
*/
private async uploadToSignedUrl(
signedUrl: string,
data: string,
headers: Record<string, string>,
credentialType: string,
): Promise<void> {
try {
const response = await fetch(signedUrl, {
method: 'PUT',
headers,
body: data,
});
if (!response.ok) {
throw new Error(
`${credentialType} upload failed: ${response.status} ${response.statusText}`,
);
}
} catch (error) {
throw new Error(`Failed to upload to ${credentialType}: ${(error as Error).message}`);
}
}
/**
* Upload data to Azure Blob Storage using SAS URI
* Uses simple PUT for all uploads since traces rarely exceed 100MB
* https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview
*/
private async uploadToAzureBlob(
sasUri: string,
data: string,
headers: Record<string, string>,
): Promise<void> {
try {
const response = await fetch(sasUri, {
method: 'PUT',
headers: {
...headers,
'x-ms-blob-type': 'BlockBlob',
'Content-Type': 'application/json',
},
body: data,
});
if (!response.ok) {
throw new Error(`Azure Blob upload failed: ${response.status} ${response.statusText}`);
}
} catch (error) {
throw new Error(`Failed to upload to Azure Blob Storage: ${(error as Error).message}`);
}
}
/**
* Upload data to Azure Data Lake Storage Gen2 using SAS URI
* https://learn.microsoft.com/en-us/rest/api/storageservices/data-lake-storage-gen2
*/
private async uploadToAzureAdlsGen2(
sasUri: string,
data: string,
headers: Record<string, string>,
): Promise<void> {
try {
const dataBuffer = new TextEncoder().encode(data);
// ADLS Gen2 uses a different API pattern - create file then append data
// Create the file
const createUrl = `${sasUri}&resource=file`;
const createResponse = await fetch(createUrl, {
method: 'PUT',
headers: {
...headers,
'Content-Length': '0',
},
});
if (!createResponse.ok) {
throw new Error(
`Azure ADLS Gen2 file creation failed: ${createResponse.status} ${createResponse.statusText}`,
);
}
// Append data to the file
const appendUrl = `${sasUri}&action=append&position=0`;
const appendResponse = await fetch(appendUrl, {
method: 'PATCH',
headers: {
...headers,
'Content-Type': 'application/octet-stream',
},
body: dataBuffer,
});
if (!appendResponse.ok) {
throw new Error(
`Azure ADLS Gen2 data append failed: ${appendResponse.status} ${appendResponse.statusText}`,
);
}
// Flush the data to complete the upload
const flushUrl = `${sasUri}&action=flush&position=${dataBuffer.length}`;
const flushResponse = await fetch(flushUrl, {
method: 'PATCH',
headers: {
...headers,
'Content-Length': '0',
},
});
if (!flushResponse.ok) {
throw new Error(
`Azure ADLS Gen2 flush failed: ${flushResponse.status} ${flushResponse.statusText}`,
);
}
} catch (error) {
throw new Error(`Failed to upload to Azure ADLS Gen2: ${(error as Error).message}`);
}
}
/**
* Download data from cloud storage using signed URL
*/
private async downloadFromSignedUrl(
credentials: ArtifactCredentialInfo,
): Promise<SerializedTraceData> {
const headers: Record<string, string> = {};
// Add headers from credentials (if they exist)
if (credentials.headers && Array.isArray(credentials.headers)) {
credentials.headers.forEach((header) => {
headers[header.name] = header.value;
});
}
try {
const response = await fetch(credentials.signed_uri, {
method: 'GET',
headers,
});
if (!response.ok) {
if (response.status === 404) {
throw new Error(`Trace data not found (404)`);
}
throw new Error(`Download failed: ${response.status} ${response.statusText}`);
}
const textData = await response.text();
try {
return JSONBig.parse(textData) as SerializedTraceData;
} catch (parseError) {
throw new Error(`Trace data corrupted: invalid JSON - ${(parseError as Error).message}`);
}
} catch (error) {
if (error instanceof Error) {
throw error;
}
throw new Error(`Failed to download trace data: ${String(error)}`);
}
}
}
/**
* HTTP header for artifact upload/download
*/
export interface HttpHeader {
name: string;
value: string;
}
/**
* Artifact credential information for upload/download
*/
export interface ArtifactCredentialInfo {
/** ID of the MLflow Run containing the artifact */
run_id?: string;
/** Relative path to the artifact */
path?: string;
/** Signed URI credential for artifact access */
signed_uri: string;
/** HTTP headers for upload/download (optional, may not be present) */
headers?: HttpHeader[];
/** Type of signed credential URI */
type: ArtifactCredentialType;
}
/**
* Enum for artifact credential types.
* This ensures type safety when handling different cloud storage providers.
*/
export type ArtifactCredentialType =
| 'AWS_PRESIGNED_URL'
| 'GCP_SIGNED_URL'
| 'AZURE_SAS_URI'
| 'AZURE_ADLS_GEN2_SAS_URI';
@@ -0,0 +1,50 @@
import { ArtifactsClient } from './base';
import { DatabricksArtifactsClient } from './databricks';
import { MlflowArtifactsClient } from './mlflow';
import { AuthProvider } from '../../auth';
/**
* Options for creating an ArtifactsClient.
*/
export interface ArtifactsClientOptions {
/**
* The tracking URI (e.g., "databricks", "http://localhost:5000")
*/
trackingUri: string;
/**
* The resolved host URL for API requests
*/
host: string;
/**
* Authentication provider
*/
authProvider: AuthProvider;
}
/**
* Get the appropriate artifacts client based on the tracking URI.
*
* @param trackingUri - The tracking URI (e.g., "databricks", "http://localhost:5000")
* @param host - The resolved host URL for API requests
* @param authProvider - The authentication provider to get tokens for authenticated requests
* @returns The appropriate artifacts client.
*/
export function getArtifactsClient({
trackingUri,
host,
authProvider,
}: {
trackingUri: string;
host: string;
authProvider: AuthProvider;
}): ArtifactsClient {
if (trackingUri.startsWith('databricks')) {
return new DatabricksArtifactsClient({ host, authProvider });
} else {
return new MlflowArtifactsClient({ host, authProvider });
}
}
export { ArtifactsClient };
@@ -0,0 +1,149 @@
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { TraceTagKey } from '../../core/constants';
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
import { TraceInfo } from '../../core/entities/trace_info';
import { JSONBig } from '../../core/utils/json';
import { makeRequest } from '../utils';
import { isLocalArtifactUri, toAbsoluteLocalPath } from '../../core/utils/artifact_uri';
import { ArtifactsClient } from './base';
import { AuthProvider, HeadersProvider } from '../../auth';
/**
* Trace data file name constant - matches Python SDK
*/
const TRACE_DATA_FILE_NAME = 'traces.json';
/**
* MLflow OSS Artifacts Client
*
* Implements artifact upload/download for OSS MLflow Tracking Server using the standard
* HTTP artifact repository endpoints. Based on Python HttpArtifactRepository.
*/
export class MlflowArtifactsClient implements ArtifactsClient {
private readonly host: string;
private headersProvider: HeadersProvider;
constructor(options: { host: string; authProvider: AuthProvider }) {
this.host = options.host;
this.headersProvider = options.authProvider.getHeadersProvider();
}
/**
* Upload trace data to MLflow artifact storage.
*
* Equivalent to Python's upload_trace_data() method which uses log_artifact()
* under the hood to upload the trace data as a JSON file.
*
* @param traceInfo The trace information containing artifact URI
* @param traceData The trace data to upload
*/
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
const artifactUri = this.getArtifactLocation(traceInfo);
if (isLocalArtifactUri(artifactUri)) {
await this.uploadTraceDataLocal(artifactUri, traceData);
return;
}
const traceDataJson = traceData.toJson();
const artifactUrl = this.resolveArtifactUri(artifactUri, TRACE_DATA_FILE_NAME);
await makeRequest<void>('PUT', artifactUrl, this.headersProvider, traceDataJson);
}
/**
* Write trace data to a local filesystem artifact location.
*/
private async uploadTraceDataLocal(artifactUri: string, traceData: TraceData): Promise<void> {
const dir = toAbsoluteLocalPath(artifactUri);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, TRACE_DATA_FILE_NAME), JSONBig.stringify(traceData.toJson()), 'utf8');
}
/**
* Download trace data from MLflow artifact storage.
*
* Equivalent to Python's download_trace_data() method which downloads
* the traces.json file and parses it back to TraceData.
*
* @param traceInfo The trace information containing artifact URI
* @returns The downloaded and parsed trace data
*/
async downloadTraceData(traceInfo: TraceInfo): Promise<TraceData> {
const artifactUri = this.getArtifactLocation(traceInfo);
if (isLocalArtifactUri(artifactUri)) {
return this.downloadTraceDataLocal(artifactUri);
}
// Download the trace data file
const artifactUrl = this.resolveArtifactUri(artifactUri, TRACE_DATA_FILE_NAME);
const traceDataJson = await makeRequest<SerializedTraceData>(
'GET',
artifactUrl,
this.headersProvider,
);
// Parse JSON back to TraceData (equivalent to Python's try_read_trace_data)
try {
return TraceData.fromJson(traceDataJson);
} catch (error) {
throw new Error(
`Failed to parse trace data JSON: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
/**
* Read trace data from a local filesystem artifact location.
*/
private async downloadTraceDataLocal(artifactUri: string): Promise<TraceData> {
const filePath = join(toAbsoluteLocalPath(artifactUri), TRACE_DATA_FILE_NAME);
const contents = await readFile(filePath, 'utf8');
try {
return TraceData.fromJson(JSONBig.parse(contents) as SerializedTraceData);
} catch (error) {
throw new Error(
`Failed to parse trace data JSON: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
/**
* Read the artifact location tag set on the trace by the backend.
*/
private getArtifactLocation(traceInfo: TraceInfo): string {
const artifactUri = traceInfo.tags[TraceTagKey.MLFLOW_ARTIFACT_LOCATION];
if (!artifactUri) {
throw new Error('Artifact location not found in trace tags');
}
return artifactUri;
}
/**
* Resolve mlflow-artifacts:// URI to HTTP endpoint.
*
* Equivalent to Python's MlflowArtifactsRepository.resolve_uri() method.
* Transforms URIs like "mlflow-artifacts:/0/traces/tr-abc123/artifacts"
* to "http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/0/traces/tr-abc123/artifacts/traces.json"
*
* @param artifactUri The mlflow-artifacts:// URI from trace tags
* @param fileName The file name to append (e.g., "traces.json")
* @returns The resolved HTTP endpoint URL
*/
private resolveArtifactUri(artifactUri: string, fileName: string): string {
const baseApiPath = '/api/2.0/mlflow-artifacts/artifacts';
const url = new URL(artifactUri);
if (url.protocol !== 'mlflow-artifacts:') {
throw new Error(`Expected mlflow-artifacts:// URI, got ${url.protocol}`);
}
// Construct the final HTTP URL
const cleanHost = this.host.replace(/\/$/, ''); // Remove trailing slash
return `${cleanHost}${baseApiPath}${url.pathname}/${fileName}`;
}
}
+413
View File
@@ -0,0 +1,413 @@
import { TraceInfo } from '../core/entities/trace_info';
import { Trace } from '../core/entities/trace';
import {
CreateExperiment,
CreateTraceInfoV4,
DeleteExperiment,
ExportOtlpTraces,
ExportOtlpTracesOss,
GetExperiment,
GetExperimentByName,
GetTraceInfoV3,
SearchTracesV3,
StartTraceV3,
} from './spec';
import { makeRawRequest, makeRequest, MlflowHttpError } from './utils';
import { TraceData } from '../core/entities/trace_data';
import { ArtifactsClient, getArtifactsClient } from './artifacts';
import { AuthProvider, HeadersProvider } from '../auth';
import { DATABRICKS_UC_TABLE_HEADER, MLFLOW_EXPERIMENT_ID_HEADER } from '../core/constants';
import { serializeTraceLocation, type TraceLocation } from '../core/entities/trace_location';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import type { ReadableSpan as OTelReadableSpan } from '@opentelemetry/sdk-trace-base';
import { ExportResultCode, type ExportResult } from '@opentelemetry/core';
export interface SearchTracesOptions {
/** Trace locations (e.g. MLflow experiments) to search over. */
locations: TraceLocation[];
/** Search filter string, e.g. `"tags.env = 'prod'"`. */
filter?: string;
/** Maximum number of traces to return in a single page. */
maxResults?: number;
/** List of order-by clauses, e.g. `['timestamp_ms DESC']`. */
orderBy?: string[];
/** Pagination token obtained from a previous searchTraces call. */
pageToken?: string;
/**
* Whether to fetch span data for each matching trace. Defaults to `true`.
* When `false`, only trace metadata is returned and the traces have empty
* span data.
*/
includeSpans?: boolean;
}
export interface SearchTracesResult {
traces: Trace[];
nextPageToken?: string;
}
/**
* Client for MLflow tracing operations.
*
* Supports multiple authentication methods:
* - Databricks: PAT tokens, OAuth M2M, Azure CLI/MSI, Google Cloud
* - OSS MLflow: Basic auth, Bearer tokens, or no auth
*/
export class MlflowClient {
/** Client implementation to upload/download trace data artifacts */
private artifactsClient: ArtifactsClient;
/** Headers provider for authenticated requests */
private headersProvider: HeadersProvider;
/** Host URL for API requests */
private hostUrl: string;
/**
* Creates a new MlflowClient.
*
* @param trackingUri - The tracking URI (e.g., "databricks", "http://localhost:5000")
* @param authProvider - The authentication provider to get tokens for authenticated requests
*/
constructor(options: { trackingUri: string; authProvider: AuthProvider }) {
this.headersProvider = options.authProvider.getHeadersProvider();
this.hostUrl = options.authProvider.getHost();
this.artifactsClient = getArtifactsClient({
trackingUri: options.trackingUri,
host: this.hostUrl,
authProvider: options.authProvider,
});
}
/**
* Get the host URL for this client.
*/
getHost(): string {
return this.hostUrl;
}
// === TRACE LOGGING METHODS ===
/**
* Create a new TraceInfo record in the backend store.
* Corresponding to the Python SDK's start_trace_v3() method.
*
* Note: the backend API is named as "Start" due to unfortunate miscommunication.
* The API is indeed called at the "end" of a trace, not the "start".
*/
async createTrace(traceInfo: TraceInfo): Promise<TraceInfo> {
const url = StartTraceV3.getEndpoint(this.hostUrl);
const payload: StartTraceV3.Request = { trace: { trace_info: traceInfo.toJson() } };
const response = await makeRequest<StartTraceV3.Response>(
'POST',
url,
this.headersProvider,
payload,
);
return TraceInfo.fromJson(response.trace.trace_info);
}
/**
* Create a new TraceInfo record in the Databricks V4 (Unity Catalog)
* trace-info endpoint. Used for UC-backed traces so that trace-level
* tags and metadata persist alongside spans uploaded via OTLP.
*
* Corresponds to Python's `DatabricksRestStore._start_trace_v4`.
*
* @param location - The UC location string ("catalog.schema" or
* "catalog.schema.table_prefix") that comes from
* the trace ID `trace:/<location>/<hex>`.
* @param otelTraceId - The hex OTel trace ID portion.
* @param traceInfo - The full TraceInfo to persist; the backend reads
* `tags`, `trace_metadata`, `state`, durations, etc.
*/
async createTraceInfoV4(
location: string,
otelTraceId: string,
traceInfo: TraceInfo,
): Promise<TraceInfo> {
const url = CreateTraceInfoV4.getEndpoint(this.hostUrl, location, otelTraceId);
const payload: CreateTraceInfoV4.Request = traceInfo.toJson();
const response = await makeRequest<CreateTraceInfoV4.Response>(
'POST',
url,
this.headersProvider,
payload,
);
return TraceInfo.fromJson(response);
}
/**
* Upload OTel spans to a Databricks Unity Catalog location via the OTLP
* HTTP+protobuf endpoint. The `spansTableName` is the fully qualified
* spans table (catalog.schema.table) and is forwarded as the
* `X-Databricks-UC-Table-Name` header used by Databricks to route the
* payload to the correct UC location.
*
* Uses `@opentelemetry/exporter-trace-otlp-proto` which serializes the
* spans to the OTLP `ExportTraceServiceRequest` protobuf wire format —
* the Databricks endpoint only accepts `application/x-protobuf`, not
* the OTLP/HTTP+JSON form.
*/
async exportOtlpSpansToUc(spans: OTelReadableSpan[], spansTableName: string): Promise<void> {
if (spans.length === 0) {
return;
}
// Fetch fresh auth headers for every export so OAuth-rotated tokens are
// honored. The exporter takes a static headers map at construction time,
// so we rebuild it per call rather than caching an instance.
const authHeaders = await this.headersProvider();
const exporter = new OTLPTraceExporter({
url: ExportOtlpTraces.getEndpoint(this.hostUrl),
headers: {
...authHeaders,
[DATABRICKS_UC_TABLE_HEADER]: spansTableName,
},
});
try {
await new Promise<void>((resolve, reject) => {
exporter.export(spans, (result: ExportResult) => {
if (result.code === ExportResultCode.SUCCESS) {
resolve();
} else {
reject(result.error ?? new Error(`OTLP span export failed with code ${result.code}`));
}
});
});
} finally {
// Drain any pending state so the exporter doesn't keep handles open.
await exporter.shutdown().catch(() => undefined);
}
}
/**
* Export OTLP spans to an OSS (non-Databricks) tracking server.
*
* `otlpProtoBytes` is an already-serialized OTLP `ExportTraceServiceRequest`
* protobuf (captured at WAL-write time), POSTed as `application/x-protobuf`
* with the experiment id forwarded via the `x-mlflow-experiment-id` header.
*/
async exportOtlpSpans(experimentId: string, otlpProtoBytes: Uint8Array): Promise<void> {
if (otlpProtoBytes.length === 0) {
return;
}
await makeRawRequest(
'POST',
ExportOtlpTracesOss.getEndpoint(this.hostUrl),
this.headersProvider,
otlpProtoBytes,
{
'Content-Type': 'application/x-protobuf',
[MLFLOW_EXPERIMENT_ID_HEADER]: experimentId,
},
);
}
// === TRACE RETRIEVAL METHODS ===
/**
* Get a single trace by ID
* Fetches both trace info and trace data from backend
* Corresponds to Python: client.get_trace()
*/
async getTrace(traceId: string): Promise<Trace> {
const traceInfo = await this.getTraceInfo(traceId);
const traceData = await this.artifactsClient.downloadTraceData(traceInfo);
return new Trace(traceInfo, traceData);
}
/**
* Get trace info using V3 API
* Endpoint: GET /api/3.0/mlflow/traces/{trace_id}
*/
async getTraceInfo(traceId: string): Promise<TraceInfo> {
const url = GetTraceInfoV3.getEndpoint(this.hostUrl, traceId);
const response = await makeRequest<GetTraceInfoV3.Response>('GET', url, this.headersProvider);
// The V3 API returns a Trace object with trace_info field
if (response.trace?.trace_info) {
return TraceInfo.fromJson(response.trace.trace_info);
}
throw new Error(`Invalid response format: missing trace_info: ${JSON.stringify(response)}`);
}
/**
* Search for traces that match the given search criteria.
* Corresponding to the Python SDK's search_traces() method.
*
* By default the span data of each matching trace is fetched with one
* download per trace; traces whose span data cannot be downloaded are
* skipped, matching the Python SDK's behavior. Pass `includeSpans: false`
* to return trace metadata only, with empty span data.
*
* Locations are passed through to the tracking server, which validates
* that it supports the requested location types.
*
* @param options.locations Trace locations (e.g. MLflow experiments) to search over.
* @param options.filter Search filter string, e.g. `"tags.env = 'prod'"`.
* @param options.maxResults Maximum number of traces to return in a single page.
* @param options.orderBy List of order-by clauses, e.g. `['timestamp_ms DESC']`.
* @param options.pageToken Pagination token obtained from a previous searchTraces call.
* @param options.includeSpans Whether to fetch span data for each trace. Defaults to `true`.
* @returns The matching traces and, when more results are available, a token for the next page.
*
* @example
* ```typescript
* const { traces, nextPageToken } = await client.searchTraces({
* locations: [
* { type: TraceLocationType.MLFLOW_EXPERIMENT, mlflowExperiment: { experimentId: '123' } },
* ],
* filter: "tags.env = 'prod'",
* maxResults: 10,
* orderBy: ['timestamp_ms DESC'],
* });
* ```
*/
async searchTraces(options: SearchTracesOptions): Promise<SearchTracesResult> {
const { locations, includeSpans = true } = options;
if (!locations?.length) {
throw new Error('At least one location must be specified for searching traces.');
}
const url = SearchTracesV3.getEndpoint(this.hostUrl);
const payload: SearchTracesV3.Request = {
locations: locations.map(serializeTraceLocation),
filter: options.filter,
max_results: options.maxResults,
order_by: options.orderBy,
page_token: options.pageToken,
};
const response = await makeRequest<SearchTracesV3.Response>(
'POST',
url,
this.headersProvider,
payload,
);
const traceInfos = (response.traces ?? []).map((trace) => TraceInfo.fromJson(trace));
let traces: Trace[];
if (includeSpans) {
const results = await Promise.all(
traceInfos.map((traceInfo) => this.downloadTraceDataForSearch(traceInfo)),
);
traces = results.filter((trace): trace is Trace => trace !== undefined);
} else {
traces = traceInfos.map((traceInfo) => new Trace(traceInfo, new TraceData()));
}
return {
traces,
nextPageToken: response.next_page_token || undefined,
};
}
/**
* Fetch span data for a search result. Returns undefined when the data
* cannot be downloaded (e.g. missing or corrupted) so the trace is dropped
* from the results with a warning, matching the Python SDK.
*/
private async downloadTraceDataForSearch(traceInfo: TraceInfo): Promise<Trace | undefined> {
try {
const traceData = await this.artifactsClient.downloadTraceData(traceInfo);
return new Trace(traceInfo, traceData);
} catch (error) {
console.warn(`Failed to download trace data for trace ${traceInfo.traceId}:`, error);
return undefined;
}
}
/**
* Upload trace data to the artifact store.
*/
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
await this.artifactsClient.uploadTraceData(traceInfo, traceData);
}
// === EXPERIMENT METHODS ===
/**
* Create a new experiment
*/
async createExperiment(
name: string,
artifactLocation?: string,
tags?: Record<string, string>,
): Promise<string> {
const url = CreateExperiment.getEndpoint(this.hostUrl);
const payload: CreateExperiment.Request = { name, artifact_location: artifactLocation, tags };
const response = await makeRequest<CreateExperiment.Response>(
'POST',
url,
this.headersProvider,
payload,
);
return response.experiment_id;
}
/**
* Get an experiment by ID, including its tags. Used to auto-resolve UC
* trace destinations from a Databricks-linked experiment.
*/
async getExperiment(
experimentId: string,
): Promise<{ experimentId: string; name: string; tags: Record<string, string> } | null> {
const url = GetExperiment.getEndpoint(this.hostUrl, experimentId);
try {
const response = await makeRequest<GetExperiment.Response>('GET', url, this.headersProvider);
const exp = response.experiment;
if (!exp?.experiment_id) {
return null;
}
const tags: Record<string, string> = {};
for (const tag of exp.tags ?? []) {
tags[tag.key] = tag.value;
}
return { experimentId: exp.experiment_id, name: exp.name, tags };
} catch (error) {
if (
error instanceof MlflowHttpError &&
(error.status === 404 || error.errorCode === 'RESOURCE_DOES_NOT_EXIST')
) {
return null;
}
throw error;
}
}
/**
* Get an experiment by name.
*/
async getExperimentByName(name: string): Promise<{ experimentId: string; name: string } | null> {
const url = GetExperimentByName.getEndpoint(this.hostUrl, name);
try {
const response = await makeRequest<GetExperimentByName.Response>(
'GET',
url,
this.headersProvider,
);
if (!response.experiment?.experiment_id || !response.experiment?.name) {
return null;
}
return {
experimentId: response.experiment.experiment_id,
name: response.experiment.name,
};
} catch (error) {
if (
error instanceof MlflowHttpError &&
(error.status === 404 || error.errorCode === 'RESOURCE_DOES_NOT_EXIST')
) {
return null;
}
throw error;
}
}
/**
* Delete an experiment
*/
async deleteExperiment(experimentId: string): Promise<void> {
const url = DeleteExperiment.getEndpoint(this.hostUrl);
const payload: DeleteExperiment.Request = { experiment_id: experimentId };
await makeRequest<void>('POST', url, this.headersProvider, payload);
}
}
@@ -0,0 +1,3 @@
export { MlflowClient } from './client';
export type { SearchTracesOptions, SearchTracesResult } from './client';
export { MlflowHttpError } from './utils';
+191
View File
@@ -0,0 +1,191 @@
/**
* MLflow API Request/Response Specifications
*
* This module defines the types and interfaces for MLflow API communication,
* including request payloads and response structures for all trace-related endpoints.
*/
import type { TraceInfo } from '../core/entities/trace_info';
import type { SerializedTraceLocation } from '../core/entities/trace_location';
import { ArtifactCredentialType } from './artifacts/databricks';
/**
* Create a new TraceInfo entity in the backend.
*/
export namespace StartTraceV3 {
export const getEndpoint = (host: string) => `${host}/api/3.0/mlflow/traces`;
export interface Request {
trace: {
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
};
}
export interface Response {
trace: {
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
};
}
}
/**
* Get the TraceInfo entity for a given trace ID.
*/
export namespace GetTraceInfoV3 {
export const getEndpoint = (host: string, traceId: string) =>
`${host}/api/3.0/mlflow/traces/${traceId}`;
export interface Response {
trace: {
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
};
}
}
/**
* Search trace metadata using the V3 traces API.
*/
export namespace SearchTracesV3 {
export const getEndpoint = (host: string) => `${host}/api/3.0/mlflow/traces/search`;
export interface Request {
locations: SerializedTraceLocation[];
filter?: string;
max_results?: number;
order_by?: string[];
page_token?: string;
}
export interface Response {
traces?: Parameters<typeof TraceInfo.fromJson>[0][];
next_page_token?: string;
}
}
/** Create Experiment (used for testing) */
export namespace CreateExperiment {
export const getEndpoint = (host: string) => `${host}/api/2.0/mlflow/experiments/create`;
export interface Request {
name?: string;
artifact_location?: string;
tags?: Record<string, string>;
}
export interface Response {
experiment_id: string;
}
}
/** Get Experiment (used for UC destination auto-resolution) */
export namespace GetExperiment {
export const getEndpoint = (host: string, experimentId: string) =>
`${host}/api/2.0/mlflow/experiments/get?experiment_id=${encodeURIComponent(experimentId)}`;
export interface Response {
experiment?: {
experiment_id: string;
name: string;
artifact_location?: string;
lifecycle_stage?: string;
tags?: { key: string; value: string }[];
};
}
}
/** Get Experiment By Name */
export namespace GetExperimentByName {
export const getEndpoint = (host: string, experimentName: string) =>
`${host}/api/2.0/mlflow/experiments/get-by-name?experiment_name=${encodeURIComponent(experimentName)}`;
export interface Response {
experiment?: {
experiment_id: string;
name: string;
};
}
}
/** Delete Experiment (used for testing) */
export namespace DeleteExperiment {
export const getEndpoint = (host: string) => `${host}/api/2.0/mlflow/experiments/delete`;
export interface Request {
experiment_id: string;
}
}
/**
* Create trace info using the V4 API path. Used for Databricks Unity Catalog
* backed traces (UC schema or UC table prefix locations).
*
* Endpoint: POST /api/4.0/mlflow/traces/{location}/{otel_trace_id}/info
* where `{location}` is "catalog.schema" or "catalog.schema.table_prefix",
* and `{otel_trace_id}` is the hex OTel trace ID (no `trace:/<location>/` prefix).
*/
export namespace CreateTraceInfoV4 {
export const getEndpoint = (host: string, location: string, otelTraceId: string) =>
`${host}/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`;
// The Databricks RPC convention extracts `location_id` and `trace_info.trace_id`
// from the URL path; the HTTP body is the serialized TraceInfo proto directly
// (not wrapped in `{ trace_info: ... }`). Mirrors Python's
// `message_to_json(trace_info.to_proto())`.
export type Request = Parameters<typeof TraceInfo.fromJson>[0];
// Backend returns a TraceInfo proto serialized as JSON.
export type Response = Parameters<typeof TraceInfo.fromJson>[0];
}
/**
* OTLP span upload endpoint for Databricks Unity Catalog backed traces.
*
* Endpoint: POST /api/2.0/otel/v1/traces
* Required header: `X-Databricks-UC-Table-Name: <fully_qualified_spans_table>`
* Content-Type: application/x-protobuf (OTLP/HTTP+protobuf); the Databricks
* endpoint does not accept the OTLP/HTTP+JSON form.
*/
export namespace ExportOtlpTraces {
export const getEndpoint = (host: string) => `${host}/api/2.0/otel/v1/traces`;
}
/**
* OTLP span upload endpoint for OSS (non-Databricks) tracking servers.
*
* Endpoint: POST /v1/traces
* Required header: `x-mlflow-experiment-id: <experiment_id>`
* Content-Type: application/x-protobuf (OTLP/HTTP+protobuf).
*/
export namespace ExportOtlpTracesOss {
export const getEndpoint = (host: string) => `${host}/v1/traces`;
}
/**
* Get credentials for uploading trace data to the artifact store. Only used for Databricks.
*/
export namespace GetCredentialsForTraceDataUpload {
export const getEndpoint = (host: string, traceId: string) =>
`${host}/api/2.0/mlflow/traces/${traceId}/credentials-for-data-upload`;
export interface Response {
credential_info: {
type: ArtifactCredentialType;
signed_uri: string;
};
}
}
/**
* Get credentials for downloading trace data from the artifact store. Only used for Databricks.
*/
export namespace GetCredentialsForTraceDataDownload {
export const getEndpoint = (host: string, traceId: string) =>
`${host}/api/2.0/mlflow/traces/${traceId}/credentials-for-data-download`;
export interface Response {
credential_info: {
type: ArtifactCredentialType;
signed_uri: string;
};
}
}
+190
View File
@@ -0,0 +1,190 @@
import { JSONBig } from '../core/utils/json';
import { HeadersProvider } from '../auth';
const MAX_ERROR_BODY_LENGTH = 1000;
/**
* Error thrown when an HTTP request to the MLflow backend returns a non-2xx response.
* Carries the parsed status code and (when present) the `error_code` field from the
* response body so callers can branch on status without matching error messages.
*/
export class MlflowHttpError extends Error {
readonly status: number;
readonly statusText: string;
readonly body: string;
readonly errorCode?: string;
constructor(status: number, statusText: string, body: string) {
super(MlflowHttpError.formatMessage(status, statusText, body));
this.name = 'MlflowHttpError';
this.status = status;
this.statusText = statusText;
this.body = body;
this.errorCode = MlflowHttpError.extractErrorCode(body);
}
private static formatMessage(status: number, statusText: string, body: string): string {
let message = `HTTP ${status}: ${statusText}`;
if (body) {
message +=
body.length > MAX_ERROR_BODY_LENGTH
? ` - ${body.substring(0, MAX_ERROR_BODY_LENGTH)}... (truncated)`
: ` - ${body}`;
}
return message;
}
private static extractErrorCode(body: string): string | undefined {
if (!body) {
return undefined;
}
try {
const parsed: unknown = JSON.parse(body);
if (
parsed &&
typeof parsed === 'object' &&
'error_code' in parsed &&
typeof (parsed as { error_code: unknown }).error_code === 'string'
) {
return (parsed as { error_code: string }).error_code;
}
} catch {
// body is not JSON
}
return undefined;
}
}
/**
* Make a request to the given URL with the given method, headers, body, and timeout.
*
* TODO: add retry logic for transient errors
*/
export async function makeRequest<T>(
method: string,
url: string,
headerProvider: HeadersProvider,
body?: any,
timeout?: number,
): Promise<T> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout ?? getDefaultTimeout());
const headers = await headerProvider();
try {
const response = await fetch(url, {
method,
headers: headers,
body: body ? JSONBig.stringify(body) : undefined,
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
let responseBody = '';
try {
responseBody = await response.text();
} catch {
// If we can't read the body, leave it empty
}
throw new MlflowHttpError(response.status, response.statusText, responseBody);
}
// Handle empty responses (like DELETE operations)
if (response.status === 204 || response.headers.get('content-length') === '0') {
return {} as T;
}
const responseText = await response.text();
return JSONBig.parse(responseText) as T;
} catch (error) {
clearTimeout(timeoutId);
if (error instanceof MlflowHttpError) {
throw error;
}
if (error instanceof Error) {
if (error.name === 'AbortError') {
throw new Error(`Request timeout after ${timeout}ms`);
}
throw new Error(`API request failed: ${error.message}`);
}
throw new Error(`API request failed: ${String(error)}`);
}
}
/**
* Send a raw (already-serialized) body — e.g. OTLP protobuf bytes — to `url`.
* `extraHeaders` merge over the auth headers (to override `Content-Type` etc).
* The response body is ignored; non-2xx throws {@link MlflowHttpError} so
* callers can branch on `status` (e.g. 501 capability gating).
*/
export async function makeRawRequest(
method: string,
url: string,
headerProvider: HeadersProvider,
body: BodyInit,
extraHeaders: Record<string, string> = {},
timeout?: number,
): Promise<void> {
const controller = new AbortController();
const effectiveTimeout = timeout ?? getDefaultTimeout();
const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout);
const headers: Record<string, string> = { ...(await headerProvider()) };
for (const [key, value] of Object.entries(extraHeaders)) {
const existingKey = Object.keys(headers).find((h) => h.toLowerCase() === key.toLowerCase());
if (existingKey) {
delete headers[existingKey];
}
headers[key] = value;
}
try {
const response = await fetch(url, {
method,
headers,
body,
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
let responseBody = '';
try {
responseBody = await response.text();
} catch {
// If we can't read the body, leave it empty
}
throw new MlflowHttpError(response.status, response.statusText, responseBody);
}
} catch (error) {
clearTimeout(timeoutId);
if (error instanceof MlflowHttpError) {
throw error;
}
if (error instanceof Error) {
if (error.name === 'AbortError') {
throw new Error(`Request timeout after ${effectiveTimeout}ms`);
}
throw new Error(`API request failed: ${error.message}`);
}
throw new Error(`API request failed: ${String(error)}`);
}
}
function getDefaultTimeout(): number {
const envTimeout = process.env.MLFLOW_HTTP_REQUEST_TIMEOUT;
if (envTimeout) {
const timeout = parseInt(envTimeout, 10);
if (!isNaN(timeout) && timeout > 0) {
return timeout;
}
}
return 30000; // Default 30 seconds
}
+643
View File
@@ -0,0 +1,643 @@
import { trace as otelTrace, context, Span as ApiSpan, INVALID_TRACEID } from '@opentelemetry/api';
import { Span as OTelSpan } from '@opentelemetry/sdk-trace-node';
import { DEFAULT_SPAN_NAME, SpanLogLevel, SpanType, TraceMetadataKey } from './constants';
import { createMlflowSpan, LiveSpan, NoOpSpan } from './entities/span';
import { getTracer } from './provider';
import { InMemoryTraceManager } from './trace_manager';
import { convertNanoSecondsToHrTime, mapArgsToObject } from './utils';
import { SpanStatusCode } from './entities/span_status';
import { isTracingEnabledInContext } from './context';
/*
* Options for starting a span
*/
export interface SpanOptions {
/**
* The name of the span.
*/
name: string;
/**
* The type of the span, e.g., `LLM`, `TOOL`, `RETRIEVER`, etc.
*/
spanType?: SpanType;
/**
* The inputs of the span.
*/
inputs?: any;
/**
* The attributes of the span.
*/
attributes?: Record<string, any>;
/**
* The start time of the span in nanoseconds. If not provided, the current time will be used.
*/
startTimeNs?: number;
/**
* The parent span object. If not provided, the span is considered a root span.
*/
parent?: LiveSpan;
/**
* Optional severity level to attach to the span. Accepts a SpanLogLevel
* enum value or its name (e.g. "INFO", "DEBUG"). If not provided, the span
* level is resolved from the span type at end time.
*/
logLevel?: SpanLogLevel | string;
}
/**
* Options for tracing a function
*/
export interface TraceOptions
extends Omit<SpanOptions, 'parent' | 'startTimeNs' | 'inputs' | 'name'> {
/**
* The name of the span.
*/
name?: string;
}
/**
* Start a new span with the given name and span type.
*
* This function does NOT attach the created span to the current context. To create
* nested spans, you need to set the parent span explicitly in the `parent` field.
* The span must be ended by calling `end` method on the returned Span object.
*
* @param options Optional span options (name, spanType, inputs, attributes, startTimeNs, parent)
* @returns The created span object.
*
* @example
* ```typescript
* const span = startSpan({
* name: 'my_span',
* spanType: 'LLM',
* inputs: {
* prompt: 'Hello, world!'
* }
* });
*
* // Do something
*
* // End the span
* span.end({
* status: 'OK',
* outputs: {
* response: 'Hello, world!'
* }
* });
* ```
*
*/
export function startSpan(options: SpanOptions): LiveSpan {
if (isTracingEnabledInContext() === false) {
return new NoOpSpan();
}
try {
const tracer = getTracer('default');
// If parent is provided, use it as the parent spanAdd commentMore actions
const parentContext = options.parent
? otelTrace.setSpan(context.active(), options.parent._span)
: context.active();
// Convert startTimeNs to OTel format
const startTime = options.startTimeNs
? convertNanoSecondsToHrTime(options.startTimeNs)
: undefined;
const otelSpan = tracer.startSpan(
options.name,
{ startTime: startTime },
parentContext,
) as OTelSpan;
// SpanProcessor should have already registered the mlflow span
return getMlflowSpan(otelSpan, options);
} catch (error) {
console.warn('Failed to start span', error);
return new NoOpSpan();
}
}
/**
* Execute a function within a span context. The span is automatically started before
* the function executes and ended after it completes (or throws an error).
*
* This function uses OpenTelemetry's active span context to automatically manage
* parent-child relationships between spans.
*
* This function supports two usage patterns:
* 1. Inline: `withSpan((span) => { ... })` - span properties set within the callback
* 2. Options: `withSpan((span) => { ... }, { name: 'test', ... })` - span properties set via options object
*
* @param callback The callback function to execute within the span context
* @param options Optional span options (name, spanType, inputs, attributes, startTimeNs)
* @returns The result of the callback function
*/
export function withSpan<T>(
callback: (span: LiveSpan) => T | Promise<T>,
options?: Omit<SpanOptions, 'parent'>,
): T | Promise<T> {
if (isTracingEnabledInContext() === false) {
return callback(new NoOpSpan());
}
const spanOptions: Omit<SpanOptions, 'parent'> = options ?? { name: DEFAULT_SPAN_NAME };
// Generate a default span name if not provided
const spanName = spanOptions.name || DEFAULT_SPAN_NAME;
const tracer = getTracer('default');
// Convert startTimeNs to OTel format if provided
const startTime = spanOptions.startTimeNs
? convertNanoSecondsToHrTime(spanOptions.startTimeNs)
: undefined;
// Use startActiveSpan to automatically manage context and parent-child relationships
return tracer.startActiveSpan(spanName, { startTime }, (otelSpan: ApiSpan) => {
let mlflowSpan: LiveSpan | NoOpSpan;
try {
// SpanProcessor should have already registered the mlflow span
mlflowSpan = getMlflowSpan(otelSpan as OTelSpan, spanOptions);
} catch (error) {
console.debug('Failed to create and register MLflow span', error);
mlflowSpan = new NoOpSpan();
}
// Expression function to handle errors consistently
const handleError = (error: Error): never => {
try {
mlflowSpan.setStatus(SpanStatusCode.ERROR, error.message);
mlflowSpan.recordException(error);
mlflowSpan.end();
} catch (error) {
console.debug('Failed to set status or record exception on MLflow span', error);
}
throw error;
};
try {
// Execute the callback with the span
const result = callback(mlflowSpan);
// Handle both sync and async results
if (result instanceof Promise) {
return result
.then((value) => {
// Set outputs if they are not already set
if (mlflowSpan.outputs === undefined) {
mlflowSpan.setOutputs(value);
}
mlflowSpan.end();
return value;
})
.catch(handleError);
} else {
// Synchronous execution
if (mlflowSpan.outputs === undefined) {
mlflowSpan.setOutputs(result);
}
mlflowSpan.end();
return result;
}
} catch (error) {
// Handle synchronous errors
return handleError(error as Error);
}
});
}
function getMlflowSpan(otelSpan: OTelSpan, options: SpanOptions): LiveSpan | NoOpSpan {
// MlflowSpanProcessor should have already registered the span
const traceManager = InMemoryTraceManager.getInstance();
const mlflowTraceId = traceManager.getMlflowTraceIdFromOtelId(otelSpan.spanContext().traceId);
const mlflowSpan =
traceManager.getSpan(mlflowTraceId, otelSpan.spanContext().spanId) || new NoOpSpan();
// Set custom properties to the span
if (options.inputs) {
mlflowSpan.setInputs(options.inputs);
}
if (options.attributes) {
mlflowSpan.setAttributes(options.attributes);
}
if (options.spanType) {
mlflowSpan.setSpanType(options.spanType);
}
if (options.logLevel !== undefined) {
mlflowSpan.setLogLevel(options.logLevel);
}
return mlflowSpan;
}
/**
* Helper function to create and register an MLflow span from an OpenTelemetry span
* @param otelSpan The OpenTelemetry span
* @param spanType The MLflow span type
* @param inputs Optional inputs to set on the span
* @param attributes Optional attributes to set on the span
* @returns The created and registered MLflow LiveSpan
*/
export function createAndRegisterMlflowSpan(
otelSpan: OTelSpan | ApiSpan,
spanType?: SpanType,
inputs?: any,
attributes?: Record<string, any>,
): LiveSpan {
// Get the MLflow trace ID from the OpenTelemetry trace ID
const otelTraceId = otelSpan.spanContext().traceId;
const traceId =
InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId) || otelTraceId;
// Create the MLflow span from the OTel span
const mlflowSpan = createMlflowSpan(otelSpan, traceId, spanType) as LiveSpan;
// Set initial properties if provided
if (inputs) {
mlflowSpan.setInputs(inputs);
}
if (attributes) {
mlflowSpan.setAttributes(attributes);
}
// Register the span with the trace manager
const traceManager = InMemoryTraceManager.getInstance();
traceManager.registerSpan(mlflowSpan);
return mlflowSpan;
}
/**
* Create a traced version of a function or decorator for tracing class methods.
*
* When used as a function wrapper, the span will automatically capture:
* - The function inputs
* - The function outputs
* - The function name as the span name
* - The function execution time
* - Any exception thrown by the function
*
* When used as a decorator, it preserves the `this` context for class methods.
*
* Note: Typescript decorator is still in experimental stage.
*
* @param func The function to trace (when used as function wrapper)
* @param options Optional trace options including name, spanType, and attributes
* @returns The traced function or method decorator
*
* @example
* // Function wrapper with no options
* const tracedFunc = trace(myFunc);
*
* @example
* // Function wrapper with options
* const tracedFunc = trace(myFunc, { name: 'custom_span_name', spanType: 'LLM' });
*
* @example
* // Decorator with no options
* class MyService {
* @trace()
* async processData(data: any) {
* return processedData;
* }
* }
*
* @example
* // Decorator with options
* class MyService {
* @trace({ name: 'custom_span', spanType: SpanType.LLM })
* async generateText(prompt: string) {
* return generatedText;
* }
* }
*/
export function trace(options?: TraceOptions): any;
export function trace<T extends (...args: any[]) => any>(func: T, options?: TraceOptions): T;
export function trace<T extends (...args: any[]) => any>(
funcOrOptions?: T | TraceOptions,
options?: TraceOptions,
): any {
// Check if this is being used as a decorator (no function provided, or options provided)
if (typeof funcOrOptions !== 'function') {
const decoratorOptions = funcOrOptions;
// Return a method decorator that supports both old and new TypeScript decorator syntax
return function (...args: any[]) {
let originalMethod: Function;
let methodName: string;
// Check if using new TypeScript 5.0+ decorator syntax
const isNewSyntax =
args.length === 2 && typeof args[1] === 'object' && !Array.isArray(args[1]);
if (isNewSyntax) {
// New syntax: (originalMethod, context)
originalMethod = args[0];
const context = args[1] as ClassMethodDecoratorContext;
methodName = String(context.name);
} else {
// Old syntax: (target, propertyKey, descriptor)
const desc = args[2] as PropertyDescriptor;
originalMethod = desc.value;
methodName = String(args[1]);
}
// Create the traced method wrapper
const tracedMethod = function (this: any, ...args: any[]) {
let inputs: any;
try {
inputs = mapArgsToObject(originalMethod, args);
} catch (error) {
console.debug('Failed to map arguments values to names', error);
inputs = args;
}
const spanOptions: Omit<SpanOptions, 'parent'> = {
name: decoratorOptions?.name || originalMethod.name || methodName,
spanType: decoratorOptions?.spanType,
attributes: decoratorOptions?.attributes,
inputs,
logLevel: decoratorOptions?.logLevel,
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return withSpan((_span) => {
// Call the original method with the preserved `this` context
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return originalMethod.apply(this, args) as ReturnType<T>;
}, spanOptions) as ReturnType<T>;
};
// Return the appropriate value based on the decorator syntax
if (isNewSyntax) {
return tracedMethod as T;
} else {
const descriptor = args[2] as PropertyDescriptor;
descriptor.value = tracedMethod;
// Preserve the original method's properties
Object.defineProperty(descriptor.value, 'length', { value: originalMethod.length });
Object.defineProperty(descriptor.value, 'name', { value: originalMethod.name });
return descriptor;
}
};
} else {
// This is the function-based usage (existing behavior)
return traceFunction(funcOrOptions, options);
}
}
/**
* Internal function to handle function-based tracing (non-decorator usage)
*/
function traceFunction<T extends (...args: any[]) => any>(func: T, options?: TraceOptions): T {
// Create a wrapper function that preserves the original function's properties
const wrapper = function (this: any, ...args: Parameters<T>): ReturnType<T> {
let inputs: any;
try {
inputs = mapArgsToObject(func, args);
} catch (error) {
console.debug('Failed to map arguments values to names', error);
inputs = args;
}
const spanOptions: Omit<SpanOptions, 'parent'> = {
name: options?.name || func.name || DEFAULT_SPAN_NAME,
spanType: options?.spanType,
attributes: options?.attributes,
inputs: inputs,
logLevel: options?.logLevel,
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return withSpan((_span) => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return func.apply(this, args);
}, spanOptions) as ReturnType<T>;
};
// Preserve function properties
Object.defineProperty(wrapper, 'length', { value: func.length });
Object.defineProperty(wrapper, 'name', { value: func.name });
// Copy any additional properties from the original function
for (const prop in func) {
if (Object.prototype.hasOwnProperty.call(func, prop)) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
(wrapper as any)[prop] = (func as any)[prop];
}
}
return wrapper as T;
}
/**
* Get the last active trace ID.
* @returns The last active trace ID.
*/
export function getLastActiveTraceId(): string | undefined {
const traceManager = InMemoryTraceManager.getInstance();
return traceManager.lastActiveTraceId;
}
/**
* Get the current active span in the global context.
*
* This only works when the span is created with fluent APIs like `@trace` or
* `withSpan`. If a span is created with the `startSpan` API, it won't be
* attached to the global context so this function will not return it.
*
* @returns The current active span if exists, otherwise null.
*
* @example
* ```typescript
* const tracedFunc = trace(() => {
* const span = getCurrentActiveSpan();
* span?.setAttribute("key", "value");
* return 0;
* });
*
* tracedFunc();
* ```
*/
export function getCurrentActiveSpan(): LiveSpan | null {
const otelSpan = otelTrace.getActiveSpan();
// If no active span or it's a NonRecordingSpan, return undefined
if (!otelSpan || otelSpan.spanContext().traceId === INVALID_TRACEID) {
return null;
}
const traceManager = InMemoryTraceManager.getInstance();
const otelTraceId = otelSpan.spanContext().traceId;
const mlflowTraceId = traceManager.getMlflowTraceIdFromOtelId(otelTraceId);
if (!mlflowTraceId) {
return null;
}
const spanId = otelSpan.spanContext().spanId;
return traceManager.getSpan(mlflowTraceId, spanId) || null;
}
/**
* Options for updating the current trace
*/
export interface UpdateCurrentTraceOptions {
/**
* A dictionary of tags to update the trace with. Tags are designed for mutable values
* that can be updated after the trace is created via MLflow UI or API.
*/
tags?: Record<string, string>;
/**
* A dictionary of metadata to update the trace with. Metadata cannot be updated
* once the trace is logged. It is suitable for recording immutable values like the
* git hash of the application version that produced the trace.
*/
metadata?: Record<string, string>;
/**
* Session ID to associate with the trace. Stored as metadata under the
* `mlflow.trace.session` key.
*/
sessionId?: string;
/**
* User identifier to associate with the trace. Stored as metadata under the
* `mlflow.trace.user` key.
*/
user?: string;
/**
* Client supplied request ID to associate with the trace. This is useful for linking
* the trace back to a specific request in your application or external system.
*/
clientRequestId?: string;
/**
* A preview of the request to be shown in the Trace list view in the UI.
* By default, MLflow will truncate the trace request naively by limiting the length.
* This parameter allows you to specify a custom preview string.
*/
requestPreview?: string;
/**
* A preview of the response to be shown in the Trace list view in the UI.
* By default, MLflow will truncate the trace response naively by limiting the length.
* This parameter allows you to specify a custom preview string.
*/
responsePreview?: string;
}
/**
* Update the current active trace with the given options.
*
* You can use this function either within a function decorated with `@trace` or
* within the scope of the `withSpan` context. If there is no active trace found,
* this function will log a warning and return.
*
* @param options Options for updating the trace
*
* @example
* Using within a function decorated with `@trace`:
* ```typescript
* const myFunc = trace((x: number) => {
* updateCurrentTrace({ tags: { fruit: "apple" }, clientRequestId: "req-12345" });
* return x + 1;
* });
* ```
*
* @example
* Using within the `withSpan` context:
* ```typescript
* withSpan((span) => {
* updateCurrentTrace({ tags: { fruit: "apple" }, clientRequestId: "req-12345" });
* }, { name: "span" });
* ```
*
* @example
* Updating user, session, and source information of the trace:
* ```typescript
* updateCurrentTrace({
* sessionId: "session-4f855da00427",
* user: "user-id-cc156f29bcfb",
* metadata: {
* "mlflow.source.name": "inference.ts",
* "mlflow.source.git.commit": "1234567890",
* "mlflow.source.git.repoURL": "https://github.com/mlflow/mlflow"
* }
* });
* ```
*/
export function updateCurrentTrace({
tags,
metadata,
sessionId,
user,
clientRequestId,
requestPreview,
responsePreview,
}: UpdateCurrentTraceOptions): void {
const activeSpan = getCurrentActiveSpan();
if (!activeSpan) {
console.warn(
'No active trace found. Please create a span using `withSpan` or ' +
'`@trace` before calling `updateCurrentTrace`.',
);
return;
}
// Validate string parameters
if (requestPreview !== undefined && typeof requestPreview !== 'string') {
throw new Error('The `requestPreview` parameter must be a string.');
}
if (responsePreview !== undefined && typeof responsePreview !== 'string') {
throw new Error('The `responsePreview` parameter must be a string.');
}
// Update trace info for the trace stored in-memory
const traceManager = InMemoryTraceManager.getInstance();
const trace = traceManager.getTrace(activeSpan.traceId);
if (!trace) {
console.warn(`Trace ${activeSpan.traceId} does not exist or already finished.`);
return;
}
// Inject sessionId and user into metadata
const mergedMetadata = { ...metadata };
if (sessionId !== undefined) {
mergedMetadata[TraceMetadataKey.TRACE_SESSION] = sessionId;
}
if (user !== undefined) {
mergedMetadata[TraceMetadataKey.TRACE_USER] = user;
}
// Update trace info properties
if (requestPreview !== undefined) {
trace.info.requestPreview = requestPreview;
}
if (responsePreview !== undefined) {
trace.info.responsePreview = responsePreview;
}
if (tags !== undefined) {
Object.assign(trace.info.tags, tags);
}
if (Object.keys(mergedMetadata).length > 0) {
Object.assign(trace.info.traceMetadata, mergedMetadata);
}
if (clientRequestId !== undefined) {
trace.info.clientRequestId = String(clientRequestId);
}
}
+334
View File
@@ -0,0 +1,334 @@
import path from 'path';
import os from 'os';
import { initializeSDK } from './provider';
import { AuthProvider, createAuthProvider, isDatabricksUri } from '../auth';
/**
* User-facing shape for specifying a Databricks Unity Catalog trace location.
*
* All three fields are required. Unlike Python's
* `mlflow.set_experiment(trace_location=UnityCatalog(...))`, the TS SDK does
* not upsert the underlying UC trace location, so it can't default
* `tablePrefix` to anything sensible - the customer must point at a
* trace location that is already provisioned in the workspace.
*/
export interface UnityCatalogLocationOptions {
catalogName: string;
schemaName: string;
tablePrefix: string;
}
/**
* Validate that a URI has a proper protocol (http or https)
* @param uri The URI to validate
* @returns true if valid, false otherwise
*/
function isValidHttpUri(uri: string): boolean {
try {
const url = new URL(uri);
return url.protocol === 'http:' || url.protocol === 'https:';
} catch {
return false;
}
}
/**
* Configuration options for the MLflow tracing SDK
*/
export interface MLflowTracingConfig {
/**
* The host where traces will be logged.
* Can be:
* - HTTP URI for MLflow trace server backend
* - "databricks" (uses default profile)
* - "databricks://profile" (uses specific profile)
*/
trackingUri: string;
/**
* The experiment ID where traces will be logged
*/
experimentId: string;
/**
* The location of the Databricks config file, default to ~/.databrickscfg
*/
databricksConfigPath?: string;
/**
* The Databricks host. If not provided, the host will be read from the Databricks config file.
* @deprecated For Databricks auth, prefer using environment variables or config profiles.
* The auth system will automatically resolve credentials.
*/
host?: string;
/**
* The Databricks token. If not provided, the token will be read from the Databricks config file.
* @deprecated For Databricks auth, prefer using environment variables or config profiles.
* The auth system will automatically resolve credentials.
*/
databricksToken?: string;
/**
* The tracking server username for basic auth.
*/
trackingServerUsername?: string;
/**
* The tracking server password for basic auth.
*/
trackingServerPassword?: string;
/**
* Bearer token for OSS MLflow tracking servers.
* Can also be set via MLFLOW_TRACKING_TOKEN environment variable.
*/
trackingServerToken?: string;
/**
* Workspace name for MLflow servers with workspaces enabled (OSS/non-Databricks only).
* Sets the X-MLFLOW-WORKSPACE header on requests to self-hosted MLflow.
* Can also be set via MLFLOW_WORKSPACE environment variable (takes precedence).
*/
workspace?: string;
/**
* Optional Databricks Unity Catalog trace location. When provided, the SDK
* generates V4 trace IDs and persists trace tags / metadata via the V4
* `CreateTraceInfo` endpoint, mirroring Python's
* `mlflow.set_experiment(trace_location=UnityCatalog(...))`. When omitted,
* traces use the V3 experiment-backed path.
*
* The UC trace location must already be provisioned in the workspace; the
* TS SDK does not upsert it.
*/
traceLocation?: UnityCatalogLocationOptions;
}
/**
* Initialization options for the MLflow tracing SDK. The trackingUri and experimentId
* can be omitted and will be resolved from environment variables when available. Since
* this is used on the client side, we need to make sure the trackingUri and experimentId
* are required.
*/
export type MLflowTracingInitOptions = Partial<MLflowTracingConfig>;
/**
* Global configuration state
*/
let globalConfig: MLflowTracingConfig | null = null;
/**
* Global authentication provider
*/
let globalAuthProvider: AuthProvider | null = null;
/**
* Configure the MLflow tracing SDK with tracking location settings.
* This must be called before using other tracing functions.
*
* Call once per process. Calling `init()` again will tear down the existing
* OpenTelemetry SDK and start a fresh one, but the underlying NodeSDK
* shutdown is asynchronous and may race with subsequent tracer registration;
* configuration is intended to be set up once at startup, not reconfigured
* mid-process.
*
* ## Authentication
*
* ### Databricks (trackingUri = "databricks" or "databricks://profile")
*
* Authentication is handled automatically by the Databricks SDK. It supports:
* - Personal Access Tokens (DATABRICKS_TOKEN env var)
* - OAuth M2M service principals (DATABRICKS_CLIENT_ID/DATABRICKS_CLIENT_SECRET)
* - Azure CLI / Azure MSI / Azure Client Secret
* - Google Cloud credentials
* - Config file profiles (~/.databrickscfg)
*
* For Databricks Apps, credentials are automatically injected by the runtime.
*
* ### OSS MLflow (trackingUri = HTTP URL)
*
* Authentication is resolved in priority order:
* 1. Basic Auth (trackingServerUsername/Password or MLFLOW_TRACKING_USERNAME/PASSWORD)
* 2. Bearer Token (trackingServerToken or MLFLOW_TRACKING_TOKEN)
* 3. No Auth
*
* @param config Configuration object with trackingUri and experimentId
*
* @example
* ```typescript
* import { init, withSpan } from '@mlflow/core';
*
* // Option 1: Use MLflow Tracking Server
* init({
* trackingUri: "http://localhost:5000",
* experimentId: "123456789"
* });
*
* // Option 2: Use default Databricks profile from ~/.databrickscfg
* init({
* trackingUri: "databricks",
* experimentId: "123456789"
* });
*
* // Option 3: Use specific Databricks profile
* init({
* trackingUri: "databricks://my-profile",
* experimentId: "123456789"
* });
*
* // Option 4: Custom config file path
* init({
* trackingUri: "databricks",
* experimentId: "123456789",
* databricksConfigPath: "/path/to/my/databrickscfg"
* });
*
* // Option 5: Override with explicit host/token (backwards compatibility)
* init({
* trackingUri: "databricks",
* experimentId: "123456789",
* host: "https://my-workspace.databricks.com",
* databricksToken: "my-token"
* });
*
* // Option 6: OSS MLflow with basic auth
* init({
* trackingUri: "http://localhost:5000",
* experimentId: "123456789",
* trackingServerUsername: "user",
* trackingServerPassword: "pass"
* });
*
* // Option 7: OSS MLflow with bearer token
* init({
* trackingUri: "http://localhost:5000",
* experimentId: "123456789",
* trackingServerToken: "my-token"
* });
*
* // Option 8: Databricks Unity Catalog trace location.
* // Mirrors Python's `mlflow.set_experiment(..., trace_location=UnityCatalog(...))`.
* // The UC trace location must already be provisioned for this workspace.
* init({
* trackingUri: "databricks",
* experimentId: "123456789",
* traceLocation: {
* catalogName: "my_catalog",
* schemaName: "my_schema",
* tablePrefix: "my_prefix",
* },
* });
*
* // Now you can use tracing functions
* function add(a: number, b: number) {
* return withSpan(
* { name: 'add', inputs: { a, b } },
* (span) => {
* const result = a + b;
* span.setOutputs({ result });
* return result;
* }
* );
* }
* ```
*/
export function init(config: MLflowTracingInitOptions): void {
const trackingUri = config.trackingUri ?? process.env.MLFLOW_TRACKING_URI;
const experimentId = config.experimentId ?? process.env.MLFLOW_EXPERIMENT_ID;
if (!trackingUri) {
throw new Error(
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
);
}
if (!experimentId) {
throw new Error(
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
);
}
if (typeof trackingUri !== 'string') {
throw new Error('trackingUri must be a string');
}
if (typeof experimentId !== 'string') {
throw new Error('experimentId must be a string');
}
const databricksConfigPath =
config.databricksConfigPath ?? path.join(os.homedir(), '.databrickscfg');
// Validate non-Databricks URIs
if (!isDatabricksUri(trackingUri) && !isValidHttpUri(trackingUri)) {
throw new Error(
`Invalid trackingUri: '${trackingUri}'. Must be a valid HTTP or HTTPS URL, or 'databricks' / 'databricks://<profile>'.`,
);
}
// Create the authentication provider - this is the single source of truth
// for credential resolution. It handles env vars, config files, OAuth, etc.
globalAuthProvider = createAuthProvider({
trackingUri,
host: config.host,
databricksToken: config.databricksToken,
databricksConfigPath,
trackingServerUsername: config.trackingServerUsername,
trackingServerPassword: config.trackingServerPassword,
trackingServerToken: config.trackingServerToken,
workspace: config.workspace,
});
// Build effective config, populating host and databricksToken from auth provider
// for backwards compatibility with code that reads these properties directly
const effectiveConfig: MLflowTracingConfig = {
...config,
trackingUri,
experimentId,
databricksConfigPath,
host: globalAuthProvider.getHost(),
databricksToken: globalAuthProvider.getDatabricksToken(),
};
// Store the config
globalConfig = { ...effectiveConfig };
// Initialize SDK with new configuration
initializeSDK();
}
/**
* Get the current configuration. Throws an error if not configured.
* @returns The current MLflow tracing configuration
*/
export function getConfig(): MLflowTracingConfig {
if (!globalConfig) {
throw new Error(
'The MLflow Tracing client is not configured. Please call init() with host and experimentId before using tracing functions.',
);
}
return globalConfig;
}
/**
* Get the current authentication provider. Throws an error if not configured.
* @returns The current authentication provider
*/
export function getAuthProvider(): AuthProvider {
if (!globalAuthProvider) {
throw new Error(
'The MLflow Tracing client is not configured. Please call init() before using tracing functions.',
);
}
return globalAuthProvider;
}
/**
* Reset the global configuration. For testing purposes only.
* @internal
*/
export function resetConfig(): void {
globalConfig = null;
globalAuthProvider = null;
}
+167
View File
@@ -0,0 +1,167 @@
/**
* Constants for MLflow Tracing
*/
/**
* Enum for span types that can be used with MLflow Tracing
*/
export enum SpanType {
LLM = 'LLM',
CHAIN = 'CHAIN',
AGENT = 'AGENT',
TOOL = 'TOOL',
CHAT_MODEL = 'CHAT_MODEL',
RETRIEVER = 'RETRIEVER',
PARSER = 'PARSER',
EMBEDDING = 'EMBEDDING',
RERANKER = 'RERANKER',
MEMORY = 'MEMORY',
UNKNOWN = 'UNKNOWN',
}
/**
* Severity level for an MLflow trace span. The public tracing API accepts a
* `SpanLogLevel` member or its string name (e.g. "INFO").
*/
export enum SpanLogLevel {
DEBUG = 10,
INFO = 20,
WARNING = 30,
ERROR = 40,
CRITICAL = 50,
}
/**
* Normalize an enum or string into a SpanLogLevel. Raw integers are not
* accepted at the type level; the runtime number branch exists only because
* `SpanLogLevel` is a numeric enum and its members arrive as primitive
* numbers.
*/
export function toSpanLogLevel(value: SpanLogLevel | string): SpanLogLevel {
if (typeof value === 'number') {
if (Object.values(SpanLogLevel).includes(value as SpanLogLevel)) {
return value as SpanLogLevel;
}
throw new Error(
`Invalid SpanLogLevel value ${value}. Expected one of ${Object.values(SpanLogLevel)
.filter((v) => typeof v === 'number')
.join(', ')}.`,
);
}
if (typeof value === 'string') {
const matched = (SpanLogLevel as Record<string, SpanLogLevel | string>)[
value.trim().toUpperCase()
];
if (typeof matched === 'number') {
return matched;
}
throw new Error(
`Invalid SpanLogLevel name ${JSON.stringify(value)}. Expected one of ${Object.keys(
SpanLogLevel,
)
.filter((k) => isNaN(Number(k)))
.join(', ')}.`,
);
}
throw new Error(`SpanLogLevel must be a SpanLogLevel or string; got ${typeof value}.`);
}
/**
* Constants for MLflow span attribute keys
*/
export const SpanAttributeKey = {
EXPERIMENT_ID: 'mlflow.experimentId',
TRACE_ID: 'mlflow.traceRequestId',
INPUTS: 'mlflow.spanInputs',
OUTPUTS: 'mlflow.spanOutputs',
SPAN_TYPE: 'mlflow.spanType',
// Severity level of the span (one of the `SpanLogLevel` members). Absent
// means the span was not classified.
LOG_LEVEL: 'mlflow.spanLogLevel',
// This attribute is used to store token usage information from LLM responses.
// Stored in {"input_tokens": int, "output_tokens": int, "total_tokens": int} format.
TOKEN_USAGE: 'mlflow.chat.tokenUsage',
// This attribute indicates which flavor/format generated the LLM span. This is
// used by downstream (e.g., UI) to determine the message format for parsing.
MESSAGE_FORMAT: 'mlflow.message.format',
};
/**
* Constants for MLflow trace metadata keys
*/
export const TraceMetadataKey = {
SOURCE_RUN: 'mlflow.sourceRun',
MODEL_ID: 'mlflow.modelId',
SIZE_BYTES: 'mlflow.trace.sizeBytes',
SCHEMA_VERSION: 'mlflow.trace_schema.version',
TOKEN_USAGE: 'mlflow.trace.tokenUsage',
TRACE_SESSION: 'mlflow.trace.session',
TRACE_USER: 'mlflow.trace.user',
// Deprecated, do not use. These fields are used for storing trace request and response
// in MLflow 2.x. In MLflow 3.x, these are replaced in favor of the request_preview and
// response_preview fields in the trace info.
// TODO: Remove this once the new trace table UI is available that is based on MLflow V3 trace.
INPUTS: 'mlflow.traceInputs',
OUTPUTS: 'mlflow.traceOutputs',
};
/**
* Constants for MLflow trace tag keys
*/
export const TraceTagKey = {
MLFLOW_ARTIFACT_LOCATION: 'mlflow.artifactLocation',
};
/**
* Current version of the MLflow trace schema
*/
export const TRACE_SCHEMA_VERSION = '3';
/**
* The prefix for MLflow trace IDs (V3 schema).
*/
export const TRACE_ID_PREFIX = 'tr-';
/**
* The prefix for V4 schema trace IDs (Databricks Unity Catalog backed
* traces). Full format: `trace:/<location>/<hex_trace_id>`.
*/
export const TRACE_ID_V4_PREFIX = 'trace:/';
/**
* HTTP header used to route OTLP span uploads to a specific Databricks
* Unity Catalog table when sending to /api/2.0/otel/v1/traces.
*/
export const DATABRICKS_UC_TABLE_HEADER = 'X-Databricks-UC-Table-Name';
/**
* HTTP header used to associate OTLP span uploads with an MLflow experiment
* when sending to the OSS tracking server's /v1/traces endpoint.
*/
export const MLFLOW_EXPERIMENT_ID_HEADER = 'x-mlflow-experiment-id';
/**
* The default name for spans if the name is not provided when starting a span
*/
export const DEFAULT_SPAN_NAME = 'span';
/**
* Trace ID for no-op spans
*/
export const NO_OP_SPAN_TRACE_ID = 'no-op-span-trace-id';
/**
* Constants for token usage keys (matching Python TokenUsageKey)
*/
export const TokenUsageKey = {
INPUT_TOKENS: 'input_tokens',
OUTPUT_TOKENS: 'output_tokens',
TOTAL_TOKENS: 'total_tokens',
CACHE_READ_INPUT_TOKENS: 'cache_read_input_tokens',
CACHE_CREATION_INPUT_TOKENS: 'cache_creation_input_tokens',
};
/**
* Max length of the request/response preview in the trace info.
*/
export const REQUEST_RESPONSE_PREVIEW_MAX_LENGTH = 1000;
+141
View File
@@ -0,0 +1,141 @@
import { AsyncLocalStorage } from 'node:async_hooks';
import { TraceMetadataKey } from './constants';
/**
* Options for the tracingContext function.
*/
export interface TracingContextOptions {
/**
* Key-value pairs to inject into the trace's request_metadata (immutable after trace creation).
*/
metadata?: Record<string, string>;
/**
* Key-value pairs to inject into the trace's tags.
*/
tags?: Record<string, string>;
/**
* Whether tracing is enabled within the scope. If false, all tracing calls within the scope
* will return NoOpSpan without creating any traces. If undefined, the value is inherited from
* the outer scope.
*/
enabled?: boolean;
/**
* Session ID to associate with traces created in this scope.
* Stored as metadata under the `mlflow.trace.session` key.
*/
sessionId?: string;
/**
* User identifier to associate with traces created in this scope.
* Stored as metadata under the `mlflow.trace.user` key.
*/
user?: string;
}
interface UserTraceContext {
metadata: Record<string, string>;
tags: Record<string, string>;
enabled: boolean | undefined;
}
const storage = new AsyncLocalStorage<UserTraceContext>();
/**
* Get the configured trace metadata from the current tracing context scope.
* Returns undefined if no context is active or metadata is empty.
*/
export function getConfiguredTraceMetadata(): Record<string, string> | undefined {
const ctx = storage.getStore();
if (!ctx || Object.keys(ctx.metadata).length === 0) {
return undefined;
}
return ctx.metadata;
}
/**
* Get the configured trace tags from the current tracing context scope.
* Returns undefined if no context is active or tags are empty.
*/
export function getConfiguredTraceTags(): Record<string, string> | undefined {
const ctx = storage.getStore();
if (!ctx || Object.keys(ctx.tags).length === 0) {
return undefined;
}
return ctx.tags;
}
/**
* Check if tracing is enabled in the current context scope.
* Returns undefined if no context is active or enabled was not set.
*/
export function isTracingEnabledInContext(): boolean | undefined {
const ctx = storage.getStore();
return ctx?.enabled;
}
/**
* Run a function within a tracing context scope that injects metadata and/or tags into any
* trace created within the scope, without creating a wrapper span. It can also be used to
* selectively disable tracing within the scope.
*
* This is useful when you need to attach trace-level information (e.g. session IDs) to traces
* produced by code you don't control like auto-instrumented libraries, or when you want to
* suppress tracing for a specific code block.
*
* The context can be nested. When the same key is specified in multiple levels, the value
* from the inner level takes precedence.
*
* @param options - Metadata, tags, and enabled flag to inject into traces
* @param fn - The function to execute within the context scope
* @returns The return value of the function
*
* @example
* ```typescript
* import * as mlflow from "@mlflow/core";
*
* // Inject session ID, user, and tags into all traces created within the scope
* mlflow.tracingContext(
* { sessionId: "session-123", user: "user-456", tags: { project: "my-project" } },
* () => {
* // Any trace created here will carry the metadata and tags
* agent.invoke("What is the capital of France?");
* }
* );
*
* // Disable tracing within a specific scope
* mlflow.tracingContext({ enabled: false }, () => {
* // No traces will be created inside this scope
* agent.invoke("This call will not be traced");
* });
* ```
*/
export function tracingContext<T>(options: TracingContextOptions, fn: () => T): T {
const current = storage.getStore();
// Inject sessionId and user into metadata
const metadata = { ...(options.metadata ?? {}) };
if (options.sessionId !== undefined) {
metadata[TraceMetadataKey.TRACE_SESSION] = options.sessionId;
}
if (options.user !== undefined) {
metadata[TraceMetadataKey.TRACE_USER] = options.user;
}
// Merge with any outer context scope (inner wins on conflict)
const mergedMetadata = { ...(current?.metadata ?? {}), ...metadata };
const mergedTags = { ...(current?.tags ?? {}), ...(options.tags ?? {}) };
const resolvedEnabled =
options.enabled !== undefined ? options.enabled : (current?.enabled ?? undefined);
const newContext: UserTraceContext = {
metadata: mergedMetadata,
tags: mergedTags,
enabled: resolvedEnabled,
};
return storage.run(newContext, fn);
}
@@ -0,0 +1,44 @@
import type { UnityCatalogLocation } from './entities/trace_location';
/**
* Databricks experiment tag keys carrying the linked UC trace location.
* Matches Python's `MLFLOW_EXPERIMENT_DATABRICKS_TRACE_*` constants in
* mlflow/utils/mlflow_tags.py.
*/
export const DATABRICKS_TRACE_DESTINATION_PATH_TAG =
'mlflow.experiment.databricksTraceDestinationPath';
export const DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG =
'mlflow.experiment.databricksTraceSpanStorageTable';
export const DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG =
'mlflow.experiment.databricksTraceLogStorageTable';
export const DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG =
'mlflow.experiment.databricksTraceAnnotationsTable';
/**
* Parse a UC table-prefix location from the Databricks experiment tags
* (`mlflow.experiment.databricksTrace*`). Returns null when the experiment is
* not linked to a UC trace destination.
*
* Mirrors Python's `Experiment._resolve_trace_location_from_tags`.
*/
export function ucLocationFromExperimentTags(
tags: Record<string, string>,
): UnityCatalogLocation | null {
const path = tags[DATABRICKS_TRACE_DESTINATION_PATH_TAG];
if (!path) {
return null;
}
const parts = path.split('.');
if (parts.length !== 3 || parts.some((p) => !p)) {
return null;
}
const [catalogName, schemaName, tablePrefix] = parts;
return {
catalogName,
schemaName,
tablePrefix,
otelSpansTableName: tags[DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG],
otelLogsTableName: tags[DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG],
annotationsTableName: tags[DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG],
};
}
@@ -0,0 +1,703 @@
import {
HrTime,
INVALID_SPANID,
INVALID_TRACEID,
SpanStatusCode as OTelSpanStatusCode,
} from '@opentelemetry/api';
import type { Span as OTelSpan } from '@opentelemetry/sdk-trace-base';
import {
SpanAttributeKey,
SpanLogLevel,
SpanType,
toSpanLogLevel,
NO_OP_SPAN_TRACE_ID,
} from '../constants';
import { defaultLogLevelForSpanType } from '../log_level';
import { SpanEvent } from './span_event';
import { SpanStatus, SpanStatusCode } from './span_status';
import {
convertHrTimeToNanoSeconds,
convertNanoSecondsToHrTime,
encodeSpanIdToBase64,
encodeTraceIdToBase64,
decodeIdFromBase64,
} from '../utils';
import { safeJsonStringify } from '../utils/json';
/**
* MLflow Span interface
*/
export interface ISpan {
/**
* The OpenTelemetry span wrapped by MLflow Span
*/
readonly _span: OTelSpan;
/**
* The trace ID
*/
readonly traceId: string;
/**
* The attributes of the span
*/
readonly attributes: Record<string, any>;
get spanId(): string;
get name(): string;
get spanType(): SpanType;
/**
* The severity level of the span, or null if it was not classified.
*/
get logLevel(): SpanLogLevel | null;
get startTime(): HrTime;
get endTime(): HrTime | null;
get parentId(): string | null;
get status(): SpanStatus;
get inputs(): any;
get outputs(): any;
/**
* Get an attribute from the span
* @param key Attribute key
* @returns Attribute value
*/
getAttribute(key: string): any;
/**
* Get events from the span
*/
get events(): SpanEvent[];
/**
* Convert this span to JSON format
* @returns JSON object representation of the span
*/
toJson(): SerializedSpan;
}
/**
* MLflow Span class that wraps the OpenTelemetry Span.
*/
export class Span implements ISpan {
readonly _span: OTelSpan;
readonly _attributesRegistry: SpanAttributesRegistry;
// Internal only flag to allow mutating the ended span. This is used to set the custom attributes
// from span processor's onEnd hook. The hook is invoked after the span is ended and OpenTelemetry
// blocks setting attributes on them by default. Set this flag to true to allow mutating the ended
// span.
allowMutatingEndedSpan: boolean = false;
/**
* Create a new MLflowSpan
* @param span OpenTelemetry span
*/
constructor(span: OTelSpan, is_mutable: boolean = false) {
this._span = span;
if (is_mutable) {
this._attributesRegistry = new SpanAttributesRegistry(span);
} else {
this._attributesRegistry = new CachedSpanAttributesRegistry(span);
}
}
get traceId(): string {
return this.getAttribute(SpanAttributeKey.TRACE_ID) as string;
}
get spanId(): string {
return this._span.spanContext().spanId;
}
get spanType(): SpanType {
return this.getAttribute(SpanAttributeKey.SPAN_TYPE) as SpanType;
}
get logLevel(): SpanLogLevel | null {
const raw = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | undefined | null;
if (raw == null) {
return null;
}
return raw as SpanLogLevel;
}
/**
* Get the parent span ID
*/
get parentId(): string | null {
return this._span.parentSpanContext?.spanId ?? null;
}
get name(): string {
return this._span.name;
}
get startTime(): HrTime {
return this._span.startTime;
}
get endTime(): HrTime | null {
return this._span.endTime;
}
get status(): SpanStatus {
return SpanStatus.fromOtelStatus(this._span.status);
}
get inputs(): any {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return this.getAttribute(SpanAttributeKey.INPUTS);
}
get outputs(): any {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return this.getAttribute(SpanAttributeKey.OUTPUTS);
}
get attributes(): Record<string, any> {
return this._attributesRegistry.getAll();
}
getAttribute(key: string): any {
return this._attributesRegistry.get(key);
}
get events(): SpanEvent[] {
return this._span.events.map((event) => {
const [seconds, nanoseconds] = event.time;
return new SpanEvent({
name: event.name,
attributes: event.attributes as Record<string, any>,
timestamp: BigInt(seconds) * 1_000_000_000n + BigInt(nanoseconds),
});
});
}
/**
* Convert this span to JSON format (OpenTelemetry format)
* @returns JSON object representation of the span
*/
toJson(): SerializedSpan {
return {
trace_id: encodeTraceIdToBase64(this.traceId),
span_id: encodeSpanIdToBase64(this.spanId),
// Use empty string for parent_span_id if it is not set, to be consistent with Python implementation.
parent_span_id: this.parentId ? encodeSpanIdToBase64(this.parentId) : '',
name: this.name,
start_time_unix_nano: convertHrTimeToNanoSeconds(this.startTime),
end_time_unix_nano: this.endTime ? convertHrTimeToNanoSeconds(this.endTime) : null,
status: {
code: this.status?.statusCode || SpanStatusCode.UNSET,
message: this.status?.description,
},
attributes: this.attributes || {},
events: this.events.map((event) => ({
name: event.name,
time_unix_nano: event.timestamp,
attributes: event.attributes || {},
})),
};
}
/**
* Create a Span from JSON data (following Python implementation)
* Converts the JSON data back into OpenTelemetry-compatible span
*/
static fromJson(json: SerializedSpan): ISpan {
// Convert the JSON data back to an OpenTelemetry-like span structure
// This is simplified compared to Python but follows the same pattern
const otelSpanData = {
name: json.name,
startTime: convertNanoSecondsToHrTime(json.start_time_unix_nano),
endTime: json.end_time_unix_nano ? convertNanoSecondsToHrTime(json.end_time_unix_nano) : null,
status: {
code: convertStatusCodeToOTel(json.status.code),
message: json.status.message,
},
// For fromJson, attributes are already in their final form (not JSON serialized)
// so we store them directly
attributes: json.attributes || {},
events: (json.events || []).map((event) => ({
name: event.name,
time: convertNanoSecondsToHrTime(event.time_unix_nano),
attributes: event.attributes || {},
})),
ended: true,
// Add spanContext() method that returns proper SpanContext
spanContext: () => ({
traceId: decodeIdFromBase64(json.trace_id),
spanId: decodeIdFromBase64(json.span_id),
traceFlags: 1, // Sampled
isRemote: false,
}),
// Add parentSpanContext for parent span ID
parentSpanContext: json.parent_span_id
? {
traceId: decodeIdFromBase64(json.trace_id),
spanId: decodeIdFromBase64(json.parent_span_id),
traceFlags: 1,
isRemote: false,
}
: undefined,
};
// Create a span that behaves like our Span class but from downloaded data
return new Span(otelSpanData as OTelSpan, false); // false = immutable
}
}
/**
* Convert MLflow status codes to OpenTelemetry status codes
* @param statusCode Status code from MLflow JSON format
* @returns OpenTelemetry compatible status code
*/
function convertStatusCodeToOTel(statusCode?: string): OTelSpanStatusCode {
if (!statusCode) {
return OTelSpanStatusCode.UNSET;
}
// Handle MLflow format -> OTel format conversion
switch (statusCode) {
case 'STATUS_CODE_OK':
return OTelSpanStatusCode.OK;
case 'STATUS_CODE_ERROR':
return OTelSpanStatusCode.ERROR;
case 'STATUS_CODE_UNSET':
return OTelSpanStatusCode.UNSET;
// Also handle OTel format directly
case 'OK':
return OTelSpanStatusCode.OK;
case 'ERROR':
return OTelSpanStatusCode.ERROR;
case 'UNSET':
return OTelSpanStatusCode.UNSET;
default:
return OTelSpanStatusCode.UNSET;
}
}
export class LiveSpan extends Span {
// Internal only flag to allow mutating the ended span
allowMutatingEndedSpan: boolean = false;
constructor(span: OTelSpan, traceId: string, span_type: SpanType) {
super(span, true);
this.setAttribute(SpanAttributeKey.TRACE_ID, traceId);
this.setAttribute(SpanAttributeKey.SPAN_TYPE, span_type);
}
/**
* Set the type of the span
* @param spanType The type of the span
*/
setSpanType(spanType: SpanType): void {
this.setAttribute(SpanAttributeKey.SPAN_TYPE, spanType);
}
/**
* Set the severity level of the span. Accepts a SpanLogLevel enum value or
* its name (e.g. "INFO").
*/
setLogLevel(level: SpanLogLevel | string): void {
const normalized = toSpanLogLevel(level);
this.setAttribute(SpanAttributeKey.LOG_LEVEL, normalized as number);
}
/**
* Set inputs for the span
* @param inputs Input data for the span
*/
setInputs(inputs: any): void {
this.setAttribute(SpanAttributeKey.INPUTS, inputs);
}
/**
* Set outputs for the span
* @param outputs Output data for the span
*/
setOutputs(outputs: any): void {
this.setAttribute(SpanAttributeKey.OUTPUTS, outputs);
}
/**
* Set an attribute on the span
* @param key Attribute key
* @param value Attribute value
*/
setAttribute(key: string, value: any): void {
this._attributesRegistry.set(key, value, this.allowMutatingEndedSpan);
}
/**
* Set multiple attributes on the span
* @param attributes Object containing key-value pairs for attributes
*/
setAttributes(attributes: Record<string, any>): void {
if (!attributes || Object.keys(attributes).length === 0) {
return;
}
Object.entries(attributes).forEach(([key, value]) => {
this.setAttribute(key, value);
});
}
/**
* Add an event to the span
* @param event Event object with name and attributes
*/
addEvent(event: SpanEvent): void {
// Convert BigInt timestamp to HrTime for OpenTelemetry
const timeInput = convertNanoSecondsToHrTime(event.timestamp);
this._span.addEvent(event.name, event.attributes, timeInput);
// Promote the span to ERROR for exception events. Preserves user-set
// CRITICAL. (Inlined rather than extracted to a private method because
// TypeScript's structural typing makes adding privates a breaking API
// change for NoOpSpan, which sits behind the same union return type.)
if (event.name === 'exception') {
const current = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | null | undefined;
if (current == null || current < (SpanLogLevel.ERROR as number)) {
this.setAttribute(SpanAttributeKey.LOG_LEVEL, SpanLogLevel.ERROR as number);
}
}
}
/**
* Record an exception event to the span
* @param error Error object
*/
recordException(error: Error): void {
this._span.recordException(error);
const current = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | null | undefined;
if (current == null || current < (SpanLogLevel.ERROR as number)) {
this.setAttribute(SpanAttributeKey.LOG_LEVEL, SpanLogLevel.ERROR as number);
}
}
/**
* Set the status of the span
* @param status Status code or SpanStatus object
* @param description Optional description for the status
*/
setStatus(status: SpanStatus | SpanStatusCode | string, description?: string): void {
if (status instanceof SpanStatus) {
this._span.setStatus(status.toOtelStatus());
} else if (typeof status === 'string') {
const spanStatus = new SpanStatus(status as SpanStatusCode, description);
this._span.setStatus(spanStatus.toOtelStatus());
}
}
/**
* End the span
*
* @param outputs Optional outputs to set before ending
* @param attributes Optional attributes to set before ending
* @param status Optional status code
* @param endTimeNs Optional end time in nanoseconds
*/
end(options?: {
outputs?: any;
attributes?: Record<string, any>;
status?: SpanStatus | SpanStatusCode;
endTimeNs?: number;
}): void {
try {
if (options?.outputs != null) {
this.setOutputs(options.outputs);
}
if (options?.attributes != null) {
this.setAttributes(options.attributes);
}
if (options?.status != null) {
this.setStatus(options.status);
}
// NB: In OpenTelemetry, status code remains UNSET if not explicitly set
// by the user. However, there is no way to set the status when using
// `trace` function wrapper. Therefore, we just automatically set the status
// to OK if it is not ERROR.
if (this.status.statusCode !== SpanStatusCode.ERROR) {
this.setStatus(SpanStatusCode.OK);
}
// Resolve the log level from the final span_type if neither
// `setLogLevel` nor an exception bump set it during the span's lifetime.
// Mirrors the Python LiveSpan.end() behavior.
if (this.getAttribute(SpanAttributeKey.LOG_LEVEL) == null) {
this.setAttribute(
SpanAttributeKey.LOG_LEVEL,
defaultLogLevelForSpanType(this.spanType) as number,
);
}
// OTel SDK default end time to current time if not provided
const endTime = options?.endTimeNs
? convertNanoSecondsToHrTime(options.endTimeNs)
: undefined;
this._span.end(endTime);
} catch (error) {
console.error(`Failed to end span ${this.spanId}: ${String(error)}.`);
}
}
}
/**
* A no-operation span implementation that doesn't record anything
*/
export class NoOpSpan implements ISpan {
readonly _span: any; // Use any for NoOp span to avoid type conflicts
readonly _attributesRegistry: SpanAttributesRegistry;
allowMutatingEndedSpan: boolean = false;
constructor(span?: any) {
// Create a minimal no-op span object
this._span = span || {
spanContext: () => ({
spanId: INVALID_SPANID,
traceId: INVALID_TRACEID,
}),
attributes: {},
events: [],
};
this._attributesRegistry = new SpanAttributesRegistry(this._span as OTelSpan);
}
get traceId(): string {
return NO_OP_SPAN_TRACE_ID;
}
get spanId(): string {
return '';
}
get parentId(): string | null {
return null;
}
get name(): string {
return '';
}
get spanType(): SpanType {
return SpanType.UNKNOWN;
}
get logLevel(): SpanLogLevel | null {
return null;
}
get startTime(): HrTime {
return [0, 0];
}
get endTime(): HrTime | null {
return null;
}
get status(): SpanStatus {
return new SpanStatus(SpanStatusCode.UNSET);
}
get inputs(): any {
return null;
}
get outputs(): any {
return null;
}
get attributes(): Record<string, any> {
return {};
}
getAttribute(_key: string): any {
return null;
}
// Implement all methods to do nothing
setSpanType(_spanType: SpanType): void {}
setLogLevel(_level: SpanLogLevel | string): void {}
setInputs(_inputs: any): void {}
setOutputs(_outputs: any): void {}
setAttribute(_key: string, _value: any): void {}
setAttributes(_attributes: Record<string, any>): void {}
setStatus(_status: SpanStatus | SpanStatusCode | string, _description?: string): void {}
addEvent(_event: SpanEvent): void {}
recordException(_error: Error): void {}
end(
_outputs?: any,
_attributes?: Record<string, any>,
_status?: SpanStatus | SpanStatusCode,
_endTimeNs?: number,
): void {}
get events(): SpanEvent[] {
return [];
}
toJson(): SerializedSpan {
return {
trace_id: NO_OP_SPAN_TRACE_ID,
span_id: '',
parent_span_id: '',
name: '',
start_time_unix_nano: 0n,
end_time_unix_nano: null,
status: { code: 'UNSET', message: '' },
attributes: {},
events: [],
};
}
}
export interface SerializedSpan {
trace_id: string;
span_id: string;
parent_span_id: string;
name: string;
// Use bigint for nanosecond timestamps to maintain precision
start_time_unix_nano: bigint;
end_time_unix_nano: bigint | null;
status: {
code: string;
message: string;
};
attributes: Record<string, any>;
events: {
name: string;
time_unix_nano: bigint;
attributes: Record<string, any>;
}[];
}
/**
* A utility class to manage the span attributes.
* In MLflow users can add arbitrary key-value pairs to the span attributes, however,
* OpenTelemetry only allows a limited set of types to be stored in the attribute values.
* Therefore, we serialize all values into JSON string before storing them in the span.
* This class provides simple getter and setter methods to interact with the span attributes
* without worrying about the serde process.
*/
class SpanAttributesRegistry {
private readonly _span: OTelSpan;
constructor(otelSpan: OTelSpan) {
this._span = otelSpan;
}
/**
* Get all attributes as a dictionary
*/
getAll(): Record<string, any> {
const result: Record<string, any> = {};
if (this._span.attributes) {
Object.keys(this._span.attributes).forEach((key) => {
result[key] = this.get(key);
});
}
return result;
}
/**
* Get a single attribute value
*/
get(key: string): any {
const serializedValue = this._span.attributes?.[key];
if (serializedValue && typeof serializedValue === 'string') {
try {
return JSON.parse(serializedValue);
} catch (e) {
// If JSON.parse fails, this might be a raw string value or
// the span was created from JSON (attributes already parsed)
// In that case, return the value as-is
return serializedValue;
}
}
return serializedValue;
}
/**
* Set a single attribute value
*/
set(key: string, value: any, allowMutatingEndedSpan: boolean = false): void {
if (typeof key !== 'string') {
console.warn(`Attribute key must be a string, but got ${typeof key}. Skipping.`);
return;
}
if (allowMutatingEndedSpan && this._span.ended) {
// Directly set the attribute value to bypass the isSpanEnded check.
this._span.attributes[key] = safeJsonStringify(value);
return;
}
// NB: OpenTelemetry attribute can store not only string but also a few primitives like
// int, float, bool, and list of them. However, we serialize all into JSON string here
// for the simplicity in deserialization process.
this._span.setAttribute(key, safeJsonStringify(value));
}
}
/**
* A cache-enabled version of the SpanAttributesRegistry.
* The caching helps to avoid the redundant deserialization of the attribute, however, it does
* not handle the value change well. Therefore, this class should only be used for the persisted
* spans that are immutable, and thus implemented as a subclass of SpanAttributesRegistry.
*/
class CachedSpanAttributesRegistry extends SpanAttributesRegistry {
private readonly _cache = new Map<string, any>();
/**
* Get a single attribute value with LRU caching (maxsize=128)
*/
get(key: string): any {
if (this._cache.has(key)) {
// Move to end (most recently used)
const value = this._cache.get(key);
this._cache.delete(key);
this._cache.set(key, value);
return value;
}
const value = super.get(key);
// Implement LRU eviction
if (this._cache.size >= 128) {
// Remove least recently used (first entry)
const firstKey = this._cache.keys().next().value;
if (firstKey != null) {
this._cache.delete(firstKey);
}
}
this._cache.set(key, value);
return value;
}
/**
* Set operation is not allowed for cached registry (immutable spans)
*/
set(_key: string, _value: any, _allowMutatingEndedSpan: boolean = false): void {
throw new Error('The attributes of the immutable span must not be updated.');
}
}
/**
* Factory function to create a span object.
*/
export function createMlflowSpan(
otelSpan: any,
traceId: string,
spanType?: string,
): NoOpSpan | Span | LiveSpan {
// NonRecordingSpan always has a spanId of '0000000000000000'
// https://github.com/open-telemetry/opentelemetry-js/blob/f2cfd1327a5b131ea795301b10877291aac4e6f5/api/src/trace/invalid-span-constants.ts#L23C32-L23C48
/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call */
if (!otelSpan || otelSpan.spanContext().spanId === INVALID_SPANID) {
return new NoOpSpan(otelSpan);
}
// If the span is completed, it should be immutable.
if (otelSpan.ended) {
return new Span(otelSpan);
}
return new LiveSpan(otelSpan, traceId, (spanType as SpanType) || SpanType.UNKNOWN);
/* eslint-enable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call */
}
@@ -0,0 +1,161 @@
/**
* MLflow Span Event Entities
*
* This module provides TypeScript implementations of MLflow span event entities,
* compatible with OpenTelemetry for recording specific occurrences during span execution.
*/
/**
* Type definition for attribute values that can be stored in span events.
* Compatible with OpenTelemetry attribute value types.
*/
export type AttributeValue = string | number | boolean | string[] | number[] | boolean[];
/**
* Parameters for creating a SpanEvent instance.
*/
export interface SpanEventParams {
/** Name of the event */
name: string;
/**
* The exact time the event occurred, measured in nanoseconds since epoch.
* If not provided, the current time will be used.
*/
timestamp?: bigint;
/**
* A collection of key-value pairs representing detailed attributes of the event,
* such as exception stack traces or other contextual information.
*/
attributes?: Record<string, AttributeValue>;
}
/**
* Represents an event that records specific occurrences or moments in time
* during a span, such as an exception being thrown. Compatible with OpenTelemetry.
*
* SpanEvents are used to capture important moments during span execution,
* providing detailed context about what happened and when.
*
* @example
* ```typescript
* // Create a custom event
* const event = new SpanEvent({
* name: 'user_action',
* attributes: {
* 'action.type': 'click',
* 'element.id': 'submit-button'
* }
* });
*
* // Create an event from an exception
* const errorEvent = SpanEvent.fromException(new Error('Something went wrong'));
* ```
*/
export class SpanEvent {
/** Name of the event */
readonly name: string;
/**
* The exact time the event occurred, measured in nanosecond since epoch.
* Defaults to current time if not provided during construction.
*/
readonly timestamp: bigint;
/**
* A collection of key-value pairs representing detailed attributes of the event.
* Attributes provide contextual information about the event.
*/
readonly attributes: Record<string, AttributeValue>;
/**
* Creates a new SpanEvent instance.
*
* @param params - Event parameters including name, optional timestamp, and attributes
*
* @example
* ```typescript
* const event = new SpanEvent({
* name: 'database_query',
* attributes: {
* 'db.statement': 'SELECT * FROM users',
* 'db.duration_ms': 150
* }
* });
* ```
*/
constructor(params: SpanEventParams) {
this.name = params.name;
this.timestamp = params.timestamp ?? this.getCurrentTimeNano();
this.attributes = params.attributes ?? {};
}
/**
* Creates a span event from an exception.
*
* This is a convenience method for creating events that represent exceptions
* or errors that occurred during span execution. The event will include
* standard exception attributes like message, type, and stack trace.
*
* @param exception - The exception to create an event from
* @returns New SpanEvent instance representing the exception
*
* @example
* ```typescript
* try {
* // Some operation that might fail
* throw new Error('Database connection failed');
* } catch (error) {
* const errorEvent = SpanEvent.fromException(error);
* span.addEvent(errorEvent);
* }
* ```
*/
static fromException(exception: Error): SpanEvent {
const stackTrace = this.getStackTrace(exception);
return new SpanEvent({
name: 'exception',
attributes: {
'exception.message': exception.message,
'exception.type': exception.name,
'exception.stacktrace': stackTrace,
},
});
}
/**
* Gets the stack trace from an error object.
*
* @param error - The error to extract stack trace from
* @returns Stack trace as a string, or error representation if stack trace unavailable
*/
private static getStackTrace(error: Error): string {
try {
return error.stack ?? String(error);
} catch {
// Fallback if stack trace extraction fails
return String(error);
}
}
/**
* Convert this SpanEvent to JSON format
* @returns JSON object representation of the span event
*/
toJson(): Record<string, string | bigint | Record<string, AttributeValue>> {
return {
name: this.name,
timestamp: this.timestamp,
attributes: this.attributes,
};
}
/**
* Gets the current time in nanoseconds since epoch.
*
* @returns Current timestamp in nanoseconds
*/
private getCurrentTimeNano(): bigint {
return BigInt(Date.now()) * BigInt(1e6);
}
}
@@ -0,0 +1,108 @@
import { SpanStatus as OTelStatus, SpanStatusCode as OTelSpanStatusCode } from '@opentelemetry/api';
/**
* MLflow Span Status module
*
* This module provides the MLflow SpanStatusCode enum and SpanStatus class,
* matching the Python MLflow implementation.
*/
/**
* Enum for status code of a span
* Uses the same set of status codes as OTLP SpanStatusCode
* https://github.com/open-telemetry/opentelemetry-proto/blob/189b2648d29aa6039aeb2
lemetry/proto/trace/v1/trace.proto#L314-L322
*/
export enum SpanStatusCode {
/** Status is unset/unspecified */
UNSET = 'STATUS_CODE_UNSET',
/** The operation completed successfully */
OK = 'STATUS_CODE_OK',
/** The operation encountered an error */
ERROR = 'STATUS_CODE_ERROR',
}
/**
* Status of the span or the trace.
*/
export class SpanStatus {
/**
* The status code of the span or the trace.
*/
readonly statusCode: SpanStatusCode;
/**
* Description of the status. This should be only set when the status is ERROR,
* otherwise it will be ignored.
*/
readonly description: string;
/**
* Create a new SpanStatus instance
* @param statusCode The status code - must be one of SpanStatusCode enum values
* @param description Optional description, typically used for ERROR status
*/
constructor(statusCode: SpanStatusCode, description: string = '') {
// If user provides a string status code, validate it and convert to enum
this.statusCode = statusCode;
this.description = description;
}
/**
* Convert SpanStatus object to OpenTelemetry status object.
*/
toOtelStatus(): OTelStatus {
let otelStatusCode: OTelSpanStatusCode;
switch (this.statusCode) {
case SpanStatusCode.OK:
otelStatusCode = OTelSpanStatusCode.OK;
break;
case SpanStatusCode.ERROR:
otelStatusCode = OTelSpanStatusCode.ERROR;
break;
case SpanStatusCode.UNSET:
default:
otelStatusCode = OTelSpanStatusCode.UNSET;
break;
}
return {
code: otelStatusCode,
message: this.description,
};
}
/**
* Convert OpenTelemetry status object to SpanStatus object.
*/
static fromOtelStatus(otelStatus: OTelStatus): SpanStatus {
let statusCode: SpanStatusCode;
switch (otelStatus.code) {
case OTelSpanStatusCode.OK:
statusCode = SpanStatusCode.OK;
break;
case OTelSpanStatusCode.ERROR:
statusCode = SpanStatusCode.ERROR;
break;
case OTelSpanStatusCode.UNSET:
default:
statusCode = SpanStatusCode.UNSET;
break;
}
return new SpanStatus(statusCode, otelStatus.message ?? '');
}
/**
* Convert this SpanStatus to JSON format
* @returns JSON object representation of the span status
*/
toJson(): Record<string, string | SpanStatusCode> {
return {
status_code: this.statusCode,
description: this.description,
};
}
}
@@ -0,0 +1,54 @@
import { SerializedTraceInfo, TraceInfo } from './trace_info';
import { SerializedTraceData, TraceData } from './trace_data';
/**
* Represents a complete trace with metadata and span data
*/
export class Trace {
/**
* Trace metadata
*/
info: TraceInfo;
/**
* Trace data containing spans
*/
data: TraceData;
/**
* Create a new Trace instance
* @param info Trace metadata
* @param data Trace data containing spans
*/
constructor(info: TraceInfo, data: TraceData) {
this.info = info;
this.data = data;
}
/**
* Convert this Trace instance to JSON format
* @returns JSON object representation of the Trace
*/
toJson(): SerializedTrace {
return {
info: this.info.toJson(),
data: this.data.toJson(),
};
}
/**
* Create a Trace instance from JSON data
* @param json JSON object containing trace data
* @returns Trace instance
*/
static fromJson(json: SerializedTrace): Trace {
const info = TraceInfo.fromJson(json.info);
const data = TraceData.fromJson(json.data);
return new Trace(info, data);
}
}
interface SerializedTrace {
info: SerializedTraceInfo;
data: SerializedTraceData;
}
@@ -0,0 +1,43 @@
import { ISpan, SerializedSpan, Span } from './span';
/**
* Represents the spans and associated data for a trace
*/
export class TraceData {
/**
* The spans that make up this trace
*/
spans: ISpan[];
/**
* Create a new TraceData instance
* @param spans List of spans
*/
constructor(spans: ISpan[] = []) {
this.spans = spans;
}
/**
* Convert this TraceData instance to JSON format
* @returns JSON object representation of the TraceData
*/
toJson(): SerializedTraceData {
return {
spans: this.spans.map((span) => span.toJson()),
};
}
/**
* Create a TraceData instance from JSON data (following Python implementation)
* @param json JSON object containing trace data
* @returns TraceData instance
*/
static fromJson(json: SerializedTraceData): TraceData {
const spans: ISpan[] = json.spans.map((spanData) => Span.fromJson(spanData));
return new TraceData(spans);
}
}
export interface SerializedTraceData {
spans: SerializedSpan[];
}
@@ -0,0 +1,199 @@
import {
deserializeTraceLocation,
serializeTraceLocation,
type SerializedTraceLocation,
type TraceLocation,
} from './trace_location';
import type { TraceState } from './trace_state';
import { TraceMetadataKey } from '../constants';
/**
* Interface for token usage information
*/
export interface TokenUsage {
input_tokens: number;
output_tokens: number;
total_tokens: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
}
/**
* Metadata about a trace, such as its ID, location, timestamp, etc.
*/
export class TraceInfo {
/**
* The primary identifier for the trace
*/
traceId: string;
/**
* The location where the trace is stored
*/
traceLocation: TraceLocation;
/**
* Start time of the trace, in milliseconds
*/
requestTime: number;
/**
* State of the trace
*/
state: TraceState;
/**
* Request to the model/agent (JSON-encoded, may be truncated)
*/
requestPreview?: string;
/**
* Response from the model/agent (JSON-encoded, may be truncated)
*/
responsePreview?: string;
/**
* Client supplied request ID associated with the trace
*/
clientRequestId?: string;
/**
* Duration of the trace, in milliseconds
*/
executionDuration?: number;
/**
* Key-value pairs associated with the trace (immutable)
*/
traceMetadata: Record<string, string>;
/**
* Tags associated with the trace (mutable)
*/
tags: Record<string, string>;
/**
* List of assessments associated with the trace.
* TODO: Assessments are not yet supported in the TypeScript SDK.
*/
assessments: any[];
/**
* Create a new TraceInfo instance
* @param params TraceInfo parameters
*/
constructor(params: {
traceId: string;
traceLocation: TraceLocation;
requestTime: number;
state: TraceState;
requestPreview?: string;
responsePreview?: string;
clientRequestId?: string;
executionDuration?: number;
traceMetadata?: Record<string, string>;
tags?: Record<string, string>;
assessments?: any[];
}) {
this.traceId = params.traceId;
this.traceLocation = params.traceLocation;
this.requestTime = params.requestTime;
this.state = params.state;
this.requestPreview = params.requestPreview;
this.responsePreview = params.responsePreview;
this.clientRequestId = params.clientRequestId;
this.executionDuration = params.executionDuration;
this.traceMetadata = params.traceMetadata || {};
this.tags = params.tags || {};
// TODO: Assessments are not yet supported in the TypeScript SDK.
this.assessments = [];
}
/**
* Convert this TraceInfo instance to JSON format
* @returns JSON object representation of the TraceInfo
*/
toJson(): SerializedTraceInfo {
return {
trace_id: this.traceId,
client_request_id: this.clientRequestId,
trace_location: serializeTraceLocation(this.traceLocation),
request_preview: this.requestPreview,
response_preview: this.responsePreview,
request_time: new Date(this.requestTime).toISOString(),
execution_duration:
this.executionDuration != null ? `${this.executionDuration / 1000}s` : undefined,
state: this.state,
trace_metadata: this.traceMetadata,
tags: this.tags,
assessments: this.assessments,
};
}
/**
* Get aggregated token usage information for this trace.
* Returns null if no token usage data is available.
* @returns Token usage object or null
*/
get tokenUsage(): TokenUsage | null {
const tokenUsageJson = this.traceMetadata[TraceMetadataKey.TOKEN_USAGE];
if (!tokenUsageJson) {
return null;
}
const usage = JSON.parse(tokenUsageJson) as TokenUsage;
const result: TokenUsage = {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
total_tokens: usage.total_tokens,
};
if (usage.cache_read_input_tokens != null) {
result.cache_read_input_tokens = usage.cache_read_input_tokens;
}
if (usage.cache_creation_input_tokens != null) {
result.cache_creation_input_tokens = usage.cache_creation_input_tokens;
}
return result;
}
/**
* Create a TraceInfo instance from JSON data
* @param json JSON object containing trace info data
* @returns TraceInfo instance
*/
static fromJson(json: SerializedTraceInfo): TraceInfo {
return new TraceInfo({
traceId: json.trace_id,
clientRequestId: json.client_request_id,
traceLocation: deserializeTraceLocation(json.trace_location),
requestPreview: json.request_preview,
responsePreview: json.response_preview,
requestTime: json.request_time != null ? new Date(json.request_time).getTime() : Date.now(),
executionDuration:
json.execution_duration != null
? parseFloat(json.execution_duration.replace('s', '')) * 1000
: undefined,
state: json.state,
traceMetadata: json.trace_metadata || {},
tags: json.tags || {},
assessments: json.assessments || [],
});
}
}
export interface SerializedTraceInfo {
trace_id: string;
client_request_id?: string;
trace_location: SerializedTraceLocation;
request_preview?: string;
response_preview?: string;
// "request_time": "2025-06-15T14:07:41.282Z"
request_time: string;
execution_duration?: string;
state: TraceState;
trace_metadata: Record<string, string>;
tags: Record<string, string>;
// TODO: Define proper type for assessments once supported
assessments: any[];
}
@@ -0,0 +1,234 @@
/**
* Types of trace locations
*/
export enum TraceLocationType {
/**
* Unspecified trace location type
*/
TRACE_LOCATION_TYPE_UNSPECIFIED = 'TRACE_LOCATION_TYPE_UNSPECIFIED',
/**
* Trace is stored in an MLflow experiment
*/
MLFLOW_EXPERIMENT = 'MLFLOW_EXPERIMENT',
/**
* Trace is stored in a Databricks inference table
*/
INFERENCE_TABLE = 'INFERENCE_TABLE',
/**
* Trace is stored under a Databricks Unity Catalog table prefix.
* The user-supplied prefix determines the span/log table names.
*/
UC_TABLE_PREFIX = 'UC_TABLE_PREFIX',
}
/**
* Interface representing an MLflow experiment location
*/
export interface MlflowExperimentLocation {
/**
* The ID of the MLflow experiment where the trace is stored
*/
experimentId: string;
}
/**
* Interface representing a Databricks inference table location
*/
export interface InferenceTableLocation {
/**
* The fully qualified name of the inference table where the trace is stored
*/
fullTableName: string;
}
/**
* Interface representing a Databricks Unity Catalog table-prefix location.
* Mirrors Python's `UnityCatalog` location.
*/
export interface UnityCatalogLocation {
catalogName: string;
schemaName: string;
/** Customer-supplied table prefix; required for trace ID location string. */
tablePrefix?: string;
/** Backend-populated fully qualified spans table name. */
otelSpansTableName?: string;
/** Backend-populated fully qualified logs table name. */
otelLogsTableName?: string;
/** Backend-populated fully qualified annotations table name. */
annotationsTableName?: string;
}
/**
* Interface representing the location where the trace is stored
*/
export interface TraceLocation {
/**
* The type of the trace location
*/
type: TraceLocationType;
/**
* The MLflow experiment location
* Set this when the location type is MLflow experiment
*/
mlflowExperiment?: MlflowExperimentLocation;
/**
* The inference table location
* Set this when the location type is Databricks Inference table
*/
inferenceTable?: InferenceTableLocation;
/**
* The Databricks UC table-prefix location. Set when type is UC_TABLE_PREFIX.
*/
ucTablePrefix?: UnityCatalogLocation;
}
export interface SerializedTraceLocation {
type: TraceLocationType;
mlflow_experiment?: { experiment_id: string };
inference_table?: { full_table_name: string };
uc_table_prefix?: {
catalog_name: string;
schema_name: string;
table_prefix?: string;
otel_spans_table_name?: string;
otel_logs_table_name?: string;
annotations_table_name?: string;
};
}
export function serializeTraceLocation(loc: TraceLocation): SerializedTraceLocation {
const out: SerializedTraceLocation = { type: loc.type };
if (loc.mlflowExperiment) {
out.mlflow_experiment = { experiment_id: loc.mlflowExperiment.experimentId };
}
if (loc.inferenceTable) {
out.inference_table = { full_table_name: loc.inferenceTable.fullTableName };
}
if (loc.ucTablePrefix) {
const uc = loc.ucTablePrefix;
out.uc_table_prefix = {
catalog_name: uc.catalogName,
schema_name: uc.schemaName,
...(uc.tablePrefix ? { table_prefix: uc.tablePrefix } : {}),
...(uc.otelSpansTableName ? { otel_spans_table_name: uc.otelSpansTableName } : {}),
...(uc.otelLogsTableName ? { otel_logs_table_name: uc.otelLogsTableName } : {}),
...(uc.annotationsTableName ? { annotations_table_name: uc.annotationsTableName } : {}),
};
}
return out;
}
export function deserializeTraceLocation(json: SerializedTraceLocation | undefined): TraceLocation {
if (!json?.type) {
throw new Error('Invalid trace location: missing type.');
}
return {
type: json.type,
mlflowExperiment: json.mlflow_experiment
? { experimentId: json.mlflow_experiment.experiment_id }
: undefined,
inferenceTable: json.inference_table
? { fullTableName: json.inference_table.full_table_name }
: undefined,
ucTablePrefix: json.uc_table_prefix
? {
catalogName: json.uc_table_prefix.catalog_name,
schemaName: json.uc_table_prefix.schema_name,
tablePrefix: json.uc_table_prefix.table_prefix,
otelSpansTableName: json.uc_table_prefix.otel_spans_table_name,
otelLogsTableName: json.uc_table_prefix.otel_logs_table_name,
annotationsTableName: json.uc_table_prefix.annotations_table_name,
}
: undefined,
};
}
/**
* Returns "catalog.schema.table_prefix" for a UC table-prefix location.
* Throws if the prefix is not set; the prefix is required for trace IDs.
*/
export function ucTablePrefixLocationString(location: UnityCatalogLocation): string {
if (!location.tablePrefix) {
throw new Error(
'Unity Catalog table_prefix is required to build a trace location string. ' +
'Provide a tablePrefix when constructing the UnityCatalog destination.',
);
}
return `${location.catalogName}.${location.schemaName}.${location.tablePrefix}`;
}
/**
* Get the location string used for V4 trace IDs and UC OTLP routing
* (i.e. the `X-Databricks-UC-Table-Name` header value).
*/
export function getUcLocationString(traceLocation: TraceLocation): string | null {
if (traceLocation.type === TraceLocationType.UC_TABLE_PREFIX && traceLocation.ucTablePrefix) {
return ucTablePrefixLocationString(traceLocation.ucTablePrefix);
}
return null;
}
/**
* Get the fully qualified OTel spans table name to use as the
* `X-Databricks-UC-Table-Name` header when exporting spans for this
* trace via OTLP.
*
* Falls back to `<catalog>.<schema>.<table_prefix>_otel_spans`, which is the
* default spans table name Databricks creates when a UC trace location is
* provisioned. Customers with a custom backend-provisioned spans table can
* override by setting `ucTablePrefix.otelSpansTableName`.
*/
export function getOtelSpansTableName(traceLocation: TraceLocation): string | null {
if (traceLocation.type === TraceLocationType.UC_TABLE_PREFIX && traceLocation.ucTablePrefix) {
const loc = traceLocation.ucTablePrefix;
if (loc.otelSpansTableName) {
return loc.otelSpansTableName;
}
if (loc.tablePrefix) {
return `${loc.catalogName}.${loc.schemaName}.${loc.tablePrefix}_otel_spans`;
}
return null;
}
return null;
}
/**
* Create a TraceLocation from an experiment ID
* @param experimentId The ID of the MLflow experiment
*/
export function createTraceLocationFromExperimentId(experimentId: string): TraceLocation {
return {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: experimentId,
},
};
}
/**
* Create a TraceLocation from a Databricks UC table-prefix location.
*/
export function createTraceLocationFromUcTablePrefix(
catalogName: string,
schemaName: string,
tablePrefix: string,
): TraceLocation {
return {
type: TraceLocationType.UC_TABLE_PREFIX,
ucTablePrefix: { catalogName, schemaName, tablePrefix },
};
}
/**
* True iff the trace is stored in a Databricks Unity Catalog location.
*/
export function isUcTraceLocation(traceLocation: TraceLocation): boolean {
return traceLocation.type === TraceLocationType.UC_TABLE_PREFIX;
}
@@ -0,0 +1,41 @@
import { SpanStatusCode } from '@opentelemetry/api';
/**
* Enum representing the state of a trace
*/
export enum TraceState {
/**
* Unspecified trace state
*/
STATE_UNSPECIFIED = 'STATE_UNSPECIFIED',
/**
* Trace successfully completed
*/
OK = 'OK',
/**
* Trace encountered an error
*/
ERROR = 'ERROR',
/**
* Trace is currently in progress
*/
IN_PROGRESS = 'IN_PROGRESS',
}
/**
* Convert OpenTelemetry status code to MLflow TraceState
* @param statusCode OpenTelemetry status code
*/
export function fromOtelStatus(statusCode: SpanStatusCode): TraceState {
switch (statusCode) {
case SpanStatusCode.OK:
return TraceState.OK;
case SpanStatusCode.ERROR:
return TraceState.ERROR;
default:
return TraceState.STATE_UNSPECIFIED;
}
}
@@ -0,0 +1,32 @@
import { SpanLogLevel, SpanType } from './constants';
/**
* Span types whose default level is INFO. These represent the user-visible
* semantic operations (model calls, tool calls, retrievals, agent turns, etc.)
* that should remain visible at the default UI threshold. Everything else --
* chain glue, output parsing, internal/custom types -- defaults to DEBUG.
*
* Mirrors the Python `_INFO_SPAN_TYPES` set in
* `mlflow/tracing/utils/default_log_level.py`. Keep the two in sync.
*/
const INFO_SPAN_TYPES: ReadonlySet<string> = new Set<string>([
SpanType.LLM,
SpanType.CHAT_MODEL,
SpanType.TOOL,
SpanType.RETRIEVER,
SpanType.AGENT,
SpanType.EMBEDDING,
]);
/**
* Return the default SpanLogLevel for a span of the given type. Used by the
* `LiveSpan` constructor so every span -- autologged or manual -- gets a
* sensible level for the trace explorer's verbosity filter without users
* annotating every call.
*/
export function defaultLogLevelForSpanType(spanType: string | undefined | null): SpanLogLevel {
if (spanType && INFO_SPAN_TYPES.has(spanType)) {
return SpanLogLevel.INFO;
}
return SpanLogLevel.DEBUG;
}
+110
View File
@@ -0,0 +1,110 @@
import { trace, Tracer } from '@opentelemetry/api';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { SpanProcessor } from '@opentelemetry/sdk-trace-base';
import { MlflowClient } from '../clients';
import { MlflowSpanExporter, MlflowSpanProcessor } from '../exporters/mlflow';
import {
DatabricksUCTableSpanExporter,
DatabricksUCTableSpanProcessor,
} from '../exporters/uc_table';
import { MlflowWalSpanExporter } from '../exporters/wal';
import { getConfig, getAuthProvider, type UnityCatalogLocationOptions } from './config';
import type { UnityCatalogLocation } from './entities/trace_location';
import { asyncExportEnabled } from './utils/env';
let sdk: NodeSDK | null = null;
// Keep a reference to the active span processor for flushing.
let processor: SpanProcessor | null = null;
/**
* Initialize the OpenTelemetry SDK and span processor.
*
* When `config.traceLocation` is provided, wires up the V4 Databricks Unity
* Catalog span processor + exporter so that `updateCurrentTrace` tags persist
* on UC-backed traces. Otherwise installs the V3 experiment-backed processor.
*
* TODO: Auto-resolve UC location from the linked Databricks experiment's
* `mlflow.experiment.databricksTrace*` tags so customers don't have to pass
* `traceLocation` explicitly. This needs `GetExperiment` to run, and Node
* has no idiomatic way to do a synchronous HTTP request from `init()`
* (the documented `worker_threads` + `Atomics.wait` pattern is for CPU-bound
* work, not for converting async I/O into sync). The likely path is a
* buffering processor that queues spans until the async fetch resolves;
* until that lands, customers configure UC explicitly via `traceLocation`.
*/
export function initializeSDK(): void {
if (sdk) {
sdk.shutdown().catch((error) => {
console.error('Error shutting down existing SDK:', error);
});
}
try {
const config = getConfig();
const authProvider = getAuthProvider();
const client = new MlflowClient({
trackingUri: config.trackingUri,
authProvider,
});
if (config.traceLocation) {
const ucLocation = resolveUcLocation(config.traceLocation);
const ucExporter = new DatabricksUCTableSpanExporter(client);
processor = new DatabricksUCTableSpanProcessor(ucExporter, ucLocation);
} else {
if (asyncExportEnabled()) {
// WAL path: the hook submits records to a daemon over IPC; the
// bundled daemon performs the actual auth setup, fsyncs the
// record to `queue.log`, and runs the HTTP upload in a separate
// process. The IPC client spawns the daemon on first use, so no
// client construction or supervisor wiring is needed here.
const mlflowWalExporter = new MlflowWalSpanExporter();
processor = new MlflowSpanProcessor(mlflowWalExporter);
} else {
const exporter = new MlflowSpanExporter(client);
processor = new MlflowSpanProcessor(exporter);
}
}
sdk = new NodeSDK({ spanProcessors: [processor] });
sdk.start();
} catch (error) {
console.error('Failed to initialize MLflow tracing:', error);
}
}
/**
* Validate the user-supplied UC location and return it as a
* `UnityCatalogLocation`. All three fields are required; the SDK does not
* upsert UC trace locations, so we cannot default any of them.
*/
function resolveUcLocation(options: UnityCatalogLocationOptions): UnityCatalogLocation {
if (!options.catalogName || !options.schemaName || !options.tablePrefix) {
throw new Error(
'traceLocation requires catalogName, schemaName, and tablePrefix. The UC ' +
'trace location must already be provisioned in the workspace.',
);
}
return {
catalogName: options.catalogName,
schemaName: options.schemaName,
tablePrefix: options.tablePrefix,
};
}
export function getTracer(module_name: string): Tracer {
return trace.getTracer(module_name);
}
/**
* Force flush all pending trace exports.
*
* In WAL mode this only awaits on-disk durability of the WAL appends,
* not the upstream HTTP upload — that's the daemon's job and the
* caller (e.g. the Claude Code Stop hook) is intentionally decoupled
* from backend latency.
*/
export async function flushTraces(): Promise<void> {
await processor?.forceFlush();
}
@@ -0,0 +1,292 @@
import { LiveSpan, Span } from './entities/span';
import { TraceInfo } from './entities/trace_info';
import { Trace } from './entities/trace';
import { TraceData } from './entities/trace_data';
import {
REQUEST_RESPONSE_PREVIEW_MAX_LENGTH,
SpanAttributeKey,
TraceMetadataKey,
} from './constants';
/**
* Internal representation to keep the state of a trace.
* Uses a Map<string, LiveSpan> instead of TraceData to allow access by span_id.
*/
class _Trace {
info: TraceInfo;
spanDict: Map<string, LiveSpan>;
constructor(info: TraceInfo) {
this.info = info;
this.spanDict = new Map<string, LiveSpan>();
}
/**
* Convert the internal trace representation to an MLflow Trace object
*/
toMlflowTrace(): Trace {
// Convert LiveSpan, mutable objects, into immutable Span objects before persisting
const traceData = new TraceData([...this.spanDict.values()] as Span[]);
const root_span = traceData.spans.find((span) => span.parentId == null);
if (root_span) {
// Only set previews if they haven't been explicitly set by updateCurrentTrace
if (!this.info.requestPreview) {
this.info.requestPreview = getTruncatedPreview(
root_span._span.attributes[SpanAttributeKey.INPUTS] as string,
'user',
);
}
if (!this.info.responsePreview) {
this.info.responsePreview = getTruncatedPreview(
root_span._span.attributes[SpanAttributeKey.OUTPUTS] as string,
'assistant',
);
}
// TODO: Remove this once the new trace table UI is available that is based on MLflow V3 trace.
// Until then, these two metadata are still used to render the "request" and "response" columns.
this.info.traceMetadata[TraceMetadataKey.INPUTS] = this.info.requestPreview;
this.info.traceMetadata[TraceMetadataKey.OUTPUTS] = this.info.responsePreview;
}
return new Trace(this.info, traceData);
}
}
/**
* Generate a truncated preview string from span inputs/outputs.
* Extracts message text content from known chat formats (OpenAI messages,
* choices, Responses API) before truncating, so the preview shows readable
* text instead of cut-off JSON.
*
* Mirrors the Python SDK's `_get_truncated_preview` in
* `mlflow/tracing/utils/truncation.py`.
*/
function getTruncatedPreview(inputsOrOutputs: string, role: string): string {
if (!inputsOrOutputs) {
return '';
}
const maxLength = REQUEST_RESPONSE_PREVIEW_MAX_LENGTH;
let content: string | null = null;
try {
const obj = JSON.parse(inputsOrOutputs);
if (obj && typeof obj === 'object') {
const messages = tryExtractMessages(obj as Record<string, unknown>);
if (messages && messages.length > 0) {
const msg = getLastMessageByRole(messages, role);
content = getTextContentFromMessage(msg);
}
}
} catch {
// Not valid JSON — use as-is
}
const preview = content || inputsOrOutputs;
if (preview.length <= maxLength) {
return preview;
}
return preview.slice(0, maxLength - 3) + '...';
}
function tryExtractMessages(obj: Record<string, unknown>): Record<string, unknown>[] | null {
// OpenAI ChatCompletion request format: { messages: [...] }
if (Array.isArray(obj.messages)) {
return obj.messages.filter(isMessage);
}
// OpenAI ChatCompletion response format: { choices: [{ message: {...} }] }
if (
Array.isArray(obj.choices) &&
obj.choices.length > 0 &&
typeof obj.choices[0] === 'object' &&
obj.choices[0] != null
) {
const msg = (obj.choices[0] as Record<string, unknown>).message;
if (isMessage(msg)) {
return [msg];
}
}
// OpenAI Responses API request format: { input: [...] }
if (Array.isArray(obj.input)) {
return obj.input.filter(isMessage);
}
// OpenAI Responses API response format: { output: [...] }
if (Array.isArray(obj.output)) {
return obj.output.filter(isMessage);
}
// ResponsesAgent input: { request: { ... } }
if (obj.request && typeof obj.request === 'object') {
return tryExtractMessages(obj.request as Record<string, unknown>);
}
return null;
}
function isMessage(item: unknown): item is Record<string, unknown> {
return (
item != null &&
typeof item === 'object' &&
'role' in (item as Record<string, unknown>) &&
'content' in (item as Record<string, unknown>)
);
}
function getLastMessageByRole(
messages: Record<string, unknown>[],
role: string,
): Record<string, unknown> {
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i].role === role) {
return messages[i];
}
}
return messages[messages.length - 1];
}
function getTextContentFromMessage(message: Record<string, unknown>): string | null {
const content = message.content;
if (typeof content === 'string') {
return content;
}
if (Array.isArray(content)) {
for (const part of content) {
if (typeof part === 'string') {
return part;
}
if (
part &&
typeof part === 'object' &&
'type' in (part as Record<string, unknown>) &&
((part as Record<string, unknown>).type === 'text' ||
(part as Record<string, unknown>).type === 'output_text')
) {
return (part as Record<string, unknown>).text as string;
}
}
}
return null;
}
/**
* Manage spans and traces created by the tracing system in memory.
* This class is implemented as a singleton.
*/
export class InMemoryTraceManager {
private static _instance: InMemoryTraceManager | undefined;
// In-memory cache to store trace_id -> _Trace mapping
// TODO: Add TTL to the trace buffer similarly to Python SDK
private _traces: Map<string, _Trace>;
// Store mapping between OpenTelemetry trace ID and MLflow trace ID
private _otelIdToMlflowTraceId: Map<string, string>;
// Store the last active trace ID
lastActiveTraceId: string | undefined;
/**
* Singleton pattern implementation
*/
static getInstance(): InMemoryTraceManager {
if (InMemoryTraceManager._instance == null) {
InMemoryTraceManager._instance = new InMemoryTraceManager();
}
return InMemoryTraceManager._instance;
}
private constructor() {
this._traces = new Map<string, _Trace>();
this._otelIdToMlflowTraceId = new Map<string, string>();
}
/**
* Register a new trace info object to the in-memory trace registry.
* @param otelTraceId The OpenTelemetry trace ID for the new trace
* @param traceInfo The trace info object to be stored
*/
registerTrace(otelTraceId: string, traceInfo: TraceInfo): void {
this._traces.set(traceInfo.traceId, new _Trace(traceInfo));
this._otelIdToMlflowTraceId.set(otelTraceId, traceInfo.traceId);
}
/**
* Store the given span in the in-memory trace data.
* @param span The span to be stored
*/
registerSpan(span: LiveSpan): void {
const trace = this._traces.get(span.traceId);
if (trace) {
trace.spanDict.set(span.spanId, span);
} else {
console.debug(`Tried to register span ${span.spanId} for trace ${span.traceId}
but trace not found. Please make sure to register the trace first.`);
}
}
/**
* Get the trace for the given trace ID.
* Returns the trace object or null if not found.
* @param traceId The trace ID to look up
*/
getTrace(traceId: string): _Trace | null {
return this._traces.get(traceId) || null;
}
/**
* Get the MLflow trace ID for the given OpenTelemetry trace ID.
* @param otelTraceId The OpenTelemetry trace ID
*/
getMlflowTraceIdFromOtelId(otelTraceId: string): string | null {
return this._otelIdToMlflowTraceId.get(otelTraceId) || null;
}
/**
* Get the span for the given trace ID and span ID.
* @param traceId The trace ID
* @param spanId The span ID
*/
getSpan(traceId?: string | null, spanId?: string | null): LiveSpan | null {
if (traceId == null || spanId == null) {
return null;
}
return this._traces.get(traceId)?.spanDict.get(spanId) || null;
}
/**
* Pop trace data for the given OpenTelemetry trace ID and return it as
* a ready-to-publish Trace object.
* @param otelTraceId The OpenTelemetry trace ID
*/
popTrace(otelTraceId: string): Trace | null {
const mlflowTraceId = this._otelIdToMlflowTraceId.get(otelTraceId);
if (!mlflowTraceId) {
console.debug(`Tried to pop trace ${otelTraceId} but no trace found.`);
return null;
}
this._otelIdToMlflowTraceId.delete(otelTraceId);
const trace = this._traces.get(mlflowTraceId);
if (trace) {
this._traces.delete(mlflowTraceId);
return trace.toMlflowTrace();
}
console.debug(`Tried to pop trace ${otelTraceId} but trace not found.`);
return null;
}
/**
* Clear all the aggregated trace data. This should only be used for testing.
*/
static reset(): void {
if (InMemoryTraceManager._instance) {
InMemoryTraceManager._instance._traces.clear();
InMemoryTraceManager._instance._otelIdToMlflowTraceId.clear();
InMemoryTraceManager._instance = undefined;
}
}
}
@@ -0,0 +1,51 @@
import { isAbsolute } from 'node:path';
import { fileURLToPath } from 'node:url';
/**
* Extract the scheme from an artifact URI.
*/
export function getArtifactUriScheme(uri: string): string {
const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):/.exec(uri);
if (!match) {
return '';
}
const scheme = match[1].toLowerCase();
// A single-letter scheme is a Windows drive designator (e.g. `C:`), not a URI
// scheme; treat it as a bare local path.
if (scheme.length === 1) {
return '';
}
return scheme;
}
/**
* Whether an artifact URI refers to a location on the local filesystem.
*/
export function isLocalArtifactUri(uri: string): boolean {
const scheme = getArtifactUriScheme(uri);
return scheme === '' || scheme === 'file';
}
/**
* Convert a local artifact URI to an filesystem path.
*/
export function artifactUriToLocalPath(uri: string): string {
if (getArtifactUriScheme(uri) === 'file') {
return fileURLToPath(uri);
}
return uri;
}
/**
* Convert a local artifact URI to a filesystem path, requiring it to be absolute.
*/
export function toAbsoluteLocalPath(uri: string): string {
const dir = artifactUriToLocalPath(uri);
if (!isAbsolute(dir)) {
throw new Error(
`Refusing to use a relative local artifact location "${uri}". ` +
`Expected an absolute filesystem path or a file:// URI.`,
);
}
return dir;
}
@@ -0,0 +1,51 @@
/**
* Read an env var and require a strictly positive integer.
*/
export const readPositiveInt = (envName: string, fallback: number): number => {
const raw = process.env[envName];
if (raw === undefined || raw === '') {
return fallback;
}
// `Number` (vs `Number.parseInt`) refuses trailing garbage and unit
// suffixes — e.g. "15000abc", "15s", and "10.9" all become NaN or a
// non-integer, so the `isInteger` check below rejects them cleanly.
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed <= 0) {
console.warn(
`[mlflow] Ignoring invalid ${envName}=${JSON.stringify(raw)}; ` +
`expected a positive integer. Falling back to default ${fallback}.`,
);
return fallback;
}
return parsed;
};
export const readNonNegativeInt = (envName: string, fallback: number): number => {
const raw = process.env[envName];
if (raw === undefined || raw === '') {
return fallback;
}
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed < 0) {
console.warn(
`[mlflow] Ignoring invalid ${envName}=${JSON.stringify(raw)}; ` +
`expected a non-negative integer. Falling back to default ${fallback}.`,
);
return fallback;
}
return parsed;
};
/**
* Whether to route trace export through an on-disk WAL queue
* drained by a background daemon, instead of the synchronous
* HTTP exporter.
*/
export function asyncExportEnabled(): boolean {
const raw = process.env.MLFLOW_ENABLE_ASYNC_TRACE_LOGGING;
if (raw === undefined || raw === '') {
return false;
}
const lower = raw.toLowerCase();
return lower === '1' || lower === 'true';
}
@@ -0,0 +1,267 @@
import type { HrTime } from '@opentelemetry/api';
import { Span } from '../entities/span';
import { SpanAttributeKey } from '../constants';
import { TokenUsage } from '../entities/trace_info';
/**
* OpenTelemetry Typescript SDK uses a unique timestamp format `HrTime` to represent
* timestamps. This function converts a timestamp in nanoseconds to an `HrTime`
* Supports both number and BigInt for large timestamps
* Ref: https://github.com/open-telemetry/opentelemetry-js/blob/a9fc600f2bd7dbf9345ec14e4421f1cc034f1f9c/api/src/common/Time.ts#L17-L30C13
* @param nanoseconds The timestamp in nanoseconds (number or BigInt)
* @returns The timestamp in `HrTime` format
*/
export function convertNanoSecondsToHrTime(nanoseconds: number | bigint): HrTime {
// Handle both number and BigInt inputs
if (typeof nanoseconds === 'bigint') {
// Use BigInt arithmetic to maintain precision
const seconds = Number(nanoseconds / 1_000_000_000n);
const nanos = Number(nanoseconds % 1_000_000_000n);
return [seconds, nanos] as HrTime;
}
// For regular numbers, use standard arithmetic
return [Math.floor(nanoseconds / 1e9), nanoseconds % 1e9] as HrTime;
}
/**
* Convert HrTime to nanoseconds as BigInt
* @param hrTime HrTime tuple [seconds, nanoseconds]
* @returns BigInt nanoseconds
*/
export function convertHrTimeToNanoSeconds(hrTime: HrTime): bigint {
return BigInt(hrTime[0]) * 1_000_000_000n + BigInt(hrTime[1]);
}
/**
* Convert HrTime to milliseconds
* @param hrTime HrTime tuple [seconds, nanoseconds]
* @returns Milliseconds
*/
export function convertHrTimeToMs(hrTime: HrTime): number {
return Math.floor(hrTime[0] * 1e3 + hrTime[1] / 1e6);
}
/**
* Convert a hex span ID to base64 format for JSON serialization
* Following Python implementation: _encode_span_id_to_byte
* @param spanId Hex string span ID (16 chars)
* @returns Base64 encoded span ID
*/
export function encodeSpanIdToBase64(spanId: string): string {
// Convert hex string to bytes (8 bytes for span ID)
const bytes = new Uint8Array(8);
// Parse hex string (add padding to 16 chars)
const hexStr = spanId.padStart(16, '0');
for (let i = 0; i < 8; i++) {
bytes[i] = parseInt(hexStr.substring(i * 2, i * 2 + 2), 16);
}
// Convert to base64
return Buffer.from(bytes).toString('base64');
}
/**
* Convert a hex span ID to base64 format for JSON serialization
* Following Python implementation: _encode_trace_id_to_byte
* @param spanId Hex string span ID (32 chars)
* @returns Base64 encoded span ID
*/
export function encodeTraceIdToBase64(traceId: string): string {
// Convert hex string to bytes (16 bytes for trace ID)
const bytes = new Uint8Array(16);
// Parse hex string (add padding to 32 chars)
const hexStr = traceId.padStart(32, '0');
for (let i = 0; i < 16; i++) {
bytes[i] = parseInt(hexStr.substring(i * 2, i * 2 + 2), 16);
}
// Convert to base64
return Buffer.from(bytes).toString('base64');
}
/**
* Convert a base64 span ID back to hex format
* Following Python implementation: _decode_id_from_byte
* @param base64SpanId Base64 encoded span ID
* @returns Hex string span ID
*/
export function decodeIdFromBase64(base64SpanId: string): string {
// Decode from base64
const bytes = Buffer.from(base64SpanId, 'base64');
// Convert to hex string
return Array.from(bytes)
.map((b) => b.toString(16).padStart(2, '0'))
.join('');
}
/**
* Map function arguments to an object with parameter names as keys
* @param func The function to extract parameter names from
* @param args The arguments passed to the function
* @returns Object mapping parameter names to argument values
*/
export function mapArgsToObject(func: Function, args: any[]): Record<string, any> {
const paramNames = getParameterNames(func);
// If we can't extract parameter names, return the args as an array
if (paramNames.length === 0) {
return args.length === 0 ? {} : { args };
}
const result: Record<string, any> = {};
paramNames.forEach((name, index) => {
if (index < args.length) {
result[name] = args[index];
}
});
return result;
}
/**
* Extract parameter names from a function using string parsing
* @param func The function to extract parameter names from
* @returns Array of parameter names
*/
function getParameterNames(func: Function): string[] {
const funcStr = func.toString();
// Handle arrow functions and regular functions
let paramMatch: RegExpMatchArray | null;
// Try arrow function pattern: (a, b) => or a =>
const arrowMatch = funcStr.match(/^[^(]*\(?([^)=]*)\)?\s*=>/);
if (arrowMatch) {
const params = arrowMatch[1].trim();
if (!params) {
return [];
}
// Handle single parameter without parentheses
if (!params.includes(',') && !funcStr.includes('(')) {
return [params.trim()];
}
paramMatch = ['', params];
} else {
// Try regular function pattern: function name(a, b) or (a, b)
paramMatch = funcStr.match(/(?:function\s*[^(]*)??\(([^)]*)\)/);
}
if (!paramMatch?.[1]) {
return [];
}
// Split parameters while handling nested brackets/braces
const params = [];
let current = '';
let depth = 0;
const paramStr = paramMatch[1];
for (let i = 0; i < paramStr.length; i++) {
const char = paramStr[i];
if (char === '{' || char === '[') {
depth++;
} else if (char === '}' || char === ']') {
depth--;
} else if (char === ',' && depth === 0) {
params.push(current.trim());
current = '';
continue;
}
current += char;
}
if (current.trim()) {
params.push(current.trim());
}
return params
.map((param) => {
let name = param.trim();
// Skip destructured parameters
if (name.includes('{') || name.includes('[')) {
return null;
}
name = name.split('=')[0].trim(); // Remove default values: a = 5
name = name.split(':')[0].trim(); // Remove type annotations: a: number
if (name.startsWith('...')) {
return null; // Ignore rest operator: ...args
}
return name;
})
.filter((name): name is string => name != null && name !== '');
}
/**
* Aggregate token usage information from all spans in a trace.
*
* This function iterates through all spans and extracts token usage from the
* SpanAttributeKey.TOKEN_USAGE attribute. It avoids double-counting token usage
* when both parent and child spans have usage data (e.g., LangChain ChatOpenAI + OpenAI tracing).
*
* @param spans - Array of spans to aggregate usage from
* @returns Aggregated token usage or null if no usage data exists
*/
export function aggregateUsageFromSpans(spans: Span[]): TokenUsage | null {
const totalUsage: TokenUsage = {
input_tokens: 0,
output_tokens: 0,
total_tokens: 0,
};
let hasUsageData = false;
// Build parent-children map for DFS traversal (mirrors Python SDK)
const spanById = new Map<string, Span>();
const childrenMap = new Map<string, Span[]>();
const roots: Span[] = [];
for (const span of spans) {
spanById.set(span.spanId, span);
if (span.parentId && spanById.has(span.parentId)) {
const children = childrenMap.get(span.parentId) || [];
children.push(span);
childrenMap.set(span.parentId, children);
} else {
roots.push(span);
}
}
function dfs(span: Span, ancestorHasData: boolean): void {
const tokenUsageAttr = span.attributes[SpanAttributeKey.TOKEN_USAGE];
const spanHasData = tokenUsageAttr != null;
if (spanHasData && !ancestorHasData) {
const tokenUsage = tokenUsageAttr as TokenUsage;
totalUsage.input_tokens += tokenUsage.input_tokens || 0;
totalUsage.output_tokens += tokenUsage.output_tokens || 0;
totalUsage.total_tokens += tokenUsage.total_tokens || 0;
// Optional cache keys — only include when present
if (tokenUsage.cache_read_input_tokens != null) {
totalUsage.cache_read_input_tokens =
(totalUsage.cache_read_input_tokens || 0) + tokenUsage.cache_read_input_tokens;
}
if (tokenUsage.cache_creation_input_tokens != null) {
totalUsage.cache_creation_input_tokens =
(totalUsage.cache_creation_input_tokens || 0) + tokenUsage.cache_creation_input_tokens;
}
hasUsageData = true;
}
for (const child of childrenMap.get(span.spanId) || []) {
dfs(child, ancestorHasData || spanHasData);
}
}
for (const root of roots) {
dfs(root, false);
}
return hasUsageData ? totalUsage : null;
}
@@ -0,0 +1,25 @@
/**
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
*
* The repository contains a critical bug fix for decimal handling, however, it has not been
* released to npm yet. This file is a copy of the source code with the bug fix applied.
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
*
* :copyright: Copyright (c) 2013 Andrey Sidorov
* :license: The MIT License (MIT)
*/
// @ts-nocheck
var json_stringify = require('./stringify.js').stringify;
var json_parse = require('./parse.js');
module.exports = function (options) {
return {
parse: json_parse(options),
stringify: json_stringify,
};
};
//create the default method members with no options applied for backwards compatibility
module.exports.parse = json_parse();
module.exports.stringify = json_stringify;
@@ -0,0 +1,455 @@
/**
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
*
* The repository contains a critical bug fix for decimal handling, however, it has not been
* released to npm yet. This file is a copy of the source code with the bug fix applied.
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
*
* :copyright: Copyright (c) 2013 Andrey Sidorov
* :license: The MIT License (MIT)
*/
// @ts-nocheck
var BigNumber = null;
// regexpxs extracted from
// (c) BSD-3-Clause
// https://github.com/fastify/secure-json-parse/graphs/contributors and https://github.com/hapijs/bourne/graphs/contributors
const suspectProtoRx =
/(?:_|\\u005[Ff])(?:_|\\u005[Ff])(?:p|\\u0070)(?:r|\\u0072)(?:o|\\u006[Ff])(?:t|\\u0074)(?:o|\\u006[Ff])(?:_|\\u005[Ff])(?:_|\\u005[Ff])/;
const suspectConstructorRx =
/(?:c|\\u0063)(?:o|\\u006[Ff])(?:n|\\u006[Ee])(?:s|\\u0073)(?:t|\\u0074)(?:r|\\u0072)(?:u|\\u0075)(?:c|\\u0063)(?:t|\\u0074)(?:o|\\u006[Ff])(?:r|\\u0072)/;
/*
json_parse.js
2012-06-20
Public Domain.
NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.
This file creates a json_parse function.
During create you can (optionally) specify some behavioural switches
require('json-bigint')(options)
The optional options parameter holds switches that drive certain
aspects of the parsing process:
* options.strict = true will warn about duplicate-key usage in the json.
The default (strict = false) will silently ignore those and overwrite
values for keys that are in duplicate use.
The resulting function follows this signature:
json_parse(text, reviver)
This method parses a JSON text to produce an object or array.
It can throw a SyntaxError exception.
The optional reviver parameter is a function that can filter and
transform the results. It receives each of the keys and values,
and its return value is used instead of the original value.
If it returns what it received, then the structure is not modified.
If it returns undefined then the member is deleted.
Example:
// Parse the text. Values that look like ISO date strings will
// be converted to Date objects.
myData = json_parse(text, function (key, value) {
var a;
if (typeof value === 'string') {
a =
/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*)?)Z$/.exec(value);
if (a) {
return new Date(Date.UTC(+a[1], +a[2] - 1, +a[3], +a[4],
+a[5], +a[6]));
}
}
return value;
});
This is a reference implementation. You are free to copy, modify, or
redistribute.
This code should be minified before deployment.
See http://javascript.crockford.com/jsmin.html
USE YOUR OWN COPY. IT IS EXTREMELY UNWISE TO LOAD CODE FROM SERVERS YOU DO
NOT CONTROL.
*/
/*members "", "\"", "\/", "\\", at, b, call, charAt, f, fromCharCode,
hasOwnProperty, message, n, name, prototype, push, r, t, text
*/
var json_parse = function (options) {
'use strict';
// This is a function that can parse a JSON text, producing a JavaScript
// data structure. It is a simple, recursive descent parser. It does not use
// eval or regular expressions, so it can be used as a model for implementing
// a JSON parser in other languages.
// We are defining the function inside of another function to avoid creating
// global variables.
// Default options one can override by passing options to the parse()
var _options = {
strict: false, // not being strict means do not generate syntax errors for "duplicate key"
storeAsString: false, // toggles whether the values should be stored as BigNumber (default) or a string
alwaysParseAsBig: false, // toggles whether all numbers should be Big
useNativeBigInt: false, // toggles whether to use native BigInt instead of bignumber.js
protoAction: 'error',
constructorAction: 'error',
};
// If there are options, then use them to override the default _options
if (options !== undefined && options !== null) {
if (options.strict === true) {
_options.strict = true;
}
if (options.storeAsString === true) {
_options.storeAsString = true;
}
_options.alwaysParseAsBig =
options.alwaysParseAsBig === true ? options.alwaysParseAsBig : false;
_options.useNativeBigInt = options.useNativeBigInt === true ? options.useNativeBigInt : false;
if (typeof options.constructorAction !== 'undefined') {
if (
options.constructorAction === 'error' ||
options.constructorAction === 'ignore' ||
options.constructorAction === 'preserve'
) {
_options.constructorAction = options.constructorAction;
} else {
throw new Error(
`Incorrect value for constructorAction option, must be "error", "ignore" or undefined but passed ${options.constructorAction}`,
);
}
}
if (typeof options.protoAction !== 'undefined') {
if (
options.protoAction === 'error' ||
options.protoAction === 'ignore' ||
options.protoAction === 'preserve'
) {
_options.protoAction = options.protoAction;
} else {
throw new Error(
`Incorrect value for protoAction option, must be "error", "ignore" or undefined but passed ${options.protoAction}`,
);
}
}
}
var at, // The index of the current character
ch, // The current character
escapee = {
'"': '"',
'\\': '\\',
'/': '/',
b: '\b',
f: '\f',
n: '\n',
r: '\r',
t: '\t',
},
text,
error = function (m) {
// Call error when something is wrong.
throw {
name: 'SyntaxError',
message: m,
at: at,
text: text,
};
},
next = function (c) {
// If a c parameter is provided, verify that it matches the current character.
if (c && c !== ch) {
error("Expected '" + c + "' instead of '" + ch + "'");
}
// Get the next character. When there are no more characters,
// return the empty string.
ch = text.charAt(at);
at += 1;
return ch;
},
number = function () {
// Parse a number value.
var number,
string = '';
if (ch === '-') {
string = '-';
next('-');
}
while (ch >= '0' && ch <= '9') {
string += ch;
next();
}
if (ch === '.') {
string += '.';
while (next() && ch >= '0' && ch <= '9') {
string += ch;
}
}
if (ch === 'e' || ch === 'E') {
string += ch;
next();
if (ch === '-' || ch === '+') {
string += ch;
next();
}
while (ch >= '0' && ch <= '9') {
string += ch;
next();
}
}
number = +string;
if (!isFinite(number)) {
error('Bad number');
} else {
if (BigNumber == null) BigNumber = require('bignumber.js');
if (Number.isSafeInteger(number))
return !_options.alwaysParseAsBig
? number
: _options.useNativeBigInt
? BigInt(number)
: new BigNumber(number);
else
// Number with fractional part should be treated as number(double) including big integers in scientific notation, i.e 1.79e+308
return _options.storeAsString
? string
: /[\.eE]/.test(string)
? number
: _options.useNativeBigInt
? BigInt(string)
: new BigNumber(string);
}
},
string = function () {
// Parse a string value.
var hex,
i,
string = '',
uffff;
// When parsing for string values, we must look for " and \ characters.
if (ch === '"') {
var startAt = at;
while (next()) {
if (ch === '"') {
if (at - 1 > startAt) string += text.substring(startAt, at - 1);
next();
return string;
}
if (ch === '\\') {
if (at - 1 > startAt) string += text.substring(startAt, at - 1);
next();
if (ch === 'u') {
uffff = 0;
for (i = 0; i < 4; i += 1) {
hex = parseInt(next(), 16);
if (!isFinite(hex)) {
break;
}
uffff = uffff * 16 + hex;
}
string += String.fromCharCode(uffff);
} else if (typeof escapee[ch] === 'string') {
string += escapee[ch];
} else {
break;
}
startAt = at;
}
}
}
error('Bad string');
},
white = function () {
// Skip whitespace.
while (ch && ch <= ' ') {
next();
}
},
word = function () {
// true, false, or null.
switch (ch) {
case 't':
next('t');
next('r');
next('u');
next('e');
return true;
case 'f':
next('f');
next('a');
next('l');
next('s');
next('e');
return false;
case 'n':
next('n');
next('u');
next('l');
next('l');
return null;
}
error("Unexpected '" + ch + "'");
},
value, // Place holder for the value function.
array = function () {
// Parse an array value.
var array = [];
if (ch === '[') {
next('[');
white();
if (ch === ']') {
next(']');
return array; // empty array
}
while (ch) {
array.push(value());
white();
if (ch === ']') {
next(']');
return array;
}
next(',');
white();
}
}
error('Bad array');
},
object = function () {
// Parse an object value.
var key,
object = Object.create(null);
if (ch === '{') {
next('{');
white();
if (ch === '}') {
next('}');
return object; // empty object
}
while (ch) {
key = string();
white();
next(':');
if (_options.strict === true && Object.hasOwnProperty.call(object, key)) {
error('Duplicate key "' + key + '"');
}
if (suspectProtoRx.test(key) === true) {
if (_options.protoAction === 'error') {
error('Object contains forbidden prototype property');
} else if (_options.protoAction === 'ignore') {
value();
} else {
object[key] = value();
}
} else if (suspectConstructorRx.test(key) === true) {
if (_options.constructorAction === 'error') {
error('Object contains forbidden constructor property');
} else if (_options.constructorAction === 'ignore') {
value();
} else {
object[key] = value();
}
} else {
object[key] = value();
}
white();
if (ch === '}') {
next('}');
return object;
}
next(',');
white();
}
}
error('Bad object');
};
value = function () {
// Parse a JSON value. It could be an object, an array, a string, a number,
// or a word.
white();
switch (ch) {
case '{':
return object();
case '[':
return array();
case '"':
return string();
case '-':
return number();
default:
return ch >= '0' && ch <= '9' ? number() : word();
}
};
// Return the json_parse function. It will have access to all of the above
// functions and variables.
return function (source, reviver) {
var result;
text = source + '';
at = 0;
ch = ' ';
result = value();
white();
if (ch) {
error('Syntax error');
}
// If there is a reviver function, we recursively walk the new structure,
// passing each name/value pair to the reviver function for possible
// transformation, starting with a temporary root object that holds the result
// in an empty key. If there is not a reviver function, we simply return the
// result.
return typeof reviver === 'function'
? (function walk(holder, key) {
var k,
v,
value = holder[key];
if (value && typeof value === 'object') {
Object.keys(value).forEach(function (k) {
v = walk(value, k);
if (v !== undefined) {
value[k] = v;
} else {
delete value[k];
}
});
}
return reviver.call(holder, key, value);
})({ '': result }, '')
: result;
};
};
module.exports = json_parse;
@@ -0,0 +1,395 @@
/**
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
*
* The repository contains a critical bug fix for decimal handling, however, it has not been
* released to npm yet. This file is a copy of the source code with the bug fix applied.
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
*
* :copyright: Copyright (c) 2013 Andrey Sidorov
* :license: The MIT License (MIT)
*/
// @ts-nocheck
var BigNumber = require('bignumber.js');
/*
json2.js
2013-05-26
Public Domain.
NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.
See http://www.JSON.org/js.html
This code should be minified before deployment.
See http://javascript.crockford.com/jsmin.html
USE YOUR OWN COPY. IT IS EXTREMELY UNWISE TO LOAD CODE FROM SERVERS YOU DO
NOT CONTROL.
This file creates a global JSON object containing two methods: stringify
and parse.
JSON.stringify(value, replacer, space)
value any JavaScript value, usually an object or array.
replacer an optional parameter that determines how object
values are stringified for objects. It can be a
function or an array of strings.
space an optional parameter that specifies the indentation
of nested structures. If it is omitted, the text will
be packed without extra whitespace. If it is a number,
it will specify the number of spaces to indent at each
level. If it is a string (such as '\t' or '&nbsp;'),
it contains the characters used to indent at each level.
This method produces a JSON text from a JavaScript value.
When an object value is found, if the object contains a toJSON
method, its toJSON method will be called and the result will be
stringified. A toJSON method does not serialize: it returns the
value represented by the name/value pair that should be serialized,
or undefined if nothing should be serialized. The toJSON method
will be passed the key associated with the value, and this will be
bound to the value
For example, this would serialize Dates as ISO strings.
Date.prototype.toJSON = function (key) {
function f(n) {
// Format integers to have at least two digits.
return n < 10 ? '0' + n : n;
}
return this.getUTCFullYear() + '-' +
f(this.getUTCMonth() + 1) + '-' +
f(this.getUTCDate()) + 'T' +
f(this.getUTCHours()) + ':' +
f(this.getUTCMinutes()) + ':' +
f(this.getUTCSeconds()) + 'Z';
};
You can provide an optional replacer method. It will be passed the
key and value of each member, with this bound to the containing
object. The value that is returned from your method will be
serialized. If your method returns undefined, then the member will
be excluded from the serialization.
If the replacer parameter is an array of strings, then it will be
used to select the members to be serialized. It filters the results
such that only members with keys listed in the replacer array are
stringified.
Values that do not have JSON representations, such as undefined or
functions, will not be serialized. Such values in objects will be
dropped; in arrays they will be replaced with null. You can use
a replacer function to replace those with JSON values.
JSON.stringify(undefined) returns undefined.
The optional space parameter produces a stringification of the
value that is filled with line breaks and indentation to make it
easier to read.
If the space parameter is a non-empty string, then that string will
be used for indentation. If the space parameter is a number, then
the indentation will be that many spaces.
Example:
text = JSON.stringify(['e', {pluribus: 'unum'}]);
// text is '["e",{"pluribus":"unum"}]'
text = JSON.stringify(['e', {pluribus: 'unum'}], null, '\t');
// text is '[\n\t"e",\n\t{\n\t\t"pluribus": "unum"\n\t}\n]'
text = JSON.stringify([new Date()], function (key, value) {
return this[key] instanceof Date ?
'Date(' + this[key] + ')' : value;
});
// text is '["Date(---current time---)"]'
JSON.parse(text, reviver)
This method parses a JSON text to produce an object or array.
It can throw a SyntaxError exception.
The optional reviver parameter is a function that can filter and
transform the results. It receives each of the keys and values,
and its return value is used instead of the original value.
If it returns what it received, then the structure is not modified.
If it returns undefined then the member is deleted.
Example:
// Parse the text. Values that look like ISO date strings will
// be converted to Date objects.
myData = JSON.parse(text, function (key, value) {
var a;
if (typeof value === 'string') {
a =
/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*)?)Z$/.exec(value);
if (a) {
return new Date(Date.UTC(+a[1], +a[2] - 1, +a[3], +a[4],
+a[5], +a[6]));
}
}
return value;
});
myData = JSON.parse('["Date(09/09/2001)"]', function (key, value) {
var d;
if (typeof value === 'string' &&
value.slice(0, 5) === 'Date(' &&
value.slice(-1) === ')') {
d = new Date(value.slice(5, -1));
if (d) {
return d;
}
}
return value;
});
This is a reference implementation. You are free to copy, modify, or
redistribute.
*/
/*jslint evil: true, regexp: true */
/*members "", "\b", "\t", "\n", "\f", "\r", "\"", JSON, "\\", apply,
call, charCodeAt, getUTCDate, getUTCFullYear, getUTCHours,
getUTCMinutes, getUTCMonth, getUTCSeconds, hasOwnProperty, join,
lastIndex, length, parse, prototype, push, replace, slice, stringify,
test, toJSON, toString, valueOf
*/
// Create a JSON object only if one does not already exist. We create the
// methods in a closure to avoid creating global variables.
var JSON = module.exports;
(function () {
'use strict';
function f(n) {
// Format integers to have at least two digits.
return n < 10 ? '0' + n : n;
}
var cx =
/[\u0000\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
escapable =
/[\\\"\x00-\x1f\x7f-\x9f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
gap,
indent,
meta = {
// table of character substitutions
'\b': '\\b',
'\t': '\\t',
'\n': '\\n',
'\f': '\\f',
'\r': '\\r',
'"': '\\"',
'\\': '\\\\',
},
rep;
function quote(string) {
// If the string contains no control characters, no quote characters, and no
// backslash characters, then we can safely slap some quotes around it.
// Otherwise we must also replace the offending characters with safe escape
// sequences.
escapable.lastIndex = 0;
return escapable.test(string)
? '"' +
string.replace(escapable, function (a) {
var c = meta[a];
return typeof c === 'string'
? c
: '\\u' + ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
}) +
'"'
: '"' + string + '"';
}
function str(key, holder) {
// Produce a string from holder[key].
var i, // The loop counter.
k, // The member key.
v, // The member value.
length,
mind = gap,
partial,
value = holder[key],
isBigNumber = value != null && (value instanceof BigNumber || BigNumber.isBigNumber(value));
// If the value has a toJSON method, call it to obtain a replacement value.
if (value && typeof value === 'object' && typeof value.toJSON === 'function') {
value = value.toJSON(key);
}
// If we were called with a replacer function, then call the replacer to
// obtain a replacement value.
if (typeof rep === 'function') {
value = rep.call(holder, key, value);
}
// What happens next depends on the value's type.
switch (typeof value) {
case 'string':
if (isBigNumber) {
return value;
} else {
return quote(value);
}
case 'number':
// JSON numbers must be finite. Encode non-finite numbers as null.
return isFinite(value) ? String(value) : 'null';
case 'boolean':
case 'null':
case 'bigint':
// If the value is a boolean or null, convert it to a string. Note:
// typeof null does not produce 'null'. The case is included here in
// the remote chance that this gets fixed someday.
return String(value);
// If the type is 'object', we might be dealing with an object or an array or
// null.
case 'object':
// Due to a specification blunder in ECMAScript, typeof null is 'object',
// so watch out for that case.
if (!value) {
return 'null';
}
// Make an array to hold the partial results of stringifying this object value.
gap += indent;
partial = [];
// Is the value an array?
if (Object.prototype.toString.apply(value) === '[object Array]') {
// The value is an array. Stringify every element. Use null as a placeholder
// for non-JSON values.
length = value.length;
for (i = 0; i < length; i += 1) {
partial[i] = str(i, value) || 'null';
}
// Join all of the elements together, separated with commas, and wrap them in
// brackets.
v =
partial.length === 0
? '[]'
: gap
? '[\n' + gap + partial.join(',\n' + gap) + '\n' + mind + ']'
: '[' + partial.join(',') + ']';
gap = mind;
return v;
}
// If the replacer is an array, use it to select the members to be stringified.
if (rep && typeof rep === 'object') {
length = rep.length;
for (i = 0; i < length; i += 1) {
if (typeof rep[i] === 'string') {
k = rep[i];
v = str(k, value);
if (v) {
partial.push(quote(k) + (gap ? ': ' : ':') + v);
}
}
}
} else {
// Otherwise, iterate through all of the keys in the object.
Object.keys(value).forEach(function (k) {
var v = str(k, value);
if (v) {
partial.push(quote(k) + (gap ? ': ' : ':') + v);
}
});
}
// Join all of the member texts together, separated with commas,
// and wrap them in braces.
v =
partial.length === 0
? '{}'
: gap
? '{\n' + gap + partial.join(',\n' + gap) + '\n' + mind + '}'
: '{' + partial.join(',') + '}';
gap = mind;
return v;
}
}
// If the JSON object does not yet have a stringify method, give it one.
if (typeof JSON.stringify !== 'function') {
JSON.stringify = function (value, replacer, space) {
// The stringify method takes a value and an optional replacer, and an optional
// space parameter, and returns a JSON text. The replacer can be a function
// that can replace values, or an array of strings that will select the keys.
// A default replacer method can be provided. Use of the space parameter can
// produce text that is more easily readable.
var i;
gap = '';
indent = '';
// If the space parameter is a number, make an indent string containing that
// many spaces.
if (typeof space === 'number') {
for (i = 0; i < space; i += 1) {
indent += ' ';
}
// If the space parameter is a string, it will be used as the indent string.
} else if (typeof space === 'string') {
indent = space;
}
// If there is a replacer, it must be a function or an array.
// Otherwise, throw an error.
rep = replacer;
if (
replacer &&
typeof replacer !== 'function' &&
(typeof replacer !== 'object' || typeof replacer.length !== 'number')
) {
throw new Error('JSON.stringify');
}
// Make a fake root object containing our value under the key of ''.
// Return the result of stringifying the value.
return str('', { '': value });
};
}
})();
@@ -0,0 +1,52 @@
import fastStringify from 'fast-safe-stringify';
// Configure json-bigint to handle large integers
type JSON = {
parse: (text: string) => any;
stringify: (value: any) => string;
};
// eslint-disable-next-line @typescript-eslint/no-var-requires
const JSONBigInt = require('./json-bigint/index.js');
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const JSONBig: JSON = JSONBigInt({
useNativeBigInt: true,
alwaysParseAsBig: false,
storeAsString: false,
});
/**
* Safely stringify a value to JSON, handling circular references and non-serializable objects.
* Uses fast-safe-stringify for optimal performance and reliability.
*
* @param value The value to stringify
* @returns JSON string representation of the value
*/
export function safeJsonStringify(value: any): string {
// MLflow-specific replacer that handles functions, undefined, and errors
const mlflowReplacer = (_key: string, val: any): any => {
if (typeof val === 'function') {
return '[Function]';
}
if (val === undefined) {
return '[Undefined]';
}
if (val instanceof Error) {
return {
name: val.name,
message: val.message,
stack: val.stack,
};
}
return val;
};
// Use fast-safe-stringify with our MLflow replacer
// The circular reference detection is handled by fast-safe-stringify itself
return fastStringify(value, mlflowReplacer);
}
export { JSONBig };
@@ -0,0 +1,39 @@
import { TRACE_ID_PREFIX, TRACE_ID_V4_PREFIX } from '../constants';
/**
* Construct a V3-schema MLflow trace ID from an OTel hex trace ID.
* Format: `tr-<hex_trace_id>`.
*/
export function generateTraceIdV3(otelHexTraceId: string): string {
return TRACE_ID_PREFIX + otelHexTraceId;
}
/**
* Construct a V4-schema MLflow trace ID for the given UC location and
* OTel hex trace ID. Format: `trace:/<location>/<hex_trace_id>`.
*/
export function constructTraceIdV4(location: string, otelHexTraceId: string): string {
return `${TRACE_ID_V4_PREFIX}${location}/${otelHexTraceId}`;
}
/**
* Parse an MLflow trace ID. For V4 IDs returns `[location, otelHexTraceId]`;
* for V3 IDs returns `[null, raw_id]`. Mirrors Python's `parse_trace_id_v4`.
*/
export function parseTraceIdV4(traceId: string | null | undefined): [string | null, string | null] {
if (!traceId) {
return [null, null];
}
if (traceId.startsWith(TRACE_ID_V4_PREFIX)) {
const rest = traceId.slice(TRACE_ID_V4_PREFIX.length);
const slash = rest.indexOf('/');
if (slash <= 0 || slash === rest.length - 1) {
throw new Error(
`Invalid trace ID format: ${traceId}. ` +
`Expected format: ${TRACE_ID_V4_PREFIX}<location>/<trace_id>`,
);
}
return [rest.slice(0, slash), rest.slice(slash + 1)];
}
return [null, traceId];
}
@@ -0,0 +1,245 @@
import { Trace } from '../core/entities/trace';
import { ExportResult } from '@opentelemetry/core';
import {
Span as OTelSpan,
SpanProcessor,
ReadableSpan as OTelReadableSpan,
SpanExporter,
} from '@opentelemetry/sdk-trace-base';
import { Context } from '@opentelemetry/api';
import { createAndRegisterMlflowSpan } from '../core/api';
import { getConfiguredTraceMetadata, getConfiguredTraceTags } from '../core/context';
import { InMemoryTraceManager } from '../core/trace_manager';
import { TraceInfo } from '../core/entities/trace_info';
import { createTraceLocationFromExperimentId } from '../core/entities/trace_location';
import { fromOtelStatus, TraceState } from '../core/entities/trace_state';
import {
SpanAttributeKey,
TRACE_ID_PREFIX,
TRACE_SCHEMA_VERSION,
TraceMetadataKey,
} from '../core/constants';
import { convertHrTimeToMs, aggregateUsageFromSpans } from '../core/utils';
import { getConfig } from '../core/config';
import { MlflowClient } from '../clients';
import { executeOnSpanEndHooks, executeOnSpanStartHooks } from './span_processor_hooks';
/**
* Generate a MLflow-compatible trace ID for the given span.
* @param span The span to generate the trace ID for
*/
function generateTraceId(span: OTelSpan): string {
// NB: trace Id is already hex string in Typescript OpenTelemetry SDK
return TRACE_ID_PREFIX + span.spanContext().traceId;
}
export class MlflowSpanProcessor implements SpanProcessor {
private _exporter: SpanExporter;
constructor(exporter: SpanExporter) {
this._exporter = exporter;
}
/**
* Called when a {@link Span} is started, if the `span.isRecording()`
* returns true.
* @param span the Span that just started.
*/
onStart(span: OTelSpan, _parentContext: Context): void {
const otelTraceId = span.spanContext().traceId;
let traceId: string;
const experimentId = getConfig().experimentId;
if (!span.parentSpanContext?.spanId) {
// This is a root span
traceId = generateTraceId(span);
// Build trace metadata, merging context-injected values
const traceMetadata: Record<string, string> = {
[TraceMetadataKey.SCHEMA_VERSION]: TRACE_SCHEMA_VERSION,
};
const ctxMetadata = getConfiguredTraceMetadata();
if (ctxMetadata) {
Object.assign(traceMetadata, ctxMetadata);
}
// Build trace tags, merging context-injected values
const tags: Record<string, string> = {};
const ctxTags = getConfiguredTraceTags();
if (ctxTags) {
Object.assign(tags, ctxTags);
}
const trace_info = new TraceInfo({
traceId: traceId,
traceLocation: createTraceLocationFromExperimentId(experimentId),
requestTime: convertHrTimeToMs(span.startTime),
executionDuration: 0,
state: TraceState.IN_PROGRESS,
traceMetadata,
tags,
assessments: [],
});
InMemoryTraceManager.getInstance().registerTrace(otelTraceId, trace_info);
} else {
traceId = InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId) || '';
if (!traceId) {
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
return;
}
}
// Set trace ID to the span
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
createAndRegisterMlflowSpan(span);
executeOnSpanStartHooks(span);
}
/**
* Called when a {@link ReadableSpan} is ended, if the `span.isRecording()`
* returns true.
* @param span the Span that just ended.
*/
onEnd(span: OTelReadableSpan): void {
const traceManager = InMemoryTraceManager.getInstance();
executeOnSpanEndHooks(span);
// Only trigger trace export for root span completion
if (span.parentSpanContext?.spanId) {
return;
}
// Update trace info
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
if (!traceId) {
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
return;
}
const trace = InMemoryTraceManager.getInstance().getTrace(traceId);
if (!trace) {
console.warn(`No trace found for span ${span.name}. Skipping.`);
return;
}
this.updateTraceInfo(trace.info, span);
// Aggregate token usage from all spans and add to trace metadata
const allSpans = Array.from(trace.spanDict.values());
const aggregatedUsage = aggregateUsageFromSpans(allSpans);
if (aggregatedUsage) {
trace.info.traceMetadata[TraceMetadataKey.TOKEN_USAGE] = JSON.stringify(aggregatedUsage);
}
this._exporter.export([span], (_) => {});
}
/**
* Update the trace info with the span end time and status.
* @param trace The trace to update
* @param span The span to update the trace with
*/
updateTraceInfo(traceInfo: TraceInfo, span: OTelReadableSpan): void {
traceInfo.executionDuration = convertHrTimeToMs(span.endTime) - traceInfo.requestTime;
let state = fromOtelStatus(span.status.code);
// NB: In OpenTelemetry, status code remains UNSET if not explicitly set
// by the user. However, there is no way to set the status when using
// `trace` function wrapper. Therefore, we just automatically set the status
// to OK if it is not ERROR.
if (state === TraceState.STATE_UNSPECIFIED) {
state = TraceState.OK;
}
traceInfo.state = state;
}
/**
* Shuts down the processor. Called when SDK is shut down. This is an
* opportunity for processor to do any cleanup required.
*/
async shutdown() {
await this._exporter.shutdown();
}
/**
* Forces to export all finished spans
*/
async forceFlush() {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
await this._exporter.forceFlush!();
}
}
export class MlflowSpanExporter implements SpanExporter {
private _client: MlflowClient;
private _pendingExports: Record<string, Promise<void>> = {}; // traceId -> export promise
constructor(client: MlflowClient) {
this._client = client;
}
export(spans: OTelReadableSpan[], _resultCallback: (result: ExportResult) => void): void {
for (const span of spans) {
// Only export root spans
if (span.parentSpanContext?.spanId) {
continue;
}
const traceManager = InMemoryTraceManager.getInstance();
const trace = traceManager.popTrace(span.spanContext().traceId);
if (!trace) {
console.warn(`No trace found for span ${span.name}. Skipping.`);
continue;
}
// Set the last active trace ID
traceManager.lastActiveTraceId = trace.info.traceId;
// Export trace to backend and track the promise
const exportPromise = this.exportTraceToBackend(trace).catch((error) => {
console.error(`Failed to export trace ${trace.info.traceId}:`, error);
});
this._pendingExports[trace.info.traceId] = exportPromise;
}
}
/**
* Export a complete trace to the MLflow backend
* Step 1: Create trace metadata via StartTraceV3 endpoint
* Step 2: Upload trace data (spans) via artifact repository pattern
*/
private async exportTraceToBackend(trace: Trace): Promise<void> {
try {
// Step 1: Create trace metadata in backend
const traceInfo = await this._client.createTrace(trace.info);
// Step 2: Upload trace data (spans) to artifact storage
await this._client.uploadTraceData(traceInfo, trace.data);
} catch (error) {
console.error(`Failed to export trace ${trace.info.traceId}:`, error);
throw error;
} finally {
// Remove the promise from the pending exports
delete this._pendingExports[trace.info.traceId];
}
}
/**
* Force flush all pending trace exports.
* Waits for all async export operations to complete.
*/
async forceFlush(): Promise<void> {
await Promise.all(Object.values(this._pendingExports));
this._pendingExports = {};
}
/**
* Shutdown the exporter.
* Waits for all pending exports to complete before shutting down.
*/
async shutdown(): Promise<void> {
await this.forceFlush();
}
}
@@ -0,0 +1,76 @@
import { Span as OTelSpan, ReadableSpan as OTelReadableSpan } from '@opentelemetry/sdk-trace-base';
import { LiveSpan } from '../core/entities/span';
import { InMemoryTraceManager } from '../core/trace_manager';
/**
* Hooks to be executed by the span processor.
* Primary used for adding custom processing to the span for autologging integrations.
*/
type OnSpanStartHook = (span: LiveSpan) => void;
type OnSpanEndHook = (span: LiveSpan) => void;
const onSpanStartHooks: Set<OnSpanStartHook> = new Set();
const onSpanEndHooks: Set<OnSpanEndHook> = new Set();
export function registerOnSpanStartHook(hook: OnSpanStartHook): void {
onSpanStartHooks.add(hook);
}
export function getOnSpanStartHooks(): OnSpanStartHook[] {
return Array.from(onSpanStartHooks);
}
export function registerOnSpanEndHook(hook: OnSpanEndHook): void {
onSpanEndHooks.add(hook);
}
export function getOnSpanEndHooks(): OnSpanEndHook[] {
return Array.from(onSpanEndHooks);
}
export function executeOnSpanStartHooks(span: OTelSpan): void {
// Execute onStart hooks for autologging integrations
const hooks = getOnSpanStartHooks();
if (hooks.length === 0) {
return;
}
const mlflowSpan = getMlflowSpan(span);
if (!mlflowSpan) {
return;
}
for (const hook of hooks) {
try {
hook(mlflowSpan);
} catch (error) {
console.debug('Error executing onStart hook:', error);
}
}
}
export function executeOnSpanEndHooks(span: OTelReadableSpan): void {
const hooks = getOnSpanEndHooks();
if (hooks.length === 0) {
return;
}
const mlflowSpan = getMlflowSpan(span);
if (!mlflowSpan) {
return;
}
for (const hook of hooks) {
try {
hook(mlflowSpan);
} catch (error) {
console.debug('Error executing onEnd hook:', error);
}
}
}
function getMlflowSpan(span: OTelSpan | OTelReadableSpan): LiveSpan | null {
const traceManager = InMemoryTraceManager.getInstance();
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
return traceManager.getSpan(traceId, span.spanContext().spanId);
}
@@ -0,0 +1,276 @@
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
import { Context } from '@opentelemetry/api';
import {
Span as OTelSpan,
ReadableSpan as OTelReadableSpan,
SpanProcessor,
SpanExporter,
} from '@opentelemetry/sdk-trace-base';
import { createAndRegisterMlflowSpan } from '../core/api';
import { MlflowClient } from '../clients';
import { SpanAttributeKey, TraceMetadataKey } from '../core/constants';
import { getConfiguredTraceMetadata, getConfiguredTraceTags } from '../core/context';
import { TraceInfo } from '../core/entities/trace_info';
import {
TraceLocation,
TraceLocationType,
UnityCatalogLocation,
getOtelSpansTableName,
getUcLocationString,
} from '../core/entities/trace_location';
import { Trace } from '../core/entities/trace';
import { fromOtelStatus, TraceState } from '../core/entities/trace_state';
import { InMemoryTraceManager } from '../core/trace_manager';
import { constructTraceIdV4, parseTraceIdV4 } from '../core/utils/trace_id';
import { aggregateUsageFromSpans, convertHrTimeToMs } from '../core/utils';
import { executeOnSpanEndHooks, executeOnSpanStartHooks } from './span_processor_hooks';
/**
* Span processor for Databricks Unity Catalog backed traces.
*
* Mirrors Python's `DatabricksUCTableSpanProcessor`:
* - generates V4 trace IDs (`trace:/<location>/<hex>`)
* - constructs TraceInfo with the UC trace location, V4 schema version, and
* any context-injected metadata/tags so that `updateCurrentTrace({ tags })`
* can mutate the in-memory record before export
* - delegates to a UC-aware SpanExporter for trace info + OTLP span upload
*
* Wired up by `init({ trackingUri, experimentId })` when the linked Databricks
* experiment carries `mlflow.experiment.databricksTrace*` tags pointing at a UC
* destination.
*/
export class DatabricksUCTableSpanProcessor implements SpanProcessor {
private _exporter: SpanExporter;
private _location: UnityCatalogLocation;
constructor(exporter: SpanExporter, location: UnityCatalogLocation) {
this._exporter = exporter;
this._location = location;
}
onStart(span: OTelSpan, _parentContext: Context): void {
const otelTraceId = span.spanContext().traceId;
if (!span.parentSpanContext?.spanId) {
// Root span: build the V4 trace ID and TraceInfo for this trace.
const traceLocation: TraceLocation = {
type: TraceLocationType.UC_TABLE_PREFIX,
ucTablePrefix: { ...this._location },
};
const locationString = getUcLocationString(traceLocation);
if (!locationString) {
console.warn(
`Unable to derive UC location string for ${JSON.stringify(this._location)}; skipping trace registration.`,
);
return;
}
const traceId = constructTraceIdV4(locationString, otelTraceId);
const traceMetadata: Record<string, string> = {
[TraceMetadataKey.SCHEMA_VERSION]: '4',
};
const ctxMetadata = getConfiguredTraceMetadata();
if (ctxMetadata) {
Object.assign(traceMetadata, ctxMetadata);
}
const tags: Record<string, string> = {};
const ctxTags = getConfiguredTraceTags();
if (ctxTags) {
Object.assign(tags, ctxTags);
}
const traceInfo = new TraceInfo({
traceId,
traceLocation,
requestTime: convertHrTimeToMs(span.startTime),
executionDuration: 0,
state: TraceState.IN_PROGRESS,
traceMetadata,
tags,
assessments: [],
});
InMemoryTraceManager.getInstance().registerTrace(otelTraceId, traceInfo);
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
} else {
const traceId = InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId);
if (!traceId) {
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
return;
}
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
}
createAndRegisterMlflowSpan(span);
executeOnSpanStartHooks(span);
}
onEnd(span: OTelReadableSpan): void {
const traceManager = InMemoryTraceManager.getInstance();
executeOnSpanEndHooks(span);
if (span.parentSpanContext?.spanId) {
return;
}
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
if (!traceId) {
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
return;
}
const trace = traceManager.getTrace(traceId);
if (!trace) {
console.warn(`No trace found for span ${span.name}. Skipping.`);
return;
}
this.updateTraceInfo(trace.info, span);
const allSpans = Array.from(trace.spanDict.values());
const aggregatedUsage = aggregateUsageFromSpans(allSpans);
if (aggregatedUsage) {
trace.info.traceMetadata[TraceMetadataKey.TOKEN_USAGE] = JSON.stringify(aggregatedUsage);
}
this._exporter.export([span], (_) => {});
}
private updateTraceInfo(traceInfo: TraceInfo, span: OTelReadableSpan): void {
traceInfo.executionDuration = convertHrTimeToMs(span.endTime) - traceInfo.requestTime;
let state = fromOtelStatus(span.status.code);
if (state === TraceState.STATE_UNSPECIFIED) {
state = TraceState.OK;
}
traceInfo.state = state;
}
async shutdown(): Promise<void> {
await this._exporter.shutdown();
}
async forceFlush(): Promise<void> {
if (typeof this._exporter.forceFlush === 'function') {
await this._exporter.forceFlush();
}
}
}
/**
* Exporter for Databricks Unity Catalog backed traces.
*
* Mirrors Python's `DatabricksUCTableSpanExporter`. Two-step export per trace:
* 1. Call the V4 `CreateTraceInfo` REST endpoint with the full `TraceInfo`
* (including `tags`, `trace_metadata`, state, durations). This is what
* makes `updateCurrentTrace({ tags })` persist as trace-level tags in
* `_traces_unified`, `get_trace`, `search_traces`, and the UI.
* 2. Upload the OTel spans for this trace to `/api/2.0/otel/v1/traces` with
* the `X-Databricks-UC-Table-Name` header pointing to the trace's UC
* spans table.
*
* Errors in either step are logged but do not surface to user code, matching
* the non-fatal behavior of `_log_spans` on the Python side.
*/
export class DatabricksUCTableSpanExporter implements SpanExporter {
private _client: MlflowClient;
private _pendingExports: Record<string, Promise<void>> = {};
private _hasRaisedMissingSpansTableWarning = false;
private _hasRaisedSpanExportError = false;
constructor(client: MlflowClient) {
this._client = client;
}
export(spans: OTelReadableSpan[], resultCallback: (result: ExportResult) => void): void {
for (const span of spans) {
if (span.parentSpanContext?.spanId) {
continue;
}
const traceManager = InMemoryTraceManager.getInstance();
const trace = traceManager.popTrace(span.spanContext().traceId);
if (!trace) {
console.warn(`No trace found for span ${span.name}. Skipping.`);
continue;
}
traceManager.lastActiveTraceId = trace.info.traceId;
const exportPromise = this.exportTraceToBackend(trace).catch((error) => {
console.error(`Failed to export UC trace ${trace.info.traceId}:`, error);
});
this._pendingExports[trace.info.traceId] = exportPromise;
}
// Fire-and-forget: backend errors are logged inside exportTraceToBackend
// and intentionally not surfaced (matches Python `_log_spans`). Resolve
// the callback synchronously so the SpanExporter contract is honored.
resultCallback({ code: ExportResultCode.SUCCESS });
}
private async exportTraceToBackend(trace: Trace): Promise<void> {
try {
const [location, otelTraceId] = parseTraceIdV4(trace.info.traceId);
if (!location || !otelTraceId) {
throw new Error(
`UC exporter received non-V4 trace ID ${trace.info.traceId}. This indicates the ` +
`processor and exporter destinations are out of sync.`,
);
}
// Step 1: persist the trace info (tags, metadata, state).
const returnedTraceInfo = await this._client.createTraceInfoV4(
location,
otelTraceId,
trace.info,
);
// The backend may populate the spans table name on the returned trace
// location. Use that when present; fall back to the location we sent.
const spansTable =
getOtelSpansTableName(returnedTraceInfo.traceLocation) ??
getOtelSpansTableName(trace.info.traceLocation);
if (!spansTable) {
// Without a spans table we can't upload spans, but the trace info
// (including tags) is already persisted in step 1.
if (!this._hasRaisedMissingSpansTableWarning) {
console.warn(
`No OTel spans table resolved for UC trace ${trace.info.traceId}; ` +
`spans will not be exported. Tags and metadata are still persisted.`,
);
this._hasRaisedMissingSpansTableWarning = true;
}
return;
}
// Step 2: upload spans via OTLP.
try {
await this._client.exportOtlpSpansToUc(
trace.data.spans.map((s) => s._span),
spansTable,
);
} catch (error) {
if (!this._hasRaisedSpanExportError) {
console.error(`Failed to export UC spans for ${trace.info.traceId}:`, error);
this._hasRaisedSpanExportError = true;
}
}
} finally {
delete this._pendingExports[trace.info.traceId];
}
}
async forceFlush(): Promise<void> {
// Each export self-removes from `_pendingExports` in its `finally` block.
// Loop so exports started during the await are still awaited, instead of
// resetting the map (which would silently drop new in-flight exports and
// make `shutdown()` lose track of them).
while (Object.keys(this._pendingExports).length > 0) {
await Promise.all(Object.values(this._pendingExports));
}
}
async shutdown(): Promise<void> {
await this.forceFlush();
}
}
@@ -0,0 +1,108 @@
/**
* Group-commit writer for WAL lines.
*/
import { open, FileHandle } from 'node:fs/promises';
import { JSONBig } from '../../core/utils/json';
import { getWalPath } from './paths';
import { ensureParentDir, queueWriter } from './storage';
import { WalLine, WalRecord } from './types';
interface BatchItem {
line: WalLine;
resolve: () => void;
reject: (err: Error) => void;
}
export class BatchingWriter {
private pending: BatchItem[] = [];
private flushScheduled = false;
/**
* Enqueue an append for `record`. Resolves once the record (and its
* batch-mates) have been fsynced to `queue.log`.
*/
submit(record: WalRecord): Promise<void> {
return this.enqueue({ type: 'append', record });
}
/**
* Enqueue a tombstone for `id`. Resolves once the tombstone (and its
* batch-mates) have been fsynced to `queue.log`.
*/
submitTombstone(id: string): Promise<void> {
return this.enqueue({ type: 'tombstone', id });
}
private enqueue(line: WalLine): Promise<void> {
return new Promise<void>((resolve, reject) => {
this.pending.push({ line, resolve, reject });
this.scheduleFlush();
});
}
private scheduleFlush(): void {
if (this.flushScheduled) {
return;
}
this.flushScheduled = true;
// setImmediate fires at the end of the current event-loop tick,
// so every `submit` call made within the same tick batches
// together.
setImmediate(() => {
this.flushScheduled = false;
if (this.pending.length === 0) {
return;
}
void queueWriter.run(() => this.drainAndFlush());
});
}
private async drainAndFlush(): Promise<void> {
const batch = this.pending;
this.pending = [];
if (batch.length === 0) {
return;
}
const path = getWalPath();
try {
await ensureParentDir(path);
} catch (err) {
const error = err as Error;
for (const item of batch) {
item.reject(error);
}
return;
}
let fh: FileHandle;
try {
fh = await open(path, 'a');
} catch (err) {
const error = err as Error;
for (const item of batch) {
item.reject(error);
}
return;
}
try {
const buffers = batch.map((item) => Buffer.from(JSONBig.stringify(item.line) + '\n', 'utf8'));
await fh.writev(buffers);
await fh.sync();
} catch (err) {
const error = err as Error;
for (const item of batch) {
item.reject(error);
}
await fh.close().catch(() => {});
return;
}
await fh.close().catch(() => {});
for (const item of batch) {
item.resolve();
}
}
}
@@ -0,0 +1,43 @@
/**
* Daemon-side `MlflowClient` cache.
*/
import { createAuthProvider } from '../../auth';
import { MlflowClient } from '../../clients/client';
const cache = new Map<string, MlflowClient>();
/**
* Return the cached `MlflowClient` for `trackingUri`, constructing it
* lazily on first access. Subsequent calls with the same URI return the
* same instance unless an explicit {@link clearClientForUri} happened
* in between (typically driven by the upload loop's auth-error retry).
*/
export function clientForUri(trackingUri: string): MlflowClient {
const cached = cache.get(trackingUri);
if (cached !== undefined) {
return cached;
}
const authProvider = createAuthProvider({ trackingUri });
const client = new MlflowClient({ trackingUri, authProvider });
cache.set(trackingUri, client);
return client;
}
/**
* Drop the cached client for `trackingUri` only, leaving other URIs'
* entries intact. Called by the upload loop on 401 / 403 so the next
* `clientForUri(uri)` call rebuilds the client with freshly-resolved
* credentials. No-op if no entry exists for `trackingUri`.
*/
export function clearClientForUri(trackingUri: string): void {
cache.delete(trackingUri);
}
/**
* Drop all cached clients. Intended for daemon shutdown paths and
* tests; not part of the steady-state batch loop.
*/
export function clearClientCache(): void {
cache.clear();
}
@@ -0,0 +1,35 @@
/**
* Tunable defaults for the WAL exporter and uploader daemon.
*/
import { readNonNegativeInt, readPositiveInt } from '../../core/utils/env';
/**
* Wall-clock budget for retrying a single record before the daemon moves it
* to the dead-letter file (`failed.log`), in milliseconds.
*/
export const RETRY_TIMEOUT_MS: number =
readNonNegativeInt('MLFLOW_ASYNC_TRACE_LOGGING_RETRY_TIMEOUT', 500) * 1_000;
/**
* Interval between daemon batch loop iterations, in milliseconds.
*/
export const BATCH_INTERVAL_MS: number = readPositiveInt(
'MLFLOW_ASYNC_TRACE_LOGGING_MAX_INTERVAL_MILLIS',
15_000,
);
/**
* Threshold for daemon idle shutdown, in milliseconds. After this much
* time with the WAL empty and no new records appended, the daemon flushes,
* releases its lock, and exits.
*/
export const IDLE_MS: number = readPositiveInt('MLFLOW_TRACE_DAEMON_IDLE_MS', 120_000);
/**
* Retention window (in UTC days) for the daily-rotated `failed.log.<date>`
* and `daemon.log.<date>` files under the WAL directory. Files older than
* `today_utc LOG_RETENTION_DAYS` are unlinked once per daemon startup;
* a value of `0` disables the sweep entirely.
*/
export const LOG_RETENTION_DAYS: number = readNonNegativeInt('MLFLOW_TRACE_LOG_RETENTION_DAYS', 14);
@@ -0,0 +1,635 @@
/**
* The trace-uploader daemon — long-lived background process that drains
* the WAL on behalf of all hooks running for this OS user.
*/
import { appendFileSync } from 'node:fs';
import { mkdir, readdir, unlink } from 'node:fs/promises';
import { createServer, Server, Socket } from 'node:net';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import { MlflowClient } from '../../clients/client';
import { MlflowHttpError } from '../../clients/utils';
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
import { SerializedTraceInfo, TraceInfo } from '../../core/entities/trace_info';
import { BatchingWriter } from './batching_writer';
import { clearClientCache, clearClientForUri, clientForUri } from './clients';
import { BATCH_INTERVAL_MS, IDLE_MS, LOG_RETENTION_DAYS, RETRY_TIMEOUT_MS } from './constants';
import { createIpcConnectionHandler } from './ipc';
import { getDaemonLogPath, getLockSocketPath, getWalDir } from './paths';
import { PidLock, tryAcquirePidLock } from './pid_lock';
import { appendDeadLetter, compact, readPending, walSize } from './storage';
import { isDaemonAlive } from './supervisor';
import { WalRecord } from './types';
const MAX_BACKOFF_MS = 5 * 60_000;
const COMPACTION_THRESHOLD_BYTES = 100_000;
function isErrnoException(err: unknown): err is NodeJS.ErrnoException {
return typeof err === 'object' && err != null && 'code' in err;
}
function formatError(err: unknown): string {
if (err instanceof Error) {
return err.stack ?? err.message;
}
return String(err);
}
/**
* Append a single timestamped line to the daemon log file. Uses sync
* I/O so two concurrent log calls cannot interleave;
*/
export function log(line: string): void {
const timestamp = new Date().toISOString();
const fullLine = `[${timestamp}] [pid=${process.pid}] ${line}\n`;
try {
appendFileSync(getDaemonLogPath(), fullLine);
} catch {
// Failures are swallowed so a broken disk never crashes the daemon mid-batch.
}
}
/** Awaits `ms` real milliseconds. Used for the batch loop's pacing. */
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Exponential backoff with a fixed cap.
*/
export function backoff(attempt: number): number {
return Math.min(2_000 * 2 ** (attempt - 1), MAX_BACKOFF_MS);
}
/**
* Default connection handler used when {@link acquireLock} is invoked
* without one (e.g. by tests that only care about the lock semantic).
*/
function defaultConnectionHandler(socket: Socket): void {
socket.end();
}
/**
* Result of a single `listen()` attempt on the lock socket. Distinguishes
* the three outcomes acquireLock has to dispatch on: bound (we won),
* EADDRINUSE (someone else holds the path — live daemon or stale file),
* or any other error (unexpected).
*/
type BindResult =
| { kind: 'bound'; server: Server }
| { kind: 'eaddrinuse' }
| { kind: 'error'; err: Error };
/**
* Try to bind a fresh server to `socketPath`. Never throws — every
* outcome is encoded in the returned discriminated union so callers can
* keep their control flow linear instead of try/catching around a
* Promise that resolves three different ways.
*/
function tryBind(socketPath: string, onConnection: (socket: Socket) => void): Promise<BindResult> {
return new Promise<BindResult>((resolve) => {
const server = createServer(onConnection);
server.once('error', (err: NodeJS.ErrnoException) => {
if (err.code === 'EADDRINUSE') {
resolve({ kind: 'eaddrinuse' });
return;
}
resolve({ kind: 'error', err });
});
server.listen(socketPath, () => resolve({ kind: 'bound', server }));
});
}
/**
* Handle returned by {@link acquireLock} on success. Pairs the bound
* server with the optional cross-process PID lock so {@link releaseLock}
* can tear both down together. `pidLock` is `null` on Windows because
* named pipes already provide kernel-refcounted cross-process
* exclusion; the file-based lock would be pure overhead there.
*/
export interface DaemonLock {
server: Server;
pidLock: PidLock | null;
}
/**
* Acquire the singleton daemon lock.
*
* Returns the {@link DaemonLock} on success, or `null` when another
* daemon is already running (the caller should `process.exit(0)`
* cleanly).
*
* ## How exclusion is enforced
*
* On POSIX, exclusion is enforced by the atomic PID lock acquired in
* {@link tryAcquirePidLock} *before* the socket bind. The PID lock uses
* `link()`, a single kernel-atomic syscall — only one process can win,
* the rest see `EEXIST` and concede. With the lock held, no sibling
* daemon is past this point at the same time as us, so the subsequent
* `bind()` runs uncontested.
*
* On Windows, named pipes are kernel-refcounted: two daemons cannot
* bind the same pipe name simultaneously regardless of the order of
* operations. We skip the PID lock there and rely on `listen()`'s
* native exclusion — the loser of a concurrent-bind race sees
* `EADDRINUSE` from {@link bindUnderPidLock}, which returns `null` so
* we concede here the same way the PID-lock-lost path does on POSIX.
*/
export async function acquireLock(
onConnection: (socket: Socket) => void = defaultConnectionHandler,
): Promise<DaemonLock | null> {
if (await isDaemonAlive()) {
return null;
}
// Cross-process exclusion gate. On POSIX this is the only mechanism
// that prevents two daemons from racing through stale-socket
// recovery and ending up both bound;
const pidLock = process.platform === 'win32' ? null : await tryAcquirePidLock();
if (process.platform !== 'win32' && pidLock == null) {
return null;
}
try {
const socketPath = getLockSocketPath();
const server = await bindUnderPidLock(socketPath, onConnection);
if (server == null) {
// Windows-only path: a sibling daemon won the named-pipe race.
// No PID lock to release (Windows path; `pidLock` is always
// null here). Concede the same way the POSIX PID-lock-lost case
// does so the caller can `process.exit(0)` cleanly.
return null;
}
return { server, pidLock };
} catch (err) {
// Bind path failed after we acquired the PID lock — release it so
// the next spawn isn't blocked by our orphaned lockfile.
await pidLock?.release();
throw err;
}
}
/**
* Bind the lock socket, with recovery semantics that depend on the
* cross-process exclusion guarantee held by the caller.
*
* Returns:
* - the bound `Server` on success.
* - `null` on Windows when a sibling daemon won the named-pipe bind
* race. POSIX handles the equivalent "sibling won" case via the
* PID lock acquisition earlier in {@link acquireLock}, so this
* return value only fires on Windows.
*
* The asymmetric handling reflects platform reality: Unix domain
* sockets leave a filesystem entry that survives the owning process,
* so POSIX needs an unlink + rebind dance for orphan recovery. Windows
* named pipes are kernel-refcounted — when the owning process dies
* the pipe disappears from the kernel namespace automatically, so
* there is no orphan recovery to do; `EADDRINUSE` there can only mean
* a live sibling.
*/
async function bindUnderPidLock(
socketPath: string,
onConnection: (socket: Socket) => void,
): Promise<Server | null> {
const first = await tryBind(socketPath, onConnection);
if (first.kind === 'bound') {
return first.server;
}
if (first.kind === 'error') {
throw first.err;
}
// EADDRINUSE on the first attempt. The right action depends on
// platform:
// - Windows: no PID lock is held (named pipes provide their own
// kernel-level exclusion), and there is no orphan pipe file to
// unlink. EADDRINUSE here means a live sibling daemon won the
// race; concede by returning null and let `acquireLock`
// propagate that as a clean "another daemon is running" signal.
// - POSIX: we hold the PID lock, so no sibling daemon is racing
// us. EADDRINUSE must be a leftover socket file from a daemon
// that crashed without cleanup. Unlink the orphan and retry.
if (process.platform === 'win32') {
return null;
}
try {
await unlink(socketPath);
} catch (err) {
if (isErrnoException(err) && err.code !== 'ENOENT') {
throw err;
}
}
const second = await tryBind(socketPath, onConnection);
if (second.kind === 'bound') {
return second.server;
}
if (second.kind === 'eaddrinuse') {
throw new Error(
`bind failed with EADDRINUSE on ${socketPath} while holding the PID lock — ` +
`another process is squatting on the daemon's socket path`,
);
}
throw second.err;
}
/**
* Release the lock acquired by {@link acquireLock}. Closes the server,
* unlinks the POSIX socket file, and (when present) releases the PID
* lock so the next daemon's `acquireLock` finds a clean slate.
*/
export async function releaseLock(server: Server, pidLock: PidLock | null = null): Promise<void> {
await new Promise<void>((resolve) => server.close(() => resolve()));
if (process.platform !== 'win32') {
try {
await unlink(getLockSocketPath());
} catch (err) {
if (isErrnoException(err) && err.code !== 'ENOENT') {
log(`releaseLock: socket unlink failed: ${formatError(err)}`);
}
}
}
if (pidLock != null) {
try {
await pidLock.release();
} catch (err) {
log(`releaseLock: pid lock release failed: ${formatError(err)}`);
}
}
}
const ROTATED_LOG_REGEX = /^(failed|daemon)\.log\.(\d{4})-(\d{2})-(\d{2})$/;
const MS_PER_DAY = 24 * 60 * 60 * 1_000;
/**
* One-shot retention sweep for the daily-rotated `failed.log.<date>` and
* `daemon.log.<date>` files in the WAL dir.
*
* Called once per daemon startup, after the liveness lock is held and
* before the batch loop starts.
*
* The sweep is best-effort: any failure (missing dir, EPERM on a file the
* user hand-edited, malformed filenames) is logged at debug and
* swallowed. Retention housekeeping must never abort daemon startup.
*/
export async function pruneOldLogs(
opts: { now?: Date; retentionDays?: number } = {},
): Promise<void> {
const retentionDays = opts.retentionDays ?? LOG_RETENTION_DAYS;
if (retentionDays <= 0) {
return;
}
const now = opts.now ?? new Date();
const todayUtcMs = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
const cutoffMs = todayUtcMs - retentionDays * MS_PER_DAY;
const walDir = getWalDir();
let entries: string[];
try {
entries = await readdir(walDir);
} catch (err) {
log(`pruneOldLogs: readdir failed for ${walDir}: ${formatError(err)}`);
return;
}
for (const name of entries) {
const match = ROTATED_LOG_REGEX.exec(name);
if (match == null) {
continue;
}
const [, , yyyy, mm, dd] = match;
const year = Number(yyyy);
const month = Number(mm);
const day = Number(dd);
const fileDate = new Date(Date.UTC(year, month - 1, day));
if (
fileDate.getUTCFullYear() !== year ||
fileDate.getUTCMonth() !== month - 1 ||
fileDate.getUTCDate() !== day
) {
continue;
}
if (fileDate.getTime() >= cutoffMs) {
continue;
}
const fullPath = join(walDir, name);
try {
await unlink(fullPath);
log(`pruneOldLogs: removed ${name} (older than ${retentionDays} day(s))`);
} catch (err) {
log(`pruneOldLogs: failed to unlink ${name}: ${formatError(err)}`);
}
}
}
/**
* HTTP statuses that {@link uploadOne} treats as "the cached
* credentials are stale; rebuild and retry once."
*/
const AUTH_REFRESH_STATUSES: ReadonlySet<number> = new Set([401, 403]);
function isAuthRefreshableError(err: unknown): boolean {
return err instanceof MlflowHttpError && AUTH_REFRESH_STATUSES.has(err.status);
}
/**
* Perform the create-trace + upload-trace-data sequence, then send
* the captured OTLP spans to the tracking-store DB.
*/
async function performUpload(client: MlflowClient, record: WalRecord): Promise<void> {
const traceInfo = TraceInfo.fromJson(record.traceInfo as unknown as SerializedTraceInfo);
const traceData = TraceData.fromJson(record.traceData as SerializedTraceData);
const resolvedInfo = await client.createTrace(traceInfo);
await client.uploadTraceData(resolvedInfo, traceData);
await sendOtlpSpans(client, record);
}
/**
* Ship the record's captured OTLP spans to the tracking store's
* span-ingestion endpoint (`exportOtlpSpans` → `/v1/traces`) so they land in the
* `SqlSpan` table that powers DB-backed span metrics (e.g. the "Tool calls"
* overview).
*/
async function sendOtlpSpans(client: MlflowClient, record: WalRecord): Promise<void> {
if (!record.otlpSpans) {
return;
}
try {
const bytes = Buffer.from(record.otlpSpans, 'base64');
await client.exportOtlpSpans(record.experimentId, bytes);
} catch (err) {
log(
`Failed to log OTLP spans for record ${record.id} to ${record.trackingUri}; ` +
`spans remain in the JSON artifact only: ${formatError(err)}`,
);
}
}
/**
* Process a single WAL record: deserialize, push to the backend,
* tombstone on success or schedule retry / dead-letter on failure.
*/
export async function uploadOne(
record: WalRecord,
client: MlflowClient,
writer: BatchingWriter,
factory: ClientFactory = clientForUri,
): Promise<void> {
// Anchor the retry-budget window the first time the daemon touches
// the record.
if (record.firstAttemptAt === undefined) {
record.firstAttemptAt = Date.now();
}
const traceId = (record.traceInfo as { trace_id?: string }).trace_id ?? '<unknown>';
try {
await performUpload(client, record);
await writer.submitTombstone(record.id);
log(`Uploaded record ${record.id} (trace ${traceId})`);
return;
} catch (err) {
if (!isAuthRefreshableError(err)) {
await handleUploadFailure(record, err, writer);
return;
}
log(
`Auth failure on record ${record.id} (trace ${traceId}) for ${record.trackingUri}; ` +
`evicting cached client and retrying once: ${formatError(err)}`,
);
clearClientForUri(record.trackingUri);
let refreshed: MlflowClient;
try {
refreshed = factory(record.trackingUri);
} catch (factoryErr) {
log(
`Failed to rebuild client for ${record.trackingUri} after auth error: ${formatError(factoryErr)}`,
);
await handleUploadFailure(record, factoryErr, writer);
return;
}
try {
await performUpload(refreshed, record);
await writer.submitTombstone(record.id);
log(`Uploaded record ${record.id} (trace ${traceId}) after credential refresh`);
} catch (retryErr) {
// The credential-refresh retry also failed — fall back to the
// normal failure machinery (backoff / dead-letter).
await handleUploadFailure(record, retryErr, writer);
}
}
}
/**
* Branch the failure path: either dead-letter the record (if the next
* retry's backoff would push the total elapsed time past
* `RETRY_TIMEOUT_MS`) or re-append it with bumped `attempts` and a
* future `nextAttemptAt`. Either way the original row is tombstoned so
* the next batch doesn't see it again.
*
* Retries use a fresh row `id` so per-attempt rows are independently
* addressable in the log; the MLflow trace id inside `traceInfo` and
* the original `firstAttemptAt` stay stable across attempts.
*/
async function handleUploadFailure(
record: WalRecord,
err: unknown,
writer: BatchingWriter,
): Promise<void> {
const nextAttempts = record.attempts + 1;
const traceId = (record.traceInfo as { trace_id?: string }).trace_id ?? '<unknown>';
const delayMs = backoff(nextAttempts);
const anchor = record.firstAttemptAt ?? record.createdAt;
const now = Date.now();
const elapsedMs = now - anchor;
if (elapsedMs + delayMs >= RETRY_TIMEOUT_MS) {
log(
`Dead-lettering record ${record.id} (trace ${traceId}) after ${record.attempts} prior ` +
`attempts, elapsed=${elapsedMs}ms; next retry would push past ${RETRY_TIMEOUT_MS}ms ` +
`budget: ${formatError(err)}`,
);
await appendDeadLetter(record);
await writer.submitTombstone(record.id);
return;
}
const retry: WalRecord = {
...record,
id: randomUUID(),
attempts: nextAttempts,
nextAttemptAt: now + delayMs,
firstAttemptAt: anchor,
};
log(
`Retry record ${record.id}${retry.id} (trace ${traceId}, attempts=${nextAttempts}, ` +
`delay=${delayMs}ms, elapsed=${elapsedMs}ms): ${formatError(err)}`,
);
// Both submissions ride the same `BatchingWriter` tick: one `writev`,
// one `fsync`, atomic. Either both lines are durable or neither is.
await Promise.all([writer.submit(retry), writer.submitTombstone(record.id)]);
}
export type ClientFactory = (trackingUri: string) => MlflowClient;
/**
* Group `records` by `trackingUri` and upload each group in parallel.
*/
export async function processBatch(
records: WalRecord[],
writer: BatchingWriter,
factory: ClientFactory = clientForUri,
): Promise<void> {
if (records.length === 0) {
return;
}
const groups = new Map<string, WalRecord[]>();
for (const record of records) {
const list = groups.get(record.trackingUri);
if (list === undefined) {
groups.set(record.trackingUri, [record]);
} else {
list.push(record);
}
}
await Promise.all(
[...groups.entries()].map(async ([trackingUri, group]) => {
let client: MlflowClient;
try {
client = factory(trackingUri);
} catch (err) {
log(`Failed to build client for ${trackingUri}: ${formatError(err)}`);
await Promise.all(group.map((record) => handleUploadFailure(record, err, writer)));
return;
}
await Promise.all(group.map((record) => uploadOne(record, client, writer, factory)));
}),
);
}
/**
* Wire SIGTERM / SIGINT handlers to abort the supplied controller so
* the daemon finishes its current batch and exits cleanly rather than
* being torn down mid-write. The abort is consulted at the top of each
* {@link runBatchLoop} iteration.
*/
function installSignalHandlers(controller: AbortController): void {
for (const sig of ['SIGTERM', 'SIGINT'] as const) {
process.on(sig, () => {
log(`Received ${sig}; will shut down after current batch.`);
controller.abort();
});
}
}
/**
* The batch loop. Broken out from {@link main} so the dependencies
* (client factory, writer, timings, shutdown signal) can be injected
* in tests without standing up the full daemon lifecycle.
*
* Pass an `AbortSignal` to request a graceful shutdown — the loop
* finishes its current iteration and returns. Without a signal the
* loop only exits via the `idleMs` timeout path (or by throwing).
*/
export async function runBatchLoop({
factory = clientForUri,
writer = new BatchingWriter(),
batchIntervalMs = BATCH_INTERVAL_MS,
idleMs = IDLE_MS,
signal,
}: {
factory?: ClientFactory;
writer?: BatchingWriter;
batchIntervalMs?: number;
idleMs?: number;
signal?: AbortSignal;
} = {}): Promise<void> {
let lastNonEmpty = Date.now();
while (signal?.aborted !== true) {
try {
const pending = await readPending();
const now = Date.now();
const due = pending.filter((r) => r.nextAttemptAt <= now);
if (due.length > 0) {
lastNonEmpty = now;
await processBatch(due, writer, factory);
} else if (now - lastNonEmpty >= idleMs) {
log('WAL idle threshold reached; shutting down.');
return;
}
if ((await walSize()) > COMPACTION_THRESHOLD_BYTES) {
await compact().catch((err) => log(`Periodic compact failed: ${formatError(err)}`));
}
} catch (err) {
// Don't let a transient FS / parse error tear down the daemon.
log(`Batch iteration error: ${formatError(err)}`);
}
await sleep(batchIntervalMs);
}
}
/**
* Daemon entry point.
*/
export async function main(): Promise<void> {
await mkdir(getWalDir(), { recursive: true });
const ipcWriter = new BatchingWriter();
const handle = await acquireLock(createIpcConnectionHandler(ipcWriter));
if (handle == null) {
log('Another daemon is already running; exiting cleanly.');
process.exit(0);
return;
}
log(`Daemon started; pid=${process.pid}, walDir=${getWalDir()}`);
const shutdown = new AbortController();
installSignalHandlers(shutdown);
let exitCode = 0;
try {
await pruneOldLogs().catch((err) => log(`Retention sweep failed: ${formatError(err)}`));
await compact().catch((err) => log(`Startup compact failed: ${formatError(err)}`));
await runBatchLoop({ writer: ipcWriter, signal: shutdown.signal });
await compact().catch((err) => log(`Final compact failed: ${formatError(err)}`));
} catch (err) {
log(`Daemon main loop crashed: ${formatError(err)}`);
exitCode = 1;
} finally {
await releaseLock(handle.server, handle.pidLock).catch((err) =>
log(`releaseLock failed: ${formatError(err)}`),
);
clearClientCache();
log(`Daemon exited with code ${exitCode}.`);
}
process.exit(exitCode);
}
// Run main() only when invoked directly (i.e. `node bundle/daemon.cjs`),
// not when this module is imported by tests.
if (require.main === module) {
main().catch((err) => {
console.error('[mlflow][wal-daemon] fatal:', err);
process.exit(1);
});
}
@@ -0,0 +1,191 @@
import { randomUUID } from 'node:crypto';
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
import { ReadableSpan as OTelReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base';
import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer';
import { getConfig, MLflowTracingConfig } from '../../core/config';
import { InMemoryTraceManager } from '../../core/trace_manager';
import { ipcRequestByteLength, MAX_REQUEST_BYTES, submitRecord } from './ipc';
import { WalRecord } from './types';
function decodeMlflowSpanAttributes(
attributes: OTelReadableSpan['attributes'],
): Record<string, unknown> {
const decoded: Record<string, unknown> = {};
for (const [key, value] of Object.entries(attributes)) {
if (typeof value === 'string' && value.startsWith('"')) {
try {
decoded[key] = JSON.parse(value);
} catch {
decoded[key] = value;
}
} else {
decoded[key] = value;
}
}
return decoded;
}
/**
* Return a view of `span` whose `attributes` are decoded (see
* {@link decodeMlflowSpanAttributes}) while every other property/method is
* delegated to the original span untouched.
*/
function spanWithDecodedAttributes(span: OTelReadableSpan): OTelReadableSpan {
const decoded = decodeMlflowSpanAttributes(span.attributes);
return new Proxy(span, {
get(target, prop) {
if (prop === 'attributes') {
return decoded;
}
const value: unknown = Reflect.get(target, prop);
return typeof value === 'function' ? (value.bind(target) as unknown) : value;
},
});
}
/**
* Serialize live OTel spans into a base64-encoded OTLP
* `ExportTraceServiceRequest` protobuf, suitable for storing on a
* {@link WalRecord} and later POSTing to the server's OTLP span-ingestion
* endpoint
*/
function serializeSpansToOtlpBase64(spans: OTelReadableSpan[]): string | undefined {
const serializable = spans.filter((s) => s?.resource != null && s?.instrumentationScope != null);
if (serializable.length === 0) {
return undefined;
}
try {
const bytes = ProtobufTraceSerializer.serializeRequest(
serializable.map(spanWithDecodedAttributes),
);
if (!bytes || bytes.length === 0) {
return undefined;
}
return Buffer.from(bytes).toString('base64');
} catch (err) {
console.warn(
'[mlflow][wal] Failed to serialize spans to OTLP protobuf; ' +
'falling back to JSON artifact upload (spans will not appear in DB-backed span metrics).',
err,
);
return undefined;
}
}
/**
* Hook-side submit function. Returns once the daemon has fsynced the
* record to `queue.log` (ack-after-fsync). Injectable so tests can
* swap in a stub without spinning up an actual daemon.
*/
export type SubmitRecord = (record: WalRecord) => Promise<void>;
/**
* Asynchronous SpanExporter that decouples the user's Stop hook from
* backend latency for MlflowSpanProcessor.
*
* {@link forceFlush} awaits only the IPC submits, *not* the upstream
* upload. That gives the Stop hook the exact "trace is durable on this
* machine, then return" semantics it needs to bound its runtime.
*/
export class MlflowWalSpanExporter implements SpanExporter {
private _pendingSubmits: Set<Promise<void>> = new Set();
private _submit: SubmitRecord;
private _shuttingDown = false;
constructor(opts: { submit?: SubmitRecord } = {}) {
this._submit = opts.submit ?? submitRecord;
}
export(spans: OTelReadableSpan[], resultCallback: (result: ExportResult) => void): void {
if (this._shuttingDown) {
resultCallback({ code: ExportResultCode.FAILED });
return;
}
let cfg: MLflowTracingConfig;
try {
cfg = getConfig();
} catch (err) {
console.warn(
'[mlflow][wal] Tracing config unavailable; Spans will be lost until config is set.',
err,
);
resultCallback({ code: ExportResultCode.FAILED });
return;
}
for (const span of spans) {
if (span.parentSpanContext?.spanId) {
continue;
}
const traceManager = InMemoryTraceManager.getInstance();
const trace = traceManager.popTrace(span.spanContext().traceId);
if (!trace) {
console.warn(`No trace found for span ${span.name}. Skipping.`);
continue;
}
traceManager.lastActiveTraceId = trace.info.traceId;
const otlpSpans = serializeSpansToOtlpBase64(trace.data.spans.map((s) => s._span));
const record: WalRecord = {
id: randomUUID(),
trackingUri: cfg.trackingUri,
experimentId: cfg.experimentId,
traceInfo: trace.info.toJson() as unknown as Record<string, unknown>,
traceData: trace.data.toJson(),
attempts: 0,
nextAttemptAt: 0,
createdAt: Date.now(),
otlpSpans,
};
const fullSizeBytes = ipcRequestByteLength(record);
if (otlpSpans !== undefined && fullSizeBytes > MAX_REQUEST_BYTES) {
console.warn(
`[mlflow][wal] WAL record for trace ${trace.info.traceId} is ${fullSizeBytes} bytes, exceeding the ` +
`${MAX_REQUEST_BYTES}-byte IPC request limit; dropping OTLP spans and attempting JSON artifact upload ` +
'(spans will not appear in DB-backed span metrics).',
);
record.otlpSpans = undefined;
}
const pending = this._submit(record).catch((err) => {
console.error(
`[mlflow][wal] Failed to submit WAL record for trace ${trace.info.traceId}:`,
err,
);
});
this._pendingSubmits.add(pending);
void pending.finally(() => this._pendingSubmits.delete(pending));
}
resultCallback({ code: ExportResultCode.SUCCESS });
}
/**
* Resolves once every IPC submit that was in flight when this method
* was called has been acknowledged (post-fsync). Matches OTel's
* "flush currently pending" semantic — concurrent `export()` calls
* during the await are not waited on, and the next `forceFlush` will
* pick them up. For "drain everything before exit" semantics, use
* {@link shutdown}.
*/
async forceFlush(): Promise<void> {
await Promise.all([...this._pendingSubmits]);
}
/**
* Stop accepting new exports and drain every in-flight IPC submit to
* convergence before resolving.
*/
async shutdown(): Promise<void> {
this._shuttingDown = true;
while (this._pendingSubmits.size > 0) {
await Promise.all([...this._pendingSubmits]);
}
}
}
@@ -0,0 +1,5 @@
/**
* Public entry point for the WAL exporter package.
*/
export { MlflowWalSpanExporter } from './exporter';
@@ -0,0 +1,288 @@
/**
* Hook ↔ daemon IPC over the daemon's lock socket / Named Pipe.
*
* Client side ({@link submitRecord}):
* 1. Connect to {@link getLockSocketPath}.
* 2. Write a single newline-terminated JSON request.
* 3. Read a single newline-terminated JSON response.
* 4. Close.
* The promise resolves only after the daemon has fsynced the record's
* byte to `queue.log`.
*
* Server side ({@link createIpcConnectionHandler}):
* - Wait for one line of data on the connection.
* - Parse it as an {@link IpcRequest}.
* - Dispatch the `append` op into the daemon's {@link BatchingWriter}.
* - Once the writer's promise settles, write a one-line
* {@link IpcResponse} and close.
* - Connections that close before sending a line are liveness probes
* (used by {@link isDaemonAlive}); we let them go silently.
*/
import { createConnection, Socket } from 'node:net';
import { JSONBig } from '../../core/utils/json';
import { BatchingWriter } from './batching_writer';
import { getLockSocketPath } from './paths';
import { IpcRequest, IpcResponse } from './protocol';
import { ensureDaemon } from './supervisor';
import { WalRecord } from './types';
/**
* Upper bound on a single submit round-trip (connect + write + ACK).
*/
const SUBMIT_TIMEOUT_MS = 10_000;
const INITIAL_CONNECT_RETRY_DELAY_MS = 50;
const CONNECT_RETRY_DELAYS_MS = Array.from(
{ length: 6 },
(_, i) => INITIAL_CONNECT_RETRY_DELAY_MS * 2 ** i,
);
const DAEMON_NO_RESPONSE_CODE = 'EDAEMONNORESPONSE';
export const MAX_REQUEST_BYTES = 32 * 1024 * 1024;
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Exact byte length of the wire payload {@link submitRecord} would send for
* `record`, including the `append` envelope and trailing newline. Use this to
* compare against {@link MAX_REQUEST_BYTES} before submitting.
*/
function buildRequestPayload(record: WalRecord): string {
return JSONBig.stringify({ op: 'append', record } satisfies IpcRequest) + '\n';
}
export function ipcRequestByteLength(record: WalRecord): number {
return Buffer.byteLength(buildRequestPayload(record), 'utf8');
}
function isConnectError(err: unknown): boolean {
if (typeof err !== 'object' || err == null) {
return false;
}
const code = (err as { code?: string }).code;
return (
code === 'ECONNREFUSED' ||
code === 'ENOENT' ||
code === 'EPIPE' ||
code === 'ECONNRESET' ||
code === DAEMON_NO_RESPONSE_CODE
);
}
/**
* Send `record` to the running daemon and wait for the post-fsync ACK.
*
* Delivery semantics: **at-least-once**.
*
* A successful resolution means the record was durably fsynced to
* `queue.log` at least once. A rejection means the daemon could not be
* acknowledged after 7 attempts (1 initial + 6 backoffs from
* `CONNECT_RETRY_DELAYS_MS`), but the daemon may still have persisted
* the record on one of those attempts before the failure surfaced.
*
* The internal retry loop retries on five connect-class codes:
* `ECONNREFUSED`, `ENOENT`, `EPIPE`, `ECONNRESET`, and
* `EDAEMONNORESPONSE`. The first two prove the request never reached a
* daemon and are safe; the last three can fire *after* the daemon has
* already fsynced (e.g. daemon crashed between fsync and ACK, or the
* socket reset post-fsync), so a single logical submit can produce
* more than one physical line in `queue.log` and more than one
* `createTrace` call to the backend.
*
*/
export async function submitRecord(record: WalRecord): Promise<void> {
const payload = buildRequestPayload(record);
let lastErr: unknown;
for (let i = 0; i <= CONNECT_RETRY_DELAYS_MS.length; i++) {
if (i > 0) {
await sleep(CONNECT_RETRY_DELAYS_MS[i - 1]);
}
try {
const response = await sendRequest(payload);
if (response.ok) {
return;
}
throw new Error(`Daemon rejected record: ${response.error}`);
} catch (err) {
lastErr = err;
if (!isConnectError(err)) {
throw err;
}
// Fire-and-forget spawn; the next loop iteration's connect will
// observe the bound socket once the daemon finishes binding.
ensureDaemon().catch(() => {});
}
}
throw lastErr as Error;
}
/**
* One IPC round-trip against the daemon's lock socket.
*/
function sendRequest(payload: string): Promise<IpcResponse> {
return new Promise<IpcResponse>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const chunks: Buffer[] = [];
let settled = false;
const finish = (act: () => void): void => {
if (settled) {
return;
}
settled = true;
socket.destroy();
act();
};
const timer = setTimeout(
() => finish(() => reject(new Error(`IPC submit timed out after ${SUBMIT_TIMEOUT_MS}ms`))),
SUBMIT_TIMEOUT_MS,
);
socket.once('connect', () => {
socket.write(payload);
});
socket.on('data', (chunk) => chunks.push(chunk));
socket.once('end', () => {
clearTimeout(timer);
const text = Buffer.concat(chunks).toString('utf8').trimEnd();
if (text === '') {
finish(() =>
reject(
Object.assign(new Error('Daemon closed connection without responding'), {
code: DAEMON_NO_RESPONSE_CODE,
}),
),
);
return;
}
try {
const parsed = JSONBig.parse(text) as IpcResponse;
finish(() => resolve(parsed));
} catch (err) {
finish(() => reject(err as Error));
}
});
socket.once('error', (err) => {
clearTimeout(timer);
finish(() => reject(err));
});
});
}
/**
* Per-connection state for an in-flight IPC request. One instance per
* accepted socket.
*/
function handleConnection(writer: BatchingWriter, socket: Socket): void {
const chunks: Buffer[] = [];
let bytesReceived = 0;
let dispatched = false;
socket.on('error', (err: NodeJS.ErrnoException) => {
if (err.code !== 'ECONNRESET' && err.code !== 'EPIPE') {
console.error(
`[mlflow][wal] ipc connection error: code=${err.code ?? 'none'} message=${err.message}`,
);
}
socket.destroy();
});
socket.on('data', (chunk: Buffer) => {
if (dispatched) {
// Extra bytes after the first line are not part of the protocol
// today; ignore them rather than reject
return;
}
bytesReceived += chunk.length;
if (bytesReceived > MAX_REQUEST_BYTES) {
dispatched = true;
socket.pause();
sendResponse(socket, {
ok: false,
error: `request exceeds max size ${MAX_REQUEST_BYTES} bytes without newline terminator`,
});
const destroyTimer = setTimeout(() => socket.destroy(), 500);
destroyTimer.unref();
socket.once('close', () => clearTimeout(destroyTimer));
return;
}
// Prior chunks contained no newline (otherwise `dispatched` would
// be true and we'd have returned above), so only this chunk needs
// scanning. `0x0A` is the byte value of '\n'.
const newlineInChunk = chunk.indexOf(0x0a);
if (newlineInChunk < 0) {
chunks.push(chunk);
return;
}
chunks.push(chunk.subarray(0, newlineInChunk));
const line = Buffer.concat(chunks).toString('utf8');
dispatched = true;
void dispatch(writer, socket, line);
});
}
async function dispatch(writer: BatchingWriter, socket: Socket, line: string): Promise<void> {
let request: IpcRequest;
try {
request = JSONBig.parse(line) as IpcRequest;
} catch (err) {
sendResponse(socket, {
ok: false,
error: `malformed request: ${(err as Error).message}`,
});
return;
}
if (request.op !== 'append') {
sendResponse(socket, {
ok: false,
error: `unknown op: ${(request as { op: string }).op}`,
});
return;
}
const record = request.record as Partial<WalRecord> | null | undefined;
if (
typeof record !== 'object' ||
record == null ||
typeof record.id !== 'string' ||
record.id === ''
) {
sendResponse(socket, {
ok: false,
error: 'invalid record: must be an object with a non-empty string id',
});
return;
}
try {
await writer.submit(request.record);
sendResponse(socket, { ok: true });
} catch (err) {
sendResponse(socket, { ok: false, error: (err as Error).message });
}
}
function sendResponse(socket: Socket, response: IpcResponse): void {
// `socket.end` does not throw synchronously for our inputs (always a
// string, never null, never on a not-yet-constructed handle). Any
// transport-level failure during the write surfaces asynchronously
// via the `'error'` event, which is handled at the top of
// handleConnection - no defensive try/catch needed here.
socket.end(JSONBig.stringify(response) + '\n');
}
/**
* Build the per-connection handler that the daemon installs on its
* lock-socket server. Bound to a {@link BatchingWriter} so the
* fsync-coalescing state persists across connections.
*/
export function createIpcConnectionHandler(writer: BatchingWriter): (socket: Socket) => void {
return (socket: Socket) => handleConnection(writer, socket);
}
@@ -0,0 +1,101 @@
/**
* Filesystem paths for the MLflow trace WAL (Write-Ahead Log) spool.
*
* A spool is shared across all Claude Code sessions
*
* The daemon's singleton lock is scoped per `(user, spool root)` on both
* platforms; its exact location depends on the platform and the resolved
* spool root's length:
* - POSIX, common case: `<spool>/daemon.sock` next to `queue.log`.
* - POSIX, long-path fallback: `<tmpdir>/mlflow-<scoped_id>.sock` when the
* natural path would exceed `sun_path`.
* - Windows: a Named Pipe whose name embeds a hash of the spool root.
*/
import { createHash } from 'node:crypto';
import { homedir, platform, tmpdir, userInfo } from 'node:os';
import { join, resolve } from 'node:path';
/**
* Maximum allowed length, in bytes, for a UNIX domain socket path.
*/
const POSIX_SOCKET_PATH_MAX_BYTES = 103;
/**
* Root directory of the per-user spool. Defaults to `~/.mlflow/wal/`.
*/
export function getWalDir(): string {
const override = process.env.MLFLOW_WAL_DIR;
const raw =
override !== undefined && override !== '' ? override : join(homedir(), '.mlflow', 'wal');
return resolve(raw);
}
/**
* Append-only Write-Ahead Log of pending trace uploads.
*/
export function getWalPath(): string {
return join(getWalDir(), 'queue.log');
}
function dateSuffix(d: Date = new Date()): string {
const yyyy = d.getUTCFullYear().toString().padStart(4, '0');
const mm = (d.getUTCMonth() + 1).toString().padStart(2, '0');
const dd = d.getUTCDate().toString().padStart(2, '0');
return `${yyyy}-${mm}-${dd}`;
}
/**
* Append-only dead-letter file containing records whose retry-budget window
* (`MLFLOW_ASYNC_TRACE_LOGGING_RETRY_TIMEOUT`) has elapsed. Records here are
* durable and inspectable / replayable by operators.
*/
export function getDeadLetterPath(now: Date = new Date()): string {
return join(getWalDir(), `failed.log.${dateSuffix(now)}`);
}
/**
* Daemon log file with diagnostic lines (retries, DLQ entries, fatal errors).
*/
export function getDaemonLogPath(now: Date = new Date()): string {
return join(getWalDir(), `daemon.log.${dateSuffix(now)}`);
}
export function getPidLockPath(): string {
return join(getWalDir(), 'daemon.pid');
}
/**
* Stable per (user, wal_dir) identifier used to scope the daemon lock
*/
function getScopedLockId(): string {
const sanitizedUser = userInfo().username.replace(/[^a-zA-Z0-9_-]/g, '_');
const walDirHash = createHash('sha256').update(getWalDir()).digest('hex').slice(0, 12);
return `${sanitizedUser}-${walDirHash}`;
}
/**
* Path used as the daemon's singleton liveness lock.
*/
export function getLockSocketPath(): string {
const scopedId = getScopedLockId();
if (platform() === 'win32') {
return `\\\\?\\pipe\\mlflow-trace-daemon-${scopedId}`;
}
const naturalPath = join(getWalDir(), 'daemon.sock');
if (Buffer.byteLength(naturalPath, 'utf8') <= POSIX_SOCKET_PATH_MAX_BYTES) {
return naturalPath;
}
// Short basename so the fallback itself stays under sun_path even on
// macOS, whose tmpdir is already ~49 chars (`/var/folders/<2>/<30>/T/`).
const fallback = join(tmpdir(), `mlflow-${scopedId}.sock`);
console.warn(
`[mlflow][wal] WAL dir produces a socket path longer than ` +
`${POSIX_SOCKET_PATH_MAX_BYTES} bytes (${Buffer.byteLength(naturalPath, 'utf8')}); ` +
`using ${fallback} for the daemon lock instead.`,
);
return fallback;
}
@@ -0,0 +1,133 @@
/**
* Cross-process exclusion lock for the daemon singleton.
*
* The lock is POSIX-only; on Windows the daemon's named-pipe path is
* kernel-refcounted and provides cross-process exclusion natively, so
* `acquireLock` skips this module on `process.platform === 'win32'`.
*/
import { randomUUID } from 'node:crypto';
import { link, readFile, unlink, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { getPidLockPath, getWalDir } from './paths';
const MAX_ACQUIRE_ATTEMPTS = 5;
function isErrnoException(err: unknown): err is NodeJS.ErrnoException {
return typeof err === 'object' && err != null && 'code' in err;
}
export interface PidLock {
release(): Promise<void>;
}
/**
* Attempt to claim the daemon's cross-process PID lock.
*
* Returns a {@link PidLock} on success, or `null` when another live
* process already holds it. Throws on unrecoverable I/O errors
* (`EACCES`, `ENOSPC`, missing WAL dir, etc.) — callers should treat
* a throw as "daemon cannot start" rather than "concede to sibling."
*/
export async function tryAcquirePidLock(): Promise<PidLock | null> {
const lockPath = getPidLockPath();
const expectedContent = `${process.pid}\n`;
for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) {
const tmpPath = join(getWalDir(), `daemon.pid.${process.pid}.${randomUUID()}.tmp`);
try {
await writeFile(tmpPath, expectedContent, { flag: 'wx' });
await link(tmpPath, lockPath);
await unlink(tmpPath).catch(() => {});
return { release: () => releaseLockFile(lockPath, expectedContent) };
} catch (err) {
await unlink(tmpPath).catch(() => {});
if (!isErrnoException(err) || err.code !== 'EEXIST') {
throw err;
}
}
// EEXIST path. Inspect the holder to decide between conceding and
// recovering from a stale lockfile left by a crashed daemon.
let content: string;
try {
content = await readFile(lockPath, 'utf8');
} catch (err) {
if (isErrnoException(err) && err.code === 'ENOENT') {
// Holder released between our link attempt and our read; the
// path is free again, loop and reattempt the link.
continue;
}
throw err;
}
const holderPid = parsePid(content);
if (holderPid != null && isProcessAlive(holderPid)) {
return null;
}
// Stale or malformed. Unlink the lockfile, but only if its content
// is still the same bytes we just read - otherwise a successor has
// already taken over and we would nuke their fresh lockfile.
const fresh = await readFile(lockPath, 'utf8').catch(() => null);
if (fresh === content) {
await unlink(lockPath).catch(() => {});
}
}
throw new Error(
`tryAcquirePidLock: gave up after ${MAX_ACQUIRE_ATTEMPTS} contention attempts on ${lockPath}`,
);
}
/**
* Read the pid recorded in the daemon's lockfile, or `null` if the
* file is missing or unparseable. Provided as a diagnostic helper for
* callers (e.g. supervisor) that want to log "who owns the lock right
* now"; not used by acquisition itself.
*/
export async function readLockHolderPid(): Promise<number | null> {
try {
const content = await readFile(getPidLockPath(), 'utf8');
return parsePid(content);
} catch (err) {
if (isErrnoException(err) && err.code === 'ENOENT') {
return null;
}
throw err;
}
}
function parsePid(content: string): number | null {
const trimmed = content.trim();
if (trimmed === '') {
return null;
}
const n = Number.parseInt(trimmed, 10);
// PIDs are positive integers; reject 0 (which `process.kill` would
// interpret as "signal every process in our process group") and
// negatives (which would target a process group).
return Number.isInteger(n) && n > 0 ? n : null;
}
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (err) {
// `EPERM` means the process exists but we lack permission to
// signal it (different uid). Treat as alive — the lockfile's
// owner is alive enough that we should not unlink their lock.
if (isErrnoException(err) && err.code === 'EPERM') {
return true;
}
return false;
}
}
async function releaseLockFile(lockPath: string, expectedContent: string): Promise<void> {
const fresh = await readFile(lockPath, 'utf8').catch(() => null);
if (fresh === expectedContent) {
await unlink(lockPath).catch(() => {});
}
}
@@ -0,0 +1,27 @@
/**
* Wire protocol for the daemon's IPC socket.
*
* op: 'append' resolves only after the record is durable in
* queue.log (the daemon's batching writer fsyncs the batch before
* acknowledging).
*/
import { WalRecord } from './types';
export interface AppendRequest {
op: 'append';
record: WalRecord;
}
export type IpcRequest = AppendRequest;
export interface OkResponse {
ok: true;
}
export interface ErrResponse {
ok: false;
error: string;
}
export type IpcResponse = OkResponse | ErrResponse;
@@ -0,0 +1,167 @@
/**
* Append-only JSONL store backing the trace upload WAL.
*/
import { createReadStream, existsSync } from 'node:fs';
import { mkdir, open, rename, stat, unlink } from 'node:fs/promises';
import { dirname } from 'node:path';
import { createInterface } from 'node:readline';
import { JSONBig } from '../../core/utils/json';
import { getDeadLetterPath, getWalPath } from './paths';
import { WalLine, WalRecord } from './types';
/**
* Promise-chained queue that serializes async tasks one at a time.
*/
class SerialQueue {
private tail: Promise<void> = Promise.resolve();
run<T>(fn: () => Promise<T>): Promise<T> {
const result = this.tail.catch(() => {}).then(fn);
this.tail = result.then(
() => {},
() => {},
);
return result;
}
}
export const queueWriter = new SerialQueue();
const deadLetterWriter = new SerialQueue();
export async function ensureParentDir(filePath: string): Promise<void> {
await mkdir(dirname(filePath), { recursive: true });
}
/**
* Append one JSONL line to `path`, fsync, and close.
*
* Encoded as a single `Buffer` so the write goes out in one `O_APPEND`
* syscall — line-sized records sit well under `PIPE_BUF`, so concurrent
* appenders never interleave their bytes.
*/
async function appendJsonLine(path: string, line: WalLine): Promise<void> {
await ensureParentDir(path);
const buf = Buffer.from(JSONBig.stringify(line) + '\n', 'utf8');
const fh = await open(path, 'a');
try {
await fh.write(buf);
await fh.sync();
} finally {
await fh.close();
}
}
/**
* Append a pending trace upload to the WAL.
*/
export function appendRecord(record: WalRecord): Promise<void> {
return queueWriter.run(() => appendJsonLine(getWalPath(), { type: 'append', record }));
}
/**
* Append a tombstone for `id` to the WAL. Logically shadows any earlier
* `{type:"append"}` line for the same id.
*/
export function appendTombstone(id: string): Promise<void> {
return queueWriter.run(() => appendJsonLine(getWalPath(), { type: 'tombstone', id }));
}
/**
* Append `record` to the dead-letter file (`failed.log.<YYYY-MM-DD>`).
*/
export function appendDeadLetter(record: WalRecord): Promise<void> {
return deadLetterWriter.run(() =>
appendJsonLine(getDeadLetterPath(), { type: 'append', record }),
);
}
/**
* Replay `queue.log` and return the set of records still considered pending.
*/
export async function readPending(): Promise<WalRecord[]> {
const path = getWalPath();
if (!existsSync(path)) {
return [];
}
const alive = new Map<string, WalRecord>();
const stream = createReadStream(path, { encoding: 'utf8' });
const rl = createInterface({ input: stream, crlfDelay: Infinity });
for await (const line of rl) {
if (line === '') {
continue;
}
let parsed: unknown;
try {
// JSONBig.parse so bigint fields written by `appendJsonLine` are
// restored as `bigint` (and not as the lossy `number` JSON.parse
// would coerce them to once stringified back through `JSONBig`).
parsed = JSONBig.parse(line);
} catch {
console.debug(`[mlflow][wal] Skipping malformed WAL line: ${line.slice(0, 120)}`);
continue;
}
if (!parsed || typeof parsed !== 'object') {
continue;
}
const obj = parsed as { type?: unknown; record?: unknown; id?: unknown };
if (obj.type === 'append') {
const record = obj.record as WalRecord | undefined;
if (record && typeof record.id === 'string') {
alive.set(record.id, record);
}
} else if (obj.type === 'tombstone' && typeof obj.id === 'string') {
alive.delete(obj.id);
}
}
return [...alive.values()];
}
/**
* Rewrite `queue.log` to contain only currently-live records.
*/
export function compact(): Promise<void> {
return queueWriter.run(async () => {
const path = getWalPath();
if (!existsSync(path)) {
return;
}
const liveRecords = await readPending();
const tmpPath = `${path}.tmp.${process.pid}`;
const tmpFh = await open(tmpPath, 'w');
try {
for (const record of liveRecords) {
const buf = Buffer.from(
JSONBig.stringify({ type: 'append', record } as WalLine) + '\n',
'utf8',
);
await tmpFh.write(buf);
}
await tmpFh.sync();
await tmpFh.close();
await rename(tmpPath, path);
} catch (err) {
await tmpFh.close().catch(() => {});
await unlink(tmpPath).catch(() => {});
throw err;
}
});
}
/**
* Size of `queue.log` in bytes, or 0 if the file does not exist.
* Used by the daemon's idle-shutdown heuristic.
*/
export async function walSize(): Promise<number> {
const path = getWalPath();
if (!existsSync(path)) {
return 0;
}
return (await stat(path)).size;
}
@@ -0,0 +1,167 @@
/**
* Daemon supervisor - handles the "is the uploader daemon alive? if not,
* spawn one" check invoked by `MlflowWalSpanExporter` on each WAL append.
*/
import { spawn } from 'node:child_process';
import { createConnection } from 'node:net';
import { createRequire } from 'node:module';
import { dirname, join } from 'node:path';
import { getLockSocketPath } from './paths';
const PROBE_TIMEOUT_MS = 1000;
/**
* Upper bound on how long {@link ensureDaemon} waits for a freshly
* spawned daemon to bind its lock socket.
*/
const SPAWN_BIND_WAIT_TIMEOUT_MS = 2000;
const SPAWN_BIND_WAIT_POLL_MS = 50;
/**
* Probe the daemon's liveness lock with a short timeout.
*
* Returns:
* - `true` if the socket/pipe at `getLockSocketPath()` is currently
* accepting connections (i.e. a daemon has bound it).
* - `false` on any error (ENOENT — no socket file; ECONNREFUSED — a
* stale socket file with no listener) or on probe timeout.
*/
export function isDaemonAlive(): Promise<boolean> {
return new Promise((resolve) => {
const socketPath = getLockSocketPath();
const socket = createConnection(socketPath);
let settled = false;
const finish = (alive: boolean): void => {
if (settled) {
return;
}
settled = true;
socket.destroy();
resolve(alive);
};
const timer = setTimeout(() => finish(false), PROBE_TIMEOUT_MS);
socket.once('connect', () => {
clearTimeout(timer);
finish(true);
});
socket.once('error', () => {
clearTimeout(timer);
finish(false);
});
});
}
/**
* Resolve the absolute path of the daemon entry point.
*
* Precedence:
* 1. `MLFLOW_TRACE_DAEMON_EXECUTABLE` env var — used by tests and
* advanced operators who want to override the bundled daemon.
* 2. The bundled daemon at `<@mlflow/core install dir>/bundle/daemon.cjs`.
*/
export function resolveDaemonEntry(): string {
const override = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
if (override !== undefined && override !== '') {
return override;
}
// DO NOT collapse this into a plain string literal. esbuild
// evaluates `require.resolve(literal)` at build time and bakes the
// resolved (developer-machine) path directly into the shipped
// bundle. On end users' machines that path does not exist, so
// `spawn` fails with ENOENT and traces silently stop uploading
// while the WAL grows unbounded. The string-concat hides the
// specifier from esbuild's AST scanner so the lookup runs at
// runtime against the user's own node_modules.
//
// This defense lives in the supervisor (not in consumers' esbuild
// configs) because every plugin-marketplace consumer must bundle
// @mlflow/core into their hook to ship as a self-contained file.
const pkgJsonModule = '@mlflow' + '/core' + '/package.json';
const requireFromHere = createRequire(__filename);
const pkgJsonPath = requireFromHere.resolve(pkgJsonModule);
return join(dirname(pkgJsonPath), 'bundle', 'daemon.cjs');
}
/**
* Fork a detached daemon process. Assumes no daemon is currently bound —
* the caller is responsible for guarding with {@link isDaemonAlive}.
*/
export function spawnDaemon(): void {
const entry = resolveDaemonEntry();
const child = spawn(process.execPath, [entry], {
detached: true,
stdio: 'ignore',
env: process.env,
});
child.once('error', (err) => {
console.error(`[mlflow][wal] daemon spawn failed (entry=${entry}):`, err);
});
child.once('exit', (code, signal) => {
if (code != null && code !== 0) {
console.error(
`[mlflow][wal] daemon exited unexpectedly: entry=${entry} code=${code} signal=${signal ?? 'none'}`,
);
}
});
child.unref();
}
let spawnInFlight: Promise<void> | null = null;
/**
* Poll {@link isDaemonAlive} until it returns true or `timeoutMs`
* elapses. Does not throw on timeout — the IPC client's own retry
* loop will surface any persistent connection failure.
*/
async function waitForDaemonBind(timeoutMs: number, pollIntervalMs: number): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const remainingMs = deadline - Date.now();
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
const result = await Promise.race<boolean | 'timeout'>([
isDaemonAlive(),
new Promise((resolve) => {
timeoutHandle = setTimeout(() => resolve('timeout'), remainingMs);
}),
]);
clearTimeout(timeoutHandle);
if (result === true || result === 'timeout') {
return;
}
const sleepMs = Math.min(pollIntervalMs, deadline - Date.now());
if (sleepMs > 0) {
await new Promise<void>((resolve) => setTimeout(resolve, sleepMs));
}
}
}
/**
* Ensure a daemon is alive; spawn one if not.
*/
export function ensureDaemon(): Promise<void> {
if (spawnInFlight != null) {
return spawnInFlight;
}
const inFlight = (async () => {
try {
if (await isDaemonAlive()) {
return;
}
spawnDaemon();
await waitForDaemonBind(SPAWN_BIND_WAIT_TIMEOUT_MS, SPAWN_BIND_WAIT_POLL_MS);
} catch (err) {
console.error('[mlflow][wal] ensureDaemon failed:', err);
} finally {
spawnInFlight = null;
}
})();
spawnInFlight = inFlight;
return inFlight;
}
@@ -0,0 +1,25 @@
/**
* Type definitions for the per-user trace upload WAL.
*
* The WAL is an append-only JSONL file. Each line is one of the variants of
* {@link WalLine}: either an append (a complete trace record) or a tombstone
* (a marker that shadows a previously-appended record by id).
*/
/**
* One queued trace upload, serialized to a single JSONL line.
*/
export interface WalRecord {
id: string;
trackingUri: string;
experimentId: string;
traceInfo: Record<string, unknown>;
traceData: unknown;
attempts: number;
nextAttemptAt: number;
createdAt: number;
firstAttemptAt?: number;
otlpSpans?: string; // base64 encoded ExportTraceServiceRequest protobuf
}
export type WalLine = { type: 'append'; record: WalRecord } | { type: 'tombstone'; id: string };
+44
View File
@@ -0,0 +1,44 @@
import { init } from './core/config';
import {
getLastActiveTraceId,
getCurrentActiveSpan,
updateCurrentTrace,
startSpan,
trace,
withSpan,
} from './core/api';
import { tracingContext } from './core/context';
import { flushTraces } from './core/provider';
import { MlflowClient, MlflowHttpError } from './clients';
import { InMemoryTraceManager } from './core/trace_manager';
import { createAuthProvider } from './auth';
export {
getLastActiveTraceId,
getCurrentActiveSpan,
updateCurrentTrace,
tracingContext,
flushTraces,
init,
startSpan,
trace,
withSpan,
MlflowClient,
MlflowHttpError,
InMemoryTraceManager,
createAuthProvider,
};
// Export entities
export * from './core/constants';
export type { LiveSpan, Span } from './core/entities/span';
export type { Trace } from './core/entities/trace';
export type { TraceInfo, TokenUsage } from './core/entities/trace_info';
export type { TraceData } from './core/entities/trace_data';
export type { TraceLocation, UnityCatalogLocation } from './core/entities/trace_location';
export type { SearchTracesOptions, SearchTracesResult } from './clients';
export { TraceLocationType } from './core/entities/trace_location';
export { SpanStatusCode } from './core/entities/span_status';
export type { UpdateCurrentTraceOptions, SpanOptions, TraceOptions } from './core/api';
export type { TracingContextOptions } from './core/context';
export { registerOnSpanStartHook, registerOnSpanEndHook } from './exporters/span_processor_hooks';
@@ -0,0 +1,461 @@
import { TraceInfo } from '../../../src/core/entities/trace_info';
import { TraceData } from '../../../src/core/entities/trace_data';
import { TraceLocationType } from '../../../src/core/entities/trace_location';
import { TraceState } from '../../../src/core/entities/trace_state';
import { DatabricksArtifactsClient } from '../../../src/clients/artifacts/databricks';
import { AuthProvider } from '../../../src/auth';
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
describe('DatabricksArtifactsClient', () => {
let client: DatabricksArtifactsClient;
const testHost = 'https://dbc-12345.cloud.databricks.com';
const testToken = 'test-token';
// Create a mock AuthProvider for testing
const mockAuthProvider: AuthProvider = {
getHost: () => testHost,
// eslint-disable-next-line require-await, @typescript-eslint/require-await
getHeadersProvider: () => async () => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${testToken}`,
}),
getDatabricksToken: () => testToken,
};
let server: ReturnType<typeof setupServer>;
beforeAll(() => {
server = setupServer();
server.listen();
});
afterAll(() => {
server.close();
});
beforeEach(() => {
client = new DatabricksArtifactsClient({ host: testHost, authProvider: mockAuthProvider });
});
describe('uploadTraceData', () => {
it('should get upload credentials and upload to AWS S3 signed URL', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-databricks-123',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 1000,
});
const traceData = new TraceData([]);
// Mock credentials response
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-databricks-123/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AWS_PRESIGNED_URL',
signed_uri:
'https://s3.amazonaws.com/bucket/traces/tr-databricks-123?signature=xyz',
headers: [{ name: 'x-amz-server-side-encryption', value: 'AES256' }],
},
});
},
),
);
// Mock successful S3 upload
server.use(
http.put('https://s3.amazonaws.com/bucket/traces/tr-databricks-123', () => {
return HttpResponse.json({}, { status: 200 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
it('should get upload credentials and upload to GCP signed URL', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-gcp-456',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '1' },
},
state: TraceState.OK,
requestTime: 2000,
});
const traceData = new TraceData([]);
// Mock GCP credentials response
server.use(
http.get(`${testHost}/api/2.0/mlflow/traces/tr-gcp-456/credentials-for-data-upload`, () => {
return HttpResponse.json({
credential_info: {
type: 'GCP_SIGNED_URL',
signed_uri: 'https://storage.googleapis.com/bucket/traces/tr-gcp-456?signature=abc',
},
});
}),
);
// Mock successful GCP upload
server.use(
http.put('https://storage.googleapis.com/bucket/traces/tr-gcp-456', () => {
return HttpResponse.json({}, { status: 200 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
it('should get upload credentials and upload to Azure Blob Storage', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-azure-789',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '2' },
},
state: TraceState.OK,
requestTime: 3000,
});
const traceData = new TraceData([]);
// Mock Azure credentials response
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-azure-789/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AZURE_SAS_URI',
signed_uri: 'https://storage.azure.com/traces/tr-azure-789?sas=token',
},
});
},
),
);
// Mock Azure Blob Storage upload
server.use(
http.put('https://storage.azure.com/traces/tr-azure-789', () => {
return new HttpResponse(null, { status: 201 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
it('should handle upload failures', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-fail-upload',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.ERROR,
requestTime: 4000,
});
const traceData = new TraceData([]);
// Mock credentials response
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-fail-upload/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AWS_PRESIGNED_URL',
signed_uri: 'https://s3.amazonaws.com/bucket/traces/tr-fail-upload?signature=xyz',
},
});
},
),
);
// Mock failed S3 upload
server.use(
http.put('https://s3.amazonaws.com/bucket/traces/tr-fail-upload', () => {
return HttpResponse.json({ error: 'Forbidden' }, { status: 403 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
'AWS_PRESIGNED_URL upload failed: 403 Forbidden',
);
});
});
describe('downloadTraceData', () => {
it('should get download credentials and download from AWS S3 signed URL', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-download-123',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '5' },
},
state: TraceState.OK,
requestTime: 5000,
});
// Mock credentials response
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-download-123/credentials-for-data-download`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AWS_PRESIGNED_URL',
signed_uri: 'https://s3.amazonaws.com/bucket/traces/tr-download-123?signature=xyz',
headers: [{ name: 'x-amz-server-side-encryption', value: 'AES256' }],
},
});
},
),
);
const mockTraceData = {
spans: [
{
span_id: 'c3Bhbi1kb3dubG9hZGVk', // base64 encoded
trace_id: 'dHItZG93bmxvYWQtMTIz', // base64 encoded
name: 'downloaded-span',
start_time: '5000000000',
end_time: '5100000000',
status: { code: 'OK' },
attributes: {},
},
],
};
// Mock successful S3 download
server.use(
http.get('https://s3.amazonaws.com/bucket/traces/tr-download-123', () => {
return HttpResponse.text(JSON.stringify(mockTraceData));
}),
);
const result = await client.downloadTraceData(traceInfo);
// Verify the result structure
expect(result).toBeInstanceOf(TraceData);
expect(result.spans).toHaveLength(1);
expect(result.spans[0].name).toBe('downloaded-span');
});
});
describe('Error Handling', () => {
it('should log and re-throw upload errors', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-error-log',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.ERROR,
requestTime: 9000,
});
const traceData = new TraceData([]);
// Mock network error
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-error-log/credentials-for-data-upload`,
() => {
return HttpResponse.json({ error: 'Network error' }, { status: 500 });
},
),
);
// Spy on console.error
const consoleSpy = jest.spyOn(console, 'error').mockImplementation();
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow();
expect(consoleSpy).toHaveBeenCalledWith(
'Trace data upload failed for tr-error-log:',
expect.any(Error),
);
consoleSpy.mockRestore();
});
it('should log and re-throw download errors', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-download-error',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 10000,
});
// Mock network error
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-download-error/credentials-for-data-download`,
() => {
return HttpResponse.json({ error: 'Connection timeout' }, { status: 500 });
},
),
);
// Spy on console.error
const consoleSpy = jest.spyOn(console, 'error').mockImplementation();
await expect(client.downloadTraceData(traceInfo)).rejects.toThrow();
expect(consoleSpy).toHaveBeenCalledWith(
'Failed to download trace data for tr-download-error:',
expect.any(Error),
);
consoleSpy.mockRestore();
});
});
describe('Credential Types', () => {
const credentialTypes = [
{ type: 'AWS_PRESIGNED_URL', url: 'https://s3.amazonaws.com/bucket/file' },
{ type: 'GCP_SIGNED_URL', url: 'https://storage.googleapis.com/bucket/file' },
];
credentialTypes.forEach(({ type, url }) => {
it(`should handle ${type} credential type`, async () => {
const traceInfo = new TraceInfo({
traceId: `tr-${type}`,
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 11000,
});
const traceData = new TraceData([]);
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-${type}/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type,
signed_uri: `${url}?signature=test`,
},
});
},
),
);
server.use(
http.put(`${url}`, () => {
return HttpResponse.json({}, { status: 200 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
});
it('should successfully upload to Azure Blob Storage with AZURE_SAS_URI', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-AZURE_SAS_URI',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 12000,
});
const traceData = new TraceData([]);
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-AZURE_SAS_URI/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AZURE_SAS_URI' as any,
signed_uri: 'https://storage.azure.com/file?sas=token',
},
});
},
),
);
// Mock Azure Blob Storage upload
server.use(
http.put('https://storage.azure.com/file', () => {
return new HttpResponse(null, { status: 201 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
it('should successfully upload to Azure ADLS Gen2 with AZURE_ADLS_GEN2_SAS_URI', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-AZURE_ADLS_GEN2_SAS_URI',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 12000,
});
const traceData = new TraceData([]);
server.use(
http.get(
`${testHost}/api/2.0/mlflow/traces/tr-AZURE_ADLS_GEN2_SAS_URI/credentials-for-data-upload`,
() => {
return HttpResponse.json({
credential_info: {
type: 'AZURE_ADLS_GEN2_SAS_URI' as any,
signed_uri: 'https://storage.azure.com/file?sas=token',
},
});
},
),
);
// Mock Azure ADLS Gen2 operations (create, append, flush)
server.use(
// Create file
http.put('https://storage.azure.com/file', ({ request }) => {
const url = new URL(request.url);
if (url.searchParams.get('resource') === 'file') {
return new HttpResponse(null, { status: 201 });
}
return new HttpResponse(null, { status: 404 });
}),
// Append data
http.patch('https://storage.azure.com/file', ({ request }) => {
const url = new URL(request.url);
if (url.searchParams.get('action') === 'append') {
return new HttpResponse(null, { status: 202 });
}
if (url.searchParams.get('action') === 'flush') {
return new HttpResponse(null, { status: 200 });
}
return new HttpResponse(null, { status: 404 });
}),
);
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
});
});
});
@@ -0,0 +1,260 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import { MlflowArtifactsClient } from '../../../src/clients/artifacts/mlflow';
import { TraceInfo } from '../../../src/core/entities/trace_info';
import { TraceData } from '../../../src/core/entities/trace_data';
import { TraceLocationType } from '../../../src/core/entities/trace_location';
import { TraceState } from '../../../src/core/entities/trace_state';
import { AuthProvider } from '../../../src/auth';
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
describe('MlflowArtifactsClient', () => {
let client: MlflowArtifactsClient;
let server: ReturnType<typeof setupServer>;
const testHost = 'http://localhost:5000';
// Create a mock AuthProvider for testing
const mockAuthProvider: AuthProvider = {
getHost: () => testHost,
// eslint-disable-next-line require-await, @typescript-eslint/require-await
getHeadersProvider: () => async () => ({
'Content-Type': 'application/json',
}),
getDatabricksToken: () => undefined,
};
beforeAll(() => {
server = setupServer();
server.listen();
});
afterAll(() => {
server.close();
});
beforeEach(() => {
client = new MlflowArtifactsClient({ host: testHost, authProvider: mockAuthProvider });
});
describe('uploadTraceData', () => {
it('should make PUT request to correct artifact URL', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-abc123',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 1000,
tags: {
'mlflow.artifactLocation': 'mlflow-artifacts:/0/traces/tr-abc123/artifacts',
},
});
const traceData = new TraceData([]);
// Mock the artifacts upload endpoint
server.use(
http.put(
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/0/traces/tr-abc123/artifacts/traces.json',
() => {
return HttpResponse.json({}, { status: 200 });
},
),
);
await client.uploadTraceData(traceInfo, traceData);
// Test passes if no errors thrown
});
it('should throw error when artifact location is missing', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-no-artifact',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 1000,
tags: {}, // No artifact location
});
const traceData = new TraceData([]);
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
'Artifact location not found in trace tags',
);
// Test passes if error is thrown as expected
});
});
describe('uploadTraceData with local artifact roots', () => {
let tmpRoot: string;
const makeTraceInfo = (artifactLocation: string) =>
new TraceInfo({
traceId: 'tr-local',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '0' },
},
state: TraceState.OK,
requestTime: 1000,
tags: {
'mlflow.artifactLocation': artifactLocation,
},
});
beforeEach(async () => {
tmpRoot = await mkdtemp(join(tmpdir(), 'mlflow-wal-test-'));
});
afterEach(async () => {
await rm(tmpRoot, { recursive: true, force: true });
});
it('writes traces.json to a bare local path, creating the directory', async () => {
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
const traceInfo = makeTraceInfo(artifactDir);
const traceData = new TraceData([]);
await client.uploadTraceData(traceInfo, traceData);
const written = await readFile(join(artifactDir, 'traces.json'), 'utf8');
expect(JSON.parse(written)).toEqual({ spans: [] });
});
it('writes traces.json for a file:// artifact location', async () => {
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
const fileUri = pathToFileURL(artifactDir).href;
const traceInfo = makeTraceInfo(fileUri);
const traceData = new TraceData([]);
await client.uploadTraceData(traceInfo, traceData);
const written = await readFile(join(artifactDir, 'traces.json'), 'utf8');
expect(JSON.parse(written)).toEqual({ spans: [] });
});
it('rejects unsupported remote schemes via resolveArtifactUri', async () => {
const traceInfo = makeTraceInfo('s3://bucket/0/traces/tr-local/artifacts');
const traceData = new TraceData([]);
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
'Expected mlflow-artifacts:// URI, got s3:',
);
});
it('refuses to write to a relative local artifact path', async () => {
const traceInfo = makeTraceInfo('relative/0/traces/tr-local/artifacts');
const traceData = new TraceData([]);
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
/Refusing to use a relative local artifact location/,
);
});
it('round-trips trace data through a bare local path (upload then download)', async () => {
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
const traceInfo = makeTraceInfo(artifactDir);
await client.uploadTraceData(traceInfo, new TraceData([]));
const result = await client.downloadTraceData(traceInfo);
expect(result).toBeInstanceOf(TraceData);
expect(result.spans).toHaveLength(0);
});
it('reads trace data from a file:// artifact location', async () => {
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
const fileUri = pathToFileURL(artifactDir).href;
const traceInfo = makeTraceInfo(fileUri);
await client.uploadTraceData(traceInfo, new TraceData([]));
const result = await client.downloadTraceData(traceInfo);
expect(result).toBeInstanceOf(TraceData);
expect(result.spans).toHaveLength(0);
});
});
describe('downloadTraceData', () => {
it('should make GET request to correct artifact URL', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-download',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '5' },
},
state: TraceState.OK,
requestTime: 4000,
tags: {
'mlflow.artifactLocation': 'mlflow-artifacts:/5/traces/tr-download/artifacts',
},
});
const mockResponse = {
spans: [
{
span_id: 'c3Bhbi1kb3dubG9hZA==',
trace_id: 'dHItZG93bmxvYWQ=',
name: 'downloaded-span',
start_time: '4000000000',
end_time: '4100000000',
status: { code: 'OK' },
attributes: {},
},
],
};
// Mock the artifacts download endpoint
server.use(
http.get(
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/5/traces/tr-download/artifacts/traces.json',
() => {
return HttpResponse.json(mockResponse);
},
),
);
const result = await client.downloadTraceData(traceInfo);
expect(result).toBeInstanceOf(TraceData);
expect(result.spans).toHaveLength(1);
expect(result.spans[0].name).toBe('downloaded-span');
});
it('should handle complex artifact paths in URL construction', async () => {
const traceInfo = new TraceInfo({
traceId: 'tr-complex-path',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '42' },
},
state: TraceState.OK,
requestTime: 5000,
tags: {
'mlflow.artifactLocation':
'mlflow-artifacts:/42/some/nested/path/traces/tr-complex-path/artifacts',
},
});
// Mock the complex path artifacts download endpoint
server.use(
http.get(
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/42/some/nested/path/traces/tr-complex-path/artifacts/traces.json',
() => {
return HttpResponse.json({ spans: [] });
},
),
);
await client.downloadTraceData(traceInfo);
// Test passes if no errors thrown
});
});
});
@@ -0,0 +1,367 @@
import { randomUUID } from 'crypto';
import * as mlflow from '../../src';
import { MlflowClient, type SearchTracesOptions } from '../../src/clients/client';
import type { ArtifactsClient } from '../../src/clients/artifacts';
import { Trace } from '../../src/core/entities/trace';
import { TraceData } from '../../src/core/entities/trace_data';
import { TraceInfo } from '../../src/core/entities/trace_info';
import { TraceLocationType } from '../../src/core/entities/trace_location';
import { TraceState } from '../../src/core/entities/trace_state';
import { createAuthProvider } from '../../src/auth';
import { TEST_TRACKING_URI } from '../helper';
describe('MlflowClient', () => {
let client: MlflowClient;
let experimentId: string;
beforeEach(async () => {
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
client = new MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
// Create a new experiment for each test
const experimentName = `test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
experimentId = await client.createExperiment(experimentName);
});
afterEach(async () => {
// Clean up: delete the experiment
try {
await client.deleteExperiment(experimentId);
} catch (error) {
console.warn('Failed to delete experiment:', error);
}
});
describe('createTrace', () => {
it('should create a trace', async () => {
const traceId = randomUUID();
const traceInfo = new TraceInfo({
traceId: traceId,
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: experimentId,
},
},
state: TraceState.OK,
requestTime: 1000,
executionDuration: 500,
requestPreview: '{"input":"test"}',
responsePreview: '{"output":"result"}',
clientRequestId: 'client-request-id',
traceMetadata: { 'meta-key': 'meta-value' },
tags: { 'tag-key': 'tag-value' },
assessments: [],
});
const createdTraceInfo = await client.createTrace(traceInfo);
expect(createdTraceInfo).toBeInstanceOf(TraceInfo);
expect(createdTraceInfo.traceId).toBe(traceId);
expect(createdTraceInfo.state).toBe(TraceState.OK);
expect(createdTraceInfo.requestTime).toBe(1000);
expect(createdTraceInfo.executionDuration).toBe(500);
expect(createdTraceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(createdTraceInfo.traceLocation.mlflowExperiment?.experimentId).toBe(experimentId);
expect(createdTraceInfo.requestPreview).toBe('{"input":"test"}');
expect(createdTraceInfo.responsePreview).toBe('{"output":"result"}');
expect(createdTraceInfo.clientRequestId).toBe('client-request-id');
expect(createdTraceInfo.traceMetadata).toMatchObject({ 'meta-key': 'meta-value' });
expect(createdTraceInfo.tags).toEqual({
'tag-key': 'tag-value',
'mlflow.artifactLocation': expect.any(String),
});
expect(createdTraceInfo.assessments).toEqual([]);
});
it('should create a trace with error state', async () => {
const traceId = randomUUID();
const traceInfo = new TraceInfo({
traceId: traceId,
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: experimentId,
},
},
state: TraceState.ERROR,
requestTime: 1000,
});
const createdTraceInfo = await client.createTrace(traceInfo);
expect(createdTraceInfo).toBeInstanceOf(TraceInfo);
expect(createdTraceInfo.traceId).toBe(traceId);
expect(createdTraceInfo.state).toBe(TraceState.ERROR);
expect(createdTraceInfo.requestTime).toBe(1000);
expect(createdTraceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(createdTraceInfo.traceLocation.mlflowExperiment?.experimentId).toBe(experimentId);
});
});
describe('getTraceInfo', () => {
it('should retrieve trace info for an existing trace', async () => {
const traceId = randomUUID();
const traceInfo = new TraceInfo({
traceId: traceId,
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: experimentId,
},
},
state: TraceState.OK,
requestTime: 1000,
});
await client.createTrace(traceInfo);
// Now retrieve it
const retrievedTraceInfo = await client.getTraceInfo(traceId);
expect(retrievedTraceInfo).toBeInstanceOf(TraceInfo);
expect(retrievedTraceInfo.traceId).toBe(traceId);
expect(retrievedTraceInfo.state).toBe(TraceState.OK);
expect(retrievedTraceInfo.requestTime).toBe(1000);
});
});
describe('searchTraces', () => {
const experimentLocation = () => ({
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: experimentId,
},
});
it('should search traces by location and page through results', async () => {
const firstTraceId = randomUUID();
const secondTraceId = randomUUID();
const testFilterTag = randomUUID();
await client.createTrace(
new TraceInfo({
traceId: firstTraceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 1000,
tags: { searchTracesPagination: testFilterTag },
}),
);
await client.createTrace(
new TraceInfo({
traceId: secondTraceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 2000,
tags: { searchTracesPagination: testFilterTag },
}),
);
const firstPage = await client.searchTraces({
locations: [experimentLocation()],
filter: `tags.searchTracesPagination = '${testFilterTag}'`,
maxResults: 1,
orderBy: ['timestamp_ms DESC'],
includeSpans: false,
});
expect(firstPage.traces).toHaveLength(1);
expect(firstPage.traces[0]).toBeInstanceOf(Trace);
expect(firstPage.traces[0].info).toBeInstanceOf(TraceInfo);
expect([firstTraceId, secondTraceId]).toContain(firstPage.traces[0].info.traceId);
expect(firstPage.nextPageToken).toBeDefined();
const secondPage = await client.searchTraces({
locations: [experimentLocation()],
filter: `tags.searchTracesPagination = '${testFilterTag}'`,
maxResults: 1,
orderBy: ['timestamp_ms DESC'],
pageToken: firstPage.nextPageToken,
includeSpans: false,
});
const returnedTraceIds = [
firstPage.traces[0].info.traceId,
...secondPage.traces.map((trace) => trace.info.traceId),
];
expect(returnedTraceIds).toEqual(expect.arrayContaining([firstTraceId, secondTraceId]));
});
it('should search traces with a filter', async () => {
const matchingTraceId = randomUUID();
const otherTraceId = randomUUID();
await client.createTrace(
new TraceInfo({
traceId: matchingTraceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 1000,
tags: { searchTracesFilter: 'match' },
}),
);
await client.createTrace(
new TraceInfo({
traceId: otherTraceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 2000,
tags: { searchTracesFilter: 'skip' },
}),
);
const result = await client.searchTraces({
locations: [experimentLocation()],
filter: "tags.searchTracesFilter = 'match'",
maxResults: 10,
includeSpans: false,
});
expect(result.traces.map((trace) => trace.info.traceId)).toContain(matchingTraceId);
expect(result.traces.map((trace) => trace.info.traceId)).not.toContain(otherTraceId);
});
it('should fetch span data for each trace by default', async () => {
mlflow.init({ trackingUri: TEST_TRACKING_URI, experimentId });
const span = mlflow.startSpan({ name: 'search-traces-span' });
span.setInputs({ question: 'What is MLflow?' });
span.end();
await mlflow.flushTraces();
const result = await client.searchTraces({
locations: [experimentLocation()],
});
expect(result.traces).toHaveLength(1);
const trace = result.traces[0];
expect(trace).toBeInstanceOf(Trace);
expect(trace.info.traceId).toBe(span.traceId);
expect(trace.data.spans).toHaveLength(1);
expect(trace.data.spans[0].name).toBe('search-traces-span');
});
it('should return metadata-only traces with empty span data when includeSpans is false', async () => {
const traceId = randomUUID();
await client.createTrace(
new TraceInfo({
traceId: traceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 1000,
}),
);
const result = await client.searchTraces({
locations: [experimentLocation()],
includeSpans: false,
});
expect(result.traces).toHaveLength(1);
expect(result.traces[0].info.traceId).toBe(traceId);
expect(result.traces[0].data.spans).toEqual([]);
});
it('should skip traces whose span data cannot be downloaded', async () => {
// createTrace registers trace metadata without uploading span data,
// so the span-data download fails and the trace is dropped.
await client.createTrace(
new TraceInfo({
traceId: randomUUID(),
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 1000,
}),
);
const result = await client.searchTraces({
locations: [experimentLocation()],
});
expect(result.traces).toEqual([]);
});
it('should warn and drop only the traces whose span data download fails', async () => {
const failingTraceId = randomUUID();
const okTraceId = randomUUID();
for (const traceId of [failingTraceId, okTraceId]) {
await client.createTrace(
new TraceInfo({
traceId: traceId,
traceLocation: experimentLocation(),
state: TraceState.OK,
requestTime: 1000,
}),
);
}
const { artifactsClient } = client as unknown as { artifactsClient: ArtifactsClient };
const downloadSpy = jest
.spyOn(artifactsClient, 'downloadTraceData')
.mockImplementation((traceInfo) =>
traceInfo.traceId === failingTraceId
? Promise.reject(new Error('download failed'))
: Promise.resolve(new TraceData([])),
);
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
try {
const result = await client.searchTraces({
locations: [experimentLocation()],
});
expect(result.traces.map((trace) => trace.info.traceId)).toEqual([okTraceId]);
expect(warnSpy).toHaveBeenCalledWith(
`Failed to download trace data for trace ${failingTraceId}:`,
expect.any(Error),
);
} finally {
downloadSpy.mockRestore();
warnSpy.mockRestore();
}
});
it('should return an empty result when no traces match', async () => {
const result = await client.searchTraces({
locations: [experimentLocation()],
filter: "tags.searchTracesFilter = 'missing'",
maxResults: 10,
});
expect(result.traces).toEqual([]);
expect(result.nextPageToken).toBeUndefined();
});
it('should reject search without locations', async () => {
await expect(client.searchTraces({ locations: [] })).rejects.toThrow(
'At least one location must be specified for searching traces.',
);
});
it('should reject search when locations is omitted', async () => {
// Plain-JS callers can omit locations despite the required type.
await expect(client.searchTraces({} as SearchTracesOptions)).rejects.toThrow(
'At least one location must be specified for searching traces.',
);
});
});
describe('getExperimentByName', () => {
it('should retrieve an existing experiment by name', async () => {
const experiment = await client.getExperimentByName(
`test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`,
);
expect(experiment).toBeNull();
const createdName = `lookup-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
const createdId = await client.createExperiment(createdName);
try {
const found = await client.getExperimentByName(createdName);
expect(found).toEqual({
experimentId: createdId,
name: createdName,
});
} finally {
await client.deleteExperiment(createdId);
}
});
});
});
@@ -0,0 +1,71 @@
import { MlflowClient } from '../../src/clients/client';
import { createAuthProvider } from '../../src/auth';
import { MlflowHttpError } from '../../src/clients/utils';
/**
* Unit tests for {@link MlflowClient.exportOtlpSpans} used for OSS (non-Databricks)
* tracking servers. These mock `global.fetch` so they exercise the request shaping
* (endpoint, headers, body) and error mapping without needing a live tracking server.
*/
describe('MlflowClient.exportOtlpSpans', () => {
const trackingUri = 'http://localhost:5000';
let client: MlflowClient;
let fetchMock: jest.Mock;
const originalFetch = global.fetch;
beforeEach(() => {
const authProvider = createAuthProvider({ trackingUri });
client = new MlflowClient({ trackingUri, authProvider });
fetchMock = jest.fn().mockResolvedValue(new Response(null, { status: 200 }));
global.fetch = fetchMock as unknown as typeof fetch;
});
afterEach(() => {
global.fetch = originalFetch;
});
it('POSTs OTLP protobuf to the OSS endpoint with the protobuf content type and experiment header', async () => {
const bytes = new Uint8Array([0x0a, 0x01, 0x02, 0x03]);
await client.exportOtlpSpans('exp-123', bytes);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
expect(url).toBe('http://localhost:5000/v1/traces');
expect(init.method).toBe('POST');
const headers = init.headers as Record<string, string>;
expect(headers['Content-Type']).toBe('application/x-protobuf');
expect(headers['x-mlflow-experiment-id']).toBe('exp-123');
expect(init.body).toBe(bytes);
});
it('does not call fetch when there are no span bytes to send', async () => {
await client.exportOtlpSpans('exp-123', new Uint8Array(0));
expect(fetchMock).not.toHaveBeenCalled();
});
it('surfaces a 501 as an MlflowHttpError so callers can fall back', async () => {
fetchMock.mockResolvedValue(
new Response('REST OTLP span logging is not supported', {
status: 501,
statusText: 'Not Implemented',
}),
);
await expect(
client.exportOtlpSpans('exp-123', new Uint8Array([0x0a, 0x01])),
).rejects.toBeInstanceOf(MlflowHttpError);
});
it('propagates the 501 status code on the thrown error', async () => {
fetchMock.mockResolvedValue(
new Response('nope', { status: 501, statusText: 'Not Implemented' }),
);
await expect(
client.exportOtlpSpans('exp-123', new Uint8Array([0x0a, 0x01])),
).rejects.toMatchObject({
status: 501,
});
});
});
@@ -0,0 +1,184 @@
import { makeRequest, MlflowHttpError } from '../../src/clients/utils';
describe('makeRequest', () => {
const mockHeaderProvider = () => Promise.resolve({ 'Content-Type': 'application/json' });
beforeEach(() => {
// Clear all mocks before each test
jest.clearAllMocks();
});
afterEach(() => {
jest.restoreAllMocks();
});
describe('error handling with response body', () => {
it('should include response body in error message when request fails', async () => {
const errorResponseBody = JSON.stringify({
error_code: 'INVALID_PARAMETER_VALUE',
message: 'Invalid experiment ID provided',
});
const mockResponse = {
ok: false,
status: 400,
statusText: 'Bad Request',
text: jest.fn().mockResolvedValue(errorResponseBody),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'GET',
'http://localhost:5000/api/2.0/mlflow/experiments/get',
mockHeaderProvider,
),
).rejects.toThrow(`HTTP 400: Bad Request - ${errorResponseBody}`);
expect(mockResponse.text).toHaveBeenCalled();
});
it('should handle empty response body in error', async () => {
const mockResponse = {
ok: false,
status: 404,
statusText: 'Not Found',
text: jest.fn().mockResolvedValue(''),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'GET',
'http://localhost:5000/api/2.0/mlflow/experiments/get',
mockHeaderProvider,
),
).rejects.toThrow('HTTP 404: Not Found');
expect(mockResponse.text).toHaveBeenCalled();
});
it('should handle error when reading response body fails', async () => {
const mockResponse = {
ok: false,
status: 500,
statusText: 'Internal Server Error',
text: jest.fn().mockRejectedValue(new Error('Failed to read body')),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'GET',
'http://localhost:5000/api/2.0/mlflow/experiments/get',
mockHeaderProvider,
),
).rejects.toThrow('HTTP 500: Internal Server Error');
expect(mockResponse.text).toHaveBeenCalled();
});
it('should include response body with plain text error', async () => {
const errorResponseBody = 'Service temporarily unavailable';
const mockResponse = {
ok: false,
status: 503,
statusText: 'Service Unavailable',
text: jest.fn().mockResolvedValue(errorResponseBody),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'POST',
'http://localhost:5000/api/2.0/mlflow/runs/create',
mockHeaderProvider,
{},
),
).rejects.toThrow(`HTTP 503: Service Unavailable - ${errorResponseBody}`);
expect(mockResponse.text).toHaveBeenCalled();
});
it('should throw an MlflowHttpError carrying status and error_code', async () => {
const errorResponseBody = JSON.stringify({
error_code: 'RESOURCE_DOES_NOT_EXIST',
message: 'Experiment not found',
});
const mockResponse = {
ok: false,
status: 404,
statusText: 'Not Found',
text: jest.fn().mockResolvedValue(errorResponseBody),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'GET',
'http://localhost:5000/api/2.0/mlflow/experiments/get-by-name',
mockHeaderProvider,
),
).rejects.toMatchObject({
name: 'MlflowHttpError',
status: 404,
statusText: 'Not Found',
body: errorResponseBody,
errorCode: 'RESOURCE_DOES_NOT_EXIST',
});
});
it('should leave errorCode undefined when body is not JSON', async () => {
const mockResponse = {
ok: false,
status: 500,
statusText: 'Internal Server Error',
text: jest.fn().mockResolvedValue('upstream timeout'),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
const err = await makeRequest(
'GET',
'http://localhost:5000/api/2.0/mlflow/experiments/get',
mockHeaderProvider,
).catch((e: unknown) => e);
expect(err).toBeInstanceOf(MlflowHttpError);
expect((err as MlflowHttpError).status).toBe(500);
expect((err as MlflowHttpError).errorCode).toBeUndefined();
});
it('should truncate large response bodies to prevent memory issues', async () => {
// Create a response body larger than 1000 characters
const largeResponseBody = 'x'.repeat(1500); // 1500 characters
const mockResponse = {
ok: false,
status: 500,
statusText: 'Internal Server Error',
text: jest.fn().mockResolvedValue(largeResponseBody),
};
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
await expect(
makeRequest(
'POST',
'http://localhost:5000/api/2.0/mlflow/runs/create',
mockHeaderProvider,
{},
),
).rejects.toThrow(/HTTP 500: Internal Server Error - x+\.\.\. \(truncated\)/);
expect(mockResponse.text).toHaveBeenCalled();
});
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,122 @@
import { createAuthProvider } from '../../src/auth';
const ENV_KEYS = [
'MLFLOW_TRACKING_TOKEN',
'MLFLOW_TRACKING_USERNAME',
'MLFLOW_TRACKING_PASSWORD',
'MLFLOW_WORKSPACE',
] as const;
describe('createAuthProvider', () => {
describe('OSS auth (createOssAuth)', () => {
const savedEnv: Record<string, string | undefined> = {};
beforeEach(() => {
for (const key of ENV_KEYS) {
savedEnv[key] = process.env[key];
delete process.env[key];
}
});
afterEach(() => {
for (const key of ENV_KEYS) {
if (savedEnv[key] === undefined) {
delete process.env[key];
} else {
process.env[key] = savedEnv[key];
}
}
});
it('should include Bearer token header when MLFLOW_TRACKING_TOKEN is set', async () => {
process.env.MLFLOW_TRACKING_TOKEN = 'test-token';
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
expect(headers['Authorization']).toBe('Bearer test-token');
expect(headers['Content-Type']).toBe('application/json');
});
it('should include Basic auth header when username and password are set', async () => {
process.env.MLFLOW_TRACKING_USERNAME = 'user';
process.env.MLFLOW_TRACKING_PASSWORD = 'pass';
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
const expected = `Basic ${Buffer.from('user:pass').toString('base64')}`;
expect(headers['Authorization']).toBe(expected);
});
it('should not include Authorization header when no credentials are set', async () => {
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
expect(headers['Authorization']).toBeUndefined();
expect(headers['Content-Type']).toBe('application/json');
});
it('should include X-MLFLOW-WORKSPACE header when MLFLOW_WORKSPACE is set', async () => {
process.env.MLFLOW_WORKSPACE = 'my-namespace';
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
expect(headers['X-MLFLOW-WORKSPACE']).toBe('my-namespace');
});
it('should not include X-MLFLOW-WORKSPACE header when MLFLOW_WORKSPACE is not set', async () => {
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
expect(headers['X-MLFLOW-WORKSPACE']).toBeUndefined();
});
it('should include both Authorization and X-MLFLOW-WORKSPACE when both are set', async () => {
process.env.MLFLOW_TRACKING_TOKEN = 'test-token';
process.env.MLFLOW_WORKSPACE = 'my-namespace';
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
const headers = await provider.getHeadersProvider()();
expect(headers['Authorization']).toBe('Bearer test-token');
expect(headers['X-MLFLOW-WORKSPACE']).toBe('my-namespace');
expect(headers['Content-Type']).toBe('application/json');
});
it('should use workspace from options when env var is not set', async () => {
const provider = createAuthProvider({
trackingUri: 'http://localhost:5000',
workspace: 'config-namespace',
});
const headers = await provider.getHeadersProvider()();
expect(headers['X-MLFLOW-WORKSPACE']).toBe('config-namespace');
});
it('should prefer MLFLOW_WORKSPACE env var over options', async () => {
process.env.MLFLOW_WORKSPACE = 'env-namespace';
const provider = createAuthProvider({
trackingUri: 'http://localhost:5000',
workspace: 'config-namespace',
});
const headers = await provider.getHeadersProvider()();
expect(headers['X-MLFLOW-WORKSPACE']).toBe('env-namespace');
});
it('should return the tracking URI as host', () => {
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
expect(provider.getHost()).toBe('http://localhost:5000');
});
it('should return undefined for getDatabricksToken', () => {
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
expect(provider.getDatabricksToken()).toBeUndefined();
});
});
});
@@ -0,0 +1,257 @@
import fs from 'fs';
import os from 'os';
import path from 'path';
import { init, getConfig } from '../../src/core/config';
describe('Config', () => {
describe('init and getConfig', () => {
describe('environment variable resolution', () => {
afterEach(() => {
delete process.env.MLFLOW_TRACKING_URI;
delete process.env.MLFLOW_EXPERIMENT_ID;
});
it('should read tracking configuration from environment variables when not provided', () => {
process.env.MLFLOW_TRACKING_URI = 'http://env-tracking-host:5000';
process.env.MLFLOW_EXPERIMENT_ID = 'env-experiment-id';
init({});
const result = getConfig();
expect(result.trackingUri).toBe('http://env-tracking-host:5000');
expect(result.experimentId).toBe('env-experiment-id');
expect(result.host).toBe('http://env-tracking-host:5000');
});
it('should throw an error when trackingUri is missing from both config and environment', () => {
process.env.MLFLOW_EXPERIMENT_ID = 'env-experiment-id';
expect(() => init({ experimentId: 'config-experiment-id' })).toThrow(
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
);
});
it('should throw an error when experimentId is missing from both config and environment', () => {
process.env.MLFLOW_TRACKING_URI = 'http://env-tracking-host:5000';
expect(() => init({ trackingUri: 'http://explicit-host:5000' })).toThrow(
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
);
});
});
it('should initialize with MLflow tracking server configuration', () => {
const config = {
trackingUri: 'http://localhost:5000',
experimentId: '123456789',
};
init(config);
const result = getConfig();
expect(result.trackingUri).toBe('http://localhost:5000');
expect(result.experimentId).toBe('123456789');
expect(result.host).toBe('http://localhost:5000');
});
it('should throw error if trackingUri is missing', () => {
const config = {
trackingUri: '',
experimentId: '123456789',
};
expect(() => init(config)).toThrow(
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
);
});
it('should throw error if experimentId is missing', () => {
const config = {
trackingUri: 'http://localhost:5000',
experimentId: '',
};
expect(() => init(config)).toThrow(
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
);
});
it('should throw error if trackingUri is not a string', () => {
const config = {
trackingUri: 123 as any,
experimentId: '123456789',
};
expect(() => init(config)).toThrow('trackingUri must be a string');
});
it('should throw error if experimentId is not a string', () => {
const config = {
trackingUri: 'http://localhost:5000',
experimentId: 123 as any,
};
expect(() => init(config)).toThrow('experimentId must be a string');
});
it('should throw error for malformed trackingUri', () => {
const config = {
trackingUri: 'not-a-valid-uri',
experimentId: '123456789',
};
expect(() => init(config)).toThrow(
"Invalid trackingUri: 'not-a-valid-uri'. Must be a valid HTTP or HTTPS URL, or 'databricks' / 'databricks://<profile>'.",
);
});
it.skip('should throw error if getConfig is called without init', () => {
// Skip this test as it interferes with other tests due to module state
expect(() => getConfig()).toThrow(
'The MLflow Tracing client is not configured. Please call init() with host and experimentId before using tracing functions.',
);
});
describe('Databricks configuration', () => {
const tempDir = path.join(os.tmpdir(), 'mlflow-databricks-test-' + Date.now());
const configPath = path.join(tempDir, '.databrickscfg');
beforeEach(() => {
fs.mkdirSync(tempDir, { recursive: true });
});
afterEach(() => {
fs.rmSync(tempDir, { recursive: true, force: true });
delete process.env.DATABRICKS_HOST;
delete process.env.DATABRICKS_TOKEN;
});
it('should throw error if host not found anywhere', () => {
const config = {
trackingUri: 'databricks',
experimentId: '123456789',
databricksConfigPath: '/nonexistent/path/.databrickscfg',
};
expect(() => init(config)).toThrow('Databricks host not found');
});
it('should read host from config file when env var not set', () => {
const configContent = `[DEFAULT]
host = https://config-workspace.databricks.com
token = dapi123456789abcdef`;
fs.writeFileSync(configPath, configContent);
const config = {
trackingUri: 'databricks',
experimentId: '123456789',
databricksConfigPath: configPath,
};
init(config);
const result = getConfig();
expect(result.host).toBe('https://config-workspace.databricks.com');
});
it('should read host from specific profile in config file', () => {
const configContent = `[DEFAULT]
host = https://default-workspace.databricks.com
token = default-token
[dev]
host = https://dev-workspace.databricks.com
token = dev-token`;
fs.writeFileSync(configPath, configContent);
const config = {
trackingUri: 'databricks://dev',
experimentId: '123456789',
databricksConfigPath: configPath,
};
init(config);
const result = getConfig();
expect(result.host).toBe('https://dev-workspace.databricks.com');
});
it('should use DATABRICKS_HOST from environment', () => {
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
process.env.DATABRICKS_TOKEN = 'env-token';
const config = {
trackingUri: 'databricks',
experimentId: '123456789',
};
init(config);
const result = getConfig();
expect(result.host).toBe('https://env-workspace.databricks.com');
expect(result.databricksToken).toBe('env-token');
});
it('should use explicit host/token over environment', () => {
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
process.env.DATABRICKS_TOKEN = 'env-token';
const config = {
trackingUri: 'databricks',
experimentId: '123456789',
host: 'https://override-workspace.databricks.com',
databricksToken: 'override-token',
};
init(config);
const result = getConfig();
expect(result.host).toBe('https://override-workspace.databricks.com');
expect(result.databricksToken).toBe('override-token');
});
it('should prefer env var over config file for host', () => {
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
const configContent = `[DEFAULT]
host = https://config-workspace.databricks.com
token = dapi123456789abcdef`;
fs.writeFileSync(configPath, configContent);
const config = {
trackingUri: 'databricks',
experimentId: '123456789',
databricksConfigPath: configPath,
};
init(config);
const result = getConfig();
// Env var takes precedence over config file
expect(result.host).toBe('https://env-workspace.databricks.com');
});
it('should handle empty profile name in URI', () => {
const configContent = `[DEFAULT]
host = https://default-workspace.databricks.com
token = dapi123456789abcdef`;
fs.writeFileSync(configPath, configContent);
const config = {
trackingUri: 'databricks://',
experimentId: '123456789',
databricksConfigPath: configPath,
};
init(config);
const result = getConfig();
expect(result.host).toBe('https://default-workspace.databricks.com');
});
});
});
});
@@ -0,0 +1,418 @@
import { trace } from '@opentelemetry/api';
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
import {
createMlflowSpan,
Span,
NoOpSpan,
type LiveSpan,
type SerializedSpan,
} from '../../../src/core/entities/span';
import { SpanEvent } from '../../../src/core/entities/span_event';
import { SpanStatus, SpanStatusCode } from '../../../src/core/entities/span_status';
import { SpanAttributeKey, SpanType } from '../../../src/core/constants';
import { convertHrTimeToNanoSeconds } from '../../../src/core/utils';
import { JSONBig } from '../../../src/core/utils/json';
// Set up a proper tracer provider
const provider = new BasicTracerProvider();
trace.setGlobalTracerProvider(provider);
const tracer = trace.getTracer('mlflow-test-tracer', '1.0.0');
describe('Span', () => {
describe('createMlflowSpan', () => {
describe('Live Span Creation', () => {
it('should create a live span from an active OpenTelemetry span', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('parent');
try {
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.LLM);
expect(mlflowSpan).toBeInstanceOf(Span);
expect(mlflowSpan.traceId).toBe(traceId);
expect(mlflowSpan.name).toBe('parent');
/* OpenTelemetry's end time default value is 0 when unset */
expect(mlflowSpan.startTime[0]).toBeGreaterThan(0); // Check seconds part of HrTime
expect(mlflowSpan.endTime?.[1]).toBe(0); // Check nanoseconds part of HrTime
expect(mlflowSpan.parentId).toBeNull();
} finally {
span.end();
}
});
it('should handle span inputs and outputs', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('test');
try {
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
mlflowSpan.setInputs({ input: 1 });
mlflowSpan.setOutputs(2);
expect(mlflowSpan.inputs).toEqual({ input: 1 });
expect(mlflowSpan.outputs).toBe(2);
} finally {
span.end();
}
});
it('should handle span attributes', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('test');
try {
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
mlflowSpan.setAttribute('key', 3);
expect(mlflowSpan.getAttribute('key')).toBe(3);
// Test complex object serialization
const complexObject = { nested: { value: 'test' } };
mlflowSpan.setAttribute('complex', complexObject);
expect(mlflowSpan.getAttribute('complex')).toEqual(complexObject);
} finally {
span.end();
}
});
it('should handle span status', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('test');
try {
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
mlflowSpan.setStatus(SpanStatusCode.OK);
expect(mlflowSpan.status).toBeInstanceOf(SpanStatus);
expect(mlflowSpan.status?.statusCode).toBe(SpanStatusCode.OK);
} finally {
span.end();
}
});
it('should handle span events', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('test');
try {
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
const event = new SpanEvent({
name: 'test_event',
timestamp: 99999n,
attributes: { foo: 'bar' },
});
mlflowSpan.addEvent(event);
const events = mlflowSpan.events;
expect(events.length).toBeGreaterThan(0);
// Find our test event
const testEvent = events.find((e) => e.name === 'test_event');
expect(testEvent).toBeDefined();
expect(testEvent?.attributes).toEqual({ foo: 'bar' });
} finally {
span.end();
}
});
});
describe('Completed Span Creation', () => {
it('should create a completed span from an active OpenTelemetry span', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('parent');
span.setAttribute(SpanAttributeKey.TRACE_ID, JSONBig.stringify(traceId));
span.setAttribute(SpanAttributeKey.INPUTS, '{"input": 1}');
span.setAttribute(SpanAttributeKey.OUTPUTS, '2');
span.setAttribute('custom_attr', 'custom_value');
span.end();
const mlflowSpan = createMlflowSpan(span, traceId);
expect(mlflowSpan).toBeInstanceOf(Span);
expect(mlflowSpan.traceId).toBe(traceId);
expect(mlflowSpan.name).toBe('parent');
expect(convertHrTimeToNanoSeconds(mlflowSpan.startTime)).toBeGreaterThan(0n);
expect(mlflowSpan.endTime).toBeDefined();
if (mlflowSpan.endTime) {
expect(convertHrTimeToNanoSeconds(mlflowSpan.endTime)).toBeGreaterThan(0n);
}
expect(mlflowSpan.parentId).toBeNull();
expect(mlflowSpan.inputs).toEqual({ input: 1 });
expect(mlflowSpan.outputs).toBe(2);
expect(mlflowSpan.getAttribute('custom_attr')).toBe('custom_value');
// Setter should not be defined for completed span
expect('setInputs' in mlflowSpan).toBe(false);
expect('setOutputs' in mlflowSpan).toBe(false);
expect('setAttribute' in mlflowSpan).toBe(false);
expect('addEvent' in mlflowSpan).toBe(false);
});
});
describe('No-Op Span Creation', () => {
it('should create a no-op span from null input', () => {
const traceId = 'tr-12345';
const span = createMlflowSpan(null, traceId);
expect(span).toBeInstanceOf(NoOpSpan);
expect(span.traceId).toBe('no-op-span-trace-id');
expect(span.spanId).toBe('');
expect(span.name).toBe('');
expect(span.startTime).toEqual([0, 0]); // HrTime format [seconds, nanoseconds]
expect(span.endTime).toBeNull();
expect(span.parentId).toBeNull();
expect(span.inputs).toBeNull();
expect(span.outputs).toBeNull();
});
it('should create a no-op span from undefined input', () => {
const traceId = 'tr-12345';
const span = createMlflowSpan(undefined, traceId);
expect(span).toBeInstanceOf(NoOpSpan);
});
});
});
describe('Exception Safety', () => {
it('should handle exceptions in span.end() gracefully', () => {
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
const traceId = 'tr-12345';
const span = tracer.startSpan('test-span');
try {
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
// Mock the underlying OTel span to throw on end()
// eslint-disable-next-line @typescript-eslint/unbound-method
const originalEnd = span.end;
span.end = jest.fn(() => {
throw new Error('OTel span.end() failed');
});
expect(() => mlflowSpan.end()).not.toThrow();
// Restore original end method
span.end = originalEnd;
} finally {
span.end();
consoleErrorSpy.mockRestore();
}
});
});
describe('Circular Reference Handling', () => {
it('should handle circular references in span attributes', () => {
const traceId = 'tr-circular';
const span = tracer.startSpan('circular-test');
try {
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.UNKNOWN) as LiveSpan;
// Create an object with circular reference
const circularObj: any = { name: 'test' };
circularObj.self = circularObj;
// This should not throw
expect(() => {
mlflowSpan.setAttribute('circular', circularObj);
}).not.toThrow();
// Verify the circular reference is replaced with placeholder
const attrValue = mlflowSpan.getAttribute('circular');
expect(attrValue.name).toBe('test');
expect(attrValue.self).toBe('[Circular]');
} finally {
span.end();
}
});
it('should handle non-serializable objects', () => {
const traceId = 'tr-non-serializable';
const span = tracer.startSpan('non-serializable-test');
try {
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.UNKNOWN) as LiveSpan;
// Test various non-serializable objects
const testData = {
func: () => console.log('test'),
undefinedVal: undefined,
error: new Error('Test error'),
date: new Date('2024-01-01'),
regex: /test/g,
map: new Map([['key', 'value']]),
set: new Set([1, 2, 3]),
};
// This should not throw
expect(() => {
mlflowSpan.setAttribute('nonSerializable', testData);
}).not.toThrow();
const attrValue = mlflowSpan.getAttribute('nonSerializable');
expect(attrValue.func).toBe('[Function]');
expect(attrValue.undefinedVal).toBe('[Undefined]');
expect(attrValue.error.name).toBe('Error');
expect(attrValue.error.message).toBe('Test error');
expect(attrValue.date).toBeDefined(); // Date should be serialized normally
} finally {
span.end();
}
});
});
describe('JSON Serialization', () => {
it('should produce correct JSON format for toJson()', () => {
const traceId = 'tr-12345';
const span = tracer.startSpan('test-span');
try {
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.LLM) as LiveSpan;
// Set inputs, outputs, and attributes to match expected format
mlflowSpan.setInputs({ x: 1 });
mlflowSpan.setOutputs(2);
mlflowSpan.setAttribute('mlflow.spanFunctionName', 'f');
mlflowSpan.setStatus(SpanStatusCode.OK);
// Add an event
const event = new SpanEvent({
name: 'test_event',
timestamp: 1500000000000n,
attributes: { eventAttr: 'value' },
});
mlflowSpan.addEvent(event);
} finally {
span.end();
}
// Get the completed span
const completedSpan = createMlflowSpan(span, traceId);
const json = completedSpan.toJson();
// Validate JSON structure matches expected format
expect(json).toHaveProperty('span_id');
expect(json).toHaveProperty('name', 'test-span');
expect(json).toHaveProperty('start_time_unix_nano');
expect(json).toHaveProperty('end_time_unix_nano');
expect(json).toHaveProperty('attributes');
expect(json).toHaveProperty('status');
expect(json).toHaveProperty('events');
// Validate that attributes exist and have correct values
expect(json.attributes['mlflow.spanInputs']).toEqual({ x: 1 });
expect(json.attributes['mlflow.spanOutputs']).toBe(2);
expect(json.attributes['mlflow.spanType']).toBe('LLM');
expect(json.attributes['mlflow.traceRequestId']).toBe(traceId);
// Validate status format
expect(json.status).toHaveProperty('code');
expect(json.status.code).toBe('STATUS_CODE_OK');
// Validate timestamps are bigint for precision
expect(typeof json.start_time_unix_nano).toBe('bigint');
expect(typeof json.end_time_unix_nano).toBe('bigint');
// Validate events structure
expect(Array.isArray(json.events)).toBe(true);
if (json.events.length > 0) {
const event = json.events[0];
expect(event).toHaveProperty('name');
expect(event).toHaveProperty('time_unix_nano');
expect(event).toHaveProperty('attributes');
}
});
it('should match expected JSON format from real MLflow data', () => {
// Create a span that matches the real MLflow data structure
const expectedSpanData = {
trace_id: 'rZo9DIws+6d2tejICXD4gw==',
span_id: 'DOD2qjZ6ZrU=',
trace_state: '',
parent_span_id: '',
name: 'python',
start_time_unix_nano: 1749996461282772491n,
end_time_unix_nano: 1749996461365717111n,
attributes: {
'mlflow.spanOutputs': '2',
'mlflow.spanType': '"LLM"',
'mlflow.spanInputs': '{"x": 1}',
'mlflow.traceRequestId': '"tr-ad9a3d0c8c2cfba776b5e8c80970f883"',
'mlflow.spanFunctionName': '"f"',
},
status: {
message: '',
code: 'STATUS_CODE_OK',
},
events: [],
};
// Convert to JSON string and parse with json-bigint to get proper bigints
const jsonString = JSONBig.stringify(expectedSpanData);
const parsedData = JSONBig.parse(jsonString) as SerializedSpan;
// Test fromJson can handle this format
const span = Span.fromJson(parsedData);
// Validate key properties
expect(span.traceId).toBe('tr-ad9a3d0c8c2cfba776b5e8c80970f883');
expect(span.spanId).toBe('0ce0f6aa367a66b5');
expect(span.name).toBe('python');
expect(span.spanType).toBe(SpanType.LLM);
expect(convertHrTimeToNanoSeconds(span.startTime)).toBe(1749996461282772491n);
expect(convertHrTimeToNanoSeconds(span.endTime!)).toBe(1749996461365717111n);
expect(span.parentId).toBeNull();
expect(span.status.statusCode).toBe(SpanStatusCode.OK);
expect(span.status.description).toBe('');
expect(span.inputs).toStrictEqual({ x: 1 });
expect(span.outputs).toBe(2);
expect(span.events).toEqual([]);
});
it('should handle round-trip JSON serialization correctly', () => {
const traceId = 'tr-test-round-trip';
const span = tracer.startSpan('round-trip-test');
try {
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.CHAIN) as LiveSpan;
// Set comprehensive test data
mlflowSpan.setInputs({ input: 'test', number: 42 });
mlflowSpan.setOutputs({ result: 'success', count: 100 });
mlflowSpan.setAttribute('custom.attribute', 'custom_value');
mlflowSpan.setAttribute('numeric.attribute', 123);
mlflowSpan.setStatus(SpanStatusCode.OK);
} finally {
span.end();
}
// Round-trip test: span -> JSON -> span -> JSON
const originalSpan = createMlflowSpan(span, traceId);
const originalJson = originalSpan.toJson();
// Create new span from JSON
const reconstructedSpan = Span.fromJson(originalJson);
// Key properties should match
expect(reconstructedSpan.name).toBe(originalSpan.name);
expect(reconstructedSpan.parentId).toBe(originalSpan.parentId);
expect(reconstructedSpan.startTime).toEqual(originalSpan.startTime);
expect(reconstructedSpan.endTime).toEqual(originalSpan.endTime);
// Attributes should be preserved
expect(reconstructedSpan.attributes).toEqual(originalSpan.attributes);
// Status should be preserved
expect(reconstructedSpan.status.statusCode).toBe(originalSpan.status.statusCode);
// Events should be preserved
expect(reconstructedSpan.events).toEqual(originalSpan.events);
});
});
});
@@ -0,0 +1,106 @@
import { SpanEvent } from '../../../src/core/entities/span_event';
describe('SpanEvent', () => {
describe('constructor', () => {
it('should create a span event with all parameters', () => {
const timestamp = BigInt(Date.now()) * 1_000_000n; // nanoseconds
const attributes = {
key1: 'value1',
key2: 42,
key3: true,
key4: ['a', 'b', 'c'],
};
const event = new SpanEvent({
name: 'test_event',
timestamp,
attributes,
});
expect(event.name).toBe('test_event');
expect(event.timestamp).toBe(timestamp);
expect(event.attributes).toEqual(attributes);
});
});
describe('fromException', () => {
it('should create a span event from a basic error', () => {
const error = new Error('Test error message');
const event = SpanEvent.fromException(error);
expect(event.name).toBe('exception');
expect(event.attributes['exception.message']).toBe('Test error message');
expect(event.attributes['exception.type']).toBe('Error');
expect(event.attributes['exception.stacktrace']).toContain('Test error message');
});
});
describe('toJson round-trip serialization', () => {
it('should serialize and maintain all properties', () => {
const originalEvent = new SpanEvent({
name: 'test_event',
timestamp: 1234567890000n,
attributes: {
string_attr: 'test_value',
number_attr: 42,
boolean_attr: true,
string_array: ['a', 'b', 'c'],
number_array: [1, 2, 3],
boolean_array: [true, false, true],
},
});
const json = originalEvent.toJson();
// Verify JSON structure
expect(json).toEqual({
name: 'test_event',
timestamp: 1234567890000n,
attributes: {
string_attr: 'test_value',
number_attr: 42,
boolean_attr: true,
string_array: ['a', 'b', 'c'],
number_array: [1, 2, 3],
boolean_array: [true, false, true],
},
});
// Create new event from JSON data
const recreatedEvent = new SpanEvent({
name: json.name as string,
timestamp: json.timestamp as bigint,
attributes: json.attributes as Record<string, any>,
});
// Verify round-trip preservation
expect(recreatedEvent.name).toBe(originalEvent.name);
expect(recreatedEvent.timestamp).toBe(originalEvent.timestamp);
expect(recreatedEvent.attributes).toEqual(originalEvent.attributes);
expect(recreatedEvent.toJson()).toEqual(originalEvent.toJson());
});
it('should handle events with minimal properties', () => {
const originalEvent = new SpanEvent({
name: 'minimal_event',
});
const json = originalEvent.toJson();
expect(json.name).toBe('minimal_event');
expect(json.timestamp).toBeGreaterThan(0);
expect(json.attributes).toEqual({});
// Recreate and verify
const recreatedEvent = new SpanEvent({
name: json.name as string,
timestamp: json.timestamp as bigint,
attributes: json.attributes as Record<string, any>,
});
expect(recreatedEvent.name).toBe(originalEvent.name);
expect(recreatedEvent.timestamp).toBe(originalEvent.timestamp);
expect(recreatedEvent.attributes).toEqual(originalEvent.attributes);
});
});
});
@@ -0,0 +1,107 @@
import { SpanStatusCode as OTelSpanStatusCode } from '@opentelemetry/api';
import { SpanStatus, SpanStatusCode } from '../../../src/core/entities/span_status';
describe('SpanStatus', () => {
describe('initialization', () => {
// Test both enum and string initialization (parameterized test equivalent)
const testCases = [
{ input: 'STATUS_CODE_OK', expected: SpanStatusCode.OK },
{ input: 'STATUS_CODE_ERROR', expected: SpanStatusCode.ERROR },
{ input: 'STATUS_CODE_UNSET', expected: SpanStatusCode.UNSET },
];
testCases.forEach(({ input, expected }) => {
it(`should initialize with status code ${input}`, () => {
const spanStatus = new SpanStatus(input as SpanStatusCode, 'test');
expect(spanStatus.statusCode).toBe(expected);
expect(spanStatus.description).toBe('test');
});
});
});
describe('OpenTelemetry status conversion', () => {
const conversionTestCases = [
{
mlflowStatus: SpanStatusCode.OK,
otelStatus: OTelSpanStatusCode.OK,
},
{
mlflowStatus: SpanStatusCode.ERROR,
otelStatus: OTelSpanStatusCode.ERROR,
},
{
mlflowStatus: SpanStatusCode.UNSET,
otelStatus: OTelSpanStatusCode.UNSET,
},
];
conversionTestCases.forEach(({ mlflowStatus, otelStatus }) => {
it(`should convert ${mlflowStatus} to OpenTelemetry status correctly`, () => {
const spanStatus = new SpanStatus(mlflowStatus);
const otelStatusResult = spanStatus.toOtelStatus();
expect(otelStatusResult.code).toBe(otelStatus);
});
});
});
describe('toJson round-trip serialization', () => {
it('should serialize and recreate status with all properties', () => {
const originalStatus = new SpanStatus(SpanStatusCode.ERROR, 'Something went wrong');
const json = originalStatus.toJson();
// Verify JSON structure
expect(json).toEqual({
status_code: SpanStatusCode.ERROR,
description: 'Something went wrong',
});
// Create new status from JSON data
const recreatedStatus = new SpanStatus(json.status_code as SpanStatusCode, json.description);
// Verify round-trip preservation
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
expect(recreatedStatus.description).toBe(originalStatus.description);
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
});
it('should handle status with different status codes', () => {
const testCases = [
{ code: SpanStatusCode.OK, description: 'All good' },
{ code: SpanStatusCode.ERROR, description: 'Failed operation' },
{ code: SpanStatusCode.UNSET, description: '' },
];
testCases.forEach(({ code, description }) => {
const originalStatus = new SpanStatus(code, description);
const json = originalStatus.toJson();
const recreatedStatus = new SpanStatus(
json.status_code as SpanStatusCode,
json.description,
);
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
expect(recreatedStatus.description).toBe(originalStatus.description);
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
});
});
it('should handle status with minimal properties', () => {
const originalStatus = new SpanStatus(SpanStatusCode.OK);
const json = originalStatus.toJson();
expect(json).toEqual({
status_code: SpanStatusCode.OK,
description: '',
});
const recreatedStatus = new SpanStatus(json.status_code as SpanStatusCode, json.description);
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
expect(recreatedStatus.description).toBe(originalStatus.description);
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
});
});
});
@@ -0,0 +1,102 @@
import { trace } from '@opentelemetry/api';
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
import { Trace } from '../../../src/core/entities/trace';
import { TraceInfo } from '../../../src/core/entities/trace_info';
import { TraceData } from '../../../src/core/entities/trace_data';
import { TraceState } from '../../../src/core/entities/trace_state';
import { createTraceLocationFromExperimentId } from '../../../src/core/entities/trace_location';
import { createMlflowSpan } from '../../../src/core/entities/span';
// Set up a proper tracer provider
const provider = new BasicTracerProvider();
trace.setGlobalTracerProvider(provider);
const tracer = trace.getTracer('mlflow-test-tracer', '1.0.0');
describe('Trace', () => {
function createMockTraceInfo(): TraceInfo {
return new TraceInfo({
traceId: 'tr-12345',
clientRequestId: 'client-request-id',
traceLocation: createTraceLocationFromExperimentId('exp-123'),
requestTime: Date.now(),
state: TraceState.OK,
executionDuration: 1000,
requestPreview: '{"prompt": "Hello"}',
responsePreview: '{"response": "Hi there"}',
traceMetadata: { key: 'value' },
tags: { env: 'test' },
assessments: [],
});
}
function createMockTraceData(): TraceData {
const span = tracer.startSpan('parent');
const mlflowSpan = createMlflowSpan(span, 'tr-12345');
span.end();
return new TraceData([mlflowSpan]);
}
describe('constructor', () => {
it('should create a Trace with info and data', () => {
const traceInfo = createMockTraceInfo();
const traceData = createMockTraceData();
const trace = new Trace(traceInfo, traceData);
expect(trace.info).toBe(traceInfo);
expect(trace.data).toBe(traceData);
});
});
describe('toJson/fromJson round-trip serialization', () => {
it('should serialize and deserialize a complete trace correctly', () => {
const originalTraceInfo = createMockTraceInfo();
const originalTraceData = createMockTraceData();
const originalTrace = new Trace(originalTraceInfo, originalTraceData);
const json = originalTrace.toJson();
// Verify JSON structure
expect(json).toHaveProperty('info');
expect(json).toHaveProperty('data');
expect(json.info).toMatchObject({
trace_id: 'tr-12345',
request_time: expect.any(String),
state: TraceState.OK,
});
expect(json.data).toMatchObject({
spans: [
{
name: 'parent',
trace_id: expect.any(String),
span_id: expect.any(String),
parent_span_id: '',
start_time_unix_nano: expect.any(BigInt),
end_time_unix_nano: expect.any(BigInt),
attributes: {
'mlflow.spanType': 'UNKNOWN',
},
status: { code: 'STATUS_CODE_UNSET' },
events: [],
},
],
});
// Round-trip test
const recreatedTrace = Trace.fromJson(json);
expect(recreatedTrace).toBeInstanceOf(Trace);
expect(recreatedTrace.info).toBeInstanceOf(TraceInfo);
expect(recreatedTrace.data).toBeInstanceOf(TraceData);
// Verify that key properties are preserved
expect(recreatedTrace.info.traceId).toBe(originalTrace.info.traceId);
expect(recreatedTrace.info.state).toBe(originalTrace.info.state);
expect(recreatedTrace.data.spans.length).toEqual(originalTrace.data.spans.length);
// Verify round-trip JSON serialization matches
expect(recreatedTrace.toJson()).toEqual(originalTrace.toJson());
});
});
});
@@ -0,0 +1,45 @@
import { TraceInfo } from '../../../src/core/entities/trace_info';
import { TraceLocationType } from '../../../src/core/entities/trace_location';
import { TraceState } from '../../../src/core/entities/trace_state';
describe('TraceInfo', () => {
const createTestTraceInfo = () => {
return new TraceInfo({
traceId: 'test-trace-id',
traceLocation: {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: {
experimentId: 'test-experiment-id',
},
},
requestTime: 1000,
state: TraceState.OK,
requestPreview: '{"input":"test"}',
responsePreview: '{"output":"result"}',
clientRequestId: 'client-request-id',
executionDuration: 500,
traceMetadata: { 'meta-key': 'meta-value' },
tags: { 'tag-key': 'tag-value' },
assessments: [],
});
};
describe('constructor', () => {
it('should create a TraceInfo instance with all properties', () => {
const traceInfo = createTestTraceInfo();
expect(traceInfo.traceId).toBe('test-trace-id');
expect(traceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(traceInfo.traceLocation.mlflowExperiment?.experimentId).toBe('test-experiment-id');
expect(traceInfo.requestTime).toBe(1000);
expect(traceInfo.state).toBe(TraceState.OK);
expect(traceInfo.requestPreview).toBe('{"input":"test"}');
expect(traceInfo.responsePreview).toBe('{"output":"result"}');
expect(traceInfo.clientRequestId).toBe('client-request-id');
expect(traceInfo.executionDuration).toBe(500);
expect(traceInfo.traceMetadata['meta-key']).toBe('meta-value');
expect(traceInfo.tags['tag-key']).toBe('tag-value');
expect(traceInfo.assessments).toEqual([]);
});
});
});
@@ -0,0 +1,95 @@
import {
TraceLocation,
TraceLocationType,
createTraceLocationFromExperimentId,
deserializeTraceLocation,
} from '../../../src/core/entities/trace_location';
describe('TraceLocation', () => {
describe('constructor and basic functionality', () => {
it('should create a TraceLocation with MLflow experiment', () => {
const traceLocation: TraceLocation = {
type: TraceLocationType.MLFLOW_EXPERIMENT,
mlflowExperiment: { experimentId: '123' },
};
expect(traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(traceLocation.mlflowExperiment?.experimentId).toBe('123');
expect(traceLocation.inferenceTable).toBeUndefined();
});
it('should create a TraceLocation with inference table', () => {
const traceLocation: TraceLocation = {
type: TraceLocationType.INFERENCE_TABLE,
inferenceTable: { fullTableName: 'a.b.c' },
};
expect(traceLocation.type).toBe(TraceLocationType.INFERENCE_TABLE);
expect(traceLocation.inferenceTable?.fullTableName).toBe('a.b.c');
expect(traceLocation.mlflowExperiment).toBeUndefined();
});
});
describe('validation', () => {
it('should validate that only one location type can be provided', () => {
// This test validates the conceptual constraint that only one location should be provided
// In TypeScript, this is more of a usage pattern validation
const invalidLocation: TraceLocation = {
type: TraceLocationType.TRACE_LOCATION_TYPE_UNSPECIFIED,
mlflowExperiment: { experimentId: '123' },
inferenceTable: { fullTableName: 'a.b.c' },
};
// Both are defined, which violates the constraint
expect(invalidLocation.mlflowExperiment).toBeDefined();
expect(invalidLocation.inferenceTable).toBeDefined();
// In a real implementation, this would throw an error during validation
});
it('should validate type matches MLflow experiment location', () => {
// This represents a mismatch: INFERENCE_TABLE type with mlflowExperiment data
const mismatchedLocation: TraceLocation = {
type: TraceLocationType.INFERENCE_TABLE,
mlflowExperiment: { experimentId: '123' },
};
expect(mismatchedLocation.type).toBe(TraceLocationType.INFERENCE_TABLE);
expect(mismatchedLocation.mlflowExperiment).toBeDefined();
expect(mismatchedLocation.inferenceTable).toBeUndefined();
// In a real implementation, this would be caught by validation
});
it('should validate type matches inference table location', () => {
// This represents a mismatch: MLFLOW_EXPERIMENT type with inferenceTable data
const mismatchedLocation: TraceLocation = {
type: TraceLocationType.MLFLOW_EXPERIMENT,
inferenceTable: { fullTableName: 'a.b.c' },
};
expect(mismatchedLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(mismatchedLocation.inferenceTable).toBeDefined();
expect(mismatchedLocation.mlflowExperiment).toBeUndefined();
// In a real implementation, this would be caught by validation
});
});
describe('createTraceLocationFromExperimentId', () => {
it('should create a TraceLocation with MLflow experiment', () => {
const experimentId = 'experiment123';
const location = createTraceLocationFromExperimentId(experimentId);
expect(location.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
expect(location.mlflowExperiment).toBeDefined();
expect(location.mlflowExperiment?.experimentId).toBe(experimentId);
expect(location.inferenceTable).toBeUndefined();
});
});
describe('deserializeTraceLocation', () => {
it('should reject missing trace location data', () => {
expect(() => deserializeTraceLocation(undefined)).toThrow(
'Invalid trace location: missing type.',
);
});
});
});
@@ -0,0 +1,30 @@
import { TraceState, fromOtelStatus } from '../../../src/core/entities/trace_state';
import { SpanStatusCode } from '@opentelemetry/api';
describe('TraceState', () => {
describe('enum values', () => {
it('should have the correct enum values', () => {
expect(TraceState.STATE_UNSPECIFIED).toBe('STATE_UNSPECIFIED');
expect(TraceState.OK).toBe('OK');
expect(TraceState.ERROR).toBe('ERROR');
expect(TraceState.IN_PROGRESS).toBe('IN_PROGRESS');
});
});
describe('fromOtelStatus', () => {
it('should convert OpenTelemetry OK status to TraceState.OK', () => {
const result = fromOtelStatus(SpanStatusCode.OK);
expect(result).toBe(TraceState.OK);
});
it('should convert OpenTelemetry ERROR status to TraceState.ERROR', () => {
const result = fromOtelStatus(SpanStatusCode.ERROR);
expect(result).toBe(TraceState.ERROR);
});
it('should convert OpenTelemetry UNSET status to TraceState.STATE_UNSPECIFIED', () => {
const result = fromOtelStatus(SpanStatusCode.UNSET);
expect(result).toBe(TraceState.STATE_UNSPECIFIED);
});
});
});
@@ -0,0 +1,167 @@
import { trace as otelTrace } from '@opentelemetry/api';
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
import { SpanAttributeKey, SpanLogLevel, SpanType, toSpanLogLevel } from '../../src/core/constants';
import { defaultLogLevelForSpanType } from '../../src/core/log_level';
import { createMlflowSpan, type LiveSpan } from '../../src/core/entities/span';
import { SpanEvent } from '../../src/core/entities/span_event';
const provider = new BasicTracerProvider();
otelTrace.setGlobalTracerProvider(provider);
const tracer = otelTrace.getTracer('mlflow-log-level-test', '1.0.0');
const newLiveSpan = (spanType: SpanType = SpanType.UNKNOWN): LiveSpan => {
const otelSpan = tracer.startSpan('test');
return createMlflowSpan(otelSpan, 'tr-test', spanType) as LiveSpan;
};
describe('toSpanLogLevel', () => {
it.each<[SpanLogLevel | string, SpanLogLevel]>([
[SpanLogLevel.DEBUG, SpanLogLevel.DEBUG],
[SpanLogLevel.CRITICAL, SpanLogLevel.CRITICAL],
['DEBUG', SpanLogLevel.DEBUG],
['info', SpanLogLevel.INFO],
['Warning', SpanLogLevel.WARNING],
[' ERROR ', SpanLogLevel.ERROR], // whitespace tolerated
])('accepts %p and yields %p', (input, expected) => {
expect(toSpanLogLevel(input)).toBe(expected);
});
it.each(['NOPE', 'TRACE', 'FATAL', 'INFOO', '', 'WARN', 'warn'])(
'rejects invalid string %p',
(value) => {
// "WARN" is no longer accepted; only the full names work.
expect(() => toSpanLogLevel(value)).toThrow(/Invalid SpanLogLevel/);
},
);
it.each([0, 7, 100, -1])('rejects unknown number %p (defensive runtime check)', (value) => {
// Raw numbers aren't part of the public type (`SpanLogLevel | string`),
// but `SpanLogLevel` is a numeric enum so members arrive as primitive
// numbers at runtime. The runtime branch validates them defensively;
// anything that isn't a known enum value throws.
expect(() => toSpanLogLevel(value as unknown as SpanLogLevel)).toThrow(/Invalid SpanLogLevel/);
});
});
describe('LiveSpan default applied at end()', () => {
it.each<[SpanType, SpanLogLevel]>([
[SpanType.LLM, SpanLogLevel.INFO],
[SpanType.CHAT_MODEL, SpanLogLevel.INFO],
[SpanType.TOOL, SpanLogLevel.INFO],
[SpanType.RETRIEVER, SpanLogLevel.INFO],
[SpanType.AGENT, SpanLogLevel.INFO],
[SpanType.EMBEDDING, SpanLogLevel.INFO],
[SpanType.CHAIN, SpanLogLevel.DEBUG],
[SpanType.PARSER, SpanLogLevel.DEBUG],
[SpanType.RERANKER, SpanLogLevel.DEBUG],
[SpanType.MEMORY, SpanLogLevel.DEBUG],
[SpanType.UNKNOWN, SpanLogLevel.DEBUG],
])('resolves default %p for span type %p', (spanType, expected) => {
const span = newLiveSpan(spanType);
// Mid-span: log level isn't resolved yet.
expect(span.logLevel).toBeNull();
span.end();
// Post-end: type-derived default is applied.
expect(span.logLevel).toBe(expected);
});
});
describe('LiveSpan log-level accessors', () => {
it.each<[SpanLogLevel | string, SpanLogLevel]>([
[SpanLogLevel.WARNING, SpanLogLevel.WARNING],
[SpanLogLevel.ERROR, SpanLogLevel.ERROR],
['info', SpanLogLevel.INFO],
['CRITICAL', SpanLogLevel.CRITICAL],
])('setLogLevel(%p) round-trips to %p', (input, expected) => {
const span = newLiveSpan();
span.setLogLevel(input);
expect(span.logLevel).toBe(expected);
// Stored as the canonical int under the reserved attribute key for portability.
expect(span.getAttribute(SpanAttributeKey.LOG_LEVEL)).toBe(expected as number);
span.end();
});
it('setLogLevel rejects invalid input', () => {
const span = newLiveSpan();
expect(() => span.setLogLevel('NOPE')).toThrow(/Invalid SpanLogLevel/);
span.end();
});
});
describe('exception event bumps log level to ERROR', () => {
it('bumps an unset span to ERROR via addEvent (DEBUG-defaulting type)', () => {
const span = newLiveSpan(SpanType.PARSER);
expect(span.logLevel).toBeNull();
span.addEvent(
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
);
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
span.end();
// Stays at ERROR after end(); the resolution branch only fires when unset.
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
});
it('bumps an unset span to ERROR via addEvent (INFO-defaulting type)', () => {
const span = newLiveSpan(SpanType.CHAT_MODEL);
expect(span.logLevel).toBeNull();
span.addEvent(
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
);
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
span.end();
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
});
it('preserves user-set CRITICAL', () => {
const span = newLiveSpan(SpanType.PARSER);
span.setLogLevel(SpanLogLevel.CRITICAL);
span.addEvent(
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
);
expect(span.logLevel).toBe(SpanLogLevel.CRITICAL);
span.end();
});
it('bumps via recordException (which bypasses our addEvent wrapper)', () => {
const span = newLiveSpan(SpanType.PARSER);
span.recordException(new Error('boom'));
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
span.end();
});
it('non-exception events do not bump the level', () => {
const span = newLiveSpan(SpanType.CHAT_MODEL);
span.addEvent(new SpanEvent({ name: 'my_event', attributes: { k: 'v' } }));
// Plain event must not pre-resolve the level; resolution waits until end().
expect(span.logLevel).toBeNull();
span.end();
expect(span.logLevel).toBe(SpanLogLevel.INFO);
});
});
describe('defaultLogLevelForSpanType', () => {
it.each<[string, SpanLogLevel]>([
// INFO set: user-visible semantic operations.
[SpanType.LLM, SpanLogLevel.INFO],
[SpanType.CHAT_MODEL, SpanLogLevel.INFO],
[SpanType.AGENT, SpanLogLevel.INFO],
[SpanType.TOOL, SpanLogLevel.INFO],
[SpanType.RETRIEVER, SpanLogLevel.INFO],
[SpanType.EMBEDDING, SpanLogLevel.INFO],
// DEBUG set: internal/glue work and unclassified types.
[SpanType.CHAIN, SpanLogLevel.DEBUG],
[SpanType.PARSER, SpanLogLevel.DEBUG],
[SpanType.RERANKER, SpanLogLevel.DEBUG],
[SpanType.MEMORY, SpanLogLevel.DEBUG],
[SpanType.UNKNOWN, SpanLogLevel.DEBUG],
['MY_CUSTOM_TYPE', SpanLogLevel.DEBUG], // custom types fall through to DEBUG
])('maps %p to %p', (spanType, expected) => {
expect(defaultLogLevelForSpanType(spanType)).toBe(expected);
});
it('maps undefined/null to DEBUG', () => {
expect(defaultLogLevelForSpanType(undefined)).toBe(SpanLogLevel.DEBUG);
expect(defaultLogLevelForSpanType(null)).toBe(SpanLogLevel.DEBUG);
});
});
@@ -0,0 +1,84 @@
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
// Mock the OTLP proto exporter (Jest can't load its dynamic http imports).
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
OTLPTraceExporter: jest
.fn()
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
exporterCtors.push(cfg);
return {
export: (_spans: unknown[], cb: (r: { code: number }) => void) => cb({ code: 0 }),
shutdown: () => Promise.resolve(),
forceFlush: () => Promise.resolve(),
};
}),
}));
import { init } from '../../src/core/config';
import { flushTraces } from '../../src/core/provider';
import { withSpan } from '../../src/core/api';
const testHost = 'https://dbc-12345.cloud.databricks.com';
// `init()` is documented as call-once-per-process: NodeSDK's background
// shutdown can race with a subsequent start and clobber the global tracer
// provider. This file therefore only exercises a single `init()` call.
// UC processor + exporter behavior is covered in detail by the unit tests
// in `tests/exporters/uc_table*.test.ts`; this file just verifies that
// `init()` wires up the UC processor when `traceLocation` is provided.
describe('init() with traceLocation wires the UC span processor', () => {
let server: ReturnType<typeof setupServer>;
const v4TraceInfoCalls: { url: string; body: any }[] = [];
beforeAll(() => {
process.env.DATABRICKS_HOST = testHost;
process.env.DATABRICKS_TOKEN = 'test-token';
server = setupServer(
http.post(
`${testHost}/api/4.0/mlflow/traces/:location/:otelTraceId/info`,
async ({ request }) => {
const body = (await request.json()) as any;
v4TraceInfoCalls.push({ url: request.url, body });
return HttpResponse.json({
trace_id: body.trace_id,
trace_location: body.trace_location,
request_time: body.request_time,
execution_duration: body.execution_duration,
state: body.state,
trace_metadata: body.trace_metadata,
tags: body.tags,
assessments: [],
});
},
),
);
server.listen();
init({
trackingUri: 'databricks',
experimentId: '4118495900667593',
traceLocation: { catalogName: 'cat', schemaName: 'sch', tablePrefix: 'agent' },
});
});
afterAll(() => {
server.close();
delete process.env.DATABRICKS_HOST;
delete process.env.DATABRICKS_TOKEN;
});
it('routes spans through the V4 endpoint with the configured UC location', async () => {
void withSpan(() => {}, { name: 'root' });
await flushTraces();
expect(v4TraceInfoCalls).toHaveLength(1);
expect(v4TraceInfoCalls[0].body.trace_id).toMatch(/^trace:\/cat\.sch\.agent\//);
expect(v4TraceInfoCalls[0].body.trace_location.uc_table_prefix).toEqual({
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'agent',
});
});
});
@@ -0,0 +1,90 @@
import { InMemoryTraceManager } from '../../src/core/trace_manager';
import { Trace } from '../../src/core/entities/trace';
import { TraceInfo } from '../../src/core/entities/trace_info';
import { createTraceLocationFromExperimentId } from '../../src/core/entities/trace_location';
import { TraceState } from '../../src/core/entities/trace_state';
import { createTestSpan } from '../helper';
import { Span } from '../../src/core/entities/span';
import { TraceMetadataKey } from '../../src/core/constants';
/**
* Helper function to create a test TraceInfo object
*/
function createTestTraceInfo(traceId: string): TraceInfo {
return new TraceInfo({
traceId,
traceLocation: createTraceLocationFromExperimentId('1'),
requestTime: Date.now(),
state: TraceState.IN_PROGRESS,
requestPreview: undefined,
responsePreview: undefined,
traceMetadata: {},
tags: {},
});
}
describe('InMemoryTraceManager', () => {
beforeEach(() => {
// Reset the singleton instance before each test
InMemoryTraceManager.reset();
});
afterEach(() => {
// Clean up after each test
InMemoryTraceManager.reset();
});
it('should return the same instance when called multiple times', () => {
const obj1 = InMemoryTraceManager.getInstance();
const obj2 = InMemoryTraceManager.getInstance();
expect(obj1).toBe(obj2);
});
it('should add spans and pop traces correctly', () => {
const traceManager = InMemoryTraceManager.getInstance();
// Add a new trace info
const traceId = 'tr-1';
const otelTraceId = '12345';
traceManager.registerTrace(otelTraceId, createTestTraceInfo(traceId));
// Add a span for a new trace
const span11 = createTestSpan('test', traceId, 'span11');
traceManager.registerSpan(span11);
expect(traceManager.getTrace(traceId)).toBeTruthy();
const trace1 = traceManager.getTrace(traceId);
expect(trace1?.info.traceId).toBe(traceId);
expect(trace1?.spanDict.size).toBe(1);
// Add more spans to the same trace
const span111 = createTestSpan('test', traceId, 'span111');
const span112 = createTestSpan('test', traceId, 'span112');
traceManager.registerSpan(span111);
traceManager.registerSpan(span112);
expect(trace1?.spanDict.size).toBe(3);
// Pop the trace data
const poppedTrace1 = traceManager.popTrace(otelTraceId);
expect(poppedTrace1).toBeInstanceOf(Trace);
expect(poppedTrace1?.info.traceId).toBe(traceId);
expect(poppedTrace1?.data.spans.length).toBe(3);
expect(traceManager.getTrace(traceId)).toBeNull();
expect(poppedTrace1?.data.spans[0]).toBeInstanceOf(Span);
});
it('should truncate the request/response preview if it exceeds the max length', () => {
const traceManager = InMemoryTraceManager.getInstance();
const traceId = 'tr-1';
const otelTraceId = '12345';
traceManager.registerTrace(otelTraceId, createTestTraceInfo(traceId));
const span = createTestSpan('test', traceId, 'span11');
span.setInputs('a'.repeat(5000));
traceManager.registerSpan(span);
const trace = traceManager.popTrace(otelTraceId);
expect(trace?.info.requestPreview).toHaveLength(1000);
expect(trace?.info.traceMetadata[TraceMetadataKey.INPUTS]).toHaveLength(1000);
});
});
@@ -0,0 +1,141 @@
import {
TraceLocationType,
createTraceLocationFromUcTablePrefix,
getOtelSpansTableName,
getUcLocationString,
isUcTraceLocation,
ucTablePrefixLocationString,
} from '../../src/core/entities/trace_location';
import {
constructTraceIdV4,
generateTraceIdV3,
parseTraceIdV4,
} from '../../src/core/utils/trace_id';
import {
DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG,
DATABRICKS_TRACE_DESTINATION_PATH_TAG,
DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG,
DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG,
ucLocationFromExperimentTags,
} from '../../src/core/destination';
import { TraceInfo } from '../../src/core/entities/trace_info';
import { TraceState } from '../../src/core/entities/trace_state';
describe('Trace ID v4 helpers', () => {
it('parses a v4 trace ID into location and otel trace ID', () => {
const [location, otelId] = parseTraceIdV4(
'trace:/cat.sch.tbl/abcdef1234567890abcdef1234567890',
);
expect(location).toBe('cat.sch.tbl');
expect(otelId).toBe('abcdef1234567890abcdef1234567890');
});
it('returns [null, raw] for v3 trace IDs', () => {
const [location, otelId] = parseTraceIdV4('tr-abcdef1234567890');
expect(location).toBeNull();
expect(otelId).toBe('tr-abcdef1234567890');
});
it('throws on a malformed v4 trace ID', () => {
expect(() => parseTraceIdV4('trace:/onlyone')).toThrow(/Invalid trace ID format/);
expect(() => parseTraceIdV4('trace://abc')).toThrow(/Invalid trace ID format/);
});
it('constructs v4 and v3 IDs in their canonical formats', () => {
expect(constructTraceIdV4('cat.sch.tbl', 'deadbeef')).toBe('trace:/cat.sch.tbl/deadbeef');
expect(generateTraceIdV3('deadbeef')).toBe('tr-deadbeef');
});
});
describe('UC trace location helpers', () => {
it('builds a UC table-prefix TraceLocation and returns its location string', () => {
const loc = createTraceLocationFromUcTablePrefix('cat', 'sch', 'agent');
expect(loc.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
expect(isUcTraceLocation(loc)).toBe(true);
expect(getUcLocationString(loc)).toBe('cat.sch.agent');
expect(ucTablePrefixLocationString(loc.ucTablePrefix!)).toBe('cat.sch.agent');
// Default spans table convention: `<prefix>_otel_spans` under the same
// catalog/schema. Customers can override by setting otelSpansTableName.
expect(getOtelSpansTableName(loc)).toBe('cat.sch.agent_otel_spans');
});
it('respects an explicit backend-populated spans table name', () => {
const loc = createTraceLocationFromUcTablePrefix('cat', 'sch', 'agent');
loc.ucTablePrefix!.otelSpansTableName = 'cat.sch.custom_spans';
expect(getOtelSpansTableName(loc)).toBe('cat.sch.custom_spans');
});
});
describe('ucLocationFromExperimentTags', () => {
it('returns null when the experiment has no Databricks trace tags', () => {
expect(ucLocationFromExperimentTags({})).toBeNull();
expect(ucLocationFromExperimentTags({ unrelated: 'x' })).toBeNull();
});
it('returns null when the destination path is not three-segment', () => {
expect(
ucLocationFromExperimentTags({
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch',
}),
).toBeNull();
});
it('returns null when any path segment is empty', () => {
expect(
ucLocationFromExperimentTags({
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch.',
}),
).toBeNull();
expect(
ucLocationFromExperimentTags({
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: '.sch.prefix',
}),
).toBeNull();
});
it('parses a UC table-prefix location and copies the spans / logs / annotations tables', () => {
const loc = ucLocationFromExperimentTags({
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch.prefix',
[DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG]: 'cat.sch.prefix_otel_spans',
[DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG]: 'cat.sch.prefix_otel_logs',
[DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG]: 'cat.sch.prefix_annotations',
});
expect(loc).not.toBeNull();
expect(loc).toEqual({
catalogName: 'cat',
schemaName: 'sch',
tablePrefix: 'prefix',
otelSpansTableName: 'cat.sch.prefix_otel_spans',
otelLogsTableName: 'cat.sch.prefix_otel_logs',
annotationsTableName: 'cat.sch.prefix_annotations',
});
});
});
describe('TraceInfo serialization with UC locations', () => {
it('serializes and deserializes a UC table-prefix TraceLocation', () => {
const info = new TraceInfo({
traceId: 'trace:/cat.sch.tbl/abc',
traceLocation: createTraceLocationFromUcTablePrefix('cat', 'sch', 'tbl'),
requestTime: 1700000000000,
state: TraceState.OK,
tags: { user_id: 'u1', family_id: 'f1' },
traceMetadata: {},
});
const json = info.toJson();
expect(json.trace_location.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
expect(json.trace_location.uc_table_prefix).toEqual({
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'tbl',
});
expect(json.tags).toEqual({ user_id: 'u1', family_id: 'f1' });
const roundTripped = TraceInfo.fromJson(json);
expect(roundTripped.traceLocation.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
expect(roundTripped.traceLocation.ucTablePrefix?.catalogName).toBe('cat');
expect(roundTripped.traceLocation.ucTablePrefix?.schemaName).toBe('sch');
expect(roundTripped.traceLocation.ucTablePrefix?.tablePrefix).toBe('tbl');
expect(roundTripped.tags).toEqual({ user_id: 'u1', family_id: 'f1' });
});
});
@@ -0,0 +1,72 @@
import {
artifactUriToLocalPath,
getArtifactUriScheme,
isLocalArtifactUri,
toAbsoluteLocalPath,
} from '../../../src/core/utils/artifact_uri';
describe('artifact_uri helpers', () => {
describe('getArtifactUriScheme', () => {
it('returns the scheme for URIs that have one', () => {
expect(getArtifactUriScheme('mlflow-artifacts:/0/traces/tr-1/artifacts')).toBe(
'mlflow-artifacts',
);
expect(getArtifactUriScheme('file:///tmp/a/b')).toBe('file');
expect(getArtifactUriScheme('s3://bucket/key')).toBe('s3');
expect(getArtifactUriScheme('S3://bucket/key')).toBe('s3');
});
it('returns an empty string for bare filesystem paths', () => {
expect(getArtifactUriScheme('/tmp/mlflow-artifacts/0/traces/tr-1/artifacts')).toBe('');
expect(getArtifactUriScheme('relative/path')).toBe('');
});
it('treats a single-letter (Windows drive) scheme as a bare path', () => {
expect(getArtifactUriScheme('C:\\tmp\\artifacts')).toBe('');
expect(getArtifactUriScheme('d:/tmp/artifacts')).toBe('');
});
});
describe('isLocalArtifactUri', () => {
it('treats bare paths and file:// URIs as local', () => {
expect(isLocalArtifactUri('/tmp/mlflow-artifacts/0')).toBe(true);
expect(isLocalArtifactUri('file:///tmp/mlflow-artifacts/0')).toBe(true);
expect(isLocalArtifactUri('C:\\tmp\\artifacts')).toBe(true);
});
it('treats mlflow-artifacts and remote schemes as non-local', () => {
expect(isLocalArtifactUri('mlflow-artifacts:/0/traces/tr-1/artifacts')).toBe(false);
expect(isLocalArtifactUri('s3://bucket/key')).toBe(false);
expect(isLocalArtifactUri('gs://bucket/key')).toBe(false);
});
});
describe('artifactUriToLocalPath', () => {
it('returns bare paths unchanged', () => {
expect(artifactUriToLocalPath('/tmp/mlflow-artifacts/0/traces/tr-1/artifacts')).toBe(
'/tmp/mlflow-artifacts/0/traces/tr-1/artifacts',
);
});
it('decodes file:// URIs to filesystem paths', () => {
expect(artifactUriToLocalPath('file:///tmp/mlflow-artifacts/0')).toBe(
'/tmp/mlflow-artifacts/0',
);
// Percent-encoded spaces are decoded.
expect(artifactUriToLocalPath('file:///tmp/with%20space/0')).toBe('/tmp/with space/0');
});
});
describe('toAbsoluteLocalPath', () => {
it('returns absolute bare paths and file:// URIs unchanged', () => {
expect(toAbsoluteLocalPath('/tmp/mlflow-artifacts/0')).toBe('/tmp/mlflow-artifacts/0');
expect(toAbsoluteLocalPath('file:///tmp/mlflow-artifacts/0')).toBe('/tmp/mlflow-artifacts/0');
});
it('throws for relative paths', () => {
expect(() => toAbsoluteLocalPath('relative/path')).toThrow(
/Refusing to use a relative local artifact location/,
);
});
});
});
@@ -0,0 +1,404 @@
import type { HrTime } from '@opentelemetry/api';
import {
convertNanoSecondsToHrTime,
convertHrTimeToNanoSeconds,
encodeSpanIdToBase64,
encodeTraceIdToBase64,
decodeIdFromBase64,
mapArgsToObject,
} from '../../../src/core/utils';
describe('utils', () => {
describe('convertNanoSecondsToHrTime', () => {
// Using table-driven tests with test.each for time conversion
test.each([
{
description: 'small nanosecond values',
input: 123456789,
expected: [0, 123456789],
},
{
description: 'values exactly at 1 second',
input: 1_000_000_000,
expected: [1, 0],
},
{
description: 'zero',
input: 0,
expected: [0, 0],
},
])('should convert $description correctly', ({ input, expected }) => {
const result = convertNanoSecondsToHrTime(input);
expect(result).toEqual(expected);
});
it('should convert large nanosecond values correctly', () => {
// Note: JavaScript loses precision with very large numbers
// The computation seconds * 1e9 + nanosInSecond loses precision
const seconds = 1234567890;
const nanosInSecond = 123456789;
const nanos = seconds * 1e9 + nanosInSecond;
const result = convertNanoSecondsToHrTime(nanos);
expect(result[0]).toBe(seconds);
// Due to precision loss in the calculation above
expect(result[1]).toBe(123456768);
});
it('should handle maximum safe integer', () => {
const maxSafeInt = Number.MAX_SAFE_INTEGER;
const result = convertNanoSecondsToHrTime(maxSafeInt);
expect(result[0]).toBe(Math.floor(maxSafeInt / 1e9));
expect(result[1]).toBe(maxSafeInt % 1e9);
});
});
describe('convertHrTimeToNanoSeconds', () => {
it('should convert HrTime with zero seconds correctly', () => {
const hrTime: HrTime = [0, 123456789];
const result = convertHrTimeToNanoSeconds(hrTime);
expect(result).toBe(123456789n);
});
it('should convert HrTime with seconds correctly', () => {
const hrTime: HrTime = [5, 500000000];
const result = convertHrTimeToNanoSeconds(hrTime);
expect(result).toBe(5_500_000_000n);
});
it('should handle zero HrTime', () => {
const hrTime: HrTime = [0, 0];
const result = convertHrTimeToNanoSeconds(hrTime);
expect(result).toBe(0n);
});
it('should handle large HrTime values', () => {
const hrTime: HrTime = [1234567890, 123456789];
const result = convertHrTimeToNanoSeconds(hrTime);
const expected = 1234567890n * 1_000_000_000n + 123456789n;
expect(result).toBe(expected);
});
it('should be reversible with convertNanoSecondsToHrTime', () => {
const testValues = [0, 123456789, 1_000_000_000, 5_500_000_000];
testValues.forEach((nanos) => {
const hrTime = convertNanoSecondsToHrTime(nanos);
const result = convertHrTimeToNanoSeconds(hrTime);
expect(result).toBe(BigInt(nanos));
});
});
});
describe('encodeSpanIdToBase64', () => {
// Using array syntax for test.each (alternative to object syntax)
test.each([
['standard 16-character hex span ID', '0123456789abcdef', 'ASNFZ4mrze8='],
['short span IDs with zero padding', 'abc', 'AAAAAAAACrw='],
['all zeros', '0000000000000000', 'AAAAAAAAAAA='],
['all F characters', 'ffffffffffffffff', '//////////8='],
])('should encode %s', (description, spanId, expected) => {
const result = encodeSpanIdToBase64(spanId);
expect(result).toBe(expected);
});
it('should handle mixed case hex strings', () => {
const spanId = 'AbCdEf1234567890';
const result = encodeSpanIdToBase64(spanId);
// Should work the same as lowercase
const lowercase = encodeSpanIdToBase64('abcdef1234567890');
expect(result).toBe(lowercase);
});
});
describe('encodeTraceIdToBase64', () => {
it('should encode a standard 32-character hex trace ID', () => {
const traceId = '0123456789abcdef0123456789abcdef';
const result = encodeTraceIdToBase64(traceId);
expect(result).toBe('ASNFZ4mrze8BI0VniavN7w==');
});
it('should pad short trace IDs with zeros', () => {
const traceId = 'abc';
const result = encodeTraceIdToBase64(traceId);
// Should be padded to '00000000000000000000000000000abc'
expect(result).toBe('AAAAAAAAAAAAAAAAAAAKvA==');
});
it('should handle all zeros', () => {
const traceId = '00000000000000000000000000000000';
const result = encodeTraceIdToBase64(traceId);
expect(result).toBe('AAAAAAAAAAAAAAAAAAAAAA==');
});
it('should handle all F characters', () => {
const traceId = 'ffffffffffffffffffffffffffffffff';
const result = encodeTraceIdToBase64(traceId);
expect(result).toBe('/////////////////////w==');
});
it('should handle mixed case hex strings', () => {
const traceId = 'AbCdEf1234567890AbCdEf1234567890';
const result = encodeTraceIdToBase64(traceId);
// Should work the same as lowercase
const lowercase = encodeTraceIdToBase64('abcdef1234567890abcdef1234567890');
expect(result).toBe(lowercase);
});
});
describe('decodeIdFromBase64', () => {
it('should decode a base64 encoded span ID back to hex', () => {
const base64Id = 'ASNFZ4mrze8=';
const result = decodeIdFromBase64(base64Id);
expect(result).toBe('0123456789abcdef');
});
it('should decode a base64 encoded trace ID back to hex', () => {
const base64Id = 'ASNFZ4mrze8BI0VniavN7w==';
const result = decodeIdFromBase64(base64Id);
expect(result).toBe('0123456789abcdef0123456789abcdef');
});
it('should handle all zeros', () => {
const spanBase64 = 'AAAAAAAAAAA=';
const traceBase64 = 'AAAAAAAAAAAAAAAAAAAAAA==';
expect(decodeIdFromBase64(spanBase64)).toBe('0000000000000000');
expect(decodeIdFromBase64(traceBase64)).toBe('00000000000000000000000000000000');
});
it('should handle all F values', () => {
const spanBase64 = '//////////8=';
const traceBase64 = '/////////////////////w==';
expect(decodeIdFromBase64(spanBase64)).toBe('ffffffffffffffff');
expect(decodeIdFromBase64(traceBase64)).toBe('ffffffffffffffffffffffffffffffff');
});
it('should be reversible with encodeSpanIdToBase64', () => {
const testSpanIds = [
'0123456789abcdef',
'0000000000000000',
'ffffffffffffffff',
'deadbeef12345678',
];
testSpanIds.forEach((spanId) => {
const encoded = encodeSpanIdToBase64(spanId);
const decoded = decodeIdFromBase64(encoded);
expect(decoded).toBe(spanId);
});
});
it('should be reversible with encodeTraceIdToBase64', () => {
const testTraceIds = [
'0123456789abcdef0123456789abcdef',
'00000000000000000000000000000000',
'ffffffffffffffffffffffffffffffff',
'deadbeef12345678deadbeef12345678',
];
testTraceIds.forEach((traceId) => {
const encoded = encodeTraceIdToBase64(traceId);
const decoded = decodeIdFromBase64(encoded);
expect(decoded).toBe(traceId);
});
});
it('should handle empty base64 string', () => {
const result = decodeIdFromBase64('');
expect(result).toBe('');
});
it('should handle single byte base64', () => {
// Base64 for single byte 0xFF
const result = decodeIdFromBase64('/w==');
expect(result).toBe('ff');
});
});
describe('Edge cases and integration', () => {
it('should handle conversion chain for span IDs', () => {
// Test that we can go from hex -> base64 -> hex without loss
const originalHex = 'a1b2c3d4e5f67890';
const base64 = encodeSpanIdToBase64(originalHex);
const decodedHex = decodeIdFromBase64(base64);
expect(decodedHex).toBe(originalHex);
});
it('should handle conversion chain for trace IDs', () => {
// Test that we can go from hex -> base64 -> hex without loss
const originalHex = 'a1b2c3d4e5f67890a1b2c3d4e5f67890';
const base64 = encodeTraceIdToBase64(originalHex);
const decodedHex = decodeIdFromBase64(base64);
expect(decodedHex).toBe(originalHex);
});
it('should handle odd-length hex strings for span ID', () => {
const oddHex = '12345'; // 5 characters
const base64 = encodeSpanIdToBase64(oddHex);
const decoded = decodeIdFromBase64(base64);
// Should be padded to '0000000000012345'
expect(decoded).toBe('0000000000012345');
});
it('should handle odd-length hex strings for trace ID', () => {
const oddHex = '12345'; // 5 characters
const base64 = encodeTraceIdToBase64(oddHex);
const decoded = decodeIdFromBase64(base64);
// Should be padded to '00000000000000000000000000012345'
expect(decoded).toBe('00000000000000000000000000012345');
});
it('should handle very long hex strings by truncation', () => {
// Span ID should only use first 16 chars
const longHex = '0123456789abcdef0123456789abcdef0123456789abcdef';
const spanBase64 = encodeSpanIdToBase64(longHex);
const decodedSpan = decodeIdFromBase64(spanBase64);
// Should only encode first 16 chars
expect(decodedSpan).toBe('0123456789abcdef');
});
it('should produce consistent results across multiple calls', () => {
const spanId = 'deadbeef12345678';
const traceId = 'deadbeef12345678deadbeef12345678';
// Multiple calls should produce identical results
const spanBase64_1 = encodeSpanIdToBase64(spanId);
const spanBase64_2 = encodeSpanIdToBase64(spanId);
expect(spanBase64_1).toBe(spanBase64_2);
const traceBase64_1 = encodeTraceIdToBase64(traceId);
const traceBase64_2 = encodeTraceIdToBase64(traceId);
expect(traceBase64_1).toBe(traceBase64_2);
});
});
});
describe('mapArgsToObject', () => {
// Basic argument mapping test cases
test.each([
{
description: 'regular functions',
func: function add(a: number, b: number) {
return a + b;
},
args: [5, 10],
expected: { a: 5, b: 10 },
},
{
description: 'arrow functions',
func: (x: number, y: number) => x * y,
args: [3, 4],
expected: { x: 3, y: 4 },
},
{
description: 'single parameter functions',
func: (value: number) => value * 2,
args: [7],
expected: { value: 7 },
},
{
description: 'functions with no parameters',
func: () => 42,
args: [],
expected: {},
},
{
description: 'functions with default parameters',
func: function withDefaults(name: string, greeting: string = 'Hello') {
return greeting + ' ' + name;
},
args: ['World'],
expected: { name: 'World' },
},
{
description: 'functions with type annotations',
func: function typed(id: number, name: string, active: boolean) {
return { id, name, active };
},
args: [123, 'John', true],
expected: { id: 123, name: 'John', active: true },
},
{
description: 'anonymous functions',
func: function (first: string, second: number) {
return first + second;
},
args: ['hello', 42],
expected: { first: 'hello', second: 42 },
},
{
description: 'fewer arguments than parameters',
func: function threeParams(a: number, b: number, c: number) {
return a + b + c;
},
args: [1, 2],
expected: { a: 1, b: 2 },
},
{
description: 'more arguments than parameters',
func: function twoParams(a: number, b: number) {
return a + b;
},
args: [1, 2, 3, 4],
expected: { a: 1, b: 2 },
},
{
description: 'complex argument types (objects, arrays)',
func: function complex(obj: object, arr: any[], str: string) {
return { obj, arr, str };
},
args: [{ key: 'value' }, [1, 2, 3], 'test'],
expected: { obj: { key: 'value' }, arr: [1, 2, 3], str: 'test' },
},
{
description: 'null and undefined arguments',
func: function nullable(a: any, b: any, c: any) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return a + b + c;
},
args: [null, undefined, 'value'],
expected: { a: null, b: undefined, c: 'value' },
},
{
description: 'functions with destructured parameters gracefully',
func: function withDestructuring({ prop }: any, normal: string) {
return prop + normal;
},
args: [{ prop: 'value' }, 'normal'],
expected: { normal: { prop: 'value' } },
},
{
description: 'fallback to args array when parameter extraction fails',
// eslint-disable-next-line @typescript-eslint/no-implied-eval
func: new Function('return arguments[0] + arguments[1];'),
args: [1, 2],
expected: { args: [1, 2] },
},
{
description: 'empty object for no parameters and no arguments',
func: () => {},
args: [],
expected: {},
},
{
description: 'edge case with only whitespace parameters',
func: () => {},
args: [],
expected: {},
},
{
description: 'functions with object destructuring parameters (JS/TS kwarg-only pattern)',
func: ({ a, b }: { a: number; b: number }) => a + b,
args: [{ a: 5, b: 10 }],
expected: { args: [{ a: 5, b: 10 }] },
},
])('should handle $description', ({ func, args, expected }) => {
const result = mapArgsToObject(func, args);
expect(result).toEqual(expected);
});
});
@@ -0,0 +1,71 @@
import { safeJsonStringify } from '../../../src/core/utils/json';
describe('safeJsonStringify', () => {
it('should stringify simple values correctly', () => {
expect(safeJsonStringify('hello')).toBe('"hello"');
expect(safeJsonStringify(123)).toBe('123');
expect(safeJsonStringify(true)).toBe('true');
expect(safeJsonStringify(null)).toBe('null');
expect(safeJsonStringify({ a: 1, b: 'test' })).toBe('{"a":1,"b":"test"}');
expect(safeJsonStringify([1, 2, 3])).toBe('[1,2,3]');
});
it('should handle circular references', () => {
const obj: any = { name: 'test' };
obj.self = obj;
const result = safeJsonStringify(obj);
const parsed = JSON.parse(result);
expect(parsed.name).toBe('test');
expect(parsed.self).toBe('[Circular]');
});
it('should handle functions', () => {
const obj = {
name: 'test',
fn: () => console.log('hello'),
method: function () {
return 42;
},
};
const result = safeJsonStringify(obj);
const parsed = JSON.parse(result);
expect(parsed.name).toBe('test');
expect(parsed.fn).toBe('[Function]');
expect(parsed.method).toBe('[Function]');
});
it('should handle undefined values', () => {
const obj = {
name: 'test',
value: undefined,
nested: { prop: undefined },
};
const result = safeJsonStringify(obj);
const parsed = JSON.parse(result);
expect(parsed.name).toBe('test');
expect(parsed.value).toBe('[Undefined]');
expect(parsed.nested.prop).toBe('[Undefined]');
});
it('should handle Error objects', () => {
const error = new Error('Test error');
const obj = {
status: 'failed',
error,
};
const result = safeJsonStringify(obj);
const parsed = JSON.parse(result);
expect(parsed.status).toBe('failed');
expect(parsed.error.name).toBe('Error');
expect(parsed.error.message).toBe('Test error');
expect(parsed.error.stack).toBeDefined();
});
});
@@ -0,0 +1,169 @@
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
import { ROOT_CONTEXT, SpanKind } from '@opentelemetry/api';
import {
BasicTracerProvider,
ReadableSpan as OTelReadableSpan,
Span as OTelSpan,
} from '@opentelemetry/sdk-trace-base';
// Mock the OTLP proto exporter. The real implementation does dynamic
// `import('http')` calls that Jest needs --experimental-vm-modules to run.
// We assert on constructor args (url + UC table header) and assume the
// upstream OTel library serializes correctly.
const mockExport = jest.fn();
const mockShutdown = jest.fn().mockResolvedValue(undefined);
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
OTLPTraceExporter: jest
.fn()
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
exporterCtors.push(cfg);
return {
export: (spans: unknown[], cb: (r: { code: number }) => void) => {
mockExport(spans);
cb({ code: 0 });
},
shutdown: mockShutdown,
};
}),
}));
import { AuthProvider } from '../../src/auth';
import { MlflowClient } from '../../src/clients/client';
import {
TraceLocationType,
createTraceLocationFromUcTablePrefix,
} from '../../src/core/entities/trace_location';
import { TraceInfo } from '../../src/core/entities/trace_info';
import { TraceState } from '../../src/core/entities/trace_state';
import { DATABRICKS_UC_TABLE_HEADER, TraceMetadataKey } from '../../src/core/constants';
const testHost = 'https://dbc-12345.cloud.databricks.com';
const testToken = 'test-token';
const mockAuthProvider: AuthProvider = {
getHost: () => testHost,
getHeadersProvider: () => () =>
Promise.resolve({
'Content-Type': 'application/json',
Authorization: `Bearer ${testToken}`,
}),
getDatabricksToken: () => testToken,
};
describe('MlflowClient UC methods', () => {
let server: ReturnType<typeof setupServer>;
let client: MlflowClient;
beforeAll(() => {
server = setupServer();
server.listen();
});
afterAll(() => {
server.close();
});
beforeEach(() => {
client = new MlflowClient({ trackingUri: 'databricks', authProvider: mockAuthProvider });
});
afterEach(() => {
server.resetHandlers();
});
it('createTraceInfoV4 POSTs trace_info to the V4 endpoint with tags preserved', async () => {
const location = 'cat.sch.tbl';
const otelTraceId = 'abcdef1234567890abcdef1234567890';
const traceInfo = new TraceInfo({
traceId: `trace:/${location}/${otelTraceId}`,
traceLocation: createTraceLocationFromUcTablePrefix('cat', 'sch', 'tbl'),
requestTime: 1_700_000_000_000,
executionDuration: 1500,
state: TraceState.OK,
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
traceMetadata: { [TraceMetadataKey.SCHEMA_VERSION]: '4' },
});
let capturedBody: unknown = null;
let capturedUrl: string | null = null;
server.use(
http.post(
`${testHost}/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`,
async ({ request }) => {
capturedUrl = request.url;
capturedBody = await request.json();
// Echo back a TraceInfo with the backend populating the spans table.
return HttpResponse.json({
trace_id: traceInfo.traceId,
trace_location: {
type: TraceLocationType.UC_TABLE_PREFIX,
uc_table_prefix: {
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'tbl',
otel_spans_table_name: 'cat.sch.tbl_otel_spans',
},
},
request_time: new Date(traceInfo.requestTime).toISOString(),
execution_duration: '1.5s',
state: TraceState.OK,
trace_metadata: traceInfo.traceMetadata,
tags: traceInfo.tags,
assessments: [],
});
},
),
);
const returned = await client.createTraceInfoV4(location, otelTraceId, traceInfo);
expect(capturedUrl).toContain(
`/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`,
);
// Body is the TraceInfo JSON directly (Databricks RPC convention).
expect(capturedBody).toMatchObject({
trace_id: traceInfo.traceId,
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
trace_metadata: { [TraceMetadataKey.SCHEMA_VERSION]: '4' },
trace_location: {
type: TraceLocationType.UC_TABLE_PREFIX,
uc_table_prefix: {
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'tbl',
},
},
});
expect(returned.traceLocation.ucTablePrefix?.otelSpansTableName).toBe('cat.sch.tbl_otel_spans');
});
it('exportOtlpSpansToUc constructs an OTLP proto exporter with the UC table header', async () => {
exporterCtors.length = 0;
mockExport.mockClear();
const provider = new BasicTracerProvider();
const tracer = provider.getTracer('uc-test');
const span = tracer.startSpan('root', { kind: SpanKind.INTERNAL }, ROOT_CONTEXT) as OTelSpan;
span.setAttribute('user.id', 'u1');
span.end();
await client.exportOtlpSpansToUc(
[span as unknown as OTelReadableSpan],
'cat.sch.tbl_otel_spans',
);
expect(exporterCtors).toHaveLength(1);
expect(exporterCtors[0].url).toBe(`${testHost}/api/2.0/otel/v1/traces`);
expect(exporterCtors[0].headers?.[DATABRICKS_UC_TABLE_HEADER]).toBe('cat.sch.tbl_otel_spans');
expect(exporterCtors[0].headers?.['Authorization']).toBe(`Bearer ${testToken}`);
expect(mockExport).toHaveBeenCalledTimes(1);
expect((mockExport.mock.calls[0][0] as unknown[])[0]).toBe(span);
});
it('exportOtlpSpansToUc short-circuits with no spans', async () => {
// No mock registered: if it tried to hit the network the call would throw.
await expect(client.exportOtlpSpansToUc([], 'cat.sch.tbl_otel_spans')).resolves.toBeUndefined();
});
});
@@ -0,0 +1,167 @@
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
import { context, SpanKind, ROOT_CONTEXT, trace as otelTrace } from '@opentelemetry/api';
import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
import {
BasicTracerProvider,
ReadableSpan as OTelReadableSpan,
Span as OTelSpan,
} from '@opentelemetry/sdk-trace-base';
// Mock the OTLP proto exporter (Jest can't load its dynamic http imports).
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
OTLPTraceExporter: jest
.fn()
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
exporterCtors.push(cfg);
return {
export: (_spans: unknown[], cb: (r: { code: number }) => void) => cb({ code: 0 }),
shutdown: () => Promise.resolve(),
};
}),
}));
import { AuthProvider } from '../../src/auth';
import { MlflowClient } from '../../src/clients/client';
import {
DatabricksUCTableSpanExporter,
DatabricksUCTableSpanProcessor,
} from '../../src/exporters/uc_table';
import { updateCurrentTrace } from '../../src/core/api';
import { TraceMetadataKey, DATABRICKS_UC_TABLE_HEADER } from '../../src/core/constants';
import { InMemoryTraceManager } from '../../src/core/trace_manager';
import { TraceLocationType, isUcTraceLocation } from '../../src/core/entities/trace_location';
const testHost = 'https://dbc-12345.cloud.databricks.com';
const mockAuthProvider: AuthProvider = {
getHost: () => testHost,
getHeadersProvider: () => () =>
Promise.resolve({
'Content-Type': 'application/json',
Authorization: 'Bearer test',
}),
getDatabricksToken: () => 'test',
};
function makeOtelRootSpan(): OTelSpan {
// Use a real tracer to get a real Span object whose context() works.
const provider = new BasicTracerProvider();
const tracer = provider.getTracer('uc-test');
const span = tracer.startSpan('root', { kind: SpanKind.INTERNAL }, ROOT_CONTEXT) as OTelSpan;
return span;
}
describe('DatabricksUCTableSpanProcessor + Exporter end-to-end', () => {
let server: ReturnType<typeof setupServer>;
let traceInfoCalls: { url: string; body: any }[];
let contextManager: AsyncLocalStorageContextManager;
beforeAll(() => {
// Default NoopContextManager makes context.with() a no-op, so
// updateCurrentTrace can't locate the active span. Wire a real manager.
contextManager = new AsyncLocalStorageContextManager();
contextManager.enable();
context.setGlobalContextManager(contextManager);
server = setupServer();
server.listen();
});
afterAll(() => {
contextManager.disable();
context.disable();
server.close();
});
beforeEach(() => {
traceInfoCalls = [];
exporterCtors.length = 0;
server.resetHandlers();
server.use(
http.post(
`${testHost}/api/4.0/mlflow/traces/:location/:otelTraceId/info`,
async ({ request, params }) => {
const body = (await request.json()) as any;
traceInfoCalls.push({ url: request.url, body });
return HttpResponse.json({
trace_id: body.trace_id,
trace_location: {
type: TraceLocationType.UC_TABLE_PREFIX,
uc_table_prefix: {
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'tbl',
otel_spans_table_name: 'cat.sch.tbl_otel_spans',
},
},
request_time: body.request_time,
execution_duration: body.execution_duration,
state: body.state,
trace_metadata: body.trace_metadata,
tags: body.tags,
assessments: [],
});
void params;
},
),
);
});
it('persists tags set via updateCurrentTrace through the V4 endpoint and uploads spans via OTLP', async () => {
const client = new MlflowClient({
trackingUri: 'databricks',
authProvider: mockAuthProvider,
});
const location = { catalogName: 'cat', schemaName: 'sch', tablePrefix: 'tbl' };
const exporter = new DatabricksUCTableSpanExporter(client);
const processor = new DatabricksUCTableSpanProcessor(exporter, location);
const span = makeOtelRootSpan();
processor.onStart(span, context.active());
const otelTraceId = span.spanContext().traceId;
const mgr = InMemoryTraceManager.getInstance();
const mlflowTraceId = mgr.getMlflowTraceIdFromOtelId(otelTraceId)!;
expect(mlflowTraceId.startsWith('trace:/cat.sch.tbl/')).toBe(true);
// Drive the public `updateCurrentTrace` API with the span as the OTel
// active context — the UC processor must be the one mutating the in-memory
// TraceInfo so the tags reach the V4 endpoint.
context.with(otelTrace.setSpan(context.active(), span), () => {
updateCurrentTrace({
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
});
});
const trace = mgr.getTrace(mlflowTraceId)!;
expect(isUcTraceLocation(trace.info.traceLocation)).toBe(true);
expect(trace.info.traceMetadata[TraceMetadataKey.SCHEMA_VERSION]).toBe('4');
expect(trace.info.tags).toMatchObject({
user_id: 'u1',
family_id: 'f1',
conversation_id: 'c1',
});
span.end();
processor.onEnd(span as unknown as OTelReadableSpan);
await processor.forceFlush();
expect(traceInfoCalls).toHaveLength(1);
// The Databricks RPC convention sends the TraceInfo JSON directly as the
// body (not wrapped in `{ trace_info: ... }`), so the body IS the trace info.
const posted = traceInfoCalls[0].body;
expect(posted.trace_id).toBe(mlflowTraceId);
expect(posted.tags).toEqual({ user_id: 'u1', family_id: 'f1', conversation_id: 'c1' });
expect(posted.trace_metadata[TraceMetadataKey.SCHEMA_VERSION]).toBe('4');
expect(posted.trace_location.uc_table_prefix).toEqual({
catalog_name: 'cat',
schema_name: 'sch',
table_prefix: 'tbl',
});
expect(exporterCtors).toHaveLength(1);
expect(exporterCtors[0].url).toBe(`${testHost}/api/2.0/otel/v1/traces`);
expect(exporterCtors[0].headers?.[DATABRICKS_UC_TABLE_HEADER]).toBe('cat.sch.tbl_otel_spans');
});
});
@@ -0,0 +1,196 @@
import * as fsPromises from 'node:fs/promises';
import { mkdtemp, readFile, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
import { getWalPath } from '../../../src/exporters/wal/paths';
import { appendTombstone, readPending } from '../../../src/exporters/wal/storage';
import type { WalRecord } from '../../../src/exporters/wal/types';
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
return {
id: `wal-${idSuffix}`,
trackingUri: 'http://localhost:5000',
experimentId: '0',
traceInfo: { trace_id: `t-${idSuffix}` },
traceData: { spans: [] },
attempts: 0,
nextAttemptAt: 0,
createdAt: Date.now(),
...overrides,
};
}
async function spyOnFileHandleSync(walDir: string): Promise<jest.SpyInstance> {
let spy: jest.SpyInstance | undefined;
try {
const probeFh = await fsPromises.open(join(walDir, '.probe'), 'w');
const fdProto = Object.getPrototypeOf(probeFh) as { sync?: () => Promise<void> };
await probeFh.close();
expect(typeof fdProto.sync).toBe('function');
spy = jest.spyOn(fdProto as { sync: () => Promise<void> }, 'sync');
return spy;
} catch (err) {
// `spy` is undefined for any throw before the install, so the
// optional chain is a no-op; if the install succeeded and a later
// line throws, this restores the prototype before propagating.
spy?.mockRestore();
throw err;
}
}
describe('wal/batching_writer', () => {
let walDir: string;
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-batch-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
it('persists a single submitted record after the ack', async () => {
const writer = new BatchingWriter();
const record = makeRecord('s1');
await writer.submit(record);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(1);
expect(JSON.parse(lines[0])).toEqual({ type: 'append', record });
});
it('coalesces concurrent submits into a single fsync (group commit)', async () => {
const syncSpy = await spyOnFileHandleSync(walDir);
try {
const writer = new BatchingWriter();
const records = Array.from({ length: 25 }, (_, i) =>
makeRecord(i.toString().padStart(2, '0')),
);
const syncsBefore = syncSpy.mock.calls.length;
// Fire all submits in the same tick before awaiting any of them.
await Promise.all(records.map((r) => writer.submit(r)));
const syncsAfter = syncSpy.mock.calls.length;
// Allow at most one extra fsync above the one we expect, in case
// a stray tick splits the batch in two; the regression we care
// about is the "N fsyncs for N submits" shape.
expect(syncsAfter - syncsBefore).toBeLessThanOrEqual(2);
expect(syncsAfter - syncsBefore).toBeGreaterThanOrEqual(1);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(records.length);
const ids = lines.map((l) => (JSON.parse(l) as { record: WalRecord }).record.id).sort();
expect(ids).toEqual(records.map((r) => r.id).sort());
} finally {
syncSpy.mockRestore();
}
});
it('resolves submit() only after the byte is durable', async () => {
// The post-submit file read must observe the record — i.e. the
// ack-after-fsync invariant. If submit resolved before write+sync
// completed this read would race and occasionally come up empty.
const writer = new BatchingWriter();
const record = makeRecord('durable');
await writer.submit(record);
const contents = await readFile(getWalPath(), 'utf8');
expect(contents).toContain(`"id":"${record.id}"`);
});
it('produces well-formed lines when batched submits interleave with direct writers', async () => {
const writer = new BatchingWriter();
const r1 = makeRecord('mixed-1');
const r2 = makeRecord('mixed-2');
const submitR1 = writer.submit(r1);
const tombstone = appendTombstone(r1.id);
const submitR2 = writer.submit(r2);
await Promise.all([submitR1, tombstone, submitR2]);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
// Two appends + one tombstone, none truncated, all valid JSON.
expect(lines).toHaveLength(3);
for (const line of lines) {
expect(() => {
JSON.parse(line);
}).not.toThrow();
}
const pending = await readPending();
expect(pending.map((r) => r.id)).toContain(r2.id);
});
it('persists a single submitted tombstone after the ack', async () => {
const writer = new BatchingWriter();
const record = makeRecord('t-single');
await writer.submit(record);
await writer.submitTombstone(record.id);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(2);
expect(JSON.parse(lines[1])).toEqual({ type: 'tombstone', id: record.id });
// Tombstone has shadowed the append: nothing should remain pending.
const pending = await readPending();
expect(pending.find((r) => r.id === record.id)).toBeUndefined();
});
it('resolves submitTombstone() only after the byte is durable', async () => {
// Symmetric to the submit() ack-after-fsync test: the post-ack file
// read must observe the tombstone, otherwise tombstones could be
// ack'd before being durable and a daemon crash mid-batch would
// resurrect already-uploaded records on the next pass.
const writer = new BatchingWriter();
const id = 'wal-t-durable';
await writer.submitTombstone(id);
const contents = await readFile(getWalPath(), 'utf8');
expect(contents).toContain(`"id":"${id}"`);
expect(contents).toContain('"type":"tombstone"');
});
it('coalesces appends and tombstones into a single fsync when submitted in the same tick', async () => {
// The whole point of routing tombstones through the BatchingWriter:
// a tombstone storm from the upload loop should batch with concurrent
// hook submits instead of paying its own fsync per call.
const syncSpy = await spyOnFileHandleSync(walDir);
try {
const writer = new BatchingWriter();
const records = Array.from({ length: 10 }, (_, i) => makeRecord(`mix-a-${i}`));
const tombstoneIds = Array.from({ length: 10 }, (_, i) => `mix-t-${i}`);
const syncsBefore = syncSpy.mock.calls.length;
await Promise.all([
...records.map((r) => writer.submit(r)),
...tombstoneIds.map((id) => writer.submitTombstone(id)),
]);
const syncsAfter = syncSpy.mock.calls.length;
// Same shape guarantee as the append-only group-commit test: a
// stray tick may split the batch into two, but never N fsyncs for
// N submits.
expect(syncsAfter - syncsBefore).toBeLessThanOrEqual(2);
expect(syncsAfter - syncsBefore).toBeGreaterThanOrEqual(1);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(records.length + tombstoneIds.length);
const tombstoneLines = lines.filter((l) => l.includes('"type":"tombstone"'));
const appendLines = lines.filter((l) => l.includes('"type":"append"'));
expect(tombstoneLines).toHaveLength(tombstoneIds.length);
expect(appendLines).toHaveLength(records.length);
} finally {
syncSpy.mockRestore();
}
});
});
@@ -0,0 +1,72 @@
import {
clearClientCache,
clearClientForUri,
clientForUri,
} from '../../../src/exporters/wal/clients';
import { MlflowClient } from '../../../src/clients/client';
describe('wal/clients', () => {
beforeEach(() => {
clearClientCache();
});
it('returns the same MlflowClient instance for repeat calls with the same URI', () => {
const a = clientForUri('http://localhost:5000');
const b = clientForUri('http://localhost:5000');
expect(a).toBeInstanceOf(MlflowClient);
expect(b).toBe(a);
});
it('returns distinct MlflowClient instances for different URIs', () => {
const local = clientForUri('http://localhost:5000');
const remote = clientForUri('http://other-host:5000');
expect(local).not.toBe(remote);
expect(local.getHost()).toBe('http://localhost:5000');
expect(remote.getHost()).toBe('http://other-host:5000');
});
it('rebuilds the client after the cache is cleared', () => {
const before = clientForUri('http://localhost:5000');
clearClientCache();
const after = clientForUri('http://localhost:5000');
expect(after).not.toBe(before);
expect(after).toBeInstanceOf(MlflowClient);
});
it('isolates state for different tracking URIs after a clear', () => {
const a1 = clientForUri('http://a:5000');
const b1 = clientForUri('http://b:5000');
clearClientCache();
const a2 = clientForUri('http://a:5000');
const b2 = clientForUri('http://b:5000');
expect(a2).not.toBe(a1);
expect(b2).not.toBe(b1);
expect(a2).not.toBe(b2);
});
describe('clearClientForUri', () => {
it('evicts only the targeted URI and rebuilds it on next access', () => {
// Targeted eviction is the upload loop's auth-error retry path:
// we want the *specific* stale client gone, not every other URI's
// working cache entry blown away as collateral damage.
const a = clientForUri('http://a:5000');
const b = clientForUri('http://b:5000');
clearClientForUri('http://a:5000');
const aAfter = clientForUri('http://a:5000');
const bAfter = clientForUri('http://b:5000');
expect(aAfter).not.toBe(a);
expect(bAfter).toBe(b);
});
it('is a silent no-op for a URI that was never cached', () => {
// The upload loop calls clearClientForUri unconditionally before
// re-resolving via the factory; it must not throw if the cache
// entry was already evicted (e.g. by a prior auth-retry in a
// sibling group, or by clearClientCache() during shutdown).
expect(() => clearClientForUri('http://never-cached:5000')).not.toThrow();
});
});
});
@@ -0,0 +1,825 @@
import { spawn } from 'child_process';
import { existsSync, readFileSync } from 'fs';
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
import {
acquireLock,
backoff,
log,
processBatch,
pruneOldLogs,
releaseLock,
runBatchLoop,
uploadOne,
} from '../../../src/exporters/wal/daemon';
import {
getDaemonLogPath,
getDeadLetterPath,
getLockSocketPath,
getPidLockPath,
getWalPath,
} from '../../../src/exporters/wal/paths';
import { appendRecord, readPending } from '../../../src/exporters/wal/storage';
import type { WalRecord } from '../../../src/exporters/wal/types';
import type { MlflowClient } from '../../../src/clients/client';
import { MlflowHttpError } from '../../../src/clients/utils';
import type { TraceInfo } from '../../../src/core/entities/trace_info';
import type { TraceData } from '../../../src/core/entities/trace_data';
/**
* Spawn a trivial child, wait for it to exit, and return its now-freed
* pid. Used so the integration tests can seed a stale PID lockfile
* with a pid that is guaranteed to fail `process.kill(pid, 0)` with
* ESRCH and trigger the stale-recovery branch in `tryAcquirePidLock`.
*/
function spawnAndReapPid(): Promise<number> {
return new Promise<number>((resolve, reject) => {
const child = spawn(process.execPath, ['-e', 'process.exit(0)']);
const pid = child.pid;
if (pid == null) {
reject(new Error('child spawn produced no pid'));
return;
}
child.once('exit', () => resolve(pid));
child.once('error', reject);
});
}
function makeRecord(overrides: Partial<WalRecord> = {}): WalRecord {
// Fields shaped so TraceInfo.fromJson / TraceData.fromJson succeed:
// execution_duration is the "Ns" string form expected by the parser
// (a raw number throws because of `.replace`), spans is an empty array.
const base: WalRecord = {
id: 'row-1',
trackingUri: 'http://localhost:5000',
experimentId: '0',
traceInfo: {
trace_id: 'tr-1',
trace_location: { type: 'MLFLOW_EXPERIMENT', mlflow_experiment: { experiment_id: '0' } },
request_time: 0,
execution_duration: '0s',
state: 'OK',
request_preview: '',
response_preview: '',
client_request_id: '',
tags: {},
trace_metadata: {},
assessments: [],
},
traceData: { spans: [] },
attempts: 0,
nextAttemptAt: 0,
createdAt: Date.now(),
};
return { ...base, ...overrides };
}
function makeClient(opts: {
createTrace?: jest.Mock;
uploadTraceData?: jest.Mock;
exportOtlpSpans?: jest.Mock;
}): MlflowClient {
return {
createTrace:
opts.createTrace ?? jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info)),
uploadTraceData:
opts.uploadTraceData ??
jest.fn().mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve()),
exportOtlpSpans:
opts.exportOtlpSpans ??
jest
.fn()
.mockImplementation((_experimentId: string, _bytes: Uint8Array) => Promise.resolve()),
getHost: () => 'http://localhost:5000',
} as unknown as MlflowClient;
}
describe('wal/daemon', () => {
let walDir: string;
// Capture the developer's pre-test value so we restore (not just unset)
// in afterEach. Mirrors the pattern in paths.test.ts so running
// `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override for later
// tests in the same worker.
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-daemon-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
describe('backoff', () => {
it('doubles on each attempt until the cap', () => {
expect(backoff(1)).toBe(2_000);
expect(backoff(2)).toBe(4_000);
expect(backoff(3)).toBe(8_000);
expect(backoff(8)).toBe(256_000);
});
it('caps at 5 minutes', () => {
expect(backoff(9)).toBe(300_000);
expect(backoff(20)).toBe(300_000);
});
});
describe('log', () => {
it('appends a timestamped line to the daemon log file', () => {
log('hello');
log('world');
const path = getDaemonLogPath();
// daemon.log is daily-rotated; assert the suffix is present so a
// regression to a single unbounded file is caught here too.
expect(path).toMatch(/[/\\]daemon\.log\.\d{4}-\d{2}-\d{2}$/);
const contents = readFileSync(path, 'utf8');
const lines = contents.trim().split('\n');
expect(lines).toHaveLength(2);
expect(lines[0]).toMatch(/^\[\d{4}-\d{2}-\d{2}T.+Z\] \[pid=\d+\] hello$/);
expect(lines[1]).toMatch(/world$/);
});
it('swallows write errors silently', () => {
// Point the log path at a directory that doesn't exist by deleting
// the WAL dir mid-test. appendFileSync will throw internally; log()
// must not propagate.
process.env.MLFLOW_WAL_DIR = join(walDir, 'nope', 'deeper');
expect(() => log('ignored')).not.toThrow();
});
});
describe('acquireLock / releaseLock', () => {
it('binds the lock socket and releases cleanly', async () => {
const handle = await acquireLock();
expect(handle).not.toBeNull();
expect(existsSync(getLockSocketPath())).toBe(true);
await releaseLock(handle!.server, handle!.pidLock);
expect(existsSync(getLockSocketPath())).toBe(false);
});
it('returns null when another daemon already holds the lock', async () => {
const first = await acquireLock();
expect(first).not.toBeNull();
try {
const second = await acquireLock();
expect(second).toBeNull();
} finally {
await releaseLock(first!.server, first!.pidLock);
}
});
it('unlinks a stale POSIX socket file before binding', async () => {
if (process.platform === 'win32') {
return; // Named pipes have no stale-file concept.
}
// Simulate a previously-crashed daemon by writing a regular file
// at the socket path. acquireLock should unlink it on its way to
// a fresh bind.
await writeFile(getLockSocketPath(), 'stale');
const handle = await acquireLock();
try {
expect(handle).not.toBeNull();
} finally {
await releaseLock(handle!.server, handle!.pidLock);
}
});
});
// POSIX-only PID-lock integration. On Windows the named pipe is
// kernel-refcounted and acquireLock returns `pidLock: null`, so
// there's nothing for these tests to assert. Use a describe-level
// skip rather than per-`it` guards so the suite reports honestly on
// CI ("skipped" instead of "passed in 0ms").
const describePosix = process.platform === 'win32' ? describe.skip : describe;
describePosix('acquireLock / releaseLock — PID lock integration', () => {
it('creates the pid lockfile with our pid and removes it on release', async () => {
const handle = await acquireLock();
try {
expect(handle).not.toBeNull();
expect(handle!.pidLock).not.toBeNull();
// Lockfile must exist alongside the socket and record our pid
// exactly — content-stamped release relies on the exact match.
expect(existsSync(getPidLockPath())).toBe(true);
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await releaseLock(handle!.server, handle!.pidLock);
}
expect(existsSync(getPidLockPath())).toBe(false);
});
it('recovers from a stale pid lockfile left behind by a crashed daemon', async () => {
// A previously-crashed daemon's pid file with a now-dead pid.
// tryAcquirePidLock's liveness probe (`process.kill(pid, 0)`)
// should return ESRCH, the content-stamped unlink should clear
// the file, and acquireLock should proceed to bind normally.
const deadPid = await spawnAndReapPid();
await writeFile(getPidLockPath(), `${deadPid}\n`);
const handle = await acquireLock();
try {
expect(handle).not.toBeNull();
expect(handle!.pidLock).not.toBeNull();
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await releaseLock(handle!.server, handle!.pidLock);
}
});
it('concedes when an alive process already holds the pid lockfile', async () => {
await writeFile(getPidLockPath(), `${process.pid}\n`);
const handle = await acquireLock();
expect(handle).toBeNull();
// Lockfile must remain untouched when we concede.
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
// And the socket must not have been bound.
expect(existsSync(getLockSocketPath())).toBe(false);
});
it('serializes concurrent acquireLock calls to a single winner', async () => {
// Both calls run inside the same Node event loop, but the
// `link()` syscall inside tryAcquirePidLock still goes through
// libuv's fs threadpool and the kernel's inode lock — so exactly
// one wins the atomic create, and the other reads our (alive)
// pid and concedes via the live-holder branch.
const [first, second] = await Promise.all([acquireLock(), acquireLock()]);
try {
const acquired = [first, second].filter((h) => h != null);
const conceded = [first, second].filter((h) => h == null);
expect(acquired).toHaveLength(1);
expect(conceded).toHaveLength(1);
// The winner owns the socket; the loser never bound anything.
expect(existsSync(getLockSocketPath())).toBe(true);
// The winner's pid is recorded in the lockfile.
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
if (first != null) {
await releaseLock(first.server, first.pidLock);
}
if (second != null) {
await releaseLock(second.server, second.pidLock);
}
}
});
it('releases the pid lockfile when both socket and pid lock are unlinked together', async () => {
// Verifies the teardown order is symmetric to acquisition:
// releaseLock must remove the socket file AND the pid lockfile
// so a subsequent acquireLock starts from a fully-clean slate.
const handle = await acquireLock();
expect(handle).not.toBeNull();
expect(existsSync(getLockSocketPath())).toBe(true);
expect(existsSync(getPidLockPath())).toBe(true);
await releaseLock(handle!.server, handle!.pidLock);
expect(existsSync(getLockSocketPath())).toBe(false);
expect(existsSync(getPidLockPath())).toBe(false);
});
it('allows a fresh acquire after a clean release', async () => {
// End-to-end smoke for the acquire → release → reacquire cycle.
// Catches regressions where releaseLock leaves stale state that
// the next acquireLock cannot recover from.
const first = await acquireLock();
expect(first).not.toBeNull();
await releaseLock(first!.server, first!.pidLock);
const second = await acquireLock();
try {
expect(second).not.toBeNull();
} finally {
await releaseLock(second!.server, second!.pidLock);
}
});
});
describe('uploadOne', () => {
it('uploads the trace and tombstones the WAL row on success', async () => {
const createTrace = jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info));
const uploadTraceData = jest
.fn()
.mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve());
const client = makeClient({ createTrace, uploadTraceData });
const record = makeRecord({ id: 'row-success' });
await uploadOne(record, client, new BatchingWriter());
expect(createTrace).toHaveBeenCalledTimes(1);
expect(uploadTraceData).toHaveBeenCalledTimes(1);
// After tombstone the record should not be in the live set.
const pending = await readPending();
expect(pending.find((r) => r.id === 'row-success')).toBeUndefined();
});
it('logs captured OTLP spans to the DB when record.otlpSpans is present', async () => {
const exportOtlpSpans = jest
.fn()
.mockImplementation((_experimentId: string, _bytes: Uint8Array) => Promise.resolve());
const client = makeClient({ exportOtlpSpans });
const otlpBytes = Buffer.from([0x0a, 0x01, 0x02, 0x03]);
const record = makeRecord({
id: 'row-otlp',
experimentId: 'exp-7',
otlpSpans: otlpBytes.toString('base64'),
});
await uploadOne(record, client, new BatchingWriter());
expect(exportOtlpSpans).toHaveBeenCalledTimes(1);
const [experimentId, bytes] = exportOtlpSpans.mock.calls[0] as [string, Uint8Array];
expect(experimentId).toBe('exp-7');
expect(Buffer.from(bytes).equals(otlpBytes)).toBe(true);
const pending = await readPending();
expect(pending.find((r) => r.id === 'row-otlp')).toBeUndefined();
});
it('skips span logging when record.otlpSpans is absent', async () => {
const exportOtlpSpans = jest.fn();
const client = makeClient({ exportOtlpSpans });
const record = makeRecord({ id: 'row-no-otlp' });
await uploadOne(record, client, new BatchingWriter());
expect(exportOtlpSpans).not.toHaveBeenCalled();
});
it('tombstones the record even when span logging fails (non-fatal)', async () => {
const exportOtlpSpans = jest.fn().mockRejectedValue(new Error('501 not supported'));
const createTrace = jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info));
const uploadTraceData = jest
.fn()
.mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve());
const client = makeClient({ createTrace, uploadTraceData, exportOtlpSpans });
const record = makeRecord({
id: 'row-otlp-fail',
otlpSpans: Buffer.from([0x0a, 0x01]).toString('base64'),
});
await uploadOne(record, client, new BatchingWriter());
expect(createTrace).toHaveBeenCalledTimes(1);
expect(uploadTraceData).toHaveBeenCalledTimes(1);
expect(exportOtlpSpans).toHaveBeenCalledTimes(1);
// The trace info + artifact already uploaded, so a span-log failure must
// not retry or dead-letter: the row is tombstoned and nothing re-appended.
const pending = await readPending();
expect(pending).toHaveLength(0);
});
it('re-appends a fresh row with bumped attempts on transient failure', async () => {
const createTrace = jest.fn().mockRejectedValue(new Error('5xx'));
const client = makeClient({ createTrace });
const record = makeRecord({ id: 'row-retry', attempts: 0 });
const before = Date.now();
await uploadOne(record, client, new BatchingWriter());
const after = Date.now();
const pending = await readPending();
const retry = pending.find((r) => r.id !== 'row-retry');
expect(pending.find((r) => r.id === 'row-retry')).toBeUndefined();
expect(retry).toBeDefined();
expect(retry!.attempts).toBe(1);
// backoff(1) = 2_000 ms; the new schedule must be in [before+2s, after+2s].
expect(retry!.nextAttemptAt).toBeGreaterThanOrEqual(before + 2_000);
expect(retry!.nextAttemptAt).toBeLessThanOrEqual(after + 2_000);
// firstAttemptAt is anchored on the first daemon touch and pinned
// onto the retry row so the wall-clock retry budget is measured
// from a stable point even across multiple retries / restarts.
expect(retry!.firstAttemptAt).toBeGreaterThanOrEqual(before);
expect(retry!.firstAttemptAt).toBeLessThanOrEqual(after);
});
it('preserves firstAttemptAt across retries (anchor survives daemon respawn)', async () => {
const createTrace = jest.fn().mockRejectedValue(new Error('5xx'));
const client = makeClient({ createTrace });
// A record a previous daemon already tried twice. A new daemon
// picking it up must preserve the original anchor, not reset it
// — otherwise a record could indefinitely outlive the retry
// budget by piggybacking on each daemon respawn's fresh clock.
const originalAnchor = Date.now() - 50_000;
const record = makeRecord({
id: 'row-restart',
attempts: 2,
firstAttemptAt: originalAnchor,
});
await uploadOne(record, client, new BatchingWriter());
const pending = await readPending();
const retry = pending.find((r) => r.id !== 'row-restart')!;
expect(retry.firstAttemptAt).toBe(originalAnchor);
expect(retry.attempts).toBe(3);
});
it('dead-letters the record when elapsed retry time exceeds RETRY_TIMEOUT_MS', async () => {
const createTrace = jest.fn().mockRejectedValue(new Error('still broken'));
const client = makeClient({ createTrace });
// Backdate firstAttemptAt by more than the default 500s budget,
// so the next backoff (~2s) plus the elapsed wall clock blows
// through the deadline and the record dead-letters immediately
// regardless of the attempts counter.
const longAgo = Date.now() - 600_000;
const record = makeRecord({
id: 'row-dlq',
attempts: 3,
firstAttemptAt: longAgo,
});
await uploadOne(record, client, new BatchingWriter());
const pending = await readPending();
expect(pending).toHaveLength(0);
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
const failedLines = failedContents.trim().split('\n');
expect(failedLines).toHaveLength(1);
const parsed = JSON.parse(failedLines[0]) as {
type: 'append';
record: WalRecord;
};
expect(parsed.type).toBe('append');
expect(parsed.record.id).toBe('row-dlq');
});
// The credential-refresh-on-401/403 retry path. The cached
// MlflowClient holds an AuthProvider snapshot from daemon start; a
// token rotation must surface as a transient failure that the daemon
// recovers from automatically rather than dead-letters forever.
describe('auth-refresh retry', () => {
it.each([
[401, 'Unauthorized'],
[403, 'Forbidden'],
] as const)(
'evicts the cached client and retries once after a %i, succeeding the second time',
async (status, statusText) => {
const staleCreateTrace = jest
.fn()
.mockRejectedValue(new MlflowHttpError(status, statusText, ''));
const freshCreateTrace = jest
.fn()
.mockImplementation((info: TraceInfo) => Promise.resolve(info));
const freshUploadTraceData = jest.fn().mockResolvedValue(undefined);
const stale = makeClient({ createTrace: staleCreateTrace });
const fresh = makeClient({
createTrace: freshCreateTrace,
uploadTraceData: freshUploadTraceData,
});
const factory = jest.fn(() => fresh);
const record = makeRecord({ id: `row-${status}-retry` });
await uploadOne(record, stale, new BatchingWriter(), factory);
// First attempt used the stale client and got the auth error;
// the auth-refresh branch then asked the factory for a fresh
// client (exactly once, for this URI) and the retry pushed
// through successfully.
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledWith(record.trackingUri);
expect(freshCreateTrace).toHaveBeenCalledTimes(1);
expect(freshUploadTraceData).toHaveBeenCalledTimes(1);
// Successful retry: tombstoned, no live row, no dead-letter.
const pending = await readPending();
expect(pending.find((r) => r.id === record.id)).toBeUndefined();
},
);
it('falls through to handleUploadFailure when both attempts hit 401', async () => {
const staleCreateTrace = jest
.fn()
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
const freshCreateTrace = jest
.fn()
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
const stale = makeClient({ createTrace: staleCreateTrace });
const fresh = makeClient({ createTrace: freshCreateTrace });
const factory = jest.fn(() => fresh);
// Backdate firstAttemptAt past the 500s budget so the post-retry
// failure dead-letters immediately rather than re-appending yet
// another retry row we'd have to sit on.
const longAgo = Date.now() - 600_000;
const record = makeRecord({
id: 'row-401-persistent',
firstAttemptAt: longAgo,
});
await uploadOne(record, stale, new BatchingWriter(), factory);
// Both attempts ran; factory was consulted exactly once for the
// refresh; the second failure routed to dead-letter (not a
// perpetual invalidate-retry loop).
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
expect(freshCreateTrace).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledTimes(1);
const pending = await readPending();
expect(pending).toHaveLength(0);
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
expect(failedContents).toContain('"id":"row-401-persistent"');
});
it('dead-letters with the factory error when refresh-time factory throws', async () => {
const staleCreateTrace = jest
.fn()
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
const stale = makeClient({ createTrace: staleCreateTrace });
const factory = jest.fn(() => {
throw new Error('credential provider blew up');
});
const longAgo = Date.now() - 600_000;
const record = makeRecord({
id: 'row-refresh-factory-throws',
firstAttemptAt: longAgo,
});
await uploadOne(record, stale, new BatchingWriter(), factory);
// Stale client was tried once and got the auth error; the
// refresh asked the factory once and that throw aborted the
// retry before any fresh client call could happen.
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledWith(record.trackingUri);
const pending = await readPending();
expect(pending).toHaveLength(0);
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
expect(failedContents).toContain('"id":"row-refresh-factory-throws"');
});
it('does not refresh the client on non-auth HTTP errors', async () => {
const createTrace = jest
.fn()
.mockRejectedValue(new MlflowHttpError(500, 'Internal Server Error', ''));
const client = makeClient({ createTrace });
const factory = jest.fn();
const record = makeRecord({ id: 'row-5xx' });
await uploadOne(record, client, new BatchingWriter(), factory);
// 5xx is not a credential problem — single attempt, no factory
// consultation, regular retry row appended by handleUploadFailure.
expect(createTrace).toHaveBeenCalledTimes(1);
expect(factory).not.toHaveBeenCalled();
const pending = await readPending();
const retry = pending.find((r) => r.id !== 'row-5xx');
expect(pending.find((r) => r.id === 'row-5xx')).toBeUndefined();
expect(retry).toBeDefined();
expect(retry!.attempts).toBe(1);
});
it('does not refresh the client on transport-level errors', async () => {
// A non-MlflowHttpError (e.g. fetch DNS / ECONNRESET / abort
// wrapped by makeRequest into `new Error('API request failed: ...')`)
// must not be confused with an auth failure — the `instanceof`
// gate in isAuthRefreshableError is what protects us here.
const createTrace = jest.fn().mockRejectedValue(new Error('ECONNRESET'));
const client = makeClient({ createTrace });
const factory = jest.fn();
const record = makeRecord({ id: 'row-transport' });
await uploadOne(record, client, new BatchingWriter(), factory);
expect(factory).not.toHaveBeenCalled();
const pending = await readPending();
const retry = pending.find((r) => r.id !== 'row-transport');
expect(pending.find((r) => r.id === 'row-transport')).toBeUndefined();
expect(retry).toBeDefined();
expect(retry!.attempts).toBe(1);
});
});
});
describe('processBatch', () => {
it('groups records by trackingUri and calls the factory once per URI', async () => {
const clientA = makeClient({});
const clientB = makeClient({});
const factory = jest.fn((uri: string) => (uri.endsWith(':5001') ? clientB : clientA));
const records = [
makeRecord({ id: 'r1', trackingUri: 'http://localhost:5000' }),
makeRecord({ id: 'r2', trackingUri: 'http://localhost:5000' }),
makeRecord({ id: 'r3', trackingUri: 'http://localhost:5001' }),
];
await processBatch(records, new BatchingWriter(), factory);
expect(factory).toHaveBeenCalledTimes(2);
expect(factory).toHaveBeenCalledWith('http://localhost:5000');
expect(factory).toHaveBeenCalledWith('http://localhost:5001');
const pending = await readPending();
// All three should have been tombstoned by their respective uploads.
expect(pending).toHaveLength(0);
});
it('no-ops on an empty input', async () => {
const factory = jest.fn();
await processBatch([], new BatchingWriter(), factory);
expect(factory).not.toHaveBeenCalled();
});
it('dead-letters records when their client factory throws', async () => {
const factory = jest.fn(() => {
throw new Error('auth provider blew up');
});
// Backdate firstAttemptAt past the default 500s retry budget so
// the synthetic factory failure dead-letters immediately instead
// of just bumping the retry counter.
const longAgo = Date.now() - 600_000;
const records = [makeRecord({ id: 'r-bad', firstAttemptAt: longAgo })];
await processBatch(records, new BatchingWriter(), factory);
const pending = await readPending();
expect(pending).toHaveLength(0);
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
expect(failedContents).toContain('"id":"r-bad"');
});
});
describe('queue.log layout after retry', () => {
it('atomically batches the retry append + original tombstone in one fsync', async () => {
const createTrace = jest.fn().mockRejectedValueOnce(new Error('first attempt fails'));
const client = makeClient({ createTrace });
const record = makeRecord({ id: 'row-order', attempts: 0 });
await uploadOne(record, client, new BatchingWriter());
const contents = await readFile(getWalPath(), 'utf8');
const lines = contents.trim().split('\n');
// `handleUploadFailure`'s retry branch enqueues both the new
// retry row and the tombstone for the original id on the same
// `BatchingWriter` tick, so `drainAndFlush` packs them into a
// single `writev` + `fsync`. The on-disk order matches the
// enqueue order (append first, then tombstone) and the pair is
// atomic — either both lines are durable or neither is, so a
// crash between them is structurally impossible.
expect(lines).toHaveLength(2);
const first = JSON.parse(lines[0]) as { type: string; record?: WalRecord };
const second = JSON.parse(lines[1]) as { type: string; id?: string };
expect(first.type).toBe('append');
expect(first.record!.attempts).toBe(1);
expect(second.type).toBe('tombstone');
expect(second.id).toBe('row-order');
});
});
describe('pruneOldLogs', () => {
// All retention tests pin `now` to a fixed UTC date so the cutoff
// arithmetic stays stable regardless of when CI happens to run.
const NOW = new Date('2026-05-20T12:00:00.000Z');
async function seed(names: string[]): Promise<void> {
for (const name of names) {
await writeFile(join(walDir, name), 'x');
}
}
function existsInWal(name: string): boolean {
return existsSync(join(walDir, name));
}
// Per-file existence checks are used throughout this block instead of
// full directory equality because `pruneOldLogs` itself calls `log()`
// when it unlinks a file, which writes to today's `daemon.log.<date>`
// and would show up in any whole-listing comparison. We assert the
// narrower property the function actually owns: which seeded files
// survive vs disappear.
it('removes files older than the retention window and keeps newer ones', async () => {
// retentionDays=14 with NOW=2026-05-20 → keep dates >= 2026-05-06.
const kept = ['failed.log.2026-05-19', 'failed.log.2026-05-06', 'daemon.log.2026-05-19'];
const removed = ['failed.log.2026-05-05', 'failed.log.2026-04-01', 'daemon.log.2026-05-05'];
await seed([...kept, ...removed]);
await pruneOldLogs({ now: NOW, retentionDays: 14 });
for (const name of kept) {
expect(existsInWal(name)).toBe(true);
}
for (const name of removed) {
expect(existsInWal(name)).toBe(false);
}
});
it('is a no-op when retention is 0 (disabled)', async () => {
const seeded = ['failed.log.2020-01-01', 'daemon.log.2020-01-01'];
await seed(seeded);
await pruneOldLogs({ now: NOW, retentionDays: 0 });
for (const name of seeded) {
expect(existsInWal(name)).toBe(true);
}
});
it('ignores files that do not match the rotated-log pattern', async () => {
// Including `queue.log`, the lock socket, files without a date
// suffix, malformed dates, and unrelated artifacts the operator
// may have dropped into the WAL dir.
const seeded = [
'queue.log',
'failed.log',
'daemon.log',
'failed.log.bogus',
'failed.log.2026-13-99',
'failed.log.2026-02-30',
'README.md',
'failed.log.2026-05-19.bak',
];
await seed(seeded);
await pruneOldLogs({ now: NOW, retentionDays: 14 });
// Every seeded entry must still be there — the regex must not
// strip-and-reparse, and Date.UTC's overflow-tolerance must not
// leak into the cutoff comparison.
for (const name of seeded) {
expect(existsInWal(name)).toBe(true);
}
});
it('preserves the active queue.log even when the WAL dir is full of expired logs', async () => {
await seed(['queue.log', 'failed.log.2020-01-01', 'daemon.log.2020-01-01']);
await pruneOldLogs({ now: NOW, retentionDays: 14 });
expect(existsInWal('queue.log')).toBe(true);
expect(existsInWal('failed.log.2020-01-01')).toBe(false);
expect(existsInWal('daemon.log.2020-01-01')).toBe(false);
});
it('does not throw when the WAL directory is missing', async () => {
// Simulate "daemon started, then someone rm -rf'd the dir before
// the sweep ran". The sweep must log-and-continue, never throw.
process.env.MLFLOW_WAL_DIR = join(walDir, 'does-not-exist');
await expect(pruneOldLogs({ now: NOW, retentionDays: 14 })).resolves.toBeUndefined();
});
});
describe('runBatchLoop idle semantics', () => {
it('shuts down via idleMs when the WAL is empty', async () => {
// Baseline: no records seeded → due.length === 0 every
// iteration → idle-shutdown branch fires on the first iteration
// after `idleMs` elapses.
const factory = jest.fn();
const start = Date.now();
await runBatchLoop({
factory,
writer: new BatchingWriter(),
batchIntervalMs: 10,
idleMs: 50,
});
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(50);
// Generous upper bound — guards against an accidental infinite
// loop while tolerating CI scheduler jitter.
expect(elapsed).toBeLessThan(2_000);
expect(factory).not.toHaveBeenCalled();
});
it('shuts down via idleMs when all pending records are future-dated', async () => {
// Regression test for the bug where `pending.length > 0` short-
// circuited the idle-shutdown check, pinning the daemon alive
// until each retry matured even though nothing was actually
// being worked on. The fix gates the idle check on
// `due.length === 0`, so a WAL populated only by future-dated
// retries is treated as not-actively-working and the loop
// winds down through the normal `idleMs` path.
await appendRecord(makeRecord({ id: 'row-future', nextAttemptAt: Date.now() + 60_000 }));
const factory = jest.fn();
const start = Date.now();
await runBatchLoop({
factory,
writer: new BatchingWriter(),
batchIntervalMs: 10,
idleMs: 50,
});
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(50);
expect(elapsed).toBeLessThan(2_000);
expect(factory).not.toHaveBeenCalled();
// The record is still durably parked in the WAL: the supervisor's
// next respawn (driven by a fresh IPC submission) will pick it
// up and either retry or dead-letter against the original
// `firstAttemptAt` budget — at-least-once delivery preserved.
expect(await readPending()).toHaveLength(1);
});
});
});
@@ -0,0 +1,621 @@
import { mkdtemp, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { ROOT_CONTEXT, SpanKind } from '@opentelemetry/api';
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
import {
BasicTracerProvider,
ReadableSpan as OTelReadableSpan,
Span as OTelSpan,
} from '@opentelemetry/sdk-trace-base';
import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer';
import { MlflowWalSpanExporter } from '../../../src/exporters/wal/exporter';
import * as ipc from '../../../src/exporters/wal/ipc';
import { init, resetConfig } from '../../../src/core/config';
import { InMemoryTraceManager } from '../../../src/core/trace_manager';
import { Trace } from '../../../src/core/entities/trace';
import { Span as MlflowSpan, NoOpSpan } from '../../../src/core/entities/span';
import { TraceInfo } from '../../../src/core/entities/trace_info';
import { TraceData } from '../../../src/core/entities/trace_data';
import { createTraceLocationFromExperimentId } from '../../../src/core/entities/trace_location';
import { TraceState } from '../../../src/core/entities/trace_state';
import { SpanAttributeKey } from '../../../src/core/constants';
import type { WalRecord } from '../../../src/exporters/wal/types';
function makeTrace(traceId: string): Trace {
const info = new TraceInfo({
traceId,
traceLocation: createTraceLocationFromExperimentId('exp-test'),
requestTime: 1_700_000_000_000,
state: TraceState.OK,
executionDuration: 42,
traceMetadata: { 'mlflow.trace.schemaVersion': '3' },
tags: {},
assessments: [],
});
const data = new TraceData([]);
return new Trace(info, data);
}
/**
* Build a trace whose `data.spans` carries real, ended OTel spans (wrapped in
* MLflow `Span`s) so the exporter's OTLP serialization runs end-to-end
*/
function makeTraceWithSpans(traceId: string, attributes: Record<string, string> = {}): Trace {
const info = new TraceInfo({
traceId,
traceLocation: createTraceLocationFromExperimentId('exp-test'),
requestTime: 1_700_000_000_000,
state: TraceState.OK,
executionDuration: 42,
traceMetadata: { 'mlflow.trace.schemaVersion': '3' },
tags: {},
assessments: [],
});
const provider = new BasicTracerProvider();
const tracer = provider.getTracer('wal-exporter-test');
const otelSpan = tracer.startSpan(
'tool-call',
{ kind: SpanKind.INTERNAL },
ROOT_CONTEXT,
) as OTelSpan;
otelSpan.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
// Attribute values are stored JSON-encoded on the OTel span, mirroring the
// SDK's `safeJsonStringify` behavior (see core/entities/span.ts).
for (const [key, value] of Object.entries(attributes)) {
otelSpan.setAttribute(key, value);
}
otelSpan.end();
return new Trace(info, new TraceData([new MlflowSpan(otelSpan)]));
}
function makeSpan(opts: { traceId: string; rootSpan: boolean; name?: string }): OTelReadableSpan {
return {
name: opts.name ?? (opts.rootSpan ? 'root' : 'child'),
spanContext: () => ({ traceId: opts.traceId, spanId: 'span-id' }),
parentSpanContext: opts.rootSpan ? undefined : { spanId: 'parent-span-id' },
} as unknown as OTelReadableSpan;
}
function captureExportResult(): {
callback: (r: ExportResult) => void;
promise: Promise<ExportResult>;
} {
let resolveFn: (r: ExportResult) => void = () => {};
const promise = new Promise<ExportResult>((resolve) => {
resolveFn = resolve;
});
return { callback: resolveFn, promise };
}
/**
* In OTLP protobuf wire format, each attribute's key and value strings are
* emitted close together. Require `expectedValue` to appear after `attributeKey`
* so a bare substring scan cannot false-positive on unrelated fields.
*/
function expectOtlpAttributeValue(
decoded: Buffer,
attributeKey: string,
expectedValue: string,
): void {
const keyOffset = decoded.indexOf(attributeKey);
expect(keyOffset).toBeGreaterThanOrEqual(0);
const valueOffset = decoded.indexOf(expectedValue, keyOffset + attributeKey.length);
expect(valueOffset).toBeGreaterThan(keyOffset);
}
function expectOtlpAttributeValueAbsent(
decoded: Buffer,
attributeKey: string,
unexpectedValue: string,
): void {
const keyOffset = decoded.indexOf(attributeKey);
expect(keyOffset).toBeGreaterThanOrEqual(0);
const valueOffset = decoded.indexOf(unexpectedValue, keyOffset + attributeKey.length);
expect(valueOffset).toBe(-1);
}
describe('wal/exporter', () => {
let walDir: string;
let popTraceSpy: jest.SpyInstance;
// Capture the developer's pre-test value so we restore (not just unset)
// in afterEach. Mirrors the pattern in paths.test.ts so running
// `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override for later
// tests in the same worker.
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-exporter-'));
process.env.MLFLOW_WAL_DIR = walDir;
init({ trackingUri: 'http://localhost:5000', experimentId: 'exp-test' });
popTraceSpy = jest.spyOn(InMemoryTraceManager.getInstance(), 'popTrace');
});
afterEach(async () => {
popTraceSpy.mockRestore();
InMemoryTraceManager.reset();
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
it('submits one WAL record per root span via the injected submitter', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-abc'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
expect(submit).toHaveBeenCalledTimes(1);
const record = submit.mock.calls[0][0];
expect(record.trackingUri).toBe('http://localhost:5000');
expect(record.experimentId).toBe('exp-test');
expect((record.traceInfo as { trace_id: string }).trace_id).toBe('tr-abc');
expect(typeof record.id).toBe('string');
expect(record.id.length).toBeGreaterThan(0);
});
it('skips non-root spans without invoking the submitter', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
const childSpan = makeSpan({ traceId: 'otel-1', rootSpan: false });
const { callback, promise } = captureExportResult();
exporter.export([childSpan], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
expect(submit).not.toHaveBeenCalled();
expect(popTraceSpy).not.toHaveBeenCalled();
await exporter.forceFlush();
});
it('warns and skips when popTrace returns null', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(null);
const span = makeSpan({ traceId: 'otel-missing', rootSpan: true, name: 'orphan' });
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
try {
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
expect(submit).not.toHaveBeenCalled();
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('No trace found for span orphan'),
);
await exporter.forceFlush();
} finally {
warnSpy.mockRestore();
}
});
it('handles a batch with mixed root and child spans', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValueOnce(makeTrace('tr-1')).mockReturnValueOnce(makeTrace('tr-2'));
const spans = [
makeSpan({ traceId: 'otel-1', rootSpan: true, name: 'root-1' }),
makeSpan({ traceId: 'otel-1', rootSpan: false, name: 'child' }),
makeSpan({ traceId: 'otel-2', rootSpan: true, name: 'root-2' }),
];
const { callback, promise } = captureExportResult();
exporter.export(spans, callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
expect(submit).toHaveBeenCalledTimes(2);
expect(popTraceSpy).toHaveBeenCalledTimes(2);
await exporter.forceFlush();
const traceIds = submit.mock.calls
.map(([record]) => (record.traceInfo as { trace_id: string }).trace_id)
.sort();
expect(traceIds).toEqual(['tr-1', 'tr-2']);
});
it('forceFlush resolves only once every in-flight submit has acked', async () => {
let release: (() => void) | undefined;
const submit = jest.fn<Promise<void>, [WalRecord]>().mockImplementation(
() =>
new Promise<void>((resolve) => {
release = resolve;
}),
);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-flush'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback } = captureExportResult();
exporter.export([span], callback);
let flushResolved = false;
const flush = exporter.forceFlush().then(() => {
flushResolved = true;
});
// The submit is parked on `release`; until we resolve it, forceFlush
// must remain pending. Yield a few microtasks to give any premature
// resolution a chance to surface.
await new Promise((resolve) => setImmediate(resolve));
await new Promise((resolve) => setImmediate(resolve));
expect(flushResolved).toBe(false);
release?.();
await flush;
expect(flushResolved).toBe(true);
});
it('shutdown awaits in-flight submits before resolving', async () => {
let release: (() => void) | undefined;
const submit = jest.fn<Promise<void>, [WalRecord]>().mockImplementation(
() =>
new Promise<void>((resolve) => {
release = resolve;
}),
);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-shutdown'));
const { callback } = captureExportResult();
exporter.export([makeSpan({ traceId: 'otel-1', rootSpan: true })], callback);
let shutdownResolved = false;
const shutdownPromise = exporter.shutdown().then(() => {
shutdownResolved = true;
});
// The submit is parked; shutdown must remain pending until we
// release it. Yield a few microtasks so any premature resolution
// surfaces here rather than silently passing.
await new Promise((resolve) => setImmediate(resolve));
await new Promise((resolve) => setImmediate(resolve));
expect(shutdownResolved).toBe(false);
release?.();
await shutdownPromise;
expect(shutdownResolved).toBe(true);
});
it('shutdown rejects subsequent export calls with FAILED per OTel spec', async () => {
// OTel SpanExporter.Shutdown spec: "After the call to Shutdown
// subsequent calls to Export are not allowed and SHOULD return
// a failure." Locks in the post-shutdown contract.
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
await exporter.shutdown();
popTraceSpy.mockReturnValue(makeTrace('tr-post-shutdown'));
const { callback, promise } = captureExportResult();
exporter.export([makeSpan({ traceId: 'otel-post', rootSpan: true })], callback);
expect((await promise).code).toBe(ExportResultCode.FAILED);
expect(submit).not.toHaveBeenCalled();
expect(popTraceSpy).not.toHaveBeenCalled();
});
it('shutdown awaits all in-flight submits, not just the first to resolve', async () => {
// Both `export()` calls run synchronously to completion before
// `shutdown()` is invoked, so both submit promises are already
// in `_pendingSubmits` when the drain loop takes its first
// snapshot. (The `_shuttingDown` flag in `export()` makes the
// "submit lands after the flag is set" scenario structurally
// impossible in single-threaded JS — once the flag flips, every
// export call short-circuits with FAILED before touching the
// Set.
let releaseFirst: (() => void) | undefined;
let releaseSecond: (() => void) | undefined;
const submit = jest
.fn<Promise<void>, [WalRecord]>()
.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
releaseFirst = resolve;
}),
)
.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
releaseSecond = resolve;
}),
);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy
.mockReturnValueOnce(makeTrace('tr-first'))
.mockReturnValueOnce(makeTrace('tr-second'));
// Two submits enter `_pendingSubmits` synchronously, parked on
// their respective `release*` resolvers.
const cb1 = captureExportResult();
exporter.export([makeSpan({ traceId: 'otel-1', rootSpan: true })], cb1.callback);
const cb2 = captureExportResult();
exporter.export([makeSpan({ traceId: 'otel-2', rootSpan: true })], cb2.callback);
let shutdownResolved = false;
const shutdownPromise = exporter.shutdown().then(() => {
shutdownResolved = true;
});
// Release submit #1 only. #2 is still parked, so `Promise.all`
// inside the drain loop must remain pending — proving shutdown
// awaits both, not just one.
await new Promise((resolve) => setImmediate(resolve));
releaseFirst?.();
await new Promise((resolve) => setImmediate(resolve));
await new Promise((resolve) => setImmediate(resolve));
expect(shutdownResolved).toBe(false);
releaseSecond?.();
await shutdownPromise;
expect(shutdownResolved).toBe(true);
expect(submit).toHaveBeenCalledTimes(2);
});
it('returns FAILED and warns when getConfig() throws (init not called)', async () => {
resetConfig();
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-no-config'));
const span = makeSpan({ traceId: 'otel-no-config', rootSpan: true });
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
try {
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
const result = await promise;
expect(result.code).toBe(ExportResultCode.FAILED);
expect(submit).not.toHaveBeenCalled();
expect(popTraceSpy).not.toHaveBeenCalled();
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('Tracing config unavailable'),
expect.any(Error),
);
} finally {
warnSpy.mockRestore();
// Restore config defensively so a downstream test that forgets
// to re-init doesn't observe the cleared global state.
init({ trackingUri: 'http://localhost:5000', experimentId: 'exp-test' });
}
});
it('logs and swallows submit failures so OTel never observes them', async () => {
const submit = jest
.fn<Promise<void>, [WalRecord]>()
.mockRejectedValue(new Error('daemon unreachable'));
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-err'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
try {
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
// OTel sees SUCCESS regardless of submit outcome — the exporter
// owns durability semantics and must not bubble per-record
// failures into the OTel result callback (which would surface as
// an SDK-level error users can't act on).
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
expect(submit).toHaveBeenCalledTimes(1);
expect(errSpy).toHaveBeenCalledWith(
expect.stringContaining('Failed to submit WAL record for trace tr-err'),
expect.any(Error),
);
} finally {
errSpy.mockRestore();
}
});
it('captures OTLP protobuf bytes on the record when the trace has spans', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-with-spans'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
const record = submit.mock.calls[0][0];
expect(typeof record.otlpSpans).toBe('string');
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
expect(decoded[0]).toBe(0x0a);
// And the span itself must be encoded in the payload, not just an empty
// envelope: its name is embedded as a UTF-8 string in the protobuf.
expect(decoded.includes(Buffer.from('tool-call'))).toBe(true);
// The JSON trace data is retained alongside the OTLP bytes (artifact fallback).
expect((record.traceData as { spans: unknown[] }).spans).toHaveLength(1);
});
it('omits otlpSpans when the trace has no spans', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTrace('tr-no-spans'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
const record = submit.mock.calls[0][0];
expect(record.otlpSpans).toBeUndefined();
});
it('filters malformed spans (missing resource/scope) and still serializes the valid ones', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
const trace = makeTraceWithSpans('tr-mixed');
trace.data.spans.push(new NoOpSpan());
popTraceSpy.mockReturnValue(trace);
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
const record = submit.mock.calls[0][0];
expect(typeof record.otlpSpans).toBe('string');
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
expect(decoded[0]).toBe(0x0a);
expect(decoded.includes(Buffer.from('tool-call'))).toBe(true);
});
it('warns and submits without otlpSpans when OTLP serialization throws', async () => {
const serializeSpy = jest
.spyOn(ProtobufTraceSerializer, 'serializeRequest')
.mockImplementation(() => {
throw new Error('boom');
});
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-serialize-fail'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
try {
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
// The record is still submitted (artifact fallback), just without spans.
expect(submit).toHaveBeenCalledTimes(1);
expect(submit.mock.calls[0][0].otlpSpans).toBeUndefined();
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('Failed to serialize spans to OTLP protobuf'),
expect.any(Error),
);
} finally {
serializeSpy.mockRestore();
warnSpy.mockRestore();
}
});
it('drops otlpSpans but still submits when the record would exceed the IPC size budget', async () => {
const sizeSpy = jest
.spyOn(ipc, 'ipcRequestByteLength')
.mockReturnValue(ipc.MAX_REQUEST_BYTES + 1);
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-too-big'));
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
try {
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
// The trace is not dropped: it is still submitted, just without the OTLP
// blob, so the artifact fallback (traceData) keeps it alive.
expect(submit).toHaveBeenCalledTimes(1);
const record = submit.mock.calls[0][0];
expect(record.otlpSpans).toBeUndefined();
expect((record.traceData as { spans: unknown[] }).spans).toHaveLength(1);
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('exceeding the'));
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('dropping OTLP spans'));
} finally {
sizeSpy.mockRestore();
warnSpy.mockRestore();
}
});
it('serializes decoded attribute values (no double JSON-encoding)', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
popTraceSpy.mockReturnValue(
makeTraceWithSpans('tr-decode', {
[SpanAttributeKey.SPAN_TYPE]: JSON.stringify('TOOL'),
tool_name: JSON.stringify('WebSearch'),
}),
);
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
const record = submit.mock.calls[0][0];
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
expectOtlpAttributeValue(decoded, SpanAttributeKey.SPAN_TYPE, 'TOOL');
expectOtlpAttributeValue(decoded, 'tool_name', 'WebSearch');
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.SPAN_TYPE, '"TOOL"');
expectOtlpAttributeValueAbsent(decoded, 'tool_name', '"WebSearch"');
});
it('keeps JSON object attributes as single-encoded strings in OTLP output', async () => {
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
const exporter = new MlflowWalSpanExporter({ submit });
const inputsJson = JSON.stringify({ prompt: 'hello' });
const outputsJson = JSON.stringify({ response: 'world' });
popTraceSpy.mockReturnValue(
makeTraceWithSpans('tr-obj-attrs', {
[SpanAttributeKey.SPAN_TYPE]: JSON.stringify('TOOL'),
[SpanAttributeKey.INPUTS]: inputsJson,
[SpanAttributeKey.OUTPUTS]: outputsJson,
}),
);
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
const { callback, promise } = captureExportResult();
exporter.export([span], callback);
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
await exporter.forceFlush();
const record = submit.mock.calls[0][0];
expect(record.otlpSpans).toBeDefined();
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
expectOtlpAttributeValue(decoded, SpanAttributeKey.INPUTS, inputsJson);
expectOtlpAttributeValue(decoded, SpanAttributeKey.OUTPUTS, outputsJson);
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.INPUTS, JSON.stringify(inputsJson));
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.OUTPUTS, JSON.stringify(outputsJson));
});
});
@@ -0,0 +1,430 @@
import { mkdtemp, readFile, rm } from 'fs/promises';
import { createServer, Server } from 'net';
import { tmpdir } from 'os';
import { join } from 'path';
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
import {
createIpcConnectionHandler,
ipcRequestByteLength,
MAX_REQUEST_BYTES,
submitRecord,
} from '../../../src/exporters/wal/ipc';
import { getLockSocketPath, getWalPath } from '../../../src/exporters/wal/paths';
import type { IpcRequest, IpcResponse } from '../../../src/exporters/wal/protocol';
import type { WalRecord } from '../../../src/exporters/wal/types';
import { JSONBig } from '../../../src/core/utils/json';
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
return {
id: `wal-${idSuffix}`,
trackingUri: 'http://localhost:5000',
experimentId: '0',
traceInfo: { trace_id: `t-${idSuffix}` },
traceData: { spans: [] },
attempts: 0,
nextAttemptAt: 0,
createdAt: Date.now(),
...overrides,
};
}
async function closeServer(server: Server): Promise<void> {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
describe('wal/ipc submitRecord', () => {
let walDir: string;
const originalWalDir = process.env.MLFLOW_WAL_DIR;
const originalDaemonExe = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-ipc-'));
process.env.MLFLOW_WAL_DIR = walDir;
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = join(walDir, 'noop-daemon.cjs');
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
if (originalDaemonExe === undefined) {
delete process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
} else {
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = originalDaemonExe;
}
await rm(walDir, { recursive: true, force: true });
});
it('round-trips an append request: request → ok ACK', async () => {
const requestChunks: Buffer[] = [];
const server = createServer((socket) => {
let responded = false;
socket.on('data', (chunk: Buffer) => {
requestChunks.push(chunk);
if (responded) {
return;
}
const buffered = Buffer.concat(requestChunks).toString('utf8');
if (!buffered.includes('\n')) {
return;
}
responded = true;
const response: IpcResponse = { ok: true };
socket.end(JSON.stringify(response) + '\n');
});
});
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
try {
const record = makeRecord('a');
await expect(submitRecord(record)).resolves.toBeUndefined();
const request = Buffer.concat(requestChunks).toString('utf8').trim();
const parsed = JSON.parse(request) as IpcRequest;
expect(parsed.op).toBe('append');
expect(parsed.record.id).toBe(record.id);
} finally {
await closeServer(server);
}
});
it('throws when the daemon responds with an error', async () => {
const server = createServer((socket) => {
socket.on('data', () => {
const response: IpcResponse = { ok: false, error: 'disk full' };
socket.end(JSON.stringify(response) + '\n');
});
});
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
try {
await expect(submitRecord(makeRecord('b'))).rejects.toThrow(/disk full/);
} finally {
await closeServer(server);
}
});
it('throws after exhausting retries when no daemon is listening', async () => {
// No server bound; submitRecord should retry, fail to reach
// anyone, and surface a connect error to the caller. The full
// CONNECT_RETRY_DELAYS_MS sleeps (~3.15s) still elapse - we
// don't currently inject those in tests - but pointing
// MLFLOW_WAL_DIR at a tmpdir guarantees each connect attempt
// fails fast with ENOENT instead of stacking on top of the
// (much longer) per-attempt kernel connect timeout that would
// fire against an unreachable bound socket. The 10s test
// timeout accommodates the configured backoff with margin; if
// retry latency ever becomes a CI bottleneck the fix is to make
// CONNECT_RETRY_DELAYS_MS injectable rather than to chase it
// here.
const r = makeRecord('c');
await expect(submitRecord(r)).rejects.toMatchObject({
// The contract we care about is that retry exhaustion surfaces
// a transport-level error code to the caller. The exact value
// varies by platform (POSIX UNIX-domain sockets vs. Windows
// Named Pipes) and by Node/libuv version, so pinning to
// ECONNREFUSED|ENOENT produced unnecessary Windows-CI
// fragility; matching any non-empty errno-style string is what
// this assertion actually wants to express.
code: expect.stringMatching(/^\S+$/),
});
}, 10_000);
});
describe('wal/ipc createIpcConnectionHandler', () => {
let walDir: string;
let server: Server | null = null;
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-ipc-handler-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (server) {
await closeServer(server);
server = null;
}
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
async function startHandlerServer(writer: BatchingWriter): Promise<Server> {
const s = createServer(createIpcConnectionHandler(writer));
await new Promise<void>((resolve) => s.listen(getLockSocketPath(), resolve));
return s;
}
it('persists the submitted record and acks with ok=true', async () => {
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
const record = makeRecord('h1');
await submitRecord(record);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(1);
const parsed = JSON.parse(lines[0]) as { type: string; record: WalRecord };
expect(parsed.type).toBe('append');
expect(parsed.record.id).toBe(record.id);
});
it('responds with ok=false for an unknown op', async () => {
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
const { createConnection } = await import('node:net');
const result = await new Promise<string>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const chunks: Buffer[] = [];
socket.on('connect', () => {
socket.write(JSON.stringify({ op: 'frobnicate', record: makeRecord('x') }) + '\n');
});
socket.on('data', (chunk) => chunks.push(chunk));
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
socket.on('error', reject);
});
const parsed = JSON.parse(result) as IpcResponse;
expect(parsed.ok).toBe(false);
if (!parsed.ok) {
expect(parsed.error).toMatch(/unknown op/);
}
});
it('rejects requests whose record is missing or has no string id', async () => {
// Exercises the structural check in dispatch() that stops bad
// payloads from being durably persisted to queue.log and then
// poisoning every subsequent batch loop iteration that tries to
// upload them. Walks each rejection branch (non-object, null,
// missing id, non-string id, empty string id) through one IPC
// round-trip per case so the WAL is never written.
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
const { createConnection } = await import('node:net');
const send = (record: unknown): Promise<IpcResponse> =>
new Promise<IpcResponse>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const chunks: Buffer[] = [];
socket.on('connect', () => {
socket.write(JSON.stringify({ op: 'append', record }) + '\n');
});
socket.on('data', (chunk) => chunks.push(chunk));
socket.on('end', () => {
try {
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8').trim()) as IpcResponse);
} catch (err) {
reject(err as Error);
}
});
socket.on('error', reject);
});
const badCases: unknown[] = [null, undefined, 42, 'not an object', {}, { id: 123 }, { id: '' }];
for (const bad of badCases) {
const response = await send(bad);
expect(response.ok).toBe(false);
if (!response.ok) {
expect(response.error).toMatch(/invalid record/);
}
}
// queue.log must never have been written: no successful submits
// and no partial lines from rejected ones.
await expect(readFile(getWalPath(), 'utf8')).rejects.toMatchObject({ code: 'ENOENT' });
});
it('rejects oversized requests without exhausting memory', async () => {
// Stream bytes well past the cap without ever sending a newline. The
// handler should respond with ok=false, pause its read side so the kernel
// buffer stops growing, and half-close the write side so we observe a clean
// `'end'` here.
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
// One MiB per chunk; send just enough chunks to overshoot the cap by 1 MiB,
// derived from MAX_REQUEST_BYTES so the test tracks the constant.
const oneMibBytes = 1024 * 1024;
const chunkCount = Math.ceil(MAX_REQUEST_BYTES / oneMibBytes) + 1;
const { createConnection } = await import('node:net');
const result = await new Promise<string>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const chunks: Buffer[] = [];
socket.on('connect', () => {
// No trailing newline. We send chunks back-to-back; once the daemon's
// cumulative buffer crosses the cap it must short-circuit and respond,
// after which it ignores any further bytes.
const oneMib = Buffer.alloc(oneMibBytes, 0x41);
const pump = (remaining: number): void => {
if (remaining === 0 || socket.destroyed) {
return;
}
if (!socket.write(oneMib)) {
socket.once('drain', () => pump(remaining - 1));
return;
}
setImmediate(() => pump(remaining - 1));
};
pump(chunkCount);
});
socket.on('data', (chunk) => chunks.push(chunk));
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
socket.on('error', (err: NodeJS.ErrnoException) => {
// EPIPE can surface from the pump's writes once the daemon
// pauses reading and our kernel send buffer eventually fills.
// That's expected; the contract we care about is that the
// daemon replied before that backpressure hit.
if (err.code === 'EPIPE') {
return;
}
reject(err);
});
});
const parsed = JSON.parse(result) as IpcResponse;
expect(parsed.ok).toBe(false);
if (!parsed.ok) {
expect(parsed.error).toMatch(/exceeds max size/);
}
}, 15_000);
it('measures the cap in wire bytes, not UTF-16 code units', async () => {
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
// Each '中' is 3 UTF-8 bytes but a single UTF-16 code unit, so a cap
// measured in code units would count this chunk at a third of its wire
// size. Send enough chunks to overshoot the (byte-measured) cap; a
// code-unit measurement would stay well under it and never trip.
const cjkChunk = Buffer.from('中'.repeat(349_525), 'utf8');
const chunkCount = Math.ceil(MAX_REQUEST_BYTES / cjkChunk.length) + 1;
const { createConnection } = await import('node:net');
const result = await new Promise<string>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const chunks: Buffer[] = [];
socket.on('connect', () => {
const pump = (remaining: number): void => {
if (remaining === 0 || socket.destroyed) {
return;
}
if (!socket.write(cjkChunk)) {
socket.once('drain', () => pump(remaining - 1));
return;
}
setImmediate(() => pump(remaining - 1));
};
pump(chunkCount);
});
socket.on('data', (chunk) => chunks.push(chunk));
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
socket.on('error', (err: NodeJS.ErrnoException) => {
if (err.code === 'EPIPE') {
return;
}
reject(err);
});
});
const parsed = JSON.parse(result) as IpcResponse;
expect(parsed.ok).toBe(false);
if (!parsed.ok) {
expect(parsed.error).toMatch(/exceeds max size/);
}
}, 15_000);
it('handles a multi-byte UTF-8 codepoint split across chunk boundaries', async () => {
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
const record = makeRecord('with-中');
const payload = Buffer.from(JSON.stringify({ op: 'append', record }) + '\n', 'utf8');
// First high-bit byte in the payload is the `0xE4` of `中` (the
// rest of the JSON is ASCII), so split immediately after it to
// straddle the codepoint.
const cjkStart = payload.indexOf(0xe4);
expect(cjkStart).toBeGreaterThan(0);
const chunkA = payload.subarray(0, cjkStart + 1);
const chunkB = payload.subarray(cjkStart + 1);
const { createConnection } = await import('node:net');
const response = await new Promise<IpcResponse>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
const respChunks: Buffer[] = [];
socket.on('connect', () => {
socket.write(chunkA, () => {
// setTimeout (rather than setImmediate) gives the kernel a
// tick to deliver chunk A as its own `'data'` event before
// chunk B arrives, otherwise the writes may coalesce and
// the test won't actually exercise the split-codepoint
// path it's meant to guard.
setTimeout(() => socket.write(chunkB), 10);
});
});
socket.on('data', (chunk) => respChunks.push(chunk));
socket.on('end', () => {
try {
resolve(JSON.parse(Buffer.concat(respChunks).toString('utf8').trim()) as IpcResponse);
} catch (err) {
reject(err as Error);
}
});
socket.on('error', reject);
});
expect(response.ok).toBe(true);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(1);
const persisted = JSON.parse(lines[0]) as { record: WalRecord };
expect(persisted.record.id).toBe('wal-with-中');
});
it('tolerates probe connections that close without sending data', async () => {
const writer = new BatchingWriter();
server = await startHandlerServer(writer);
const { createConnection } = await import('node:net');
await new Promise<void>((resolve, reject) => {
const socket = createConnection(getLockSocketPath());
socket.on('connect', () => {
socket.destroy();
resolve();
});
socket.on('error', reject);
});
// A subsequent real submit must still succeed: the probe-and-tear
// pattern must not have left the handler in a bad state.
await submitRecord(makeRecord('after-probe'));
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(1);
});
});
describe('wal/ipc ipcRequestByteLength', () => {
it('measures the real wire size of a record, including BigInt fields', () => {
const record = makeRecord('measure', {
traceData: { spans: [{ startTimeUnixNano: 1_700_000_000_000_000_000n }] },
});
const expected = Buffer.byteLength(JSONBig.stringify({ op: 'append', record }) + '\n', 'utf8');
expect(ipcRequestByteLength(record)).toBe(expected);
});
it('returns a value above the cap for a genuinely oversized otlpSpans payload', () => {
// A baseline record sits comfortably under the cap...
expect(ipcRequestByteLength(makeRecord('small'))).toBeLessThan(MAX_REQUEST_BYTES);
// ...while a record carrying an over-cap otlpSpans blob measures over it.
const oversized = makeRecord('oversized', {
otlpSpans: 'A'.repeat(MAX_REQUEST_BYTES + 1),
});
expect(ipcRequestByteLength(oversized)).toBeGreaterThan(MAX_REQUEST_BYTES);
});
});
@@ -0,0 +1,176 @@
import { mkdtemp, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join, resolve } from 'path';
import {
getDaemonLogPath,
getDeadLetterPath,
getLockSocketPath,
getWalDir,
getWalPath,
} from '../../../src/exporters/wal/paths';
describe('wal/paths', () => {
const originalEnv = process.env.MLFLOW_WAL_DIR;
afterEach(() => {
if (originalEnv === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalEnv;
}
});
describe('getWalDir', () => {
it('honors MLFLOW_WAL_DIR and returns an absolute, resolved path', () => {
process.env.MLFLOW_WAL_DIR = '/tmp/foo/';
expect(getWalDir()).toBe(resolve('/tmp/foo'));
});
it('treats empty MLFLOW_WAL_DIR as unset', () => {
process.env.MLFLOW_WAL_DIR = '';
expect(getWalDir()).toMatch(/[/\\]\.mlflow[/\\]wal$/);
});
it('produces the same value for /tmp/foo and /tmp/foo/', () => {
process.env.MLFLOW_WAL_DIR = '/tmp/foo';
const a = getWalDir();
process.env.MLFLOW_WAL_DIR = '/tmp/foo/';
const b = getWalDir();
expect(a).toBe(b);
});
});
describe('getLockSocketPath', () => {
let scratchDir: string;
beforeEach(async () => {
scratchDir = await mkdtemp(join(tmpdir(), 'mlflow-paths-'));
process.env.MLFLOW_WAL_DIR = scratchDir;
});
afterEach(async () => {
await rm(scratchDir, { recursive: true, force: true });
});
const isWindows = process.platform === 'win32';
(isWindows ? it.skip : it)('POSIX: returns <wal_dir>/daemon.sock for short WAL dirs', () => {
const lock = getLockSocketPath();
expect(lock).toBe(join(scratchDir, 'daemon.sock'));
});
(isWindows ? it.skip : it)(
'POSIX: falls back to tmpdir-based path when wal_dir would exceed sun_path',
() => {
// Construct a long WAL dir that pushes <wal_dir>/daemon.sock past 103
// bytes. We do not need this directory to exist; getLockSocketPath
// only inspects path lengths.
const longWalDir = '/tmp/' + 'x'.repeat(150);
process.env.MLFLOW_WAL_DIR = longWalDir;
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
try {
const lock = getLockSocketPath();
expect(lock.startsWith(tmpdir())).toBe(true);
expect(lock).toMatch(/[/\\]mlflow-[A-Za-z0-9._-]+-[0-9a-f]{12}\.sock$/);
expect(Buffer.byteLength(lock, 'utf8')).toBeLessThanOrEqual(103);
expect(warnSpy).toHaveBeenCalledTimes(1);
expect(warnSpy.mock.calls[0]?.[0]).toContain('longer than 103 bytes');
} finally {
warnSpy.mockRestore();
}
},
);
(isWindows ? it.skip : it)(
'POSIX: fallback identifier is stable for the same wal_dir and differs for distinct wal_dirs',
() => {
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
try {
process.env.MLFLOW_WAL_DIR = '/tmp/' + 'a'.repeat(150);
const lockA1 = getLockSocketPath();
const lockA2 = getLockSocketPath();
process.env.MLFLOW_WAL_DIR = '/tmp/' + 'b'.repeat(150);
const lockB = getLockSocketPath();
expect(lockA1).toBe(lockA2);
expect(lockA1).not.toBe(lockB);
} finally {
warnSpy.mockRestore();
}
},
);
(isWindows ? it : it.skip)(
'Windows: returns a Named Pipe path scoped by user + wal_dir hash',
() => {
const lock = getLockSocketPath();
expect(lock).toMatch(/^\\\\\?\\pipe\\mlflow-trace-daemon-[A-Za-z0-9._-]+-[0-9a-f]{12}$/);
},
);
(isWindows ? it : it.skip)('Windows: pipe name differs for distinct wal_dirs', () => {
process.env.MLFLOW_WAL_DIR = 'C:\\tmp\\walA';
const lockA = getLockSocketPath();
process.env.MLFLOW_WAL_DIR = 'C:\\tmp\\walB';
const lockB = getLockSocketPath();
expect(lockA).not.toBe(lockB);
});
});
describe('getWalPath', () => {
it('lives inside the resolved WAL dir', () => {
process.env.MLFLOW_WAL_DIR = '/tmp/foo';
expect(getWalPath()).toBe(join(resolve('/tmp/foo'), 'queue.log'));
});
});
describe('daily rotation of failed.log / daemon.log', () => {
beforeEach(() => {
process.env.MLFLOW_WAL_DIR = '/tmp/wal-rotation';
});
it('suffixes failed.log with the UTC date of the supplied Date', () => {
const d = new Date('2026-05-20T12:34:56.000Z');
expect(getDeadLetterPath(d)).toBe(
join(resolve('/tmp/wal-rotation'), 'failed.log.2026-05-20'),
);
});
it('suffixes daemon.log with the UTC date of the supplied Date', () => {
const d = new Date('2026-05-20T12:34:56.000Z');
expect(getDaemonLogPath(d)).toBe(join(resolve('/tmp/wal-rotation'), 'daemon.log.2026-05-20'));
});
it('rolls failed.log to the next day across midnight UTC', () => {
const lastSecondOfDay = new Date('2026-05-20T23:59:59.999Z');
const firstInstantOfNextDay = new Date('2026-05-21T00:00:00.000Z');
expect(getDeadLetterPath(lastSecondOfDay)).toMatch(/failed\.log\.2026-05-20$/);
expect(getDeadLetterPath(firstInstantOfNextDay)).toMatch(/failed\.log\.2026-05-21$/);
});
it('rolls daemon.log to the next day across midnight UTC', () => {
const lastSecondOfDay = new Date('2026-05-20T23:59:59.999Z');
const firstInstantOfNextDay = new Date('2026-05-21T00:00:00.000Z');
expect(getDaemonLogPath(lastSecondOfDay)).toMatch(/daemon\.log\.2026-05-20$/);
expect(getDaemonLogPath(firstInstantOfNextDay)).toMatch(/daemon\.log\.2026-05-21$/);
});
it('uses UTC date, not local date, regardless of the process timezone', () => {
// 2026-05-20T18:00:00 UTC = 2026-05-21T02:00:00 in UTC+8 (e.g. Singapore).
// The suffix must reflect UTC, not the host TZ.
const d = new Date('2026-05-20T18:00:00.000Z');
expect(getDaemonLogPath(d)).toMatch(/daemon\.log\.2026-05-20$/);
expect(getDeadLetterPath(d)).toMatch(/failed\.log\.2026-05-20$/);
});
it('zero-pads month and day', () => {
const earlyDay = new Date('2026-01-03T12:00:00.000Z');
expect(getDaemonLogPath(earlyDay)).toMatch(/daemon\.log\.2026-01-03$/);
});
it('defaults to "now" when called without an argument', () => {
expect(getDaemonLogPath()).toMatch(/daemon\.log\.\d{4}-\d{2}-\d{2}$/);
expect(getDeadLetterPath()).toMatch(/failed\.log\.\d{4}-\d{2}-\d{2}$/);
});
});
});
@@ -0,0 +1,223 @@
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { readLockHolderPid, tryAcquirePidLock } from '../../../src/exporters/wal/pid_lock';
import { getPidLockPath } from '../../../src/exporters/wal/paths';
// The lock is a POSIX-only mechanism — Windows daemons rely on
// kernel-refcounted named pipes for cross-process exclusion, so the
// entire suite is a no-op there. Wrap with a describe variant that
// skips on Windows to keep the test report honest about coverage.
const describePosix = process.platform === 'win32' ? describe.skip : describe;
/**
* Spawn a trivial child, wait for it to exit, and return its now-freed
* pid. Used to obtain a pid that is guaranteed to fail
* `process.kill(pid, 0)` with ESRCH so the stale-recovery path can be
* exercised deterministically. The PID could in principle be recycled
* by the OS between the child exiting and our liveness check, but in a
* single-digit-millisecond test window the probability is negligible
* — and even if it happened, the test would falsely fail rather than
* silently pass, which is the safe failure mode.
*/
function spawnAndReapPid(): Promise<number> {
return new Promise<number>((resolve, reject) => {
const child = spawn(process.execPath, ['-e', 'process.exit(0)']);
const pid = child.pid;
if (pid == null) {
reject(new Error('child spawn produced no pid'));
return;
}
child.once('exit', () => resolve(pid));
child.once('error', reject);
});
}
describePosix('wal/pid_lock', () => {
let walDir: string;
// Mirror daemon.test.ts: capture the developer's pre-test value so
// running `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override
// for later tests in the same worker.
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-pidlock-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
describe('tryAcquirePidLock', () => {
it('claims the lock and writes our pid when no lockfile exists', async () => {
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await lock?.release();
}
});
it('returns null when an alive process already holds the lock', async () => {
// Seed the lockfile with our own pid — guaranteed alive — so the
// liveness probe inside tryAcquirePidLock has to detect us and
// concede rather than steal the lock from ourselves.
await writeFile(getPidLockPath(), `${process.pid}\n`);
const lock = await tryAcquirePidLock();
expect(lock).toBeNull();
// Lockfile must remain untouched after a conceding acquisition.
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
});
it('recovers from a stale lockfile whose pid is dead', async () => {
const deadPid = await spawnAndReapPid();
await writeFile(getPidLockPath(), `${deadPid}\n`);
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await lock?.release();
}
});
it('recovers from a malformed lockfile (non-numeric content)', async () => {
await writeFile(getPidLockPath(), 'this is not a pid\n');
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await lock?.release();
}
});
it('recovers from an empty lockfile', async () => {
await writeFile(getPidLockPath(), '');
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
} finally {
await lock?.release();
}
});
it('rejects a lockfile that records pid 0 (process-group signal target)', async () => {
// pid 0 would make process.kill(0, 0) signal our entire process
// group; parsePid must screen it out before the liveness probe
// ever runs. Recovery should treat it as malformed and reclaim.
await writeFile(getPidLockPath(), '0\n');
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe(`${process.pid}\n`);
} finally {
await lock?.release();
}
});
it('rejects a lockfile that records a negative pid (process-group signal target)', async () => {
await writeFile(getPidLockPath(), '-12345\n');
const lock = await tryAcquirePidLock();
try {
expect(lock).not.toBeNull();
} finally {
await lock?.release();
}
});
it('serializes concurrent acquirers in the same event loop to a single winner', async () => {
// Both acquirers run in the same Node process so they see the
// same `process.pid`. The link() atomicity is what arbitrates;
// the loser observes EEXIST, reads our own (alive) pid, and
// concedes via the live-holder branch.
const [first, second] = await Promise.all([tryAcquirePidLock(), tryAcquirePidLock()]);
try {
const acquired = [first, second].filter((l) => l != null);
const conceded = [first, second].filter((l) => l == null);
expect(acquired).toHaveLength(1);
expect(conceded).toHaveLength(1);
} finally {
await first?.release();
await second?.release();
}
});
it('cleans up the temp file on a successful acquisition', async () => {
const lock = await tryAcquirePidLock();
try {
// No `daemon.pid.<pid>.<uuid>.tmp` should survive a successful
// acquire — the implementation unlinks it after the link()
// succeeds so the WAL dir doesn't accumulate per-spawn garbage.
const { readdir } = await import('fs/promises');
const entries = await readdir(walDir);
const tmps = entries.filter((e) => e.startsWith('daemon.pid.') && e.endsWith('.tmp'));
expect(tmps).toEqual([]);
} finally {
await lock?.release();
}
});
});
describe('PidLock.release', () => {
it('unlinks the lockfile when content still matches our pid', async () => {
const lock = await tryAcquirePidLock();
expect(lock).not.toBeNull();
await lock!.release();
expect(existsSync(getPidLockPath())).toBe(false);
});
it('leaves the lockfile intact when content has been hijacked', async () => {
const lock = await tryAcquirePidLock();
expect(lock).not.toBeNull();
// Simulate an external rewrite (operator intervention, or a
// successor that erroneously stole the path while we were still
// running). release() must not blind-unlink the file in this
// case — that would erase the successor's claim.
await writeFile(getPidLockPath(), '99999\n');
await lock!.release();
expect(existsSync(getPidLockPath())).toBe(true);
const content = await readFile(getPidLockPath(), 'utf8');
expect(content).toBe('99999\n');
});
it('is idempotent', async () => {
const lock = await tryAcquirePidLock();
expect(lock).not.toBeNull();
await lock!.release();
// Second release: lockfile already gone, must be a no-op.
await expect(lock!.release()).resolves.toBeUndefined();
expect(existsSync(getPidLockPath())).toBe(false);
});
});
describe('readLockHolderPid', () => {
it('returns the recorded pid when a lockfile exists', async () => {
await writeFile(getPidLockPath(), '4242\n');
await expect(readLockHolderPid()).resolves.toBe(4242);
});
it('returns null when no lockfile exists', async () => {
await expect(readLockHolderPid()).resolves.toBeNull();
});
it('returns null when the lockfile is malformed', async () => {
await writeFile(getPidLockPath(), 'garbage\n');
await expect(readLockHolderPid()).resolves.toBeNull();
});
});
});
@@ -0,0 +1,249 @@
// Mock node:fs/promises with a passthrough so individual tests can flip a
// single method (e.g. `rename`) to fail/inject without breaking the
// dozens of real fs operations the rest of the suite needs. Hoisted by
// ts-jest above the storage.ts import so storage's
// `import { ... } from 'node:fs/promises'` resolves to the mocked bindings.
jest.mock('node:fs/promises', () => {
const actual = jest.requireActual<typeof import('node:fs/promises')>('node:fs/promises');
return {
...actual,
rename: jest.fn(actual.rename),
};
});
import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import * as fsPromises from 'node:fs/promises';
import {
appendDeadLetter,
appendRecord,
appendTombstone,
compact,
readPending,
walSize,
} from '../../../src/exporters/wal/storage';
import { getDeadLetterPath, getWalPath } from '../../../src/exporters/wal/paths';
import type { WalRecord } from '../../../src/exporters/wal/types';
const renameMock = fsPromises.rename as jest.MockedFunction<typeof fsPromises.rename>;
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
return {
id: `wal-${idSuffix}`,
trackingUri: 'http://localhost:5000',
experimentId: '0',
traceInfo: { trace_id: `t-${idSuffix}` },
traceData: { spans: [] },
attempts: 0,
nextAttemptAt: 0,
createdAt: Date.now(),
...overrides,
};
}
describe('wal/storage', () => {
let walDir: string;
const originalWalDir = process.env.MLFLOW_WAL_DIR;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
await rm(walDir, { recursive: true, force: true });
});
it('returns an empty array when no WAL file exists yet', async () => {
expect(await readPending()).toEqual([]);
expect(await walSize()).toBe(0);
});
it('appends a record and reads it back', async () => {
const record = makeRecord('a');
await appendRecord(record);
const pending = await readPending();
expect(pending).toHaveLength(1);
expect(pending[0]).toEqual(record);
expect(await walSize()).toBeGreaterThan(0);
});
it('round-trips bigint fields (regression for SerializedSpan timestamps)', async () => {
const startNs = 1_750_000_000_123_456_789n;
const record = makeRecord('bigint', {
traceData: {
spans: [
{
trace_id: 't-bigint',
start_time_unix_nano: startNs,
end_time_unix_nano: startNs + 5_000_000n,
},
],
},
});
await appendRecord(record);
const pending = await readPending();
expect(pending).toHaveLength(1);
const restored = (pending[0].traceData as { spans: Array<Record<string, unknown>> }).spans[0];
expect(restored.start_time_unix_nano).toBe(startNs);
expect(restored.end_time_unix_nano).toBe(startNs + 5_000_000n);
});
it('hides a record once it has been tombstoned', async () => {
const r1 = makeRecord('a');
const r2 = makeRecord('b');
await appendRecord(r1);
await appendRecord(r2);
await appendTombstone(r1.id);
const pending = await readPending();
expect(pending.map((r) => r.id)).toEqual(['wal-b']);
});
it('treats a tombstone before its corresponding append as a no-op', async () => {
// Records use fresh ids per attempt, so a tombstone never legitimately
// precedes its append; if it does (e.g. corrupt log), the later append
// must still win.
const r = makeRecord('a');
await appendTombstone(r.id);
await appendRecord(r);
const pending = await readPending();
expect(pending).toHaveLength(1);
expect(pending[0]?.id).toBe(r.id);
});
it('accumulates entries in the dead-letter file without touching queue.log', async () => {
const live = makeRecord('live');
await appendRecord(live);
const dead1 = makeRecord('dead-1', { attempts: 10 });
const dead2 = makeRecord('dead-2', { attempts: 10 });
await appendDeadLetter(dead1);
await appendDeadLetter(dead2);
const dlqPath = getDeadLetterPath();
// failed.log is daily-rotated; the resolved path must include a
// YYYY-MM-DD suffix so a regression to a single unbounded file is
// caught in CI.
expect(dlqPath).toMatch(/[/\\]failed\.log\.\d{4}-\d{2}-\d{2}$/);
const dlqContents = await readFile(dlqPath, 'utf8');
const dlqLines = dlqContents.split('\n').filter((l) => l.length > 0);
expect(dlqLines).toHaveLength(2);
expect(JSON.parse(dlqLines[0])).toEqual({ type: 'append', record: dead1 });
expect(JSON.parse(dlqLines[1])).toEqual({ type: 'append', record: dead2 });
const pending = await readPending();
expect(pending).toEqual([live]);
});
it('preserves live records and shrinks the file when compacting', async () => {
const r1 = makeRecord('a');
const r2 = makeRecord('b');
const r3 = makeRecord('c');
await appendRecord(r1);
await appendRecord(r2);
await appendRecord(r3);
await appendTombstone(r2.id);
const sizeBefore = await walSize();
await compact();
const sizeAfter = await walSize();
expect(sizeAfter).toBeLessThan(sizeBefore);
const pending = await readPending();
expect(pending.map((r) => r.id).sort()).toEqual(['wal-a', 'wal-c']);
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(2);
for (const line of lines) {
expect(JSON.parse(line).type).toBe('append');
}
});
it('is a no-op when compacting a non-existent WAL', async () => {
await expect(compact()).resolves.toBeUndefined();
await expect(stat(getWalPath())).rejects.toMatchObject({ code: 'ENOENT' });
});
it('cleans up the tmp file when rename throws after a successful write', async () => {
// Regression: an earlier shape had `close()` and `rename()` outside
// the try/catch, so a flaky FS at the rename step would orphan one
// `queue.log.tmp.<pid>` per failed compaction. The fix moved both
// into the try so the catch's `unlink(tmpPath)` always runs on
// failure regardless of which step blew up.
await appendRecord(makeRecord('a'));
renameMock.mockRejectedValueOnce(
Object.assign(new Error('EBUSY: simulated rename failure'), { code: 'EBUSY' }),
);
await expect(compact()).rejects.toThrow(/simulated rename failure/);
// After the failed compaction, the WAL itself is unchanged and no
// `.tmp.<pid>` orphan remains in the spool dir.
const entries = await readdir(walDir);
expect(entries.some((e) => /\.tmp\.\d+$/.test(e))).toBe(false);
const pending = await readPending();
expect(pending.map((r) => r.id)).toEqual(['wal-a']);
});
it('skips malformed lines but keeps surrounding records', async () => {
const r1 = makeRecord('a');
const r2 = makeRecord('b');
await appendRecord(r1);
await writeFile(getWalPath(), '{not valid json}\n', { flag: 'a' });
await appendRecord(r2);
const debugSpy = jest.spyOn(console, 'debug').mockImplementation(() => {});
try {
const pending = await readPending();
expect(pending.map((r) => r.id).sort()).toEqual(['wal-a', 'wal-b']);
expect(debugSpy).toHaveBeenCalled();
} finally {
debugSpy.mockRestore();
}
});
it('survives 50 concurrent appends with no torn or lost lines', async () => {
const records = Array.from({ length: 50 }, (_, i) => makeRecord(i.toString().padStart(2, '0')));
await Promise.all(records.map((r) => appendRecord(r)));
const contents = await readFile(getWalPath(), 'utf8');
const lines = contents.split('\n').filter((l) => l.length > 0);
expect(lines).toHaveLength(50);
for (const line of lines) {
expect(() => {
JSON.parse(line);
}).not.toThrow();
}
const pending = await readPending();
expect(pending).toHaveLength(50);
const ids = new Set(pending.map((r) => r.id));
for (const r of records) {
expect(ids.has(r.id)).toBe(true);
}
});
it('records nonzero walSize after an append', async () => {
expect(await walSize()).toBe(0);
await appendRecord(makeRecord('a'));
const size = await walSize();
expect(size).toBeGreaterThan(0);
const fsSize = (await stat(getWalPath())).size;
expect(size).toBe(fsSize);
});
});
@@ -0,0 +1,294 @@
import * as childProcess from 'node:child_process';
import { existsSync } from 'fs';
import { mkdtemp, readFile, rm, unlink, writeFile } from 'fs/promises';
import { createServer, Server } from 'net';
import { tmpdir } from 'os';
import { join } from 'path';
import {
ensureDaemon,
isDaemonAlive,
resolveDaemonEntry,
} from '../../../src/exporters/wal/supervisor';
import { getLockSocketPath } from '../../../src/exporters/wal/paths';
jest.mock('node:child_process', () => {
const actual = jest.requireActual<typeof import('node:child_process')>('node:child_process');
return {
...actual,
spawn: jest.fn((...args: Parameters<typeof actual.spawn>) => actual.spawn(...args)),
};
});
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function waitFor(
predicate: () => boolean | Promise<boolean>,
timeoutMs = 2000,
): Promise<void> {
const start = Date.now();
for (;;) {
if (await predicate()) {
return;
}
if (Date.now() - start > timeoutMs) {
throw new Error(`waitFor: predicate did not become true within ${timeoutMs}ms`);
}
await sleep(20);
}
}
describe('wal/supervisor', () => {
let walDir: string;
const originalWalDir = process.env.MLFLOW_WAL_DIR;
const originalDaemonExecutable = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
beforeEach(async () => {
walDir = await mkdtemp(join(tmpdir(), 'mlflow-supervisor-'));
process.env.MLFLOW_WAL_DIR = walDir;
});
afterEach(async () => {
if (originalWalDir === undefined) {
delete process.env.MLFLOW_WAL_DIR;
} else {
process.env.MLFLOW_WAL_DIR = originalWalDir;
}
if (originalDaemonExecutable === undefined) {
delete process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
} else {
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = originalDaemonExecutable;
}
await rm(walDir, { recursive: true, force: true });
});
describe('isDaemonAlive', () => {
it('returns false when no socket is bound', async () => {
expect(await isDaemonAlive()).toBe(false);
});
it('returns true when a server is bound to the lock socket', async () => {
const server: Server = createServer((s) => s.end());
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
try {
expect(await isDaemonAlive()).toBe(true);
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
});
describe('resolveDaemonEntry', () => {
it('returns the value of MLFLOW_TRACE_DAEMON_EXECUTABLE when set', () => {
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = '/custom/path/to/daemon.cjs';
expect(resolveDaemonEntry()).toBe('/custom/path/to/daemon.cjs');
});
it('treats an empty MLFLOW_TRACE_DAEMON_EXECUTABLE as unset', () => {
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = '';
// With the env var empty, the resolver falls through to package
// resolution. We can't predict the absolute path, but it must end
// in `bundle/daemon.cjs` per the documented contract.
const resolved = resolveDaemonEntry();
expect(resolved.endsWith(join('bundle', 'daemon.cjs'))).toBe(true);
});
});
describe('ensureDaemon', () => {
let stubScript: string;
let counterFile: string;
let pidFile: string;
beforeEach(async () => {
stubScript = join(walDir, 'stub-daemon.js');
counterFile = join(walDir, 'spawns.log');
pidFile = join(walDir, 'pids.log');
// Minimal CJS script: record that we spawned, bind the lock
// socket, and self-terminate after a short window so a failing
// test cannot leak indefinitely. fs writes happen *before* the
// listen() call so the counter / pid record is durable even if
// the parent has already moved on by the time we bind.
const lockPath = getLockSocketPath();
const stubSource = [
"const net = require('net');",
"const fs = require('fs');",
`fs.appendFileSync(${JSON.stringify(counterFile)}, '+');`,
`fs.appendFileSync(${JSON.stringify(pidFile)}, process.pid + '\\n');`,
'const server = net.createServer((s) => s.end());',
`server.listen(${JSON.stringify(lockPath)}, () => {`,
' setTimeout(() => process.exit(0), 3000);',
'});',
].join('\n');
await writeFile(stubScript, stubSource);
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = stubScript;
});
afterEach(async () => {
// Kill any stub daemons that haven't self-destructed yet so
// subsequent tests don't see stale processes / sockets.
if (existsSync(pidFile)) {
const pids = (await readFile(pidFile, 'utf8')).split('\n').filter((l) => l.length > 0);
for (const pid of pids) {
try {
process.kill(Number.parseInt(pid, 10), 'SIGKILL');
} catch {
// Already exited.
}
}
}
if (process.platform !== 'win32') {
try {
await unlink(getLockSocketPath());
} catch {
// ENOENT (no stale file from this test) or any other error —
// best-effort cleanup; the next test's beforeEach reseeds
// everything regardless.
}
}
});
it('spawns a daemon when none is alive', async () => {
await ensureDaemon();
await waitFor(() => existsSync(counterFile));
expect(await readFile(counterFile, 'utf8')).toBe('+');
});
it('does not spawn a second daemon when one is already alive', async () => {
await ensureDaemon();
await waitFor(() => existsSync(counterFile));
// Wait until the first stub has actually bound the socket; only
// after that will the second probe see "alive".
await waitFor(() => isDaemonAlive(), 3000);
await ensureDaemon();
// Give any (errant) second spawn time to write to the counter.
await sleep(200);
expect(await readFile(counterFile, 'utf8')).toBe('+');
});
it('swallows broken-entry failures and surfaces a diagnostic to stderr', async () => {
// Broken-install failures surface asynchronously, *after*
// `ensureDaemon` has already resolved (the spawn itself is
// fire-and-forget by design). The listener in `spawnDaemon` must:
// 1. let `ensureDaemon` resolve cleanly (no upstream crash);
// 2. log a breadcrumb mentioning the broken entry so a broken
// install (missing `bundle/daemon.cjs`, bad
// `MLFLOW_TRACE_DAEMON_EXECUTABLE`) doesn't degrade into
// ~3.85 s of opaque ECONNREFUSED retries in the WAL exporter
// with zero indication of cause.
//
// Two child events can carry the diagnostic and the listener
// hooks both: `'error'` fires when libuv can't start the binary
// at all (we can't easily trigger this here because `spawnDaemon`
// runs `spawn(process.execPath, [entry], ...)` and the node
// binary always exists), and `'exit'` fires with a non-zero code
// when the binary starts but the entry script fails to load —
// which is what pointing `MLFLOW_TRACE_DAEMON_EXECUTABLE` at a
// nonexistent `.js` file actually triggers. The test accepts
// either: both equally serve the reviewer-visible "broken
// install" purpose.
const missing = join(walDir, 'does-not-exist.js');
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = missing;
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
try {
await expect(ensureDaemon()).resolves.toBeUndefined();
// Generous timeout: node cold-start + script-resolution failure
// + exit can comfortably exceed 1 s on a busy CI box.
await waitFor(
() =>
errSpy.mock.calls.some(
([msg]) =>
typeof msg === 'string' &&
msg.includes(missing) &&
/daemon (spawn failed|exited unexpectedly)/.test(msg),
),
5000,
);
const match = errSpy.mock.calls.find(
([msg]) =>
typeof msg === 'string' &&
msg.includes(missing) &&
/daemon (spawn failed|exited unexpectedly)/.test(msg),
);
expect(match).toBeDefined();
expect(match![0]).toContain(missing);
} finally {
errSpy.mockRestore();
}
});
it('catches synchronous spawn failures via the outer ensureDaemon catch', async () => {
const spawnMock = childProcess.spawn as jest.MockedFunction<typeof childProcess.spawn>;
spawnMock.mockImplementationOnce(() => {
throw new Error('synthetic synchronous spawn failure');
});
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
try {
// The outer catch must convert the synchronous throw into a
// clean resolution + a diagnostic log line. Anything else
// (rejection, unhandled crash, no log) is a regression in the
// defense-in-depth path.
await expect(ensureDaemon()).resolves.toBeUndefined();
expect(spawnMock).toHaveBeenCalled();
expect(
errSpy.mock.calls.some(
([msg]) => typeof msg === 'string' && msg.includes('ensureDaemon failed'),
),
).toBe(true);
} finally {
errSpy.mockRestore();
}
});
it('coalesces concurrent calls into a single spawn', async () => {
// The realistic trigger is multiple `submitRecord` calls in one
// hook all hitting ECONNREFUSED during a daemon cold-start. Each
// would otherwise spawn its own child; the in-flight latch should
// dedupe them down to a single `spawn()` invocation.
const concurrent = 5;
await Promise.all(Array.from({ length: concurrent }, () => ensureDaemon()));
await waitFor(() => existsSync(counterFile));
// Give any (errant) extra spawns time to write to the counter.
await sleep(200);
const counterContents = await readFile(counterFile, 'utf8');
expect(counterContents).toBe('+');
});
it('dedupes a second ensureDaemon call during the spawn -> bind window', async () => {
const bindDelayMs = 300;
const lockPath = getLockSocketPath();
const slowBindStub = [
"const net = require('net');",
"const fs = require('fs');",
`fs.appendFileSync(${JSON.stringify(counterFile)}, '+');`,
`fs.appendFileSync(${JSON.stringify(pidFile)}, process.pid + '\\n');`,
'const server = net.createServer((s) => s.end());',
// Delay listen() to widen the spawn→bind window the fix is meant to cover.
`setTimeout(() => {`,
` server.listen(${JSON.stringify(lockPath)}, () => {`,
` setTimeout(() => process.exit(0), 3000);`,
` });`,
`}, ${bindDelayMs});`,
].join('\n');
await writeFile(stubScript, slowBindStub);
const spawnMock = childProcess.spawn as jest.MockedFunction<typeof childProcess.spawn>;
spawnMock.mockClear();
const first = ensureDaemon();
// Land the second call mid-bind-window: after spawnDaemon() has
// returned (~ms) but well before the stub's 300ms listen() fires.
await sleep(100);
const second = ensureDaemon();
await Promise.all([first, second]);
expect(await readFile(counterFile, 'utf8')).toBe('+');
expect(spawnMock).toHaveBeenCalledTimes(1);
});
});
});
+70
View File
@@ -0,0 +1,70 @@
import { LiveSpan } from '../src/core/entities/span';
import { SpanType } from '../src/core/constants';
/**
* Port and tracking URI for the local MLflow server used for testing.
* If the server is not running, jest.global-setup.ts will start it.
*/
export const TEST_PORT = 5000;
export const TEST_TRACKING_URI = `http://localhost:${TEST_PORT}`;
/**
* Mock OpenTelemetry span class for testing
*/
export class MockOtelSpan {
name: string;
attributes: Record<string, any>;
spanId: string;
traceId: string;
constructor(
name: string = 'test-span',
spanId: string = 'test-span-id',
traceId: string = 'test-trace-id',
) {
this.name = name;
this.spanId = spanId;
this.traceId = traceId;
this.attributes = {};
}
getAttribute(key: string): any {
return this.attributes[key];
}
setAttribute(key: string, value: any): void {
this.attributes[key] = value;
}
spanContext() {
return {
spanId: this.spanId,
traceId: this.traceId,
};
}
}
/**
* Create a mock OpenTelemetry span with the given parameters
*/
export function createMockOtelSpan(
name: string = 'test-span',
spanId: string = 'test-span-id',
traceId: string = 'test-trace-id',
): MockOtelSpan {
return new MockOtelSpan(name, spanId, traceId);
}
/**
* Create a test LiveSpan with mock OpenTelemetry span
*/
export function createTestSpan(
name: string = 'test-span',
traceId: string = 'test-trace-id',
spanId: string = 'test-span-id',
spanType: SpanType = SpanType.UNKNOWN,
): LiveSpan {
const mockOtelSpan = createMockOtelSpan(name, spanId, traceId);
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
return new LiveSpan(mockOtelSpan as any, traceId, spanType);
}
+16
View File
@@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "es2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"allowJs": true,
"checkJs": true,
"declaration": true,
"outDir": "./dist",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": ["src/**/*"]
}
@@ -0,0 +1,65 @@
# MLflow Typescript SDK - Anthropic
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with Anthropic to automatically trace your Claude API calls.
| Package | NPM | Description |
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
| [@mlflow/anthropic](./) | [![npm package](https://img.shields.io/npm/v/%40mlflow%2Fanthropic?style=flat-square)](https://www.npmjs.com/package/@mlflow/anthropic) | Auto-instrumentation integration for Anthropic. |
## Installation
```bash
npm install @mlflow/anthropic
```
The package includes the [`@mlflow/core`](https://github.com/mlflow/mlflow/tree/main/libs/typescript) package and `@anthropic-ai/sdk` package as peer dependencies. Depending on your package manager, you may need to install these two packages separately.
## Quickstart
Start MLflow Tracking Server if you don't have one already:
```bash
pip install mlflow
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
```
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
Instantiate MLflow SDK in your application:
```typescript
import * as mlflow from '@mlflow/core';
mlflow.init({
trackingUri: 'http://localhost:5000',
experimentId: '<experiment-id>',
});
```
Create a trace for Anthropic Claude:
```typescript
import Anthropic from '@anthropic-ai/sdk';
import { tracedAnthropic } from '@mlflow/anthropic';
const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
const client = tracedAnthropic(anthropic);
const response = await client.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello Claude' }],
});
```
View traces in MLflow UI:
![MLflow Tracing UI](https://github.com/mlflow/mlflow/blob/master/docs/static/images/llms/anthropic/anthropic-tracing.png?raw=True)
## Documentation 📘
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
## License
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
@@ -0,0 +1,16 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
roots: ['<rootDir>/tests', '<rootDir>/src'],
testMatch: ['**/*.test.ts'],
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
transform: {
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
},
globalSetup: '<rootDir>/../../jest.global-server-setup.ts',
globalTeardown: '<rootDir>/../../jest.global-server-teardown.ts',
testTimeout: 30000,
forceExit: true,
detectOpenHandles: true,
};
@@ -0,0 +1,54 @@
{
"name": "@mlflow/anthropic",
"version": "0.3.0",
"description": "Anthropic integration package for MLflow Tracing",
"repository": {
"type": "git",
"url": "https://github.com/mlflow/mlflow.git"
},
"homepage": "https://mlflow.org/",
"author": {
"name": "MLflow",
"url": "https://mlflow.org/"
},
"bugs": {
"url": "https://github.com/mlflow/mlflow/issues"
},
"license": "Apache-2.0",
"keywords": [
"mlflow",
"tracing",
"observability",
"opentelemetry",
"llm",
"anthropic",
"claude",
"javascript",
"typescript"
],
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"prepublishOnly": "npm run build",
"test": "jest",
"lint": "eslint . --ext .ts --max-warnings 0",
"lint:fix": "eslint . --ext .ts --fix",
"format": "prettier --write .",
"format:check": "prettier --check ."
},
"peerDependencies": {
"@mlflow/core": "^0.3.0",
"@anthropic-ai/sdk": "^0.71.0"
},
"devDependencies": {
"jest": "^29.6.2",
"typescript": "^5.8.3"
},
"engines": {
"node": ">=18"
},
"files": [
"dist/"
]
}
@@ -0,0 +1,324 @@
/**
* Main tracedAnthropic wrapper function for MLflow tracing integration
*/
import {
startSpan,
getCurrentActiveSpan,
SpanAttributeKey,
SpanType,
SpanStatusCode,
TokenUsage,
LiveSpan,
withSpan,
} from '@mlflow/core';
const SUPPORTED_MODULES = ['Messages'];
const SUPPORTED_METHODS = ['create', 'stream'];
/**
* Create a traced version of Anthropic client with MLflow tracing
* @param anthropicClient - The Anthropic client instance to trace
* @returns Traced Anthropic client with tracing capabilities
*/
export function tracedAnthropic<T = any>(anthropicClient: T): T {
const tracedClient = new Proxy(anthropicClient as any, {
get(target, prop, receiver) {
const original = Reflect.get(target, prop, receiver);
const moduleName = (target as object).constructor?.name;
if (typeof original === 'function') {
if (shouldTraceMethod(moduleName, String(prop))) {
const methodName = String(prop);
if (methodName === 'stream') {
// eslint-disable-next-line @typescript-eslint/ban-types
return wrapStreamWithTracing(original as Function, moduleName) as T;
}
// eslint-disable-next-line @typescript-eslint/ban-types
return wrapWithTracing(original as Function, moduleName) as T;
}
// eslint-disable-next-line @typescript-eslint/ban-types
return (original as Function).bind(target) as T;
}
if (
original &&
!Array.isArray(original) &&
!(original instanceof Date) &&
typeof original === 'object'
) {
return tracedAnthropic(original) as T;
}
return original as T;
},
});
return tracedClient as T;
}
function shouldTraceMethod(moduleName: string | undefined, methodName: string): boolean {
if (!moduleName) {
return false;
}
return SUPPORTED_MODULES.includes(moduleName) && SUPPORTED_METHODS.includes(methodName);
}
// eslint-disable-next-line @typescript-eslint/ban-types
function wrapWithTracing(fn: Function, moduleName: string): Function {
const spanType = getSpanType(moduleName);
const name = moduleName;
return function (this: any, ...args: any[]) {
if (!spanType) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return fn.apply(this, args);
}
// Skip tracing for create() calls with stream: true
// The Anthropic SDK's stream() method internally calls create() with stream: true,
// and we don't want to create a duplicate trace - the streaming wrapper handles it
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (args[0]?.stream === true) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return fn.apply(this, args);
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return withSpan(
async (span: LiveSpan) => {
span.setInputs(args[0]);
const result = await fn.apply(this, args);
span.setOutputs(result);
try {
const usage = extractTokenUsage(result);
if (usage) {
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
}
} catch (error) {
console.debug('Error extracting token usage', error);
}
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return result;
},
{ name, spanType },
);
};
}
function getSpanType(moduleName: string): SpanType | undefined {
switch (moduleName) {
case 'Messages':
return SpanType.LLM;
default:
return undefined;
}
}
// eslint-disable-next-line @typescript-eslint/ban-types
function wrapStreamWithTracing(fn: Function, moduleName: string): Function {
const spanType = getSpanType(moduleName);
const name = moduleName;
return function (this: any, ...args: any[]) {
if (!spanType) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return fn.apply(this, args);
}
// Create the stream (synchronous call)
const stream = fn.apply(this, args);
// Return a proxy that wraps the stream with tracing
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return wrapMessageStream(stream, args[0], name, spanType);
};
}
function wrapMessageStream(stream: any, inputs: any, name: string, spanType: SpanType): any {
// Use a flag that is set synchronously on first access to prevent duplicate spans.
// It is claimed either when the async iterator getter is invoked or when the wrapped
// finalMessage function is called, before any asynchronous work begins, so only one
// of these access paths will perform tracing for a given stream instance.
let tracingClaimed = false;
return new Proxy(stream, {
get(target, prop, receiver) {
const original = Reflect.get(target, prop, receiver);
// Wrap finalMessage() to add tracing
if (prop === 'finalMessage') {
return async function () {
if (tracingClaimed) {
// Already traced via async iteration, just return the message
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return await target.finalMessage();
}
tracingClaimed = true;
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return withSpan(
async (span: LiveSpan) => {
span.setInputs(inputs);
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const message = await target.finalMessage();
span.setOutputs(message);
try {
const usage = extractTokenUsage(message);
if (usage) {
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
}
} catch (error) {
console.debug('Error extracting token usage', error);
}
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return message;
},
{ name, spanType },
);
};
}
// Wrap async iterator for `for await (const event of stream)` pattern
if (prop === Symbol.asyncIterator) {
return function () {
if (tracingClaimed) {
// In practice, MessageStreams are typically consumed once and iterating again would
// yield no events, so this may not be a real issue but for completeness we return the
// unwrapped iterator in this case.
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call
return target[Symbol.asyncIterator]();
}
tracingClaimed = true;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument
return wrapAsyncIterator(target[Symbol.asyncIterator](), target, inputs, name, spanType);
};
}
if (typeof original === 'function') {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return original.bind(target);
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return original;
},
});
}
async function* wrapAsyncIterator(
iterator: AsyncIterator<any>,
stream: any,
inputs: any,
name: string,
spanType: SpanType,
): AsyncGenerator<any> {
// Use startSpan for manual lifecycle management since withSpan doesn't support async generators
const parentSpan = getCurrentActiveSpan();
const span = startSpan({ name, spanType, parent: parentSpan ?? undefined });
span.setInputs(inputs);
let iterationError: Error | undefined;
try {
while (true) {
const { value, done } = await iterator.next();
if (done) {
break;
}
yield value;
}
} catch (error) {
iterationError = error as Error;
throw error;
} finally {
if (iterationError) {
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
span.setStatus(SpanStatusCode.ERROR, iterationError.message);
span.end();
} else {
// After iteration completes (or early termination via break/return),
// get the final message for outputs and token usage
try {
// Prefer a proxy-aware finalMessage if available on the iterator, and fall back to the stream.
let finalMessage: unknown;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
const iteratorAny = iterator as any;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (iteratorAny && typeof iteratorAny.finalMessage === 'function') {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
finalMessage = await iteratorAny.finalMessage();
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
} else if (stream && typeof stream.finalMessage === 'function') {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
finalMessage = await stream.finalMessage();
}
if (finalMessage !== undefined) {
span.setOutputs(finalMessage);
const usage = extractTokenUsage(finalMessage);
if (usage) {
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
}
}
} catch (e) {
// Stream may have completed without finalMessage available
console.debug('Could not get final message from stream', e);
span.setAttribute('mlflow.tracing.token_usage_capture_failed', true);
span.setAttribute(
'mlflow.tracing.token_usage_capture_error',
e instanceof Error ? e.message : String(e),
);
}
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
span.end();
}
}
}
function extractTokenUsage(response: any): TokenUsage | undefined {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const usage = response?.usage;
if (!usage) {
return undefined;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const inputTokens = usage.input_tokens;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const outputTokens = usage.output_tokens;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const totalTokens = usage.total_tokens ?? (inputTokens ?? 0) + (outputTokens ?? 0);
const result: TokenUsage = {
input_tokens: inputTokens ?? 0,
output_tokens: outputTokens ?? 0,
total_tokens: totalTokens ?? 0,
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (usage.cache_read_input_tokens != null) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
result.cache_read_input_tokens = usage.cache_read_input_tokens;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (usage.cache_creation_input_tokens != null) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
result.cache_creation_input_tokens = usage.cache_creation_input_tokens;
}
return result;
}
@@ -0,0 +1,429 @@
/**
* Tests for MLflow Anthropic integration with MSW mock server
*/
import * as mlflow from '@mlflow/core';
import Anthropic from '@anthropic-ai/sdk';
import { tracedAnthropic } from '../src';
import { http, HttpResponse } from 'msw';
import { setupServer } from 'msw/node';
import { anthropicMockHandlers, createStreamingErrorHandler } from './mockAnthropicServer';
import { createAuthProvider } from '@mlflow/core/src/auth';
const TEST_TRACKING_URI = 'http://localhost:5000';
describe('tracedAnthropic', () => {
let experimentId: string;
let client: mlflow.MlflowClient;
let server: ReturnType<typeof setupServer>;
beforeAll(async () => {
server = setupServer(...anthropicMockHandlers);
server.listen();
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
client = new mlflow.MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
const experimentName = `anthropic-test-${Date.now()}-${Math.random().toString(36).slice(2)}`;
experimentId = await client.createExperiment(experimentName);
mlflow.init({
trackingUri: TEST_TRACKING_URI,
experimentId,
});
});
afterAll(async () => {
server.close();
await client.deleteExperiment(experimentId);
});
const getLastActiveTrace = async (): Promise<mlflow.Trace> => {
await mlflow.flushTraces();
const traceId = mlflow.getLastActiveTraceId();
const trace = await client.getTrace(traceId!);
return trace;
};
beforeEach(() => {
server.resetHandlers();
});
afterEach(() => {
jest.clearAllMocks();
});
it('should trace messages.create()', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const result = await wrappedAnthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello Claude!' }],
});
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
const tokenUsage = trace.info.tokenUsage;
expect(tokenUsage).toBeDefined();
expect(typeof tokenUsage?.input_tokens).toBe('number');
expect(typeof tokenUsage?.output_tokens).toBe('number');
expect(typeof tokenUsage?.total_tokens).toBe('number');
expect(tokenUsage?.cache_read_input_tokens).toBe(64);
expect(tokenUsage?.cache_creation_input_tokens).toBe(32);
const span = trace.data.spans[0];
expect(span.name).toBe('Messages');
expect(span.spanType).toBe(mlflow.SpanType.LLM);
expect(span.logLevel).toBe(mlflow.SpanLogLevel.INFO);
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
expect(span.inputs).toEqual({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello Claude!' }],
});
expect(span.outputs).toEqual(result);
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
expect(spanTokenUsage).toBeDefined();
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
expect(messageFormat).toBe('anthropic');
});
it('should handle Anthropic errors properly', async () => {
server.use(
http.post('https://api.anthropic.com/v1/messages', () =>
HttpResponse.json(
{
error: {
type: 'rate_limit',
message: 'Rate limit exceeded',
},
},
{ status: 429 },
),
),
);
// Disable retries to prevent the SDK from retrying on errors.
// SDK 0.50+ calls CancelReadableStream on error responses before retrying,
// which hangs indefinitely with MSW mock responses.
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
const wrappedAnthropic = tracedAnthropic(anthropic);
await expect(
wrappedAnthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 128,
messages: [{ role: 'user', content: 'This request should fail.' }],
}),
).rejects.toThrow();
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('ERROR');
const span = trace.data.spans[0];
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
expect(span.inputs).toEqual({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 128,
messages: [{ role: 'user', content: 'This request should fail.' }],
});
expect(span.outputs).toBeUndefined();
});
it('should trace Anthropic request wrapped in a parent span', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const result = await mlflow.withSpan(
async (_span) => {
const response = await wrappedAnthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 128,
messages: [{ role: 'user', content: 'Hello from parent span.' }],
});
return response.content[0];
},
{
name: 'predict',
spanType: mlflow.SpanType.CHAIN,
inputs: 'Hello from parent span.',
},
);
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
expect(trace.data.spans.length).toBe(2);
const parentSpan = trace.data.spans[0];
expect(parentSpan.name).toBe('predict');
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
expect(parentSpan.outputs).toEqual(result);
const childSpan = trace.data.spans[1];
expect(childSpan.name).toBe('Messages');
expect(childSpan.spanType).toBe(mlflow.SpanType.LLM);
expect(childSpan.inputs).toEqual({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 128,
messages: [{ role: 'user', content: 'Hello from parent span.' }],
});
expect(childSpan.outputs).toBeDefined();
const messageFormat = childSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
expect(messageFormat).toBe('anthropic');
});
it('should trace messages.stream() with async iteration', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const events: any[] = [];
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello streaming!' }],
});
for await (const event of stream) {
events.push(event);
}
expect(events.length).toBeGreaterThan(0);
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
const span = trace.data.spans[0];
expect(span.name).toBe('Messages');
expect(span.spanType).toBe(mlflow.SpanType.LLM);
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
expect(span.inputs).toMatchObject({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello streaming!' }],
});
// Verify outputs contain actual message content, not a stream object
expect(span.outputs).toBeDefined();
expect(span.outputs).toHaveProperty('id');
expect(span.outputs).toHaveProperty('type', 'message');
expect(span.outputs).toHaveProperty('content');
expect(span.outputs.content[0]).toHaveProperty('type', 'text');
// Verify token usage is captured for streaming
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
expect(spanTokenUsage).toBeDefined();
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
expect(messageFormat).toBe('anthropic');
});
it('should trace messages.stream() with finalMessage()', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
});
const finalMessage = await stream.finalMessage();
expect(finalMessage).toBeDefined();
expect(finalMessage.content).toBeDefined();
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
const span = trace.data.spans[0];
expect(span.name).toBe('Messages');
expect(span.spanType).toBe(mlflow.SpanType.LLM);
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
expect(span.inputs).toEqual({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
});
expect(span.outputs).toEqual(finalMessage);
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
expect(messageFormat).toBe('anthropic');
});
it('should handle errors during streaming async iteration', async () => {
server.use(createStreamingErrorHandler());
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
const wrappedAnthropic = tracedAnthropic(anthropic);
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello streaming error!' }],
});
await expect(async () => {
for await (const _event of stream) {
// consume events until error
}
}).rejects.toThrow();
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('ERROR');
const span = trace.data.spans[0];
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
expect(span.inputs).toMatchObject({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello streaming error!' }],
});
});
it('should handle errors during streaming finalMessage()', async () => {
server.use(createStreamingErrorHandler());
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
const wrappedAnthropic = tracedAnthropic(anthropic);
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
});
await expect(stream.finalMessage()).rejects.toThrow();
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('ERROR');
const span = trace.data.spans[0];
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
expect(span.inputs).toEqual({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
});
});
it('should create only one span when async iteration is followed by finalMessage()', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello dedup!' }],
});
// First consume via async iteration (claims tracing)
for await (const _event of stream) {
// consume all events
}
// Then call finalMessage() on the same stream — should NOT create a second span
const finalMessage = await stream.finalMessage();
expect(finalMessage).toBeDefined();
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
// Only one LLM span should exist (no duplicate from finalMessage)
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
expect(llmSpans.length).toBe(1);
});
it('should not trace messages.create() when stream: true is passed directly', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
// Calling create() with stream: true is skipped by the tracing wrapper
// because the SDK's stream() method internally calls create({ stream: true }),
// and tracing is handled by the stream wrapper instead.
// Users should use messages.stream() for traced streaming.
const response = await wrappedAnthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 256,
messages: [{ role: 'user', content: 'Hello direct stream!' }],
stream: true,
});
// The SDK returns a Stream object when stream: true
expect(response).toBeDefined();
// Consume the stream to completion
for await (const _event of response as any) {
// consume
}
// No traced span should have been created by the create() wrapper
await mlflow.flushTraces();
const traceId = mlflow.getLastActiveTraceId();
// The last active trace should be from a previous test, not this call,
// OR if it is from this call, it should not have a Messages LLM span
// created by wrapWithTracing (since create with stream:true is skipped).
if (traceId) {
const trace = await client.getTrace(traceId);
const matchingSpans = trace.data.spans.filter(
(s) =>
s.spanType === mlflow.SpanType.LLM &&
s.inputs?.messages?.[0]?.content === 'Hello direct stream!',
);
expect(matchingSpans.length).toBe(0);
}
});
it('should trace streaming request wrapped in a parent span', async () => {
const anthropic = new Anthropic({ apiKey: 'test-key' });
const wrappedAnthropic = tracedAnthropic(anthropic);
const result = await mlflow.withSpan(
async (_span) => {
const stream = wrappedAnthropic.messages.stream({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 128,
messages: [{ role: 'user', content: 'Hello streaming parent!' }],
});
const message = await stream.finalMessage();
return message.content[0];
},
{
name: 'streaming-predict',
spanType: mlflow.SpanType.CHAIN,
inputs: 'Hello streaming parent!',
},
);
const trace = await getLastActiveTrace();
expect(trace.info.state).toBe('OK');
// At least 2 spans: parent chain span and child LLM span
expect(trace.data.spans.length).toBeGreaterThanOrEqual(2);
const parentSpan = trace.data.spans[0];
expect(parentSpan.name).toBe('streaming-predict');
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
expect(parentSpan.outputs).toEqual(result);
// Find child spans with LLM type
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
expect(llmSpans.length).toBeGreaterThanOrEqual(1);
const llmSpan = llmSpans[0];
expect(llmSpan.outputs).toBeDefined();
const messageFormat = llmSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
expect(messageFormat).toBe('anthropic');
});
});
@@ -0,0 +1,187 @@
/**
* MSW-based Anthropic API mock server for testing
*/
import { http, HttpResponse } from 'msw';
interface MessagesCreateRequest {
model: string;
messages: Array<{
role: string;
content: string | Array<{ type: string; text: string }>;
}>;
max_tokens?: number;
system?: string | Array<{ type: string; text: string }>;
stream?: boolean;
}
function createMessageResponse(request: MessagesCreateRequest) {
return {
id: `msg_${Math.random().toString(36).slice(2)}`,
type: 'message',
role: 'assistant',
model: request.model,
stop_reason: 'end_turn',
stop_sequence: null,
usage: {
input_tokens: 128,
output_tokens: 256,
total_tokens: 384,
cache_read_input_tokens: 64,
cache_creation_input_tokens: 32,
},
content: [
{
type: 'text',
text: 'This is a mocked Anthropic response.',
},
],
};
}
function createStreamingEvents(request: MessagesCreateRequest): string[] {
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
const responseText = 'This is a mocked streaming response.';
return [
`event: message_start\ndata: ${JSON.stringify({
type: 'message_start',
message: {
id: messageId,
type: 'message',
role: 'assistant',
model: request.model,
content: [],
stop_reason: null,
stop_sequence: null,
usage: {
input_tokens: 128,
output_tokens: 0,
},
},
})}\n\n`,
`event: content_block_start\ndata: ${JSON.stringify({
type: 'content_block_start',
index: 0,
content_block: {
type: 'text',
text: '',
},
})}\n\n`,
`event: content_block_delta\ndata: ${JSON.stringify({
type: 'content_block_delta',
index: 0,
delta: {
type: 'text_delta',
text: responseText,
},
})}\n\n`,
`event: content_block_stop\ndata: ${JSON.stringify({
type: 'content_block_stop',
index: 0,
})}\n\n`,
`event: message_delta\ndata: ${JSON.stringify({
type: 'message_delta',
delta: {
stop_reason: 'end_turn',
stop_sequence: null,
},
usage: {
output_tokens: 256,
},
})}\n\n`,
`event: message_stop\ndata: ${JSON.stringify({
type: 'message_stop',
})}\n\n`,
];
}
export function createStreamingErrorHandler() {
return http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
const body = (await request.json()) as MessagesCreateRequest;
if (body.stream) {
// Send a partial stream that ends with an error event
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
const events = [
`event: message_start\ndata: ${JSON.stringify({
type: 'message_start',
message: {
id: messageId,
type: 'message',
role: 'assistant',
model: body.model,
content: [],
stop_reason: null,
stop_sequence: null,
usage: { input_tokens: 128, output_tokens: 0 },
},
})}\n\n`,
`event: error\ndata: ${JSON.stringify({
type: 'error',
error: {
type: 'overloaded_error',
message: 'Overloaded',
},
})}\n\n`,
];
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
for (const event of events) {
controller.enqueue(encoder.encode(event));
await new Promise((resolve) => setTimeout(resolve, 10));
}
controller.close();
},
});
return new HttpResponse(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
},
});
}
return HttpResponse.json(
{ error: { type: 'overloaded_error', message: 'Overloaded' } },
{ status: 529 },
);
});
}
export const anthropicMockHandlers = [
http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
const body = (await request.json()) as MessagesCreateRequest;
// Check if streaming is requested
if (body.stream) {
const events = createStreamingEvents(body);
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
for (const event of events) {
controller.enqueue(encoder.encode(event));
// Small delay to simulate streaming
await new Promise((resolve) => setTimeout(resolve, 10));
}
controller.close();
},
});
return new HttpResponse(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
},
});
}
return HttpResponse.json(createMessageResponse(body));
}),
];

Some files were not shown because too many files have changed in this diff Show More