chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended",
|
||||
"plugin:@typescript-eslint/recommended-requiring-type-checking"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2022,
|
||||
"sourceType": "module",
|
||||
"project": "./tsconfig.json"
|
||||
},
|
||||
"plugins": ["@typescript-eslint"],
|
||||
"ignorePatterns": [
|
||||
"dist/",
|
||||
"build/",
|
||||
"bundle/",
|
||||
"node_modules/",
|
||||
"*.d.ts",
|
||||
"jest.config.js",
|
||||
"jest.config.cjs"
|
||||
],
|
||||
"rules": {
|
||||
// TypeScript-specific rules
|
||||
"@typescript-eslint/no-unused-vars": [
|
||||
"error",
|
||||
{
|
||||
"argsIgnorePattern": "^_",
|
||||
"varsIgnorePattern": "^_"
|
||||
}
|
||||
],
|
||||
"@typescript-eslint/no-non-null-assertion": "warn",
|
||||
"@typescript-eslint/prefer-optional-chain": "error",
|
||||
"@typescript-eslint/no-unnecessary-type-assertion": "error",
|
||||
"@typescript-eslint/no-loss-of-precision": "error",
|
||||
// We cannot type everything like user inputs, outputs, json, etc.
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-unsafe-assignment": "off",
|
||||
// Namespaces are useful for organizing API specifications
|
||||
"@typescript-eslint/no-namespace": "off",
|
||||
"@typescript-eslint/ban-types": "off",
|
||||
|
||||
// General JavaScript/TypeScript rules
|
||||
"no-console": ["warn", { "allow": ["debug", "warn", "error"] }],
|
||||
"no-debugger": "error",
|
||||
"no-alert": "error",
|
||||
"prefer-const": "error",
|
||||
"no-var": "error",
|
||||
"eqeqeq": ["error", "always", { "null": "never" }],
|
||||
"curly": ["error", "all"],
|
||||
"no-trailing-spaces": "error",
|
||||
"no-multiple-empty-lines": ["error", { "max": 2 }],
|
||||
|
||||
// Import/Export rules
|
||||
"no-duplicate-imports": "error",
|
||||
|
||||
// Promise/Async rules
|
||||
"no-async-promise-executor": "error",
|
||||
"require-await": "error"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"files": ["*.test.ts", "*.spec.ts", "examples/**/*.ts", "jest.*.ts"],
|
||||
"rules": {
|
||||
"no-console": "off",
|
||||
"@typescript-eslint/explicit-function-return-type": "off",
|
||||
"@typescript-eslint/no-unsafe-member-access": "off",
|
||||
"@typescript-eslint/no-unsafe-argument": "off",
|
||||
"@typescript-eslint/no-non-null-assertion": "off"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["scripts/**/*.ts"],
|
||||
"rules": {
|
||||
"no-console": "off"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["integrations/vercel/tests/*.test.ts"],
|
||||
"rules": {
|
||||
// AI SDK does not expose the type of createOpenAI return value
|
||||
"@typescript-eslint/no-unsafe-call": "off"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["integrations/opencode/tests/*.test.ts"],
|
||||
"rules": {
|
||||
// OpenCode plugin types are mocked and not fully typed
|
||||
"@typescript-eslint/no-unsafe-call": "off",
|
||||
"@typescript-eslint/unbound-method": "off"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["integrations/claude-code/tests/*.test.ts"],
|
||||
"rules": {
|
||||
// Claude Code transcript types are mocked and not fully typed
|
||||
"@typescript-eslint/no-unsafe-call": "off",
|
||||
"@typescript-eslint/unbound-method": "off"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
min-release-age=7
|
||||
omit-lockfile-registry-resolved=true
|
||||
@@ -0,0 +1,3 @@
|
||||
**/dist/
|
||||
**/build/
|
||||
**/bundle/
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"semi": true,
|
||||
"singleQuote": true,
|
||||
"printWidth": 100,
|
||||
"tabWidth": 2,
|
||||
"useTabs": false,
|
||||
"bracketSpacing": true,
|
||||
"bracketSameLine": false,
|
||||
"arrowParens": "always",
|
||||
"endOfLine": "lf"
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
<h1 align="center" style="border-bottom: none">
|
||||
<div>
|
||||
<a href="https://mlflow.org/"><picture>
|
||||
<img alt="MLflow Logo" src="https://raw.githubusercontent.com/mlflow/mlflow/refs/heads/master/assets/logo.svg" width="200" />
|
||||
</picture></a>
|
||||
<br>
|
||||
MLflow TypeScript SDK
|
||||
</div>
|
||||
</h1>
|
||||
<h2 align="center" style="border-bottom: none"></h2>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mlflow/mlflow"><img src="https://img.shields.io/github/stars/mlflow/mlflow?style=social" alt="stars"></a>
|
||||
<a href="https://www.npmjs.com/package/@mlflow/core"><img src="https://img.shields.io/npm/v/%40mlflow%2Fcore.svg" alt="version"></a>
|
||||
<a href="https://www.npmjs.com/package/@mlflow/core"><img src="https://img.shields.io/npm/dt/%40mlflow%2Fcore.svg" alt="downloads"></a>
|
||||
<a href="https://github.com/mlflow/mlflow/blob/master/LICENSE.txt"><img src="https://img.shields.io/github/license/mlflow/mlflow" alt="license"></a>
|
||||
</p>
|
||||
|
||||
MLflow Typescript SDK is a variant of the [MLflow Python SDK](https://github.com/mlflow/mlflow) that provides a TypeScript API for MLflow.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> MLflow Typescript SDK is catching up with the Python SDK. Currently only support [Tracing]() and [Feedback Collection]() features. Please raise an issue in Github if you need a feature that is not supported.
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | NPM | Description |
|
||||
| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| [@mlflow/core](./core) | [](https://www.npmjs.com/package/@mlflow/core) | The core tracing functionality and manual instrumentation. |
|
||||
| [@mlflow/openai](./integrations/openai) | [](https://www.npmjs.com/package/@mlflow/openai) | Auto-instrumentation integration for OpenAI. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/core
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> MLflow Typescript SDK requires Node.js 20 or higher.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server if you don't have one already:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
### Configure with environment variables
|
||||
|
||||
The SDK can also read configuration from environment variables so you can avoid
|
||||
hard-coding connection details. If `MLFLOW_TRACKING_URI` and
|
||||
`MLFLOW_EXPERIMENT_ID` are set, you can initialize the client without passing
|
||||
any arguments:
|
||||
|
||||
```bash
|
||||
export MLFLOW_TRACKING_URI=http://localhost:5000
|
||||
export MLFLOW_EXPERIMENT_ID=123456789
|
||||
```
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init(); // Uses the values from the environment
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
||||
For MLflow tracking servers that require authentication, the SDK supports:
|
||||
|
||||
1. **Basic Auth** (username/password):
|
||||
|
||||
```typescript
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '123456789',
|
||||
trackingServerUsername: 'user',
|
||||
trackingServerPassword: 'pass',
|
||||
});
|
||||
```
|
||||
|
||||
Or via environment variables:
|
||||
|
||||
```bash
|
||||
export MLFLOW_TRACKING_USERNAME=user
|
||||
export MLFLOW_TRACKING_PASSWORD=pass
|
||||
```
|
||||
|
||||
2. **Bearer Token**:
|
||||
|
||||
```typescript
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '123456789',
|
||||
trackingServerToken: 'my-token',
|
||||
});
|
||||
```
|
||||
|
||||
Or via environment variable:
|
||||
|
||||
```bash
|
||||
export MLFLOW_TRACKING_TOKEN=my-token
|
||||
```
|
||||
|
||||
3. **No authentication** (default for local development)
|
||||
|
||||
Create a trace:
|
||||
|
||||
```typescript
|
||||
// Wrap a function with mlflow.trace to generate a span when the function is called.
|
||||
// MLflow will automatically record the function name, arguments, return value,
|
||||
// latency, and exception information to the span.
|
||||
const getWeather = mlflow.trace(
|
||||
(city: string) => {
|
||||
return `The weather in ${city} is sunny`;
|
||||
},
|
||||
// Pass options to set span name. See https://mlflow.org/docs/latest/genai/tracing/quickstart
|
||||
// for the full list of options.
|
||||
{ name: 'get-weather' },
|
||||
);
|
||||
getWeather('San Francisco');
|
||||
|
||||
// Alternatively, start and end span manually
|
||||
const span = mlflow.startSpan({ name: 'my-span' });
|
||||
span.end();
|
||||
```
|
||||
|
||||
Tag spans with a severity level so users (or you) can filter by **Minimum log level** in the trace UI:
|
||||
|
||||
```typescript
|
||||
import { SpanLogLevel } from '@mlflow/core';
|
||||
|
||||
const tracedAnswer = mlflow.trace((query: string) => llm.generate(query), {
|
||||
name: 'answer',
|
||||
spanType: mlflow.SpanType.CHAT_MODEL,
|
||||
logLevel: SpanLogLevel.INFO,
|
||||
});
|
||||
|
||||
// The string form works too:
|
||||
mlflow.startSpan({ name: 'plumbing', logLevel: 'DEBUG' });
|
||||
```
|
||||
|
||||
When you use one of the autolog integrations (`@mlflow/openai`, `@mlflow/anthropic`, `@mlflow/gemini`, etc.), MLflow stamps a sensible default level on every span based on its type — you don't need to annotate manually.
|
||||
|
||||
View traces in MLflow UI:
|
||||
|
||||

|
||||
|
||||
## Publishing
|
||||
|
||||
1. Run `yarn bump-version --version <new_version>` from this directory to bump the package versions appropriately
|
||||
2. `cd` into `core` and run `npm publish`, and repeat for `integrations/openai`
|
||||
|
||||
## Adding New Integrations
|
||||
|
||||
The TypeScript SDK supports pluggable auto-instrumentation packages under [`integrations/`](./integrations). To add a new integration:
|
||||
|
||||
1. Create a new workspace package (for example, `integrations/<provider>`), modeled after the [OpenAI integration](./integrations/openai).
|
||||
2. Implement the instrumentation entry points in `src/`, exporting a `register()` helper that configures tracing for the target client library.
|
||||
3. Add package metadata (`package.json`, `tsconfig.json`, and optional `README.md`) so the integration can be built and published.
|
||||
4. Add unit and/or integration tests under `tests/` that exercise the new instrumentation.
|
||||
5. Update the root [`package.json`](./package.json) `build:integrations` and `test:integrations` scripts if your package requires additional build or test commands.
|
||||
|
||||
Once your integration package is ready, run the local workflow outlined in [Running the SDK after changes](#running-the-sdk-after-changes) and open a pull request that describes the new provider support.
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions of new features, bug fixes, and documentation improvements. To contribute:
|
||||
|
||||
1. Review the project-wide [contribution guidelines](../../CONTRIBUTING.md) and follow the MLflow [Code of Conduct](../../CODE_OF_CONDUCT.rst).
|
||||
2. Discuss larger proposals in a GitHub issue or the MLflow community channels before investing significant effort.
|
||||
3. Fork the repository (or use a feature branch) and make your changes with clear, well-structured commits.
|
||||
4. Ensure your code includes tests and documentation updates where appropriate.
|
||||
5. Submit a pull request that summarizes the motivation, implementation details, and validation steps. The MLflow team will review and provide feedback.
|
||||
|
||||
## Running the SDK after Changes
|
||||
|
||||
The TypeScript workspace uses npm workspaces. After modifying the core SDK or any integration:
|
||||
|
||||
```bash
|
||||
npm install # Install or update workspace dependencies
|
||||
npm run build # Build the core package and all integrations
|
||||
npm run test # Execute the test suites for the core SDK and integrations
|
||||
```
|
||||
|
||||
You can run package-specific scripts from their respective directories (for example, `cd core && npm run test`) when iterating on a particular feature. Remember to rebuild before consuming the SDK from another project so that the latest TypeScript output is emitted to `dist/`.
|
||||
|
||||
## Trace Usage
|
||||
|
||||
MLflow Tracing empowers you throughout the end-to-end lifecycle of your application. Here's how it helps you at each step of the workflow, click on each section to learn more:
|
||||
|
||||
<details>
|
||||
<summary><strong>🔍 Build & Debug</strong></summary>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="60%">
|
||||
|
||||
#### Smooth Debugging Experience
|
||||
|
||||
MLflow's tracing capabilities provide deep insights into what happens beneath the abstractions of your application, helping you precisely identify where issues occur.
|
||||
|
||||
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/observe-with-traces/ui)
|
||||
|
||||
</td>
|
||||
<td width="40%">
|
||||
|
||||

|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>💬 Human Feedback</strong></summary>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="60%">
|
||||
|
||||
#### Track Annotation and User Feedback Attached to Traces
|
||||
|
||||
Collecting and managing feedback is essential for improving your application. MLflow Tracing allows you to attach user feedback and annotations directly to traces, creating a rich dataset for analysis.
|
||||
|
||||
This feedback data helps you understand user satisfaction, identify areas for improvement, and build better evaluation datasets based on real user interactions.
|
||||
|
||||
[Learn more →](https://mlflow.org/docs/latest/genai/assessments/feedback)
|
||||
|
||||
</td>
|
||||
<td width="40%">
|
||||
|
||||

|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>📊 Evaluation</strong></summary>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="60%">
|
||||
|
||||
#### Systematic Quality Assessment Throughout Your Application
|
||||
|
||||
Evaluating the performance of your application is crucial, but creating a reliable evaluation process can be challenging. Traces serve as a rich data source, helping you assess quality with precise metrics for all components.
|
||||
|
||||
When combined with MLflow's evaluation capabilities, you get a seamless experience for assessing and improving your application's performance.
|
||||
|
||||
[Learn more →](https://mlflow.org/docs/latest/genai/eval-monitor)
|
||||
|
||||
</td>
|
||||
<td width="40%">
|
||||
|
||||

|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>🚀 Production Monitoring</strong></summary>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="60%">
|
||||
|
||||
#### Monitor Applications with Your Favorite Observability Stack
|
||||
|
||||
Machine learning projects don't end with the first launch. Continuous monitoring and incremental improvement are critical to long-term success.
|
||||
|
||||
Integrated with various observability platforms such as Databricks, Datadog, Grafana, and Prometheus, MLflow Tracing provides a comprehensive solution for monitoring your applications in production.
|
||||
|
||||
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/prod-tracing)
|
||||
|
||||
</td>
|
||||
<td width="40%">
|
||||
|
||||

|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>📦 Dataset Collection</strong></summary>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="60%">
|
||||
|
||||
#### Create High-Quality Evaluation Datasets from Production Traces
|
||||
|
||||
Traces from production are invaluable for building comprehensive evaluation datasets. By capturing real user interactions and their outcomes, you can create test cases that truly represent your application's usage patterns.
|
||||
|
||||
This comprehensive data capture enables you to create realistic test scenarios, validate model performance on actual usage patterns, and continuously improve your evaluation datasets.
|
||||
|
||||
[Learn more →](https://mlflow.org/docs/latest/genai/tracing/search-traces#creating-evaluation-datasets)
|
||||
|
||||
</td>
|
||||
<td width="40%">
|
||||
|
||||

|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</details>
|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,64 @@
|
||||
# MLflow Typescript SDK - Core
|
||||
|
||||
This is the core package of the [MLflow Typescript SDK](https://github.com/mlflow/mlflow/tree/main/libs/typescript). It is a skinny package that includes the core tracing functionality and manual instrumentation.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| [@mlflow/core](./) | [](https://www.npmjs.com/package/@mlflow/core) | The core tracing functionality and manual instrumentation. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/core
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server. If you have a local Python environment, you can run the following command:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
If you don't have Python environment locally, MLflow also supports Docker deployment or managed services. See [Self-Hosting Guide](https://mlflow.org/docs/latest/self-hosting/index.html) for getting started.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
Create a trace:
|
||||
|
||||
```typescript
|
||||
// Wrap a function with mlflow.trace to generate a span when the function is called.
|
||||
// MLflow will automatically record the function name, arguments, return value,
|
||||
// latency, and exception information to the span.
|
||||
const getWeather = mlflow.trace(
|
||||
(city: string) => {
|
||||
return `The weather in ${city} is sunny`;
|
||||
},
|
||||
// Pass options to set span name. See https://mlflow.org/docs/latest/genai/tracing/quickstart
|
||||
// for the full list of options.
|
||||
{ name: 'get-weather' },
|
||||
);
|
||||
getWeather('San Francisco');
|
||||
|
||||
// Alternatively, start and end span manually
|
||||
const span = mlflow.startSpan({ name: 'my-span' });
|
||||
span.end();
|
||||
```
|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Bundles the WAL uploader daemon into a single, executable CJS file.
|
||||
*
|
||||
* Why a bundle:
|
||||
* - The daemon is spawned as a child process by the supervisor (see
|
||||
* `src/exporters/wal/supervisor.ts`), which resolves it at runtime
|
||||
* via `@mlflow/core/package.json` + `bundle/daemon.cjs`. Shipping a
|
||||
* single self-contained file means we don't have to worry about
|
||||
* transitive `dist/` files or `node_modules` layout in the target
|
||||
* environment.
|
||||
* - The `#!/usr/bin/env node` banner + `chmod 0o755` makes the bundle
|
||||
* directly executable, which is how the npm `bin` field invokes it.
|
||||
*/
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { chmodSync } from 'node:fs';
|
||||
|
||||
const outfile = 'bundle/daemon.cjs';
|
||||
|
||||
await build({
|
||||
entryPoints: ['dist/exporters/wal/daemon.js'],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'cjs',
|
||||
outfile,
|
||||
external: ['node:*'],
|
||||
banner: { js: '#!/usr/bin/env node' },
|
||||
});
|
||||
|
||||
chmodSync(outfile, 0o755);
|
||||
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests', '<rootDir>/src'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
|
||||
},
|
||||
globalSetup: '<rootDir>/../jest.global-server-setup.ts',
|
||||
globalTeardown: '<rootDir>/../jest.global-server-teardown.ts',
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"name": "@mlflow/core",
|
||||
"version": "0.3.0",
|
||||
"description": "TypeScript implementation of MLflow Tracing SDK for LLM observability",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"opentelemetry",
|
||||
"llm",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"bin": {
|
||||
"mlflow-trace-daemon": "./bundle/daemon.cjs"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && node esbuild.config.mjs",
|
||||
"test": "jest",
|
||||
"lint": "eslint . --ext .ts --max-warnings 0",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@databricks/sdk-experimental": "0.15.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "^0.205.0",
|
||||
"@opentelemetry/otlp-transformer": "^0.205.0",
|
||||
"@opentelemetry/sdk-node": "^0.205.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.1.0",
|
||||
"fast-safe-stringify": "^2.1.1",
|
||||
"bignumber.js": "^9.0.0",
|
||||
"ini": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/ini": "^4.1.1",
|
||||
"@types/jest": "^29.5.3",
|
||||
"@types/node": "^20.4.5",
|
||||
"@typescript-eslint/eslint-plugin": "^6.21.0",
|
||||
"@typescript-eslint/parser": "^6.21.0",
|
||||
"esbuild": "^0.25.0",
|
||||
"eslint": "^8.57.1",
|
||||
"jest": "^29.6.2",
|
||||
"msw": "^2.10.3",
|
||||
"prettier": "^3.5.3",
|
||||
"ts-jest": "^29.1.1",
|
||||
"tsx": "^4.7.0",
|
||||
"typescript": "^5.8.3",
|
||||
"whatwg-fetch": "^3.6.20"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"files": [
|
||||
"dist/",
|
||||
"bundle/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* MLflow TypeScript SDK Authentication Module
|
||||
*
|
||||
* This module provides authentication for MLflow tracking servers:
|
||||
*
|
||||
* ## Databricks-hosted MLflow
|
||||
*
|
||||
* For `databricks://` URIs, authentication is delegated entirely to the
|
||||
* Databricks SDK which supports multiple authentication methods:
|
||||
* - Personal Access Tokens (DATABRICKS_TOKEN)
|
||||
* - OAuth M2M / Service Principals (DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET)
|
||||
* - Azure CLI, Azure MSI, Azure Client Secret
|
||||
* - Google Cloud credentials
|
||||
* - Config file profiles (~/.databrickscfg)
|
||||
*
|
||||
* Host resolution order:
|
||||
* 1. Explicit `host` option
|
||||
* 2. DATABRICKS_HOST environment variable
|
||||
* 3. Host from ~/.databrickscfg (for the specified profile)
|
||||
*
|
||||
* ## Self-hosted MLflow (OSS)
|
||||
*
|
||||
* For HTTP(S) URIs, authentication is resolved in order:
|
||||
* 1. Basic Auth (MLFLOW_TRACKING_USERNAME + MLFLOW_TRACKING_PASSWORD)
|
||||
* 2. Bearer Token (MLFLOW_TRACKING_TOKEN)
|
||||
* 3. No authentication
|
||||
*
|
||||
* Workspace support (servers with workspaces enabled):
|
||||
* - MLFLOW_WORKSPACE: sets the `X-MLFLOW-WORKSPACE` header for workspace-scoped requests
|
||||
*
|
||||
* @module auth
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { Config } from '@databricks/sdk-experimental';
|
||||
|
||||
/**
|
||||
* Function that provides HTTP headers for authenticated requests.
|
||||
* Called before each request to support token refresh for OAuth.
|
||||
*/
|
||||
export type HeadersProvider = () => Promise<Record<string, string>>;
|
||||
|
||||
/**
|
||||
* Authentication provider interface used by MLflow clients.
|
||||
* Provides methods to get the host URL and authentication headers.
|
||||
*/
|
||||
export interface AuthProvider {
|
||||
/** Get the host URL for API requests */
|
||||
getHost(): string;
|
||||
|
||||
/** Get the function that provides authentication headers */
|
||||
getHeadersProvider(): HeadersProvider;
|
||||
|
||||
/**
|
||||
* Get the Databricks token if available (for backwards compatibility).
|
||||
* Returns undefined for non-Databricks or OAuth-only authentication.
|
||||
*/
|
||||
getDatabricksToken(): string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration options for authentication resolution.
|
||||
*/
|
||||
export interface AuthOptions {
|
||||
/** Tracking URI: "databricks", "databricks://profile", or HTTP(S) URL */
|
||||
trackingUri: string;
|
||||
|
||||
/** Explicit host override (takes precedence over environment) */
|
||||
host?: string;
|
||||
|
||||
/** Explicit Databricks token override */
|
||||
databricksToken?: string;
|
||||
|
||||
/** Path to Databricks config file (default: ~/.databrickscfg) */
|
||||
databricksConfigPath?: string;
|
||||
|
||||
/** Basic auth username for OSS MLflow */
|
||||
trackingServerUsername?: string;
|
||||
|
||||
/** Basic auth password for OSS MLflow */
|
||||
trackingServerPassword?: string;
|
||||
|
||||
/** Bearer token for OSS MLflow */
|
||||
trackingServerToken?: string;
|
||||
|
||||
/** Workspace name for MLflow servers with workspaces enabled (sets X-MLFLOW-WORKSPACE header) */
|
||||
workspace?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tracking URI targets Databricks.
|
||||
*/
|
||||
export function isDatabricksUri(trackingUri: string): boolean {
|
||||
return trackingUri === 'databricks' || trackingUri.startsWith('databricks://');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse profile name from a Databricks tracking URI.
|
||||
*
|
||||
* @example
|
||||
* parseDatabricksProfile('databricks') // returns undefined (use DEFAULT)
|
||||
* parseDatabricksProfile('databricks://my-profile') // returns 'my-profile'
|
||||
*/
|
||||
export function parseDatabricksProfile(trackingUri: string): string | undefined {
|
||||
if (trackingUri === 'databricks') {
|
||||
return undefined;
|
||||
}
|
||||
const match = trackingUri.match(/^databricks:\/\/(.+)$/);
|
||||
return match?.[1] || undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an authentication provider for the given configuration.
|
||||
*
|
||||
* This is the main entry point for authentication. It returns an AuthProvider
|
||||
* with methods to get the host URL and authentication headers.
|
||||
*
|
||||
* @param options - Authentication configuration options
|
||||
* @returns AuthProvider for making authenticated requests
|
||||
* @throws Error if required configuration is missing
|
||||
*/
|
||||
export function createAuthProvider(options: AuthOptions): AuthProvider {
|
||||
if (isDatabricksUri(options.trackingUri)) {
|
||||
return createDatabricksAuth(options);
|
||||
}
|
||||
return createOssAuth(options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the host value from a Databricks config file for a specific profile.
|
||||
*
|
||||
* The config file uses INI format:
|
||||
* ```
|
||||
* [DEFAULT]
|
||||
* host = https://my-workspace.databricks.com
|
||||
* token = dapi123...
|
||||
*
|
||||
* [profile-name]
|
||||
* host = https://other-workspace.databricks.com
|
||||
* ```
|
||||
*
|
||||
* @param configPath - Path to the config file
|
||||
* @param profile - Profile name (undefined for DEFAULT)
|
||||
* @returns The host value or undefined if not found
|
||||
*/
|
||||
function readHostFromConfigFile(configPath: string, profile?: string): string | undefined {
|
||||
try {
|
||||
if (!fs.existsSync(configPath)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(configPath, 'utf-8');
|
||||
const targetSection = profile || 'DEFAULT';
|
||||
|
||||
// Simple INI parser - find the section and extract host
|
||||
const lines = content.split('\n');
|
||||
let inTargetSection = false;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
// Check for section header
|
||||
const sectionMatch = trimmed.match(/^\[(.+)\]$/);
|
||||
if (sectionMatch) {
|
||||
inTargetSection = sectionMatch[1] === targetSection;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look for host in current section
|
||||
if (inTargetSection) {
|
||||
const hostMatch = trimmed.match(/^host\s*=\s*(.+)$/);
|
||||
if (hostMatch) {
|
||||
return hostMatch[1].trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create authentication for Databricks-hosted MLflow.
|
||||
*
|
||||
* Host resolution order:
|
||||
* 1. Explicit `host` option
|
||||
* 2. DATABRICKS_HOST environment variable
|
||||
* 3. Host from config file (~/.databrickscfg)
|
||||
*
|
||||
* All credential resolution is delegated to the Databricks SDK.
|
||||
* The SDK handles env vars, config files, OAuth, and cloud-specific auth.
|
||||
*/
|
||||
function createDatabricksAuth(options: AuthOptions): AuthProvider {
|
||||
const profile = parseDatabricksProfile(options.trackingUri);
|
||||
const configPath = options.databricksConfigPath ?? path.join(os.homedir(), '.databrickscfg');
|
||||
|
||||
// Resolve host - must be available synchronously
|
||||
// Priority: explicit option > env var > config file
|
||||
const host =
|
||||
options.host || process.env.DATABRICKS_HOST || readHostFromConfigFile(configPath, profile);
|
||||
|
||||
if (!host) {
|
||||
throw new Error(
|
||||
'Databricks host not found. Please either:\n' +
|
||||
' 1. Set the DATABRICKS_HOST environment variable, or\n' +
|
||||
` 2. Add a host to your config file at ${configPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Create Databricks SDK Config - it handles all credential resolution
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
|
||||
const databricksConfig = new Config({
|
||||
host: options.host,
|
||||
token: options.databricksToken,
|
||||
profile,
|
||||
configFile: options.databricksConfigPath,
|
||||
});
|
||||
|
||||
// Headers provider delegates entirely to the SDK
|
||||
const headersProvider: HeadersProvider = async () => {
|
||||
// SDK resolves credentials from env vars, config file, OAuth, etc.
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
await databricksConfig.ensureResolved();
|
||||
|
||||
const headers = new Headers();
|
||||
headers.set('Content-Type', 'application/json');
|
||||
|
||||
// SDK adds appropriate auth headers (Bearer token, OAuth, etc.)
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
await databricksConfig.authenticate(headers);
|
||||
|
||||
// Convert Headers to plain object
|
||||
const result: Record<string, string> = {};
|
||||
headers.forEach((value, key) => {
|
||||
result[key] = value;
|
||||
});
|
||||
return result;
|
||||
};
|
||||
|
||||
// Token for backwards compatibility (may be undefined for OAuth)
|
||||
const databricksToken = options.databricksToken || process.env.DATABRICKS_TOKEN;
|
||||
|
||||
return {
|
||||
getHost: () => host,
|
||||
getHeadersProvider: () => headersProvider,
|
||||
getDatabricksToken: () => databricksToken,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create authentication for self-hosted (OSS) MLflow.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Explicit options (username/password or token)
|
||||
* 2. Environment variables (MLFLOW_TRACKING_*)
|
||||
* 3. No authentication
|
||||
*/
|
||||
function createOssAuth(options: AuthOptions): AuthProvider {
|
||||
const host = options.trackingUri;
|
||||
|
||||
// Resolve credentials from options or environment
|
||||
const username = options.trackingServerUsername || process.env.MLFLOW_TRACKING_USERNAME;
|
||||
const password = options.trackingServerPassword || process.env.MLFLOW_TRACKING_PASSWORD;
|
||||
const token = options.trackingServerToken || process.env.MLFLOW_TRACKING_TOKEN;
|
||||
|
||||
// Pre-compute auth header since credentials don't change
|
||||
let authHeader: string | undefined;
|
||||
if (username && password) {
|
||||
const encoded = Buffer.from(`${username}:${password}`).toString('base64');
|
||||
authHeader = `Basic ${encoded}`;
|
||||
} else if (token) {
|
||||
authHeader = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
const workspace = process.env.MLFLOW_WORKSPACE ?? options.workspace;
|
||||
|
||||
// Headers provider for OSS MLflow
|
||||
// eslint-disable-next-line require-await, @typescript-eslint/require-await
|
||||
const headersProvider: HeadersProvider = async () => {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader;
|
||||
}
|
||||
if (workspace) {
|
||||
headers['X-MLFLOW-WORKSPACE'] = workspace;
|
||||
}
|
||||
return headers;
|
||||
};
|
||||
|
||||
return {
|
||||
getHost: () => host,
|
||||
getHeadersProvider: () => headersProvider,
|
||||
getDatabricksToken: () => undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { TraceData } from '../../core/entities/trace_data';
|
||||
import { TraceInfo } from '../../core/entities/trace_info';
|
||||
|
||||
export interface ArtifactsClient {
|
||||
uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void>;
|
||||
downloadTraceData(traceInfo: TraceInfo): Promise<TraceData>;
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
|
||||
import { TraceInfo } from '../../core/entities/trace_info';
|
||||
import { JSONBig } from '../../core/utils/json';
|
||||
import { GetCredentialsForTraceDataDownload, GetCredentialsForTraceDataUpload } from '../spec';
|
||||
import { makeRequest } from '../utils';
|
||||
import { ArtifactsClient } from './base';
|
||||
import { AuthProvider, HeadersProvider } from '../../auth';
|
||||
|
||||
/**
|
||||
* Options for creating a DatabricksArtifactsClient.
|
||||
*/
|
||||
export interface DatabricksArtifactsClientOptions {
|
||||
/**
|
||||
* The Databricks workspace host URL
|
||||
*/
|
||||
host: string;
|
||||
|
||||
/**
|
||||
* Authentication provider
|
||||
*/
|
||||
authProvider: AuthProvider;
|
||||
}
|
||||
|
||||
export class DatabricksArtifactsClient implements ArtifactsClient {
|
||||
private host: string;
|
||||
private headersProvider: HeadersProvider;
|
||||
|
||||
constructor(options: DatabricksArtifactsClientOptions) {
|
||||
this.host = options.host;
|
||||
this.headersProvider = options.authProvider.getHeadersProvider();
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload trace data (spans) to the backend using artifact repository pattern.
|
||||
*
|
||||
* 1. Get credentials for upload
|
||||
* 2. Serialize trace data to JSON
|
||||
* 3. Upload to cloud storage using the credentials
|
||||
*/
|
||||
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
|
||||
try {
|
||||
const credentials = await this.getCredentialsForTraceDataUpload(traceInfo.traceId);
|
||||
const traceDataJson = JSONBig.stringify(traceData.toJson());
|
||||
await this.uploadToCloudStorage(credentials, traceDataJson);
|
||||
} catch (error) {
|
||||
console.error(`Trace data upload failed for ${traceInfo.traceId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download trace data (spans) from cloud storage
|
||||
* Uses artifact repository pattern with signed URLs
|
||||
*/
|
||||
async downloadTraceData(traceInfo: TraceInfo): Promise<TraceData> {
|
||||
try {
|
||||
const credentials = await this.getCredentialsForTraceDataDownload(traceInfo.traceId);
|
||||
const traceDataJson = await this.downloadFromSignedUrl(credentials);
|
||||
return TraceData.fromJson(traceDataJson);
|
||||
} catch (error) {
|
||||
console.error(`Failed to download trace data for ${traceInfo.traceId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get credentials for uploading trace data
|
||||
* Endpoint: GET /api/2.0/mlflow/traces/{request_id}/credentials-for-data-upload
|
||||
*/
|
||||
private async getCredentialsForTraceDataUpload(traceId: string): Promise<ArtifactCredentialInfo> {
|
||||
const url = GetCredentialsForTraceDataUpload.getEndpoint(this.host, traceId);
|
||||
const response = await makeRequest<GetCredentialsForTraceDataUpload.Response>(
|
||||
'GET',
|
||||
url,
|
||||
this.headersProvider,
|
||||
);
|
||||
return response.credential_info;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get credentials for downloading trace data
|
||||
* Endpoint: GET /mlflow/traces/{trace_id}/credentials-for-data-download
|
||||
*/
|
||||
private async getCredentialsForTraceDataDownload(
|
||||
traceId: string,
|
||||
): Promise<ArtifactCredentialInfo> {
|
||||
const url = GetCredentialsForTraceDataDownload.getEndpoint(this.host, traceId);
|
||||
const response = await makeRequest<GetCredentialsForTraceDataDownload.Response>(
|
||||
'GET',
|
||||
url,
|
||||
this.headersProvider,
|
||||
);
|
||||
|
||||
if (response.credential_info) {
|
||||
return response.credential_info;
|
||||
} else {
|
||||
throw new Error('Invalid response format: missing credential_info');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload data to cloud storage using the provided credentials
|
||||
*/
|
||||
private async uploadToCloudStorage(
|
||||
credentials: ArtifactCredentialInfo,
|
||||
data: string,
|
||||
): Promise<void> {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
|
||||
// Add headers from credentials (if they exist)
|
||||
if (credentials.headers && Array.isArray(credentials.headers)) {
|
||||
credentials.headers.forEach((header) => {
|
||||
headers[header.name] = header.value;
|
||||
});
|
||||
}
|
||||
|
||||
switch (credentials.type) {
|
||||
case 'AWS_PRESIGNED_URL':
|
||||
case 'GCP_SIGNED_URL':
|
||||
await this.uploadToSignedUrl(credentials.signed_uri, data, headers, credentials.type);
|
||||
break;
|
||||
case 'AZURE_SAS_URI':
|
||||
await this.uploadToAzureBlob(credentials.signed_uri, data, headers);
|
||||
break;
|
||||
case 'AZURE_ADLS_GEN2_SAS_URI':
|
||||
await this.uploadToAzureAdlsGen2(credentials.signed_uri, data, headers);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported credential type: ${credentials.type as string}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload data to cloud storage using signed URL (AWS S3 or GCP Storage)
|
||||
*/
|
||||
private async uploadToSignedUrl(
|
||||
signedUrl: string,
|
||||
data: string,
|
||||
headers: Record<string, string>,
|
||||
credentialType: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const response = await fetch(signedUrl, {
|
||||
method: 'PUT',
|
||||
headers,
|
||||
body: data,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`${credentialType} upload failed: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to upload to ${credentialType}: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload data to Azure Blob Storage using SAS URI
|
||||
* Uses simple PUT for all uploads since traces rarely exceed 100MB
|
||||
* https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview
|
||||
*/
|
||||
private async uploadToAzureBlob(
|
||||
sasUri: string,
|
||||
data: string,
|
||||
headers: Record<string, string>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const response = await fetch(sasUri, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
...headers,
|
||||
'x-ms-blob-type': 'BlockBlob',
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: data,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Azure Blob upload failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to upload to Azure Blob Storage: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload data to Azure Data Lake Storage Gen2 using SAS URI
|
||||
* https://learn.microsoft.com/en-us/rest/api/storageservices/data-lake-storage-gen2
|
||||
*/
|
||||
private async uploadToAzureAdlsGen2(
|
||||
sasUri: string,
|
||||
data: string,
|
||||
headers: Record<string, string>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const dataBuffer = new TextEncoder().encode(data);
|
||||
|
||||
// ADLS Gen2 uses a different API pattern - create file then append data
|
||||
// Create the file
|
||||
const createUrl = `${sasUri}&resource=file`;
|
||||
const createResponse = await fetch(createUrl, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
...headers,
|
||||
'Content-Length': '0',
|
||||
},
|
||||
});
|
||||
|
||||
if (!createResponse.ok) {
|
||||
throw new Error(
|
||||
`Azure ADLS Gen2 file creation failed: ${createResponse.status} ${createResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Append data to the file
|
||||
const appendUrl = `${sasUri}&action=append&position=0`;
|
||||
const appendResponse = await fetch(appendUrl, {
|
||||
method: 'PATCH',
|
||||
headers: {
|
||||
...headers,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
},
|
||||
body: dataBuffer,
|
||||
});
|
||||
|
||||
if (!appendResponse.ok) {
|
||||
throw new Error(
|
||||
`Azure ADLS Gen2 data append failed: ${appendResponse.status} ${appendResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Flush the data to complete the upload
|
||||
const flushUrl = `${sasUri}&action=flush&position=${dataBuffer.length}`;
|
||||
const flushResponse = await fetch(flushUrl, {
|
||||
method: 'PATCH',
|
||||
headers: {
|
||||
...headers,
|
||||
'Content-Length': '0',
|
||||
},
|
||||
});
|
||||
|
||||
if (!flushResponse.ok) {
|
||||
throw new Error(
|
||||
`Azure ADLS Gen2 flush failed: ${flushResponse.status} ${flushResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to upload to Azure ADLS Gen2: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download data from cloud storage using signed URL
|
||||
*/
|
||||
private async downloadFromSignedUrl(
|
||||
credentials: ArtifactCredentialInfo,
|
||||
): Promise<SerializedTraceData> {
|
||||
const headers: Record<string, string> = {};
|
||||
|
||||
// Add headers from credentials (if they exist)
|
||||
if (credentials.headers && Array.isArray(credentials.headers)) {
|
||||
credentials.headers.forEach((header) => {
|
||||
headers[header.name] = header.value;
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(credentials.signed_uri, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
throw new Error(`Trace data not found (404)`);
|
||||
}
|
||||
throw new Error(`Download failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const textData = await response.text();
|
||||
try {
|
||||
return JSONBig.parse(textData) as SerializedTraceData;
|
||||
} catch (parseError) {
|
||||
throw new Error(`Trace data corrupted: invalid JSON - ${(parseError as Error).message}`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`Failed to download trace data: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP header for artifact upload/download
|
||||
*/
|
||||
export interface HttpHeader {
|
||||
name: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Artifact credential information for upload/download
|
||||
*/
|
||||
export interface ArtifactCredentialInfo {
|
||||
/** ID of the MLflow Run containing the artifact */
|
||||
run_id?: string;
|
||||
|
||||
/** Relative path to the artifact */
|
||||
path?: string;
|
||||
|
||||
/** Signed URI credential for artifact access */
|
||||
signed_uri: string;
|
||||
|
||||
/** HTTP headers for upload/download (optional, may not be present) */
|
||||
headers?: HttpHeader[];
|
||||
|
||||
/** Type of signed credential URI */
|
||||
type: ArtifactCredentialType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enum for artifact credential types.
|
||||
* This ensures type safety when handling different cloud storage providers.
|
||||
*/
|
||||
export type ArtifactCredentialType =
|
||||
| 'AWS_PRESIGNED_URL'
|
||||
| 'GCP_SIGNED_URL'
|
||||
| 'AZURE_SAS_URI'
|
||||
| 'AZURE_ADLS_GEN2_SAS_URI';
|
||||
@@ -0,0 +1,50 @@
|
||||
import { ArtifactsClient } from './base';
|
||||
import { DatabricksArtifactsClient } from './databricks';
|
||||
import { MlflowArtifactsClient } from './mlflow';
|
||||
import { AuthProvider } from '../../auth';
|
||||
|
||||
/**
|
||||
* Options for creating an ArtifactsClient.
|
||||
*/
|
||||
export interface ArtifactsClientOptions {
|
||||
/**
|
||||
* The tracking URI (e.g., "databricks", "http://localhost:5000")
|
||||
*/
|
||||
trackingUri: string;
|
||||
|
||||
/**
|
||||
* The resolved host URL for API requests
|
||||
*/
|
||||
host: string;
|
||||
|
||||
/**
|
||||
* Authentication provider
|
||||
*/
|
||||
authProvider: AuthProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate artifacts client based on the tracking URI.
|
||||
*
|
||||
* @param trackingUri - The tracking URI (e.g., "databricks", "http://localhost:5000")
|
||||
* @param host - The resolved host URL for API requests
|
||||
* @param authProvider - The authentication provider to get tokens for authenticated requests
|
||||
* @returns The appropriate artifacts client.
|
||||
*/
|
||||
export function getArtifactsClient({
|
||||
trackingUri,
|
||||
host,
|
||||
authProvider,
|
||||
}: {
|
||||
trackingUri: string;
|
||||
host: string;
|
||||
authProvider: AuthProvider;
|
||||
}): ArtifactsClient {
|
||||
if (trackingUri.startsWith('databricks')) {
|
||||
return new DatabricksArtifactsClient({ host, authProvider });
|
||||
} else {
|
||||
return new MlflowArtifactsClient({ host, authProvider });
|
||||
}
|
||||
}
|
||||
|
||||
export { ArtifactsClient };
|
||||
@@ -0,0 +1,149 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { TraceTagKey } from '../../core/constants';
|
||||
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
|
||||
import { TraceInfo } from '../../core/entities/trace_info';
|
||||
import { JSONBig } from '../../core/utils/json';
|
||||
import { makeRequest } from '../utils';
|
||||
import { isLocalArtifactUri, toAbsoluteLocalPath } from '../../core/utils/artifact_uri';
|
||||
import { ArtifactsClient } from './base';
|
||||
import { AuthProvider, HeadersProvider } from '../../auth';
|
||||
|
||||
/**
|
||||
* Trace data file name constant - matches Python SDK
|
||||
*/
|
||||
const TRACE_DATA_FILE_NAME = 'traces.json';
|
||||
|
||||
/**
|
||||
* MLflow OSS Artifacts Client
|
||||
*
|
||||
* Implements artifact upload/download for OSS MLflow Tracking Server using the standard
|
||||
* HTTP artifact repository endpoints. Based on Python HttpArtifactRepository.
|
||||
*/
|
||||
export class MlflowArtifactsClient implements ArtifactsClient {
|
||||
private readonly host: string;
|
||||
private headersProvider: HeadersProvider;
|
||||
|
||||
constructor(options: { host: string; authProvider: AuthProvider }) {
|
||||
this.host = options.host;
|
||||
this.headersProvider = options.authProvider.getHeadersProvider();
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload trace data to MLflow artifact storage.
|
||||
*
|
||||
* Equivalent to Python's upload_trace_data() method which uses log_artifact()
|
||||
* under the hood to upload the trace data as a JSON file.
|
||||
*
|
||||
* @param traceInfo The trace information containing artifact URI
|
||||
* @param traceData The trace data to upload
|
||||
*/
|
||||
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
|
||||
const artifactUri = this.getArtifactLocation(traceInfo);
|
||||
|
||||
if (isLocalArtifactUri(artifactUri)) {
|
||||
await this.uploadTraceDataLocal(artifactUri, traceData);
|
||||
return;
|
||||
}
|
||||
|
||||
const traceDataJson = traceData.toJson();
|
||||
|
||||
const artifactUrl = this.resolveArtifactUri(artifactUri, TRACE_DATA_FILE_NAME);
|
||||
await makeRequest<void>('PUT', artifactUrl, this.headersProvider, traceDataJson);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write trace data to a local filesystem artifact location.
|
||||
*/
|
||||
private async uploadTraceDataLocal(artifactUri: string, traceData: TraceData): Promise<void> {
|
||||
const dir = toAbsoluteLocalPath(artifactUri);
|
||||
await mkdir(dir, { recursive: true });
|
||||
await writeFile(join(dir, TRACE_DATA_FILE_NAME), JSONBig.stringify(traceData.toJson()), 'utf8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Download trace data from MLflow artifact storage.
|
||||
*
|
||||
* Equivalent to Python's download_trace_data() method which downloads
|
||||
* the traces.json file and parses it back to TraceData.
|
||||
*
|
||||
* @param traceInfo The trace information containing artifact URI
|
||||
* @returns The downloaded and parsed trace data
|
||||
*/
|
||||
async downloadTraceData(traceInfo: TraceInfo): Promise<TraceData> {
|
||||
const artifactUri = this.getArtifactLocation(traceInfo);
|
||||
|
||||
if (isLocalArtifactUri(artifactUri)) {
|
||||
return this.downloadTraceDataLocal(artifactUri);
|
||||
}
|
||||
|
||||
// Download the trace data file
|
||||
const artifactUrl = this.resolveArtifactUri(artifactUri, TRACE_DATA_FILE_NAME);
|
||||
const traceDataJson = await makeRequest<SerializedTraceData>(
|
||||
'GET',
|
||||
artifactUrl,
|
||||
this.headersProvider,
|
||||
);
|
||||
|
||||
// Parse JSON back to TraceData (equivalent to Python's try_read_trace_data)
|
||||
try {
|
||||
return TraceData.fromJson(traceDataJson);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Failed to parse trace data JSON: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read trace data from a local filesystem artifact location.
|
||||
*/
|
||||
private async downloadTraceDataLocal(artifactUri: string): Promise<TraceData> {
|
||||
const filePath = join(toAbsoluteLocalPath(artifactUri), TRACE_DATA_FILE_NAME);
|
||||
const contents = await readFile(filePath, 'utf8');
|
||||
try {
|
||||
return TraceData.fromJson(JSONBig.parse(contents) as SerializedTraceData);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Failed to parse trace data JSON: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the artifact location tag set on the trace by the backend.
|
||||
*/
|
||||
private getArtifactLocation(traceInfo: TraceInfo): string {
|
||||
const artifactUri = traceInfo.tags[TraceTagKey.MLFLOW_ARTIFACT_LOCATION];
|
||||
|
||||
if (!artifactUri) {
|
||||
throw new Error('Artifact location not found in trace tags');
|
||||
}
|
||||
|
||||
return artifactUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve mlflow-artifacts:// URI to HTTP endpoint.
|
||||
*
|
||||
* Equivalent to Python's MlflowArtifactsRepository.resolve_uri() method.
|
||||
* Transforms URIs like "mlflow-artifacts:/0/traces/tr-abc123/artifacts"
|
||||
* to "http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/0/traces/tr-abc123/artifacts/traces.json"
|
||||
*
|
||||
* @param artifactUri The mlflow-artifacts:// URI from trace tags
|
||||
* @param fileName The file name to append (e.g., "traces.json")
|
||||
* @returns The resolved HTTP endpoint URL
|
||||
*/
|
||||
private resolveArtifactUri(artifactUri: string, fileName: string): string {
|
||||
const baseApiPath = '/api/2.0/mlflow-artifacts/artifacts';
|
||||
const url = new URL(artifactUri);
|
||||
|
||||
if (url.protocol !== 'mlflow-artifacts:') {
|
||||
throw new Error(`Expected mlflow-artifacts:// URI, got ${url.protocol}`);
|
||||
}
|
||||
|
||||
// Construct the final HTTP URL
|
||||
const cleanHost = this.host.replace(/\/$/, ''); // Remove trailing slash
|
||||
return `${cleanHost}${baseApiPath}${url.pathname}/${fileName}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
import { TraceInfo } from '../core/entities/trace_info';
|
||||
import { Trace } from '../core/entities/trace';
|
||||
import {
|
||||
CreateExperiment,
|
||||
CreateTraceInfoV4,
|
||||
DeleteExperiment,
|
||||
ExportOtlpTraces,
|
||||
ExportOtlpTracesOss,
|
||||
GetExperiment,
|
||||
GetExperimentByName,
|
||||
GetTraceInfoV3,
|
||||
SearchTracesV3,
|
||||
StartTraceV3,
|
||||
} from './spec';
|
||||
import { makeRawRequest, makeRequest, MlflowHttpError } from './utils';
|
||||
import { TraceData } from '../core/entities/trace_data';
|
||||
import { ArtifactsClient, getArtifactsClient } from './artifacts';
|
||||
import { AuthProvider, HeadersProvider } from '../auth';
|
||||
import { DATABRICKS_UC_TABLE_HEADER, MLFLOW_EXPERIMENT_ID_HEADER } from '../core/constants';
|
||||
import { serializeTraceLocation, type TraceLocation } from '../core/entities/trace_location';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import type { ReadableSpan as OTelReadableSpan } from '@opentelemetry/sdk-trace-base';
|
||||
import { ExportResultCode, type ExportResult } from '@opentelemetry/core';
|
||||
|
||||
export interface SearchTracesOptions {
|
||||
/** Trace locations (e.g. MLflow experiments) to search over. */
|
||||
locations: TraceLocation[];
|
||||
/** Search filter string, e.g. `"tags.env = 'prod'"`. */
|
||||
filter?: string;
|
||||
/** Maximum number of traces to return in a single page. */
|
||||
maxResults?: number;
|
||||
/** List of order-by clauses, e.g. `['timestamp_ms DESC']`. */
|
||||
orderBy?: string[];
|
||||
/** Pagination token obtained from a previous searchTraces call. */
|
||||
pageToken?: string;
|
||||
/**
|
||||
* Whether to fetch span data for each matching trace. Defaults to `true`.
|
||||
* When `false`, only trace metadata is returned and the traces have empty
|
||||
* span data.
|
||||
*/
|
||||
includeSpans?: boolean;
|
||||
}
|
||||
|
||||
export interface SearchTracesResult {
|
||||
traces: Trace[];
|
||||
nextPageToken?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Client for MLflow tracing operations.
|
||||
*
|
||||
* Supports multiple authentication methods:
|
||||
* - Databricks: PAT tokens, OAuth M2M, Azure CLI/MSI, Google Cloud
|
||||
* - OSS MLflow: Basic auth, Bearer tokens, or no auth
|
||||
*/
|
||||
export class MlflowClient {
|
||||
/** Client implementation to upload/download trace data artifacts */
|
||||
private artifactsClient: ArtifactsClient;
|
||||
|
||||
/** Headers provider for authenticated requests */
|
||||
private headersProvider: HeadersProvider;
|
||||
|
||||
/** Host URL for API requests */
|
||||
private hostUrl: string;
|
||||
|
||||
/**
|
||||
* Creates a new MlflowClient.
|
||||
*
|
||||
* @param trackingUri - The tracking URI (e.g., "databricks", "http://localhost:5000")
|
||||
* @param authProvider - The authentication provider to get tokens for authenticated requests
|
||||
*/
|
||||
constructor(options: { trackingUri: string; authProvider: AuthProvider }) {
|
||||
this.headersProvider = options.authProvider.getHeadersProvider();
|
||||
this.hostUrl = options.authProvider.getHost();
|
||||
|
||||
this.artifactsClient = getArtifactsClient({
|
||||
trackingUri: options.trackingUri,
|
||||
host: this.hostUrl,
|
||||
authProvider: options.authProvider,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the host URL for this client.
|
||||
*/
|
||||
getHost(): string {
|
||||
return this.hostUrl;
|
||||
}
|
||||
|
||||
// === TRACE LOGGING METHODS ===
|
||||
/**
|
||||
* Create a new TraceInfo record in the backend store.
|
||||
* Corresponding to the Python SDK's start_trace_v3() method.
|
||||
*
|
||||
* Note: the backend API is named as "Start" due to unfortunate miscommunication.
|
||||
* The API is indeed called at the "end" of a trace, not the "start".
|
||||
*/
|
||||
async createTrace(traceInfo: TraceInfo): Promise<TraceInfo> {
|
||||
const url = StartTraceV3.getEndpoint(this.hostUrl);
|
||||
const payload: StartTraceV3.Request = { trace: { trace_info: traceInfo.toJson() } };
|
||||
const response = await makeRequest<StartTraceV3.Response>(
|
||||
'POST',
|
||||
url,
|
||||
this.headersProvider,
|
||||
payload,
|
||||
);
|
||||
return TraceInfo.fromJson(response.trace.trace_info);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new TraceInfo record in the Databricks V4 (Unity Catalog)
|
||||
* trace-info endpoint. Used for UC-backed traces so that trace-level
|
||||
* tags and metadata persist alongside spans uploaded via OTLP.
|
||||
*
|
||||
* Corresponds to Python's `DatabricksRestStore._start_trace_v4`.
|
||||
*
|
||||
* @param location - The UC location string ("catalog.schema" or
|
||||
* "catalog.schema.table_prefix") that comes from
|
||||
* the trace ID `trace:/<location>/<hex>`.
|
||||
* @param otelTraceId - The hex OTel trace ID portion.
|
||||
* @param traceInfo - The full TraceInfo to persist; the backend reads
|
||||
* `tags`, `trace_metadata`, `state`, durations, etc.
|
||||
*/
|
||||
async createTraceInfoV4(
|
||||
location: string,
|
||||
otelTraceId: string,
|
||||
traceInfo: TraceInfo,
|
||||
): Promise<TraceInfo> {
|
||||
const url = CreateTraceInfoV4.getEndpoint(this.hostUrl, location, otelTraceId);
|
||||
const payload: CreateTraceInfoV4.Request = traceInfo.toJson();
|
||||
const response = await makeRequest<CreateTraceInfoV4.Response>(
|
||||
'POST',
|
||||
url,
|
||||
this.headersProvider,
|
||||
payload,
|
||||
);
|
||||
return TraceInfo.fromJson(response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload OTel spans to a Databricks Unity Catalog location via the OTLP
|
||||
* HTTP+protobuf endpoint. The `spansTableName` is the fully qualified
|
||||
* spans table (catalog.schema.table) and is forwarded as the
|
||||
* `X-Databricks-UC-Table-Name` header used by Databricks to route the
|
||||
* payload to the correct UC location.
|
||||
*
|
||||
* Uses `@opentelemetry/exporter-trace-otlp-proto` which serializes the
|
||||
* spans to the OTLP `ExportTraceServiceRequest` protobuf wire format —
|
||||
* the Databricks endpoint only accepts `application/x-protobuf`, not
|
||||
* the OTLP/HTTP+JSON form.
|
||||
*/
|
||||
async exportOtlpSpansToUc(spans: OTelReadableSpan[], spansTableName: string): Promise<void> {
|
||||
if (spans.length === 0) {
|
||||
return;
|
||||
}
|
||||
// Fetch fresh auth headers for every export so OAuth-rotated tokens are
|
||||
// honored. The exporter takes a static headers map at construction time,
|
||||
// so we rebuild it per call rather than caching an instance.
|
||||
const authHeaders = await this.headersProvider();
|
||||
const exporter = new OTLPTraceExporter({
|
||||
url: ExportOtlpTraces.getEndpoint(this.hostUrl),
|
||||
headers: {
|
||||
...authHeaders,
|
||||
[DATABRICKS_UC_TABLE_HEADER]: spansTableName,
|
||||
},
|
||||
});
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
exporter.export(spans, (result: ExportResult) => {
|
||||
if (result.code === ExportResultCode.SUCCESS) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(result.error ?? new Error(`OTLP span export failed with code ${result.code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
// Drain any pending state so the exporter doesn't keep handles open.
|
||||
await exporter.shutdown().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export OTLP spans to an OSS (non-Databricks) tracking server.
|
||||
*
|
||||
* `otlpProtoBytes` is an already-serialized OTLP `ExportTraceServiceRequest`
|
||||
* protobuf (captured at WAL-write time), POSTed as `application/x-protobuf`
|
||||
* with the experiment id forwarded via the `x-mlflow-experiment-id` header.
|
||||
*/
|
||||
async exportOtlpSpans(experimentId: string, otlpProtoBytes: Uint8Array): Promise<void> {
|
||||
if (otlpProtoBytes.length === 0) {
|
||||
return;
|
||||
}
|
||||
await makeRawRequest(
|
||||
'POST',
|
||||
ExportOtlpTracesOss.getEndpoint(this.hostUrl),
|
||||
this.headersProvider,
|
||||
otlpProtoBytes,
|
||||
{
|
||||
'Content-Type': 'application/x-protobuf',
|
||||
[MLFLOW_EXPERIMENT_ID_HEADER]: experimentId,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// === TRACE RETRIEVAL METHODS ===
|
||||
/**
|
||||
* Get a single trace by ID
|
||||
* Fetches both trace info and trace data from backend
|
||||
* Corresponds to Python: client.get_trace()
|
||||
*/
|
||||
async getTrace(traceId: string): Promise<Trace> {
|
||||
const traceInfo = await this.getTraceInfo(traceId);
|
||||
const traceData = await this.artifactsClient.downloadTraceData(traceInfo);
|
||||
return new Trace(traceInfo, traceData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get trace info using V3 API
|
||||
* Endpoint: GET /api/3.0/mlflow/traces/{trace_id}
|
||||
*/
|
||||
async getTraceInfo(traceId: string): Promise<TraceInfo> {
|
||||
const url = GetTraceInfoV3.getEndpoint(this.hostUrl, traceId);
|
||||
const response = await makeRequest<GetTraceInfoV3.Response>('GET', url, this.headersProvider);
|
||||
|
||||
// The V3 API returns a Trace object with trace_info field
|
||||
if (response.trace?.trace_info) {
|
||||
return TraceInfo.fromJson(response.trace.trace_info);
|
||||
}
|
||||
|
||||
throw new Error(`Invalid response format: missing trace_info: ${JSON.stringify(response)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for traces that match the given search criteria.
|
||||
* Corresponding to the Python SDK's search_traces() method.
|
||||
*
|
||||
* By default the span data of each matching trace is fetched with one
|
||||
* download per trace; traces whose span data cannot be downloaded are
|
||||
* skipped, matching the Python SDK's behavior. Pass `includeSpans: false`
|
||||
* to return trace metadata only, with empty span data.
|
||||
*
|
||||
* Locations are passed through to the tracking server, which validates
|
||||
* that it supports the requested location types.
|
||||
*
|
||||
* @param options.locations Trace locations (e.g. MLflow experiments) to search over.
|
||||
* @param options.filter Search filter string, e.g. `"tags.env = 'prod'"`.
|
||||
* @param options.maxResults Maximum number of traces to return in a single page.
|
||||
* @param options.orderBy List of order-by clauses, e.g. `['timestamp_ms DESC']`.
|
||||
* @param options.pageToken Pagination token obtained from a previous searchTraces call.
|
||||
* @param options.includeSpans Whether to fetch span data for each trace. Defaults to `true`.
|
||||
* @returns The matching traces and, when more results are available, a token for the next page.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const { traces, nextPageToken } = await client.searchTraces({
|
||||
* locations: [
|
||||
* { type: TraceLocationType.MLFLOW_EXPERIMENT, mlflowExperiment: { experimentId: '123' } },
|
||||
* ],
|
||||
* filter: "tags.env = 'prod'",
|
||||
* maxResults: 10,
|
||||
* orderBy: ['timestamp_ms DESC'],
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
async searchTraces(options: SearchTracesOptions): Promise<SearchTracesResult> {
|
||||
const { locations, includeSpans = true } = options;
|
||||
if (!locations?.length) {
|
||||
throw new Error('At least one location must be specified for searching traces.');
|
||||
}
|
||||
|
||||
const url = SearchTracesV3.getEndpoint(this.hostUrl);
|
||||
const payload: SearchTracesV3.Request = {
|
||||
locations: locations.map(serializeTraceLocation),
|
||||
filter: options.filter,
|
||||
max_results: options.maxResults,
|
||||
order_by: options.orderBy,
|
||||
page_token: options.pageToken,
|
||||
};
|
||||
const response = await makeRequest<SearchTracesV3.Response>(
|
||||
'POST',
|
||||
url,
|
||||
this.headersProvider,
|
||||
payload,
|
||||
);
|
||||
const traceInfos = (response.traces ?? []).map((trace) => TraceInfo.fromJson(trace));
|
||||
|
||||
let traces: Trace[];
|
||||
if (includeSpans) {
|
||||
const results = await Promise.all(
|
||||
traceInfos.map((traceInfo) => this.downloadTraceDataForSearch(traceInfo)),
|
||||
);
|
||||
traces = results.filter((trace): trace is Trace => trace !== undefined);
|
||||
} else {
|
||||
traces = traceInfos.map((traceInfo) => new Trace(traceInfo, new TraceData()));
|
||||
}
|
||||
|
||||
return {
|
||||
traces,
|
||||
nextPageToken: response.next_page_token || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch span data for a search result. Returns undefined when the data
|
||||
* cannot be downloaded (e.g. missing or corrupted) so the trace is dropped
|
||||
* from the results with a warning, matching the Python SDK.
|
||||
*/
|
||||
private async downloadTraceDataForSearch(traceInfo: TraceInfo): Promise<Trace | undefined> {
|
||||
try {
|
||||
const traceData = await this.artifactsClient.downloadTraceData(traceInfo);
|
||||
return new Trace(traceInfo, traceData);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to download trace data for trace ${traceInfo.traceId}:`, error);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload trace data to the artifact store.
|
||||
*/
|
||||
async uploadTraceData(traceInfo: TraceInfo, traceData: TraceData): Promise<void> {
|
||||
await this.artifactsClient.uploadTraceData(traceInfo, traceData);
|
||||
}
|
||||
|
||||
// === EXPERIMENT METHODS ===
|
||||
/**
|
||||
* Create a new experiment
|
||||
*/
|
||||
async createExperiment(
|
||||
name: string,
|
||||
artifactLocation?: string,
|
||||
tags?: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const url = CreateExperiment.getEndpoint(this.hostUrl);
|
||||
const payload: CreateExperiment.Request = { name, artifact_location: artifactLocation, tags };
|
||||
const response = await makeRequest<CreateExperiment.Response>(
|
||||
'POST',
|
||||
url,
|
||||
this.headersProvider,
|
||||
payload,
|
||||
);
|
||||
return response.experiment_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an experiment by ID, including its tags. Used to auto-resolve UC
|
||||
* trace destinations from a Databricks-linked experiment.
|
||||
*/
|
||||
async getExperiment(
|
||||
experimentId: string,
|
||||
): Promise<{ experimentId: string; name: string; tags: Record<string, string> } | null> {
|
||||
const url = GetExperiment.getEndpoint(this.hostUrl, experimentId);
|
||||
try {
|
||||
const response = await makeRequest<GetExperiment.Response>('GET', url, this.headersProvider);
|
||||
const exp = response.experiment;
|
||||
if (!exp?.experiment_id) {
|
||||
return null;
|
||||
}
|
||||
const tags: Record<string, string> = {};
|
||||
for (const tag of exp.tags ?? []) {
|
||||
tags[tag.key] = tag.value;
|
||||
}
|
||||
return { experimentId: exp.experiment_id, name: exp.name, tags };
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof MlflowHttpError &&
|
||||
(error.status === 404 || error.errorCode === 'RESOURCE_DOES_NOT_EXIST')
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an experiment by name.
|
||||
*/
|
||||
async getExperimentByName(name: string): Promise<{ experimentId: string; name: string } | null> {
|
||||
const url = GetExperimentByName.getEndpoint(this.hostUrl, name);
|
||||
try {
|
||||
const response = await makeRequest<GetExperimentByName.Response>(
|
||||
'GET',
|
||||
url,
|
||||
this.headersProvider,
|
||||
);
|
||||
if (!response.experiment?.experiment_id || !response.experiment?.name) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
experimentId: response.experiment.experiment_id,
|
||||
name: response.experiment.name,
|
||||
};
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof MlflowHttpError &&
|
||||
(error.status === 404 || error.errorCode === 'RESOURCE_DOES_NOT_EXIST')
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an experiment
|
||||
*/
|
||||
async deleteExperiment(experimentId: string): Promise<void> {
|
||||
const url = DeleteExperiment.getEndpoint(this.hostUrl);
|
||||
const payload: DeleteExperiment.Request = { experiment_id: experimentId };
|
||||
await makeRequest<void>('POST', url, this.headersProvider, payload);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export { MlflowClient } from './client';
|
||||
export type { SearchTracesOptions, SearchTracesResult } from './client';
|
||||
export { MlflowHttpError } from './utils';
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* MLflow API Request/Response Specifications
|
||||
*
|
||||
* This module defines the types and interfaces for MLflow API communication,
|
||||
* including request payloads and response structures for all trace-related endpoints.
|
||||
*/
|
||||
|
||||
import type { TraceInfo } from '../core/entities/trace_info';
|
||||
import type { SerializedTraceLocation } from '../core/entities/trace_location';
|
||||
import { ArtifactCredentialType } from './artifacts/databricks';
|
||||
|
||||
/**
|
||||
* Create a new TraceInfo entity in the backend.
|
||||
*/
|
||||
export namespace StartTraceV3 {
|
||||
export const getEndpoint = (host: string) => `${host}/api/3.0/mlflow/traces`;
|
||||
|
||||
export interface Request {
|
||||
trace: {
|
||||
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
|
||||
};
|
||||
}
|
||||
|
||||
export interface Response {
|
||||
trace: {
|
||||
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the TraceInfo entity for a given trace ID.
|
||||
*/
|
||||
export namespace GetTraceInfoV3 {
|
||||
export const getEndpoint = (host: string, traceId: string) =>
|
||||
`${host}/api/3.0/mlflow/traces/${traceId}`;
|
||||
|
||||
export interface Response {
|
||||
trace: {
|
||||
trace_info: Parameters<typeof TraceInfo.fromJson>[0];
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search trace metadata using the V3 traces API.
|
||||
*/
|
||||
export namespace SearchTracesV3 {
|
||||
export const getEndpoint = (host: string) => `${host}/api/3.0/mlflow/traces/search`;
|
||||
|
||||
export interface Request {
|
||||
locations: SerializedTraceLocation[];
|
||||
filter?: string;
|
||||
max_results?: number;
|
||||
order_by?: string[];
|
||||
page_token?: string;
|
||||
}
|
||||
|
||||
export interface Response {
|
||||
traces?: Parameters<typeof TraceInfo.fromJson>[0][];
|
||||
next_page_token?: string;
|
||||
}
|
||||
}
|
||||
|
||||
/** Create Experiment (used for testing) */
|
||||
export namespace CreateExperiment {
|
||||
export const getEndpoint = (host: string) => `${host}/api/2.0/mlflow/experiments/create`;
|
||||
|
||||
export interface Request {
|
||||
name?: string;
|
||||
artifact_location?: string;
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface Response {
|
||||
experiment_id: string;
|
||||
}
|
||||
}
|
||||
|
||||
/** Get Experiment (used for UC destination auto-resolution) */
|
||||
export namespace GetExperiment {
|
||||
export const getEndpoint = (host: string, experimentId: string) =>
|
||||
`${host}/api/2.0/mlflow/experiments/get?experiment_id=${encodeURIComponent(experimentId)}`;
|
||||
|
||||
export interface Response {
|
||||
experiment?: {
|
||||
experiment_id: string;
|
||||
name: string;
|
||||
artifact_location?: string;
|
||||
lifecycle_stage?: string;
|
||||
tags?: { key: string; value: string }[];
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Get Experiment By Name */
|
||||
export namespace GetExperimentByName {
|
||||
export const getEndpoint = (host: string, experimentName: string) =>
|
||||
`${host}/api/2.0/mlflow/experiments/get-by-name?experiment_name=${encodeURIComponent(experimentName)}`;
|
||||
|
||||
export interface Response {
|
||||
experiment?: {
|
||||
experiment_id: string;
|
||||
name: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete Experiment (used for testing) */
|
||||
export namespace DeleteExperiment {
|
||||
export const getEndpoint = (host: string) => `${host}/api/2.0/mlflow/experiments/delete`;
|
||||
|
||||
export interface Request {
|
||||
experiment_id: string;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create trace info using the V4 API path. Used for Databricks Unity Catalog
|
||||
* backed traces (UC schema or UC table prefix locations).
|
||||
*
|
||||
* Endpoint: POST /api/4.0/mlflow/traces/{location}/{otel_trace_id}/info
|
||||
* where `{location}` is "catalog.schema" or "catalog.schema.table_prefix",
|
||||
* and `{otel_trace_id}` is the hex OTel trace ID (no `trace:/<location>/` prefix).
|
||||
*/
|
||||
export namespace CreateTraceInfoV4 {
|
||||
export const getEndpoint = (host: string, location: string, otelTraceId: string) =>
|
||||
`${host}/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`;
|
||||
|
||||
// The Databricks RPC convention extracts `location_id` and `trace_info.trace_id`
|
||||
// from the URL path; the HTTP body is the serialized TraceInfo proto directly
|
||||
// (not wrapped in `{ trace_info: ... }`). Mirrors Python's
|
||||
// `message_to_json(trace_info.to_proto())`.
|
||||
export type Request = Parameters<typeof TraceInfo.fromJson>[0];
|
||||
|
||||
// Backend returns a TraceInfo proto serialized as JSON.
|
||||
export type Response = Parameters<typeof TraceInfo.fromJson>[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* OTLP span upload endpoint for Databricks Unity Catalog backed traces.
|
||||
*
|
||||
* Endpoint: POST /api/2.0/otel/v1/traces
|
||||
* Required header: `X-Databricks-UC-Table-Name: <fully_qualified_spans_table>`
|
||||
* Content-Type: application/x-protobuf (OTLP/HTTP+protobuf); the Databricks
|
||||
* endpoint does not accept the OTLP/HTTP+JSON form.
|
||||
*/
|
||||
export namespace ExportOtlpTraces {
|
||||
export const getEndpoint = (host: string) => `${host}/api/2.0/otel/v1/traces`;
|
||||
}
|
||||
|
||||
/**
|
||||
* OTLP span upload endpoint for OSS (non-Databricks) tracking servers.
|
||||
*
|
||||
* Endpoint: POST /v1/traces
|
||||
* Required header: `x-mlflow-experiment-id: <experiment_id>`
|
||||
* Content-Type: application/x-protobuf (OTLP/HTTP+protobuf).
|
||||
*/
|
||||
export namespace ExportOtlpTracesOss {
|
||||
export const getEndpoint = (host: string) => `${host}/v1/traces`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get credentials for uploading trace data to the artifact store. Only used for Databricks.
|
||||
*/
|
||||
export namespace GetCredentialsForTraceDataUpload {
|
||||
export const getEndpoint = (host: string, traceId: string) =>
|
||||
`${host}/api/2.0/mlflow/traces/${traceId}/credentials-for-data-upload`;
|
||||
|
||||
export interface Response {
|
||||
credential_info: {
|
||||
type: ArtifactCredentialType;
|
||||
signed_uri: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get credentials for downloading trace data from the artifact store. Only used for Databricks.
|
||||
*/
|
||||
export namespace GetCredentialsForTraceDataDownload {
|
||||
export const getEndpoint = (host: string, traceId: string) =>
|
||||
`${host}/api/2.0/mlflow/traces/${traceId}/credentials-for-data-download`;
|
||||
|
||||
export interface Response {
|
||||
credential_info: {
|
||||
type: ArtifactCredentialType;
|
||||
signed_uri: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { JSONBig } from '../core/utils/json';
|
||||
import { HeadersProvider } from '../auth';
|
||||
|
||||
const MAX_ERROR_BODY_LENGTH = 1000;
|
||||
|
||||
/**
|
||||
* Error thrown when an HTTP request to the MLflow backend returns a non-2xx response.
|
||||
* Carries the parsed status code and (when present) the `error_code` field from the
|
||||
* response body so callers can branch on status without matching error messages.
|
||||
*/
|
||||
export class MlflowHttpError extends Error {
|
||||
readonly status: number;
|
||||
readonly statusText: string;
|
||||
readonly body: string;
|
||||
readonly errorCode?: string;
|
||||
|
||||
constructor(status: number, statusText: string, body: string) {
|
||||
super(MlflowHttpError.formatMessage(status, statusText, body));
|
||||
this.name = 'MlflowHttpError';
|
||||
this.status = status;
|
||||
this.statusText = statusText;
|
||||
this.body = body;
|
||||
this.errorCode = MlflowHttpError.extractErrorCode(body);
|
||||
}
|
||||
|
||||
private static formatMessage(status: number, statusText: string, body: string): string {
|
||||
let message = `HTTP ${status}: ${statusText}`;
|
||||
if (body) {
|
||||
message +=
|
||||
body.length > MAX_ERROR_BODY_LENGTH
|
||||
? ` - ${body.substring(0, MAX_ERROR_BODY_LENGTH)}... (truncated)`
|
||||
: ` - ${body}`;
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
private static extractErrorCode(body: string): string | undefined {
|
||||
if (!body) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(body);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === 'object' &&
|
||||
'error_code' in parsed &&
|
||||
typeof (parsed as { error_code: unknown }).error_code === 'string'
|
||||
) {
|
||||
return (parsed as { error_code: string }).error_code;
|
||||
}
|
||||
} catch {
|
||||
// body is not JSON
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a request to the given URL with the given method, headers, body, and timeout.
|
||||
*
|
||||
* TODO: add retry logic for transient errors
|
||||
*/
|
||||
export async function makeRequest<T>(
|
||||
method: string,
|
||||
url: string,
|
||||
headerProvider: HeadersProvider,
|
||||
body?: any,
|
||||
timeout?: number,
|
||||
): Promise<T> {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout ?? getDefaultTimeout());
|
||||
const headers = await headerProvider();
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers: headers,
|
||||
body: body ? JSONBig.stringify(body) : undefined,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
let responseBody = '';
|
||||
try {
|
||||
responseBody = await response.text();
|
||||
} catch {
|
||||
// If we can't read the body, leave it empty
|
||||
}
|
||||
throw new MlflowHttpError(response.status, response.statusText, responseBody);
|
||||
}
|
||||
|
||||
// Handle empty responses (like DELETE operations)
|
||||
if (response.status === 204 || response.headers.get('content-length') === '0') {
|
||||
return {} as T;
|
||||
}
|
||||
|
||||
const responseText = await response.text();
|
||||
return JSONBig.parse(responseText) as T;
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (error instanceof MlflowHttpError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error(`Request timeout after ${timeout}ms`);
|
||||
}
|
||||
throw new Error(`API request failed: ${error.message}`);
|
||||
}
|
||||
throw new Error(`API request failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a raw (already-serialized) body — e.g. OTLP protobuf bytes — to `url`.
|
||||
* `extraHeaders` merge over the auth headers (to override `Content-Type` etc).
|
||||
* The response body is ignored; non-2xx throws {@link MlflowHttpError} so
|
||||
* callers can branch on `status` (e.g. 501 capability gating).
|
||||
*/
|
||||
export async function makeRawRequest(
|
||||
method: string,
|
||||
url: string,
|
||||
headerProvider: HeadersProvider,
|
||||
body: BodyInit,
|
||||
extraHeaders: Record<string, string> = {},
|
||||
timeout?: number,
|
||||
): Promise<void> {
|
||||
const controller = new AbortController();
|
||||
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
||||
const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout);
|
||||
|
||||
const headers: Record<string, string> = { ...(await headerProvider()) };
|
||||
for (const [key, value] of Object.entries(extraHeaders)) {
|
||||
const existingKey = Object.keys(headers).find((h) => h.toLowerCase() === key.toLowerCase());
|
||||
if (existingKey) {
|
||||
delete headers[existingKey];
|
||||
}
|
||||
headers[key] = value;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers,
|
||||
body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
let responseBody = '';
|
||||
try {
|
||||
responseBody = await response.text();
|
||||
} catch {
|
||||
// If we can't read the body, leave it empty
|
||||
}
|
||||
throw new MlflowHttpError(response.status, response.statusText, responseBody);
|
||||
}
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (error instanceof MlflowHttpError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error(`Request timeout after ${effectiveTimeout}ms`);
|
||||
}
|
||||
throw new Error(`API request failed: ${error.message}`);
|
||||
}
|
||||
throw new Error(`API request failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getDefaultTimeout(): number {
|
||||
const envTimeout = process.env.MLFLOW_HTTP_REQUEST_TIMEOUT;
|
||||
if (envTimeout) {
|
||||
const timeout = parseInt(envTimeout, 10);
|
||||
if (!isNaN(timeout) && timeout > 0) {
|
||||
return timeout;
|
||||
}
|
||||
}
|
||||
return 30000; // Default 30 seconds
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
import { trace as otelTrace, context, Span as ApiSpan, INVALID_TRACEID } from '@opentelemetry/api';
|
||||
import { Span as OTelSpan } from '@opentelemetry/sdk-trace-node';
|
||||
import { DEFAULT_SPAN_NAME, SpanLogLevel, SpanType, TraceMetadataKey } from './constants';
|
||||
import { createMlflowSpan, LiveSpan, NoOpSpan } from './entities/span';
|
||||
import { getTracer } from './provider';
|
||||
import { InMemoryTraceManager } from './trace_manager';
|
||||
import { convertNanoSecondsToHrTime, mapArgsToObject } from './utils';
|
||||
import { SpanStatusCode } from './entities/span_status';
|
||||
import { isTracingEnabledInContext } from './context';
|
||||
|
||||
/*
|
||||
* Options for starting a span
|
||||
*/
|
||||
export interface SpanOptions {
|
||||
/**
|
||||
* The name of the span.
|
||||
*/
|
||||
name: string;
|
||||
|
||||
/**
|
||||
* The type of the span, e.g., `LLM`, `TOOL`, `RETRIEVER`, etc.
|
||||
*/
|
||||
spanType?: SpanType;
|
||||
|
||||
/**
|
||||
* The inputs of the span.
|
||||
*/
|
||||
inputs?: any;
|
||||
|
||||
/**
|
||||
* The attributes of the span.
|
||||
*/
|
||||
attributes?: Record<string, any>;
|
||||
|
||||
/**
|
||||
* The start time of the span in nanoseconds. If not provided, the current time will be used.
|
||||
*/
|
||||
startTimeNs?: number;
|
||||
|
||||
/**
|
||||
* The parent span object. If not provided, the span is considered a root span.
|
||||
*/
|
||||
parent?: LiveSpan;
|
||||
|
||||
/**
|
||||
* Optional severity level to attach to the span. Accepts a SpanLogLevel
|
||||
* enum value or its name (e.g. "INFO", "DEBUG"). If not provided, the span
|
||||
* level is resolved from the span type at end time.
|
||||
*/
|
||||
logLevel?: SpanLogLevel | string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for tracing a function
|
||||
*/
|
||||
export interface TraceOptions
|
||||
extends Omit<SpanOptions, 'parent' | 'startTimeNs' | 'inputs' | 'name'> {
|
||||
/**
|
||||
* The name of the span.
|
||||
*/
|
||||
name?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new span with the given name and span type.
|
||||
*
|
||||
* This function does NOT attach the created span to the current context. To create
|
||||
* nested spans, you need to set the parent span explicitly in the `parent` field.
|
||||
* The span must be ended by calling `end` method on the returned Span object.
|
||||
*
|
||||
* @param options Optional span options (name, spanType, inputs, attributes, startTimeNs, parent)
|
||||
* @returns The created span object.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const span = startSpan({
|
||||
* name: 'my_span',
|
||||
* spanType: 'LLM',
|
||||
* inputs: {
|
||||
* prompt: 'Hello, world!'
|
||||
* }
|
||||
* });
|
||||
*
|
||||
* // Do something
|
||||
*
|
||||
* // End the span
|
||||
* span.end({
|
||||
* status: 'OK',
|
||||
* outputs: {
|
||||
* response: 'Hello, world!'
|
||||
* }
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
*/
|
||||
export function startSpan(options: SpanOptions): LiveSpan {
|
||||
if (isTracingEnabledInContext() === false) {
|
||||
return new NoOpSpan();
|
||||
}
|
||||
|
||||
try {
|
||||
const tracer = getTracer('default');
|
||||
|
||||
// If parent is provided, use it as the parent spanAdd commentMore actions
|
||||
const parentContext = options.parent
|
||||
? otelTrace.setSpan(context.active(), options.parent._span)
|
||||
: context.active();
|
||||
|
||||
// Convert startTimeNs to OTel format
|
||||
const startTime = options.startTimeNs
|
||||
? convertNanoSecondsToHrTime(options.startTimeNs)
|
||||
: undefined;
|
||||
|
||||
const otelSpan = tracer.startSpan(
|
||||
options.name,
|
||||
{ startTime: startTime },
|
||||
parentContext,
|
||||
) as OTelSpan;
|
||||
|
||||
// SpanProcessor should have already registered the mlflow span
|
||||
return getMlflowSpan(otelSpan, options);
|
||||
} catch (error) {
|
||||
console.warn('Failed to start span', error);
|
||||
return new NoOpSpan();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function within a span context. The span is automatically started before
|
||||
* the function executes and ended after it completes (or throws an error).
|
||||
*
|
||||
* This function uses OpenTelemetry's active span context to automatically manage
|
||||
* parent-child relationships between spans.
|
||||
*
|
||||
* This function supports two usage patterns:
|
||||
* 1. Inline: `withSpan((span) => { ... })` - span properties set within the callback
|
||||
* 2. Options: `withSpan((span) => { ... }, { name: 'test', ... })` - span properties set via options object
|
||||
*
|
||||
* @param callback The callback function to execute within the span context
|
||||
* @param options Optional span options (name, spanType, inputs, attributes, startTimeNs)
|
||||
* @returns The result of the callback function
|
||||
*/
|
||||
export function withSpan<T>(
|
||||
callback: (span: LiveSpan) => T | Promise<T>,
|
||||
options?: Omit<SpanOptions, 'parent'>,
|
||||
): T | Promise<T> {
|
||||
if (isTracingEnabledInContext() === false) {
|
||||
return callback(new NoOpSpan());
|
||||
}
|
||||
|
||||
const spanOptions: Omit<SpanOptions, 'parent'> = options ?? { name: DEFAULT_SPAN_NAME };
|
||||
|
||||
// Generate a default span name if not provided
|
||||
const spanName = spanOptions.name || DEFAULT_SPAN_NAME;
|
||||
|
||||
const tracer = getTracer('default');
|
||||
|
||||
// Convert startTimeNs to OTel format if provided
|
||||
const startTime = spanOptions.startTimeNs
|
||||
? convertNanoSecondsToHrTime(spanOptions.startTimeNs)
|
||||
: undefined;
|
||||
|
||||
// Use startActiveSpan to automatically manage context and parent-child relationships
|
||||
return tracer.startActiveSpan(spanName, { startTime }, (otelSpan: ApiSpan) => {
|
||||
let mlflowSpan: LiveSpan | NoOpSpan;
|
||||
|
||||
try {
|
||||
// SpanProcessor should have already registered the mlflow span
|
||||
mlflowSpan = getMlflowSpan(otelSpan as OTelSpan, spanOptions);
|
||||
} catch (error) {
|
||||
console.debug('Failed to create and register MLflow span', error);
|
||||
mlflowSpan = new NoOpSpan();
|
||||
}
|
||||
|
||||
// Expression function to handle errors consistently
|
||||
const handleError = (error: Error): never => {
|
||||
try {
|
||||
mlflowSpan.setStatus(SpanStatusCode.ERROR, error.message);
|
||||
mlflowSpan.recordException(error);
|
||||
mlflowSpan.end();
|
||||
} catch (error) {
|
||||
console.debug('Failed to set status or record exception on MLflow span', error);
|
||||
}
|
||||
throw error;
|
||||
};
|
||||
|
||||
try {
|
||||
// Execute the callback with the span
|
||||
const result = callback(mlflowSpan);
|
||||
|
||||
// Handle both sync and async results
|
||||
if (result instanceof Promise) {
|
||||
return result
|
||||
.then((value) => {
|
||||
// Set outputs if they are not already set
|
||||
if (mlflowSpan.outputs === undefined) {
|
||||
mlflowSpan.setOutputs(value);
|
||||
}
|
||||
mlflowSpan.end();
|
||||
return value;
|
||||
})
|
||||
.catch(handleError);
|
||||
} else {
|
||||
// Synchronous execution
|
||||
if (mlflowSpan.outputs === undefined) {
|
||||
mlflowSpan.setOutputs(result);
|
||||
}
|
||||
mlflowSpan.end();
|
||||
return result;
|
||||
}
|
||||
} catch (error) {
|
||||
// Handle synchronous errors
|
||||
return handleError(error as Error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getMlflowSpan(otelSpan: OTelSpan, options: SpanOptions): LiveSpan | NoOpSpan {
|
||||
// MlflowSpanProcessor should have already registered the span
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const mlflowTraceId = traceManager.getMlflowTraceIdFromOtelId(otelSpan.spanContext().traceId);
|
||||
const mlflowSpan =
|
||||
traceManager.getSpan(mlflowTraceId, otelSpan.spanContext().spanId) || new NoOpSpan();
|
||||
|
||||
// Set custom properties to the span
|
||||
if (options.inputs) {
|
||||
mlflowSpan.setInputs(options.inputs);
|
||||
}
|
||||
if (options.attributes) {
|
||||
mlflowSpan.setAttributes(options.attributes);
|
||||
}
|
||||
if (options.spanType) {
|
||||
mlflowSpan.setSpanType(options.spanType);
|
||||
}
|
||||
if (options.logLevel !== undefined) {
|
||||
mlflowSpan.setLogLevel(options.logLevel);
|
||||
}
|
||||
return mlflowSpan;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to create and register an MLflow span from an OpenTelemetry span
|
||||
* @param otelSpan The OpenTelemetry span
|
||||
* @param spanType The MLflow span type
|
||||
* @param inputs Optional inputs to set on the span
|
||||
* @param attributes Optional attributes to set on the span
|
||||
* @returns The created and registered MLflow LiveSpan
|
||||
*/
|
||||
export function createAndRegisterMlflowSpan(
|
||||
otelSpan: OTelSpan | ApiSpan,
|
||||
spanType?: SpanType,
|
||||
inputs?: any,
|
||||
attributes?: Record<string, any>,
|
||||
): LiveSpan {
|
||||
// Get the MLflow trace ID from the OpenTelemetry trace ID
|
||||
const otelTraceId = otelSpan.spanContext().traceId;
|
||||
const traceId =
|
||||
InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId) || otelTraceId;
|
||||
|
||||
// Create the MLflow span from the OTel span
|
||||
const mlflowSpan = createMlflowSpan(otelSpan, traceId, spanType) as LiveSpan;
|
||||
|
||||
// Set initial properties if provided
|
||||
if (inputs) {
|
||||
mlflowSpan.setInputs(inputs);
|
||||
}
|
||||
|
||||
if (attributes) {
|
||||
mlflowSpan.setAttributes(attributes);
|
||||
}
|
||||
|
||||
// Register the span with the trace manager
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
traceManager.registerSpan(mlflowSpan);
|
||||
|
||||
return mlflowSpan;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a traced version of a function or decorator for tracing class methods.
|
||||
*
|
||||
* When used as a function wrapper, the span will automatically capture:
|
||||
* - The function inputs
|
||||
* - The function outputs
|
||||
* - The function name as the span name
|
||||
* - The function execution time
|
||||
* - Any exception thrown by the function
|
||||
*
|
||||
* When used as a decorator, it preserves the `this` context for class methods.
|
||||
*
|
||||
* Note: Typescript decorator is still in experimental stage.
|
||||
*
|
||||
* @param func The function to trace (when used as function wrapper)
|
||||
* @param options Optional trace options including name, spanType, and attributes
|
||||
* @returns The traced function or method decorator
|
||||
*
|
||||
* @example
|
||||
* // Function wrapper with no options
|
||||
* const tracedFunc = trace(myFunc);
|
||||
*
|
||||
* @example
|
||||
* // Function wrapper with options
|
||||
* const tracedFunc = trace(myFunc, { name: 'custom_span_name', spanType: 'LLM' });
|
||||
*
|
||||
* @example
|
||||
* // Decorator with no options
|
||||
* class MyService {
|
||||
* @trace()
|
||||
* async processData(data: any) {
|
||||
* return processedData;
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* @example
|
||||
* // Decorator with options
|
||||
* class MyService {
|
||||
* @trace({ name: 'custom_span', spanType: SpanType.LLM })
|
||||
* async generateText(prompt: string) {
|
||||
* return generatedText;
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
export function trace(options?: TraceOptions): any;
|
||||
export function trace<T extends (...args: any[]) => any>(func: T, options?: TraceOptions): T;
|
||||
export function trace<T extends (...args: any[]) => any>(
|
||||
funcOrOptions?: T | TraceOptions,
|
||||
options?: TraceOptions,
|
||||
): any {
|
||||
// Check if this is being used as a decorator (no function provided, or options provided)
|
||||
if (typeof funcOrOptions !== 'function') {
|
||||
const decoratorOptions = funcOrOptions;
|
||||
|
||||
// Return a method decorator that supports both old and new TypeScript decorator syntax
|
||||
return function (...args: any[]) {
|
||||
let originalMethod: Function;
|
||||
let methodName: string;
|
||||
|
||||
// Check if using new TypeScript 5.0+ decorator syntax
|
||||
const isNewSyntax =
|
||||
args.length === 2 && typeof args[1] === 'object' && !Array.isArray(args[1]);
|
||||
|
||||
if (isNewSyntax) {
|
||||
// New syntax: (originalMethod, context)
|
||||
originalMethod = args[0];
|
||||
const context = args[1] as ClassMethodDecoratorContext;
|
||||
methodName = String(context.name);
|
||||
} else {
|
||||
// Old syntax: (target, propertyKey, descriptor)
|
||||
const desc = args[2] as PropertyDescriptor;
|
||||
originalMethod = desc.value;
|
||||
methodName = String(args[1]);
|
||||
}
|
||||
|
||||
// Create the traced method wrapper
|
||||
const tracedMethod = function (this: any, ...args: any[]) {
|
||||
let inputs: any;
|
||||
try {
|
||||
inputs = mapArgsToObject(originalMethod, args);
|
||||
} catch (error) {
|
||||
console.debug('Failed to map arguments values to names', error);
|
||||
inputs = args;
|
||||
}
|
||||
|
||||
const spanOptions: Omit<SpanOptions, 'parent'> = {
|
||||
name: decoratorOptions?.name || originalMethod.name || methodName,
|
||||
spanType: decoratorOptions?.spanType,
|
||||
attributes: decoratorOptions?.attributes,
|
||||
inputs,
|
||||
logLevel: decoratorOptions?.logLevel,
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan((_span) => {
|
||||
// Call the original method with the preserved `this` context
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return originalMethod.apply(this, args) as ReturnType<T>;
|
||||
}, spanOptions) as ReturnType<T>;
|
||||
};
|
||||
|
||||
// Return the appropriate value based on the decorator syntax
|
||||
if (isNewSyntax) {
|
||||
return tracedMethod as T;
|
||||
} else {
|
||||
const descriptor = args[2] as PropertyDescriptor;
|
||||
descriptor.value = tracedMethod;
|
||||
// Preserve the original method's properties
|
||||
Object.defineProperty(descriptor.value, 'length', { value: originalMethod.length });
|
||||
Object.defineProperty(descriptor.value, 'name', { value: originalMethod.name });
|
||||
|
||||
return descriptor;
|
||||
}
|
||||
};
|
||||
} else {
|
||||
// This is the function-based usage (existing behavior)
|
||||
return traceFunction(funcOrOptions, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal function to handle function-based tracing (non-decorator usage)
|
||||
*/
|
||||
function traceFunction<T extends (...args: any[]) => any>(func: T, options?: TraceOptions): T {
|
||||
// Create a wrapper function that preserves the original function's properties
|
||||
const wrapper = function (this: any, ...args: Parameters<T>): ReturnType<T> {
|
||||
let inputs: any;
|
||||
try {
|
||||
inputs = mapArgsToObject(func, args);
|
||||
} catch (error) {
|
||||
console.debug('Failed to map arguments values to names', error);
|
||||
inputs = args;
|
||||
}
|
||||
|
||||
const spanOptions: Omit<SpanOptions, 'parent'> = {
|
||||
name: options?.name || func.name || DEFAULT_SPAN_NAME,
|
||||
spanType: options?.spanType,
|
||||
attributes: options?.attributes,
|
||||
inputs: inputs,
|
||||
logLevel: options?.logLevel,
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan((_span) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return func.apply(this, args);
|
||||
}, spanOptions) as ReturnType<T>;
|
||||
};
|
||||
|
||||
// Preserve function properties
|
||||
Object.defineProperty(wrapper, 'length', { value: func.length });
|
||||
Object.defineProperty(wrapper, 'name', { value: func.name });
|
||||
|
||||
// Copy any additional properties from the original function
|
||||
for (const prop in func) {
|
||||
if (Object.prototype.hasOwnProperty.call(func, prop)) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
(wrapper as any)[prop] = (func as any)[prop];
|
||||
}
|
||||
}
|
||||
|
||||
return wrapper as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last active trace ID.
|
||||
* @returns The last active trace ID.
|
||||
*/
|
||||
export function getLastActiveTraceId(): string | undefined {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
return traceManager.lastActiveTraceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current active span in the global context.
|
||||
*
|
||||
* This only works when the span is created with fluent APIs like `@trace` or
|
||||
* `withSpan`. If a span is created with the `startSpan` API, it won't be
|
||||
* attached to the global context so this function will not return it.
|
||||
*
|
||||
* @returns The current active span if exists, otherwise null.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const tracedFunc = trace(() => {
|
||||
* const span = getCurrentActiveSpan();
|
||||
* span?.setAttribute("key", "value");
|
||||
* return 0;
|
||||
* });
|
||||
*
|
||||
* tracedFunc();
|
||||
* ```
|
||||
*/
|
||||
export function getCurrentActiveSpan(): LiveSpan | null {
|
||||
const otelSpan = otelTrace.getActiveSpan();
|
||||
|
||||
// If no active span or it's a NonRecordingSpan, return undefined
|
||||
if (!otelSpan || otelSpan.spanContext().traceId === INVALID_TRACEID) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const otelTraceId = otelSpan.spanContext().traceId;
|
||||
const mlflowTraceId = traceManager.getMlflowTraceIdFromOtelId(otelTraceId);
|
||||
|
||||
if (!mlflowTraceId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const spanId = otelSpan.spanContext().spanId;
|
||||
return traceManager.getSpan(mlflowTraceId, spanId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for updating the current trace
|
||||
*/
|
||||
export interface UpdateCurrentTraceOptions {
|
||||
/**
|
||||
* A dictionary of tags to update the trace with. Tags are designed for mutable values
|
||||
* that can be updated after the trace is created via MLflow UI or API.
|
||||
*/
|
||||
tags?: Record<string, string>;
|
||||
|
||||
/**
|
||||
* A dictionary of metadata to update the trace with. Metadata cannot be updated
|
||||
* once the trace is logged. It is suitable for recording immutable values like the
|
||||
* git hash of the application version that produced the trace.
|
||||
*/
|
||||
metadata?: Record<string, string>;
|
||||
|
||||
/**
|
||||
* Session ID to associate with the trace. Stored as metadata under the
|
||||
* `mlflow.trace.session` key.
|
||||
*/
|
||||
sessionId?: string;
|
||||
|
||||
/**
|
||||
* User identifier to associate with the trace. Stored as metadata under the
|
||||
* `mlflow.trace.user` key.
|
||||
*/
|
||||
user?: string;
|
||||
|
||||
/**
|
||||
* Client supplied request ID to associate with the trace. This is useful for linking
|
||||
* the trace back to a specific request in your application or external system.
|
||||
*/
|
||||
clientRequestId?: string;
|
||||
|
||||
/**
|
||||
* A preview of the request to be shown in the Trace list view in the UI.
|
||||
* By default, MLflow will truncate the trace request naively by limiting the length.
|
||||
* This parameter allows you to specify a custom preview string.
|
||||
*/
|
||||
requestPreview?: string;
|
||||
|
||||
/**
|
||||
* A preview of the response to be shown in the Trace list view in the UI.
|
||||
* By default, MLflow will truncate the trace response naively by limiting the length.
|
||||
* This parameter allows you to specify a custom preview string.
|
||||
*/
|
||||
responsePreview?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the current active trace with the given options.
|
||||
*
|
||||
* You can use this function either within a function decorated with `@trace` or
|
||||
* within the scope of the `withSpan` context. If there is no active trace found,
|
||||
* this function will log a warning and return.
|
||||
*
|
||||
* @param options Options for updating the trace
|
||||
*
|
||||
* @example
|
||||
* Using within a function decorated with `@trace`:
|
||||
* ```typescript
|
||||
* const myFunc = trace((x: number) => {
|
||||
* updateCurrentTrace({ tags: { fruit: "apple" }, clientRequestId: "req-12345" });
|
||||
* return x + 1;
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* @example
|
||||
* Using within the `withSpan` context:
|
||||
* ```typescript
|
||||
* withSpan((span) => {
|
||||
* updateCurrentTrace({ tags: { fruit: "apple" }, clientRequestId: "req-12345" });
|
||||
* }, { name: "span" });
|
||||
* ```
|
||||
*
|
||||
* @example
|
||||
* Updating user, session, and source information of the trace:
|
||||
* ```typescript
|
||||
* updateCurrentTrace({
|
||||
* sessionId: "session-4f855da00427",
|
||||
* user: "user-id-cc156f29bcfb",
|
||||
* metadata: {
|
||||
* "mlflow.source.name": "inference.ts",
|
||||
* "mlflow.source.git.commit": "1234567890",
|
||||
* "mlflow.source.git.repoURL": "https://github.com/mlflow/mlflow"
|
||||
* }
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function updateCurrentTrace({
|
||||
tags,
|
||||
metadata,
|
||||
sessionId,
|
||||
user,
|
||||
clientRequestId,
|
||||
requestPreview,
|
||||
responsePreview,
|
||||
}: UpdateCurrentTraceOptions): void {
|
||||
const activeSpan = getCurrentActiveSpan();
|
||||
|
||||
if (!activeSpan) {
|
||||
console.warn(
|
||||
'No active trace found. Please create a span using `withSpan` or ' +
|
||||
'`@trace` before calling `updateCurrentTrace`.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate string parameters
|
||||
if (requestPreview !== undefined && typeof requestPreview !== 'string') {
|
||||
throw new Error('The `requestPreview` parameter must be a string.');
|
||||
}
|
||||
if (responsePreview !== undefined && typeof responsePreview !== 'string') {
|
||||
throw new Error('The `responsePreview` parameter must be a string.');
|
||||
}
|
||||
|
||||
// Update trace info for the trace stored in-memory
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.getTrace(activeSpan.traceId);
|
||||
|
||||
if (!trace) {
|
||||
console.warn(`Trace ${activeSpan.traceId} does not exist or already finished.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Inject sessionId and user into metadata
|
||||
const mergedMetadata = { ...metadata };
|
||||
if (sessionId !== undefined) {
|
||||
mergedMetadata[TraceMetadataKey.TRACE_SESSION] = sessionId;
|
||||
}
|
||||
if (user !== undefined) {
|
||||
mergedMetadata[TraceMetadataKey.TRACE_USER] = user;
|
||||
}
|
||||
|
||||
// Update trace info properties
|
||||
if (requestPreview !== undefined) {
|
||||
trace.info.requestPreview = requestPreview;
|
||||
}
|
||||
if (responsePreview !== undefined) {
|
||||
trace.info.responsePreview = responsePreview;
|
||||
}
|
||||
if (tags !== undefined) {
|
||||
Object.assign(trace.info.tags, tags);
|
||||
}
|
||||
if (Object.keys(mergedMetadata).length > 0) {
|
||||
Object.assign(trace.info.traceMetadata, mergedMetadata);
|
||||
}
|
||||
if (clientRequestId !== undefined) {
|
||||
trace.info.clientRequestId = String(clientRequestId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { initializeSDK } from './provider';
|
||||
import { AuthProvider, createAuthProvider, isDatabricksUri } from '../auth';
|
||||
|
||||
/**
|
||||
* User-facing shape for specifying a Databricks Unity Catalog trace location.
|
||||
*
|
||||
* All three fields are required. Unlike Python's
|
||||
* `mlflow.set_experiment(trace_location=UnityCatalog(...))`, the TS SDK does
|
||||
* not upsert the underlying UC trace location, so it can't default
|
||||
* `tablePrefix` to anything sensible - the customer must point at a
|
||||
* trace location that is already provisioned in the workspace.
|
||||
*/
|
||||
export interface UnityCatalogLocationOptions {
|
||||
catalogName: string;
|
||||
schemaName: string;
|
||||
tablePrefix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that a URI has a proper protocol (http or https)
|
||||
* @param uri The URI to validate
|
||||
* @returns true if valid, false otherwise
|
||||
*/
|
||||
function isValidHttpUri(uri: string): boolean {
|
||||
try {
|
||||
const url = new URL(uri);
|
||||
return url.protocol === 'http:' || url.protocol === 'https:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration options for the MLflow tracing SDK
|
||||
*/
|
||||
export interface MLflowTracingConfig {
|
||||
/**
|
||||
* The host where traces will be logged.
|
||||
* Can be:
|
||||
* - HTTP URI for MLflow trace server backend
|
||||
* - "databricks" (uses default profile)
|
||||
* - "databricks://profile" (uses specific profile)
|
||||
*/
|
||||
trackingUri: string;
|
||||
|
||||
/**
|
||||
* The experiment ID where traces will be logged
|
||||
*/
|
||||
experimentId: string;
|
||||
|
||||
/**
|
||||
* The location of the Databricks config file, default to ~/.databrickscfg
|
||||
*/
|
||||
databricksConfigPath?: string;
|
||||
|
||||
/**
|
||||
* The Databricks host. If not provided, the host will be read from the Databricks config file.
|
||||
* @deprecated For Databricks auth, prefer using environment variables or config profiles.
|
||||
* The auth system will automatically resolve credentials.
|
||||
*/
|
||||
host?: string;
|
||||
|
||||
/**
|
||||
* The Databricks token. If not provided, the token will be read from the Databricks config file.
|
||||
* @deprecated For Databricks auth, prefer using environment variables or config profiles.
|
||||
* The auth system will automatically resolve credentials.
|
||||
*/
|
||||
databricksToken?: string;
|
||||
|
||||
/**
|
||||
* The tracking server username for basic auth.
|
||||
*/
|
||||
trackingServerUsername?: string;
|
||||
|
||||
/**
|
||||
* The tracking server password for basic auth.
|
||||
*/
|
||||
trackingServerPassword?: string;
|
||||
|
||||
/**
|
||||
* Bearer token for OSS MLflow tracking servers.
|
||||
* Can also be set via MLFLOW_TRACKING_TOKEN environment variable.
|
||||
*/
|
||||
trackingServerToken?: string;
|
||||
|
||||
/**
|
||||
* Workspace name for MLflow servers with workspaces enabled (OSS/non-Databricks only).
|
||||
* Sets the X-MLFLOW-WORKSPACE header on requests to self-hosted MLflow.
|
||||
* Can also be set via MLFLOW_WORKSPACE environment variable (takes precedence).
|
||||
*/
|
||||
workspace?: string;
|
||||
|
||||
/**
|
||||
* Optional Databricks Unity Catalog trace location. When provided, the SDK
|
||||
* generates V4 trace IDs and persists trace tags / metadata via the V4
|
||||
* `CreateTraceInfo` endpoint, mirroring Python's
|
||||
* `mlflow.set_experiment(trace_location=UnityCatalog(...))`. When omitted,
|
||||
* traces use the V3 experiment-backed path.
|
||||
*
|
||||
* The UC trace location must already be provisioned in the workspace; the
|
||||
* TS SDK does not upsert it.
|
||||
*/
|
||||
traceLocation?: UnityCatalogLocationOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization options for the MLflow tracing SDK. The trackingUri and experimentId
|
||||
* can be omitted and will be resolved from environment variables when available. Since
|
||||
* this is used on the client side, we need to make sure the trackingUri and experimentId
|
||||
* are required.
|
||||
*/
|
||||
export type MLflowTracingInitOptions = Partial<MLflowTracingConfig>;
|
||||
|
||||
/**
|
||||
* Global configuration state
|
||||
*/
|
||||
let globalConfig: MLflowTracingConfig | null = null;
|
||||
|
||||
/**
|
||||
* Global authentication provider
|
||||
*/
|
||||
let globalAuthProvider: AuthProvider | null = null;
|
||||
|
||||
/**
|
||||
* Configure the MLflow tracing SDK with tracking location settings.
|
||||
* This must be called before using other tracing functions.
|
||||
*
|
||||
* Call once per process. Calling `init()` again will tear down the existing
|
||||
* OpenTelemetry SDK and start a fresh one, but the underlying NodeSDK
|
||||
* shutdown is asynchronous and may race with subsequent tracer registration;
|
||||
* configuration is intended to be set up once at startup, not reconfigured
|
||||
* mid-process.
|
||||
*
|
||||
* ## Authentication
|
||||
*
|
||||
* ### Databricks (trackingUri = "databricks" or "databricks://profile")
|
||||
*
|
||||
* Authentication is handled automatically by the Databricks SDK. It supports:
|
||||
* - Personal Access Tokens (DATABRICKS_TOKEN env var)
|
||||
* - OAuth M2M service principals (DATABRICKS_CLIENT_ID/DATABRICKS_CLIENT_SECRET)
|
||||
* - Azure CLI / Azure MSI / Azure Client Secret
|
||||
* - Google Cloud credentials
|
||||
* - Config file profiles (~/.databrickscfg)
|
||||
*
|
||||
* For Databricks Apps, credentials are automatically injected by the runtime.
|
||||
*
|
||||
* ### OSS MLflow (trackingUri = HTTP URL)
|
||||
*
|
||||
* Authentication is resolved in priority order:
|
||||
* 1. Basic Auth (trackingServerUsername/Password or MLFLOW_TRACKING_USERNAME/PASSWORD)
|
||||
* 2. Bearer Token (trackingServerToken or MLFLOW_TRACKING_TOKEN)
|
||||
* 3. No Auth
|
||||
*
|
||||
* @param config Configuration object with trackingUri and experimentId
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { init, withSpan } from '@mlflow/core';
|
||||
*
|
||||
* // Option 1: Use MLflow Tracking Server
|
||||
* init({
|
||||
* trackingUri: "http://localhost:5000",
|
||||
* experimentId: "123456789"
|
||||
* });
|
||||
*
|
||||
* // Option 2: Use default Databricks profile from ~/.databrickscfg
|
||||
* init({
|
||||
* trackingUri: "databricks",
|
||||
* experimentId: "123456789"
|
||||
* });
|
||||
*
|
||||
* // Option 3: Use specific Databricks profile
|
||||
* init({
|
||||
* trackingUri: "databricks://my-profile",
|
||||
* experimentId: "123456789"
|
||||
* });
|
||||
*
|
||||
* // Option 4: Custom config file path
|
||||
* init({
|
||||
* trackingUri: "databricks",
|
||||
* experimentId: "123456789",
|
||||
* databricksConfigPath: "/path/to/my/databrickscfg"
|
||||
* });
|
||||
*
|
||||
* // Option 5: Override with explicit host/token (backwards compatibility)
|
||||
* init({
|
||||
* trackingUri: "databricks",
|
||||
* experimentId: "123456789",
|
||||
* host: "https://my-workspace.databricks.com",
|
||||
* databricksToken: "my-token"
|
||||
* });
|
||||
*
|
||||
* // Option 6: OSS MLflow with basic auth
|
||||
* init({
|
||||
* trackingUri: "http://localhost:5000",
|
||||
* experimentId: "123456789",
|
||||
* trackingServerUsername: "user",
|
||||
* trackingServerPassword: "pass"
|
||||
* });
|
||||
*
|
||||
* // Option 7: OSS MLflow with bearer token
|
||||
* init({
|
||||
* trackingUri: "http://localhost:5000",
|
||||
* experimentId: "123456789",
|
||||
* trackingServerToken: "my-token"
|
||||
* });
|
||||
*
|
||||
* // Option 8: Databricks Unity Catalog trace location.
|
||||
* // Mirrors Python's `mlflow.set_experiment(..., trace_location=UnityCatalog(...))`.
|
||||
* // The UC trace location must already be provisioned for this workspace.
|
||||
* init({
|
||||
* trackingUri: "databricks",
|
||||
* experimentId: "123456789",
|
||||
* traceLocation: {
|
||||
* catalogName: "my_catalog",
|
||||
* schemaName: "my_schema",
|
||||
* tablePrefix: "my_prefix",
|
||||
* },
|
||||
* });
|
||||
*
|
||||
* // Now you can use tracing functions
|
||||
* function add(a: number, b: number) {
|
||||
* return withSpan(
|
||||
* { name: 'add', inputs: { a, b } },
|
||||
* (span) => {
|
||||
* const result = a + b;
|
||||
* span.setOutputs({ result });
|
||||
* return result;
|
||||
* }
|
||||
* );
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export function init(config: MLflowTracingInitOptions): void {
|
||||
const trackingUri = config.trackingUri ?? process.env.MLFLOW_TRACKING_URI;
|
||||
const experimentId = config.experimentId ?? process.env.MLFLOW_EXPERIMENT_ID;
|
||||
|
||||
if (!trackingUri) {
|
||||
throw new Error(
|
||||
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
|
||||
);
|
||||
}
|
||||
|
||||
if (!experimentId) {
|
||||
throw new Error(
|
||||
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof trackingUri !== 'string') {
|
||||
throw new Error('trackingUri must be a string');
|
||||
}
|
||||
|
||||
if (typeof experimentId !== 'string') {
|
||||
throw new Error('experimentId must be a string');
|
||||
}
|
||||
|
||||
const databricksConfigPath =
|
||||
config.databricksConfigPath ?? path.join(os.homedir(), '.databrickscfg');
|
||||
|
||||
// Validate non-Databricks URIs
|
||||
if (!isDatabricksUri(trackingUri) && !isValidHttpUri(trackingUri)) {
|
||||
throw new Error(
|
||||
`Invalid trackingUri: '${trackingUri}'. Must be a valid HTTP or HTTPS URL, or 'databricks' / 'databricks://<profile>'.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Create the authentication provider - this is the single source of truth
|
||||
// for credential resolution. It handles env vars, config files, OAuth, etc.
|
||||
globalAuthProvider = createAuthProvider({
|
||||
trackingUri,
|
||||
host: config.host,
|
||||
databricksToken: config.databricksToken,
|
||||
databricksConfigPath,
|
||||
trackingServerUsername: config.trackingServerUsername,
|
||||
trackingServerPassword: config.trackingServerPassword,
|
||||
trackingServerToken: config.trackingServerToken,
|
||||
workspace: config.workspace,
|
||||
});
|
||||
|
||||
// Build effective config, populating host and databricksToken from auth provider
|
||||
// for backwards compatibility with code that reads these properties directly
|
||||
const effectiveConfig: MLflowTracingConfig = {
|
||||
...config,
|
||||
trackingUri,
|
||||
experimentId,
|
||||
databricksConfigPath,
|
||||
host: globalAuthProvider.getHost(),
|
||||
databricksToken: globalAuthProvider.getDatabricksToken(),
|
||||
};
|
||||
|
||||
// Store the config
|
||||
globalConfig = { ...effectiveConfig };
|
||||
|
||||
// Initialize SDK with new configuration
|
||||
initializeSDK();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current configuration. Throws an error if not configured.
|
||||
* @returns The current MLflow tracing configuration
|
||||
*/
|
||||
export function getConfig(): MLflowTracingConfig {
|
||||
if (!globalConfig) {
|
||||
throw new Error(
|
||||
'The MLflow Tracing client is not configured. Please call init() with host and experimentId before using tracing functions.',
|
||||
);
|
||||
}
|
||||
return globalConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current authentication provider. Throws an error if not configured.
|
||||
* @returns The current authentication provider
|
||||
*/
|
||||
export function getAuthProvider(): AuthProvider {
|
||||
if (!globalAuthProvider) {
|
||||
throw new Error(
|
||||
'The MLflow Tracing client is not configured. Please call init() before using tracing functions.',
|
||||
);
|
||||
}
|
||||
return globalAuthProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the global configuration. For testing purposes only.
|
||||
* @internal
|
||||
*/
|
||||
export function resetConfig(): void {
|
||||
globalConfig = null;
|
||||
globalAuthProvider = null;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Constants for MLflow Tracing
|
||||
*/
|
||||
|
||||
/**
|
||||
* Enum for span types that can be used with MLflow Tracing
|
||||
*/
|
||||
export enum SpanType {
|
||||
LLM = 'LLM',
|
||||
CHAIN = 'CHAIN',
|
||||
AGENT = 'AGENT',
|
||||
TOOL = 'TOOL',
|
||||
CHAT_MODEL = 'CHAT_MODEL',
|
||||
RETRIEVER = 'RETRIEVER',
|
||||
PARSER = 'PARSER',
|
||||
EMBEDDING = 'EMBEDDING',
|
||||
RERANKER = 'RERANKER',
|
||||
MEMORY = 'MEMORY',
|
||||
UNKNOWN = 'UNKNOWN',
|
||||
}
|
||||
|
||||
/**
|
||||
* Severity level for an MLflow trace span. The public tracing API accepts a
|
||||
* `SpanLogLevel` member or its string name (e.g. "INFO").
|
||||
*/
|
||||
export enum SpanLogLevel {
|
||||
DEBUG = 10,
|
||||
INFO = 20,
|
||||
WARNING = 30,
|
||||
ERROR = 40,
|
||||
CRITICAL = 50,
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize an enum or string into a SpanLogLevel. Raw integers are not
|
||||
* accepted at the type level; the runtime number branch exists only because
|
||||
* `SpanLogLevel` is a numeric enum and its members arrive as primitive
|
||||
* numbers.
|
||||
*/
|
||||
export function toSpanLogLevel(value: SpanLogLevel | string): SpanLogLevel {
|
||||
if (typeof value === 'number') {
|
||||
if (Object.values(SpanLogLevel).includes(value as SpanLogLevel)) {
|
||||
return value as SpanLogLevel;
|
||||
}
|
||||
throw new Error(
|
||||
`Invalid SpanLogLevel value ${value}. Expected one of ${Object.values(SpanLogLevel)
|
||||
.filter((v) => typeof v === 'number')
|
||||
.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const matched = (SpanLogLevel as Record<string, SpanLogLevel | string>)[
|
||||
value.trim().toUpperCase()
|
||||
];
|
||||
if (typeof matched === 'number') {
|
||||
return matched;
|
||||
}
|
||||
throw new Error(
|
||||
`Invalid SpanLogLevel name ${JSON.stringify(value)}. Expected one of ${Object.keys(
|
||||
SpanLogLevel,
|
||||
)
|
||||
.filter((k) => isNaN(Number(k)))
|
||||
.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
throw new Error(`SpanLogLevel must be a SpanLogLevel or string; got ${typeof value}.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constants for MLflow span attribute keys
|
||||
*/
|
||||
export const SpanAttributeKey = {
|
||||
EXPERIMENT_ID: 'mlflow.experimentId',
|
||||
TRACE_ID: 'mlflow.traceRequestId',
|
||||
INPUTS: 'mlflow.spanInputs',
|
||||
OUTPUTS: 'mlflow.spanOutputs',
|
||||
SPAN_TYPE: 'mlflow.spanType',
|
||||
// Severity level of the span (one of the `SpanLogLevel` members). Absent
|
||||
// means the span was not classified.
|
||||
LOG_LEVEL: 'mlflow.spanLogLevel',
|
||||
// This attribute is used to store token usage information from LLM responses.
|
||||
// Stored in {"input_tokens": int, "output_tokens": int, "total_tokens": int} format.
|
||||
TOKEN_USAGE: 'mlflow.chat.tokenUsage',
|
||||
// This attribute indicates which flavor/format generated the LLM span. This is
|
||||
// used by downstream (e.g., UI) to determine the message format for parsing.
|
||||
MESSAGE_FORMAT: 'mlflow.message.format',
|
||||
};
|
||||
|
||||
/**
|
||||
* Constants for MLflow trace metadata keys
|
||||
*/
|
||||
export const TraceMetadataKey = {
|
||||
SOURCE_RUN: 'mlflow.sourceRun',
|
||||
MODEL_ID: 'mlflow.modelId',
|
||||
SIZE_BYTES: 'mlflow.trace.sizeBytes',
|
||||
SCHEMA_VERSION: 'mlflow.trace_schema.version',
|
||||
TOKEN_USAGE: 'mlflow.trace.tokenUsage',
|
||||
TRACE_SESSION: 'mlflow.trace.session',
|
||||
TRACE_USER: 'mlflow.trace.user',
|
||||
// Deprecated, do not use. These fields are used for storing trace request and response
|
||||
// in MLflow 2.x. In MLflow 3.x, these are replaced in favor of the request_preview and
|
||||
// response_preview fields in the trace info.
|
||||
// TODO: Remove this once the new trace table UI is available that is based on MLflow V3 trace.
|
||||
INPUTS: 'mlflow.traceInputs',
|
||||
OUTPUTS: 'mlflow.traceOutputs',
|
||||
};
|
||||
|
||||
/**
|
||||
* Constants for MLflow trace tag keys
|
||||
*/
|
||||
export const TraceTagKey = {
|
||||
MLFLOW_ARTIFACT_LOCATION: 'mlflow.artifactLocation',
|
||||
};
|
||||
|
||||
/**
|
||||
* Current version of the MLflow trace schema
|
||||
*/
|
||||
export const TRACE_SCHEMA_VERSION = '3';
|
||||
|
||||
/**
|
||||
* The prefix for MLflow trace IDs (V3 schema).
|
||||
*/
|
||||
export const TRACE_ID_PREFIX = 'tr-';
|
||||
|
||||
/**
|
||||
* The prefix for V4 schema trace IDs (Databricks Unity Catalog backed
|
||||
* traces). Full format: `trace:/<location>/<hex_trace_id>`.
|
||||
*/
|
||||
export const TRACE_ID_V4_PREFIX = 'trace:/';
|
||||
|
||||
/**
|
||||
* HTTP header used to route OTLP span uploads to a specific Databricks
|
||||
* Unity Catalog table when sending to /api/2.0/otel/v1/traces.
|
||||
*/
|
||||
export const DATABRICKS_UC_TABLE_HEADER = 'X-Databricks-UC-Table-Name';
|
||||
|
||||
/**
|
||||
* HTTP header used to associate OTLP span uploads with an MLflow experiment
|
||||
* when sending to the OSS tracking server's /v1/traces endpoint.
|
||||
*/
|
||||
export const MLFLOW_EXPERIMENT_ID_HEADER = 'x-mlflow-experiment-id';
|
||||
|
||||
/**
|
||||
* The default name for spans if the name is not provided when starting a span
|
||||
*/
|
||||
export const DEFAULT_SPAN_NAME = 'span';
|
||||
|
||||
/**
|
||||
* Trace ID for no-op spans
|
||||
*/
|
||||
export const NO_OP_SPAN_TRACE_ID = 'no-op-span-trace-id';
|
||||
|
||||
/**
|
||||
* Constants for token usage keys (matching Python TokenUsageKey)
|
||||
*/
|
||||
export const TokenUsageKey = {
|
||||
INPUT_TOKENS: 'input_tokens',
|
||||
OUTPUT_TOKENS: 'output_tokens',
|
||||
TOTAL_TOKENS: 'total_tokens',
|
||||
CACHE_READ_INPUT_TOKENS: 'cache_read_input_tokens',
|
||||
CACHE_CREATION_INPUT_TOKENS: 'cache_creation_input_tokens',
|
||||
};
|
||||
|
||||
/**
|
||||
* Max length of the request/response preview in the trace info.
|
||||
*/
|
||||
export const REQUEST_RESPONSE_PREVIEW_MAX_LENGTH = 1000;
|
||||
@@ -0,0 +1,141 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
|
||||
import { TraceMetadataKey } from './constants';
|
||||
|
||||
/**
|
||||
* Options for the tracingContext function.
|
||||
*/
|
||||
export interface TracingContextOptions {
|
||||
/**
|
||||
* Key-value pairs to inject into the trace's request_metadata (immutable after trace creation).
|
||||
*/
|
||||
metadata?: Record<string, string>;
|
||||
|
||||
/**
|
||||
* Key-value pairs to inject into the trace's tags.
|
||||
*/
|
||||
tags?: Record<string, string>;
|
||||
|
||||
/**
|
||||
* Whether tracing is enabled within the scope. If false, all tracing calls within the scope
|
||||
* will return NoOpSpan without creating any traces. If undefined, the value is inherited from
|
||||
* the outer scope.
|
||||
*/
|
||||
enabled?: boolean;
|
||||
|
||||
/**
|
||||
* Session ID to associate with traces created in this scope.
|
||||
* Stored as metadata under the `mlflow.trace.session` key.
|
||||
*/
|
||||
sessionId?: string;
|
||||
|
||||
/**
|
||||
* User identifier to associate with traces created in this scope.
|
||||
* Stored as metadata under the `mlflow.trace.user` key.
|
||||
*/
|
||||
user?: string;
|
||||
}
|
||||
|
||||
interface UserTraceContext {
|
||||
metadata: Record<string, string>;
|
||||
tags: Record<string, string>;
|
||||
enabled: boolean | undefined;
|
||||
}
|
||||
|
||||
const storage = new AsyncLocalStorage<UserTraceContext>();
|
||||
|
||||
/**
|
||||
* Get the configured trace metadata from the current tracing context scope.
|
||||
* Returns undefined if no context is active or metadata is empty.
|
||||
*/
|
||||
export function getConfiguredTraceMetadata(): Record<string, string> | undefined {
|
||||
const ctx = storage.getStore();
|
||||
if (!ctx || Object.keys(ctx.metadata).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return ctx.metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured trace tags from the current tracing context scope.
|
||||
* Returns undefined if no context is active or tags are empty.
|
||||
*/
|
||||
export function getConfiguredTraceTags(): Record<string, string> | undefined {
|
||||
const ctx = storage.getStore();
|
||||
if (!ctx || Object.keys(ctx.tags).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return ctx.tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tracing is enabled in the current context scope.
|
||||
* Returns undefined if no context is active or enabled was not set.
|
||||
*/
|
||||
export function isTracingEnabledInContext(): boolean | undefined {
|
||||
const ctx = storage.getStore();
|
||||
return ctx?.enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a function within a tracing context scope that injects metadata and/or tags into any
|
||||
* trace created within the scope, without creating a wrapper span. It can also be used to
|
||||
* selectively disable tracing within the scope.
|
||||
*
|
||||
* This is useful when you need to attach trace-level information (e.g. session IDs) to traces
|
||||
* produced by code you don't control like auto-instrumented libraries, or when you want to
|
||||
* suppress tracing for a specific code block.
|
||||
*
|
||||
* The context can be nested. When the same key is specified in multiple levels, the value
|
||||
* from the inner level takes precedence.
|
||||
*
|
||||
* @param options - Metadata, tags, and enabled flag to inject into traces
|
||||
* @param fn - The function to execute within the context scope
|
||||
* @returns The return value of the function
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import * as mlflow from "@mlflow/core";
|
||||
*
|
||||
* // Inject session ID, user, and tags into all traces created within the scope
|
||||
* mlflow.tracingContext(
|
||||
* { sessionId: "session-123", user: "user-456", tags: { project: "my-project" } },
|
||||
* () => {
|
||||
* // Any trace created here will carry the metadata and tags
|
||||
* agent.invoke("What is the capital of France?");
|
||||
* }
|
||||
* );
|
||||
*
|
||||
* // Disable tracing within a specific scope
|
||||
* mlflow.tracingContext({ enabled: false }, () => {
|
||||
* // No traces will be created inside this scope
|
||||
* agent.invoke("This call will not be traced");
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function tracingContext<T>(options: TracingContextOptions, fn: () => T): T {
|
||||
const current = storage.getStore();
|
||||
|
||||
// Inject sessionId and user into metadata
|
||||
const metadata = { ...(options.metadata ?? {}) };
|
||||
if (options.sessionId !== undefined) {
|
||||
metadata[TraceMetadataKey.TRACE_SESSION] = options.sessionId;
|
||||
}
|
||||
if (options.user !== undefined) {
|
||||
metadata[TraceMetadataKey.TRACE_USER] = options.user;
|
||||
}
|
||||
|
||||
// Merge with any outer context scope (inner wins on conflict)
|
||||
const mergedMetadata = { ...(current?.metadata ?? {}), ...metadata };
|
||||
const mergedTags = { ...(current?.tags ?? {}), ...(options.tags ?? {}) };
|
||||
const resolvedEnabled =
|
||||
options.enabled !== undefined ? options.enabled : (current?.enabled ?? undefined);
|
||||
|
||||
const newContext: UserTraceContext = {
|
||||
metadata: mergedMetadata,
|
||||
tags: mergedTags,
|
||||
enabled: resolvedEnabled,
|
||||
};
|
||||
|
||||
return storage.run(newContext, fn);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { UnityCatalogLocation } from './entities/trace_location';
|
||||
|
||||
/**
|
||||
* Databricks experiment tag keys carrying the linked UC trace location.
|
||||
* Matches Python's `MLFLOW_EXPERIMENT_DATABRICKS_TRACE_*` constants in
|
||||
* mlflow/utils/mlflow_tags.py.
|
||||
*/
|
||||
export const DATABRICKS_TRACE_DESTINATION_PATH_TAG =
|
||||
'mlflow.experiment.databricksTraceDestinationPath';
|
||||
export const DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG =
|
||||
'mlflow.experiment.databricksTraceSpanStorageTable';
|
||||
export const DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG =
|
||||
'mlflow.experiment.databricksTraceLogStorageTable';
|
||||
export const DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG =
|
||||
'mlflow.experiment.databricksTraceAnnotationsTable';
|
||||
|
||||
/**
|
||||
* Parse a UC table-prefix location from the Databricks experiment tags
|
||||
* (`mlflow.experiment.databricksTrace*`). Returns null when the experiment is
|
||||
* not linked to a UC trace destination.
|
||||
*
|
||||
* Mirrors Python's `Experiment._resolve_trace_location_from_tags`.
|
||||
*/
|
||||
export function ucLocationFromExperimentTags(
|
||||
tags: Record<string, string>,
|
||||
): UnityCatalogLocation | null {
|
||||
const path = tags[DATABRICKS_TRACE_DESTINATION_PATH_TAG];
|
||||
if (!path) {
|
||||
return null;
|
||||
}
|
||||
const parts = path.split('.');
|
||||
if (parts.length !== 3 || parts.some((p) => !p)) {
|
||||
return null;
|
||||
}
|
||||
const [catalogName, schemaName, tablePrefix] = parts;
|
||||
return {
|
||||
catalogName,
|
||||
schemaName,
|
||||
tablePrefix,
|
||||
otelSpansTableName: tags[DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG],
|
||||
otelLogsTableName: tags[DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG],
|
||||
annotationsTableName: tags[DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,703 @@
|
||||
import {
|
||||
HrTime,
|
||||
INVALID_SPANID,
|
||||
INVALID_TRACEID,
|
||||
SpanStatusCode as OTelSpanStatusCode,
|
||||
} from '@opentelemetry/api';
|
||||
import type { Span as OTelSpan } from '@opentelemetry/sdk-trace-base';
|
||||
import {
|
||||
SpanAttributeKey,
|
||||
SpanLogLevel,
|
||||
SpanType,
|
||||
toSpanLogLevel,
|
||||
NO_OP_SPAN_TRACE_ID,
|
||||
} from '../constants';
|
||||
import { defaultLogLevelForSpanType } from '../log_level';
|
||||
import { SpanEvent } from './span_event';
|
||||
import { SpanStatus, SpanStatusCode } from './span_status';
|
||||
import {
|
||||
convertHrTimeToNanoSeconds,
|
||||
convertNanoSecondsToHrTime,
|
||||
encodeSpanIdToBase64,
|
||||
encodeTraceIdToBase64,
|
||||
decodeIdFromBase64,
|
||||
} from '../utils';
|
||||
import { safeJsonStringify } from '../utils/json';
|
||||
/**
|
||||
* MLflow Span interface
|
||||
*/
|
||||
|
||||
export interface ISpan {
|
||||
/**
|
||||
* The OpenTelemetry span wrapped by MLflow Span
|
||||
*/
|
||||
readonly _span: OTelSpan;
|
||||
|
||||
/**
|
||||
* The trace ID
|
||||
*/
|
||||
readonly traceId: string;
|
||||
|
||||
/**
|
||||
* The attributes of the span
|
||||
*/
|
||||
readonly attributes: Record<string, any>;
|
||||
|
||||
get spanId(): string;
|
||||
get name(): string;
|
||||
get spanType(): SpanType;
|
||||
/**
|
||||
* The severity level of the span, or null if it was not classified.
|
||||
*/
|
||||
get logLevel(): SpanLogLevel | null;
|
||||
get startTime(): HrTime;
|
||||
get endTime(): HrTime | null;
|
||||
get parentId(): string | null;
|
||||
get status(): SpanStatus;
|
||||
get inputs(): any;
|
||||
get outputs(): any;
|
||||
|
||||
/**
|
||||
* Get an attribute from the span
|
||||
* @param key Attribute key
|
||||
* @returns Attribute value
|
||||
*/
|
||||
getAttribute(key: string): any;
|
||||
|
||||
/**
|
||||
* Get events from the span
|
||||
*/
|
||||
get events(): SpanEvent[];
|
||||
|
||||
/**
|
||||
* Convert this span to JSON format
|
||||
* @returns JSON object representation of the span
|
||||
*/
|
||||
toJson(): SerializedSpan;
|
||||
}
|
||||
|
||||
/**
|
||||
* MLflow Span class that wraps the OpenTelemetry Span.
|
||||
*/
|
||||
export class Span implements ISpan {
|
||||
readonly _span: OTelSpan;
|
||||
readonly _attributesRegistry: SpanAttributesRegistry;
|
||||
|
||||
// Internal only flag to allow mutating the ended span. This is used to set the custom attributes
|
||||
// from span processor's onEnd hook. The hook is invoked after the span is ended and OpenTelemetry
|
||||
// blocks setting attributes on them by default. Set this flag to true to allow mutating the ended
|
||||
// span.
|
||||
allowMutatingEndedSpan: boolean = false;
|
||||
|
||||
/**
|
||||
* Create a new MLflowSpan
|
||||
* @param span OpenTelemetry span
|
||||
*/
|
||||
constructor(span: OTelSpan, is_mutable: boolean = false) {
|
||||
this._span = span;
|
||||
|
||||
if (is_mutable) {
|
||||
this._attributesRegistry = new SpanAttributesRegistry(span);
|
||||
} else {
|
||||
this._attributesRegistry = new CachedSpanAttributesRegistry(span);
|
||||
}
|
||||
}
|
||||
|
||||
get traceId(): string {
|
||||
return this.getAttribute(SpanAttributeKey.TRACE_ID) as string;
|
||||
}
|
||||
|
||||
get spanId(): string {
|
||||
return this._span.spanContext().spanId;
|
||||
}
|
||||
|
||||
get spanType(): SpanType {
|
||||
return this.getAttribute(SpanAttributeKey.SPAN_TYPE) as SpanType;
|
||||
}
|
||||
|
||||
get logLevel(): SpanLogLevel | null {
|
||||
const raw = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | undefined | null;
|
||||
if (raw == null) {
|
||||
return null;
|
||||
}
|
||||
return raw as SpanLogLevel;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the parent span ID
|
||||
*/
|
||||
get parentId(): string | null {
|
||||
return this._span.parentSpanContext?.spanId ?? null;
|
||||
}
|
||||
|
||||
get name(): string {
|
||||
return this._span.name;
|
||||
}
|
||||
|
||||
get startTime(): HrTime {
|
||||
return this._span.startTime;
|
||||
}
|
||||
|
||||
get endTime(): HrTime | null {
|
||||
return this._span.endTime;
|
||||
}
|
||||
|
||||
get status(): SpanStatus {
|
||||
return SpanStatus.fromOtelStatus(this._span.status);
|
||||
}
|
||||
|
||||
get inputs(): any {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return this.getAttribute(SpanAttributeKey.INPUTS);
|
||||
}
|
||||
|
||||
get outputs(): any {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return this.getAttribute(SpanAttributeKey.OUTPUTS);
|
||||
}
|
||||
|
||||
get attributes(): Record<string, any> {
|
||||
return this._attributesRegistry.getAll();
|
||||
}
|
||||
|
||||
getAttribute(key: string): any {
|
||||
return this._attributesRegistry.get(key);
|
||||
}
|
||||
|
||||
get events(): SpanEvent[] {
|
||||
return this._span.events.map((event) => {
|
||||
const [seconds, nanoseconds] = event.time;
|
||||
return new SpanEvent({
|
||||
name: event.name,
|
||||
attributes: event.attributes as Record<string, any>,
|
||||
timestamp: BigInt(seconds) * 1_000_000_000n + BigInt(nanoseconds),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this span to JSON format (OpenTelemetry format)
|
||||
* @returns JSON object representation of the span
|
||||
*/
|
||||
toJson(): SerializedSpan {
|
||||
return {
|
||||
trace_id: encodeTraceIdToBase64(this.traceId),
|
||||
span_id: encodeSpanIdToBase64(this.spanId),
|
||||
// Use empty string for parent_span_id if it is not set, to be consistent with Python implementation.
|
||||
parent_span_id: this.parentId ? encodeSpanIdToBase64(this.parentId) : '',
|
||||
name: this.name,
|
||||
start_time_unix_nano: convertHrTimeToNanoSeconds(this.startTime),
|
||||
end_time_unix_nano: this.endTime ? convertHrTimeToNanoSeconds(this.endTime) : null,
|
||||
status: {
|
||||
code: this.status?.statusCode || SpanStatusCode.UNSET,
|
||||
message: this.status?.description,
|
||||
},
|
||||
attributes: this.attributes || {},
|
||||
events: this.events.map((event) => ({
|
||||
name: event.name,
|
||||
time_unix_nano: event.timestamp,
|
||||
attributes: event.attributes || {},
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Span from JSON data (following Python implementation)
|
||||
* Converts the JSON data back into OpenTelemetry-compatible span
|
||||
*/
|
||||
static fromJson(json: SerializedSpan): ISpan {
|
||||
// Convert the JSON data back to an OpenTelemetry-like span structure
|
||||
// This is simplified compared to Python but follows the same pattern
|
||||
|
||||
const otelSpanData = {
|
||||
name: json.name,
|
||||
startTime: convertNanoSecondsToHrTime(json.start_time_unix_nano),
|
||||
endTime: json.end_time_unix_nano ? convertNanoSecondsToHrTime(json.end_time_unix_nano) : null,
|
||||
status: {
|
||||
code: convertStatusCodeToOTel(json.status.code),
|
||||
message: json.status.message,
|
||||
},
|
||||
// For fromJson, attributes are already in their final form (not JSON serialized)
|
||||
// so we store them directly
|
||||
attributes: json.attributes || {},
|
||||
events: (json.events || []).map((event) => ({
|
||||
name: event.name,
|
||||
time: convertNanoSecondsToHrTime(event.time_unix_nano),
|
||||
attributes: event.attributes || {},
|
||||
})),
|
||||
ended: true,
|
||||
// Add spanContext() method that returns proper SpanContext
|
||||
spanContext: () => ({
|
||||
traceId: decodeIdFromBase64(json.trace_id),
|
||||
spanId: decodeIdFromBase64(json.span_id),
|
||||
traceFlags: 1, // Sampled
|
||||
isRemote: false,
|
||||
}),
|
||||
// Add parentSpanContext for parent span ID
|
||||
parentSpanContext: json.parent_span_id
|
||||
? {
|
||||
traceId: decodeIdFromBase64(json.trace_id),
|
||||
spanId: decodeIdFromBase64(json.parent_span_id),
|
||||
traceFlags: 1,
|
||||
isRemote: false,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
|
||||
// Create a span that behaves like our Span class but from downloaded data
|
||||
return new Span(otelSpanData as OTelSpan, false); // false = immutable
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert MLflow status codes to OpenTelemetry status codes
|
||||
* @param statusCode Status code from MLflow JSON format
|
||||
* @returns OpenTelemetry compatible status code
|
||||
*/
|
||||
function convertStatusCodeToOTel(statusCode?: string): OTelSpanStatusCode {
|
||||
if (!statusCode) {
|
||||
return OTelSpanStatusCode.UNSET;
|
||||
}
|
||||
|
||||
// Handle MLflow format -> OTel format conversion
|
||||
switch (statusCode) {
|
||||
case 'STATUS_CODE_OK':
|
||||
return OTelSpanStatusCode.OK;
|
||||
case 'STATUS_CODE_ERROR':
|
||||
return OTelSpanStatusCode.ERROR;
|
||||
case 'STATUS_CODE_UNSET':
|
||||
return OTelSpanStatusCode.UNSET;
|
||||
// Also handle OTel format directly
|
||||
case 'OK':
|
||||
return OTelSpanStatusCode.OK;
|
||||
case 'ERROR':
|
||||
return OTelSpanStatusCode.ERROR;
|
||||
case 'UNSET':
|
||||
return OTelSpanStatusCode.UNSET;
|
||||
default:
|
||||
return OTelSpanStatusCode.UNSET;
|
||||
}
|
||||
}
|
||||
|
||||
export class LiveSpan extends Span {
|
||||
// Internal only flag to allow mutating the ended span
|
||||
allowMutatingEndedSpan: boolean = false;
|
||||
|
||||
constructor(span: OTelSpan, traceId: string, span_type: SpanType) {
|
||||
super(span, true);
|
||||
this.setAttribute(SpanAttributeKey.TRACE_ID, traceId);
|
||||
this.setAttribute(SpanAttributeKey.SPAN_TYPE, span_type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the type of the span
|
||||
* @param spanType The type of the span
|
||||
*/
|
||||
setSpanType(spanType: SpanType): void {
|
||||
this.setAttribute(SpanAttributeKey.SPAN_TYPE, spanType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the severity level of the span. Accepts a SpanLogLevel enum value or
|
||||
* its name (e.g. "INFO").
|
||||
*/
|
||||
setLogLevel(level: SpanLogLevel | string): void {
|
||||
const normalized = toSpanLogLevel(level);
|
||||
this.setAttribute(SpanAttributeKey.LOG_LEVEL, normalized as number);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set inputs for the span
|
||||
* @param inputs Input data for the span
|
||||
*/
|
||||
setInputs(inputs: any): void {
|
||||
this.setAttribute(SpanAttributeKey.INPUTS, inputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set outputs for the span
|
||||
* @param outputs Output data for the span
|
||||
*/
|
||||
setOutputs(outputs: any): void {
|
||||
this.setAttribute(SpanAttributeKey.OUTPUTS, outputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set an attribute on the span
|
||||
* @param key Attribute key
|
||||
* @param value Attribute value
|
||||
*/
|
||||
setAttribute(key: string, value: any): void {
|
||||
this._attributesRegistry.set(key, value, this.allowMutatingEndedSpan);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set multiple attributes on the span
|
||||
* @param attributes Object containing key-value pairs for attributes
|
||||
*/
|
||||
setAttributes(attributes: Record<string, any>): void {
|
||||
if (!attributes || Object.keys(attributes).length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
Object.entries(attributes).forEach(([key, value]) => {
|
||||
this.setAttribute(key, value);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an event to the span
|
||||
* @param event Event object with name and attributes
|
||||
*/
|
||||
addEvent(event: SpanEvent): void {
|
||||
// Convert BigInt timestamp to HrTime for OpenTelemetry
|
||||
const timeInput = convertNanoSecondsToHrTime(event.timestamp);
|
||||
this._span.addEvent(event.name, event.attributes, timeInput);
|
||||
// Promote the span to ERROR for exception events. Preserves user-set
|
||||
// CRITICAL. (Inlined rather than extracted to a private method because
|
||||
// TypeScript's structural typing makes adding privates a breaking API
|
||||
// change for NoOpSpan, which sits behind the same union return type.)
|
||||
if (event.name === 'exception') {
|
||||
const current = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | null | undefined;
|
||||
if (current == null || current < (SpanLogLevel.ERROR as number)) {
|
||||
this.setAttribute(SpanAttributeKey.LOG_LEVEL, SpanLogLevel.ERROR as number);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an exception event to the span
|
||||
* @param error Error object
|
||||
*/
|
||||
recordException(error: Error): void {
|
||||
this._span.recordException(error);
|
||||
const current = this.getAttribute(SpanAttributeKey.LOG_LEVEL) as number | null | undefined;
|
||||
if (current == null || current < (SpanLogLevel.ERROR as number)) {
|
||||
this.setAttribute(SpanAttributeKey.LOG_LEVEL, SpanLogLevel.ERROR as number);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the status of the span
|
||||
* @param status Status code or SpanStatus object
|
||||
* @param description Optional description for the status
|
||||
*/
|
||||
setStatus(status: SpanStatus | SpanStatusCode | string, description?: string): void {
|
||||
if (status instanceof SpanStatus) {
|
||||
this._span.setStatus(status.toOtelStatus());
|
||||
} else if (typeof status === 'string') {
|
||||
const spanStatus = new SpanStatus(status as SpanStatusCode, description);
|
||||
this._span.setStatus(spanStatus.toOtelStatus());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* End the span
|
||||
*
|
||||
* @param outputs Optional outputs to set before ending
|
||||
* @param attributes Optional attributes to set before ending
|
||||
* @param status Optional status code
|
||||
* @param endTimeNs Optional end time in nanoseconds
|
||||
*/
|
||||
end(options?: {
|
||||
outputs?: any;
|
||||
attributes?: Record<string, any>;
|
||||
status?: SpanStatus | SpanStatusCode;
|
||||
endTimeNs?: number;
|
||||
}): void {
|
||||
try {
|
||||
if (options?.outputs != null) {
|
||||
this.setOutputs(options.outputs);
|
||||
}
|
||||
|
||||
if (options?.attributes != null) {
|
||||
this.setAttributes(options.attributes);
|
||||
}
|
||||
|
||||
if (options?.status != null) {
|
||||
this.setStatus(options.status);
|
||||
}
|
||||
|
||||
// NB: In OpenTelemetry, status code remains UNSET if not explicitly set
|
||||
// by the user. However, there is no way to set the status when using
|
||||
// `trace` function wrapper. Therefore, we just automatically set the status
|
||||
// to OK if it is not ERROR.
|
||||
if (this.status.statusCode !== SpanStatusCode.ERROR) {
|
||||
this.setStatus(SpanStatusCode.OK);
|
||||
}
|
||||
|
||||
// Resolve the log level from the final span_type if neither
|
||||
// `setLogLevel` nor an exception bump set it during the span's lifetime.
|
||||
// Mirrors the Python LiveSpan.end() behavior.
|
||||
if (this.getAttribute(SpanAttributeKey.LOG_LEVEL) == null) {
|
||||
this.setAttribute(
|
||||
SpanAttributeKey.LOG_LEVEL,
|
||||
defaultLogLevelForSpanType(this.spanType) as number,
|
||||
);
|
||||
}
|
||||
|
||||
// OTel SDK default end time to current time if not provided
|
||||
const endTime = options?.endTimeNs
|
||||
? convertNanoSecondsToHrTime(options.endTimeNs)
|
||||
: undefined;
|
||||
this._span.end(endTime);
|
||||
} catch (error) {
|
||||
console.error(`Failed to end span ${this.spanId}: ${String(error)}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A no-operation span implementation that doesn't record anything
|
||||
*/
|
||||
export class NoOpSpan implements ISpan {
|
||||
readonly _span: any; // Use any for NoOp span to avoid type conflicts
|
||||
readonly _attributesRegistry: SpanAttributesRegistry;
|
||||
|
||||
allowMutatingEndedSpan: boolean = false;
|
||||
|
||||
constructor(span?: any) {
|
||||
// Create a minimal no-op span object
|
||||
this._span = span || {
|
||||
spanContext: () => ({
|
||||
spanId: INVALID_SPANID,
|
||||
traceId: INVALID_TRACEID,
|
||||
}),
|
||||
attributes: {},
|
||||
events: [],
|
||||
};
|
||||
this._attributesRegistry = new SpanAttributesRegistry(this._span as OTelSpan);
|
||||
}
|
||||
|
||||
get traceId(): string {
|
||||
return NO_OP_SPAN_TRACE_ID;
|
||||
}
|
||||
get spanId(): string {
|
||||
return '';
|
||||
}
|
||||
get parentId(): string | null {
|
||||
return null;
|
||||
}
|
||||
get name(): string {
|
||||
return '';
|
||||
}
|
||||
get spanType(): SpanType {
|
||||
return SpanType.UNKNOWN;
|
||||
}
|
||||
get logLevel(): SpanLogLevel | null {
|
||||
return null;
|
||||
}
|
||||
get startTime(): HrTime {
|
||||
return [0, 0];
|
||||
}
|
||||
get endTime(): HrTime | null {
|
||||
return null;
|
||||
}
|
||||
get status(): SpanStatus {
|
||||
return new SpanStatus(SpanStatusCode.UNSET);
|
||||
}
|
||||
get inputs(): any {
|
||||
return null;
|
||||
}
|
||||
get outputs(): any {
|
||||
return null;
|
||||
}
|
||||
get attributes(): Record<string, any> {
|
||||
return {};
|
||||
}
|
||||
|
||||
getAttribute(_key: string): any {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Implement all methods to do nothing
|
||||
setSpanType(_spanType: SpanType): void {}
|
||||
setLogLevel(_level: SpanLogLevel | string): void {}
|
||||
setInputs(_inputs: any): void {}
|
||||
setOutputs(_outputs: any): void {}
|
||||
setAttribute(_key: string, _value: any): void {}
|
||||
setAttributes(_attributes: Record<string, any>): void {}
|
||||
setStatus(_status: SpanStatus | SpanStatusCode | string, _description?: string): void {}
|
||||
addEvent(_event: SpanEvent): void {}
|
||||
recordException(_error: Error): void {}
|
||||
end(
|
||||
_outputs?: any,
|
||||
_attributes?: Record<string, any>,
|
||||
_status?: SpanStatus | SpanStatusCode,
|
||||
_endTimeNs?: number,
|
||||
): void {}
|
||||
|
||||
get events(): SpanEvent[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
toJson(): SerializedSpan {
|
||||
return {
|
||||
trace_id: NO_OP_SPAN_TRACE_ID,
|
||||
span_id: '',
|
||||
parent_span_id: '',
|
||||
name: '',
|
||||
start_time_unix_nano: 0n,
|
||||
end_time_unix_nano: null,
|
||||
status: { code: 'UNSET', message: '' },
|
||||
attributes: {},
|
||||
events: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export interface SerializedSpan {
|
||||
trace_id: string;
|
||||
span_id: string;
|
||||
parent_span_id: string;
|
||||
name: string;
|
||||
// Use bigint for nanosecond timestamps to maintain precision
|
||||
start_time_unix_nano: bigint;
|
||||
end_time_unix_nano: bigint | null;
|
||||
status: {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
attributes: Record<string, any>;
|
||||
events: {
|
||||
name: string;
|
||||
time_unix_nano: bigint;
|
||||
attributes: Record<string, any>;
|
||||
}[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A utility class to manage the span attributes.
|
||||
* In MLflow users can add arbitrary key-value pairs to the span attributes, however,
|
||||
* OpenTelemetry only allows a limited set of types to be stored in the attribute values.
|
||||
* Therefore, we serialize all values into JSON string before storing them in the span.
|
||||
* This class provides simple getter and setter methods to interact with the span attributes
|
||||
* without worrying about the serde process.
|
||||
*/
|
||||
class SpanAttributesRegistry {
|
||||
private readonly _span: OTelSpan;
|
||||
|
||||
constructor(otelSpan: OTelSpan) {
|
||||
this._span = otelSpan;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all attributes as a dictionary
|
||||
*/
|
||||
getAll(): Record<string, any> {
|
||||
const result: Record<string, any> = {};
|
||||
if (this._span.attributes) {
|
||||
Object.keys(this._span.attributes).forEach((key) => {
|
||||
result[key] = this.get(key);
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single attribute value
|
||||
*/
|
||||
get(key: string): any {
|
||||
const serializedValue = this._span.attributes?.[key];
|
||||
if (serializedValue && typeof serializedValue === 'string') {
|
||||
try {
|
||||
return JSON.parse(serializedValue);
|
||||
} catch (e) {
|
||||
// If JSON.parse fails, this might be a raw string value or
|
||||
// the span was created from JSON (attributes already parsed)
|
||||
// In that case, return the value as-is
|
||||
return serializedValue;
|
||||
}
|
||||
}
|
||||
return serializedValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a single attribute value
|
||||
*/
|
||||
set(key: string, value: any, allowMutatingEndedSpan: boolean = false): void {
|
||||
if (typeof key !== 'string') {
|
||||
console.warn(`Attribute key must be a string, but got ${typeof key}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (allowMutatingEndedSpan && this._span.ended) {
|
||||
// Directly set the attribute value to bypass the isSpanEnded check.
|
||||
this._span.attributes[key] = safeJsonStringify(value);
|
||||
return;
|
||||
}
|
||||
|
||||
// NB: OpenTelemetry attribute can store not only string but also a few primitives like
|
||||
// int, float, bool, and list of them. However, we serialize all into JSON string here
|
||||
// for the simplicity in deserialization process.
|
||||
this._span.setAttribute(key, safeJsonStringify(value));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A cache-enabled version of the SpanAttributesRegistry.
|
||||
* The caching helps to avoid the redundant deserialization of the attribute, however, it does
|
||||
* not handle the value change well. Therefore, this class should only be used for the persisted
|
||||
* spans that are immutable, and thus implemented as a subclass of SpanAttributesRegistry.
|
||||
*/
|
||||
class CachedSpanAttributesRegistry extends SpanAttributesRegistry {
|
||||
private readonly _cache = new Map<string, any>();
|
||||
|
||||
/**
|
||||
* Get a single attribute value with LRU caching (maxsize=128)
|
||||
*/
|
||||
get(key: string): any {
|
||||
if (this._cache.has(key)) {
|
||||
// Move to end (most recently used)
|
||||
const value = this._cache.get(key);
|
||||
this._cache.delete(key);
|
||||
this._cache.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
const value = super.get(key);
|
||||
|
||||
// Implement LRU eviction
|
||||
if (this._cache.size >= 128) {
|
||||
// Remove least recently used (first entry)
|
||||
const firstKey = this._cache.keys().next().value;
|
||||
if (firstKey != null) {
|
||||
this._cache.delete(firstKey);
|
||||
}
|
||||
}
|
||||
|
||||
this._cache.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set operation is not allowed for cached registry (immutable spans)
|
||||
*/
|
||||
set(_key: string, _value: any, _allowMutatingEndedSpan: boolean = false): void {
|
||||
throw new Error('The attributes of the immutable span must not be updated.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory function to create a span object.
|
||||
*/
|
||||
export function createMlflowSpan(
|
||||
otelSpan: any,
|
||||
traceId: string,
|
||||
spanType?: string,
|
||||
): NoOpSpan | Span | LiveSpan {
|
||||
// NonRecordingSpan always has a spanId of '0000000000000000'
|
||||
// https://github.com/open-telemetry/opentelemetry-js/blob/f2cfd1327a5b131ea795301b10877291aac4e6f5/api/src/trace/invalid-span-constants.ts#L23C32-L23C48
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call */
|
||||
if (!otelSpan || otelSpan.spanContext().spanId === INVALID_SPANID) {
|
||||
return new NoOpSpan(otelSpan);
|
||||
}
|
||||
|
||||
// If the span is completed, it should be immutable.
|
||||
if (otelSpan.ended) {
|
||||
return new Span(otelSpan);
|
||||
}
|
||||
|
||||
return new LiveSpan(otelSpan, traceId, (spanType as SpanType) || SpanType.UNKNOWN);
|
||||
/* eslint-enable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-call */
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* MLflow Span Event Entities
|
||||
*
|
||||
* This module provides TypeScript implementations of MLflow span event entities,
|
||||
* compatible with OpenTelemetry for recording specific occurrences during span execution.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Type definition for attribute values that can be stored in span events.
|
||||
* Compatible with OpenTelemetry attribute value types.
|
||||
*/
|
||||
export type AttributeValue = string | number | boolean | string[] | number[] | boolean[];
|
||||
|
||||
/**
|
||||
* Parameters for creating a SpanEvent instance.
|
||||
*/
|
||||
export interface SpanEventParams {
|
||||
/** Name of the event */
|
||||
name: string;
|
||||
/**
|
||||
* The exact time the event occurred, measured in nanoseconds since epoch.
|
||||
* If not provided, the current time will be used.
|
||||
*/
|
||||
timestamp?: bigint;
|
||||
/**
|
||||
* A collection of key-value pairs representing detailed attributes of the event,
|
||||
* such as exception stack traces or other contextual information.
|
||||
*/
|
||||
attributes?: Record<string, AttributeValue>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an event that records specific occurrences or moments in time
|
||||
* during a span, such as an exception being thrown. Compatible with OpenTelemetry.
|
||||
*
|
||||
* SpanEvents are used to capture important moments during span execution,
|
||||
* providing detailed context about what happened and when.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Create a custom event
|
||||
* const event = new SpanEvent({
|
||||
* name: 'user_action',
|
||||
* attributes: {
|
||||
* 'action.type': 'click',
|
||||
* 'element.id': 'submit-button'
|
||||
* }
|
||||
* });
|
||||
*
|
||||
* // Create an event from an exception
|
||||
* const errorEvent = SpanEvent.fromException(new Error('Something went wrong'));
|
||||
* ```
|
||||
*/
|
||||
export class SpanEvent {
|
||||
/** Name of the event */
|
||||
readonly name: string;
|
||||
|
||||
/**
|
||||
* The exact time the event occurred, measured in nanosecond since epoch.
|
||||
* Defaults to current time if not provided during construction.
|
||||
*/
|
||||
readonly timestamp: bigint;
|
||||
|
||||
/**
|
||||
* A collection of key-value pairs representing detailed attributes of the event.
|
||||
* Attributes provide contextual information about the event.
|
||||
*/
|
||||
readonly attributes: Record<string, AttributeValue>;
|
||||
|
||||
/**
|
||||
* Creates a new SpanEvent instance.
|
||||
*
|
||||
* @param params - Event parameters including name, optional timestamp, and attributes
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const event = new SpanEvent({
|
||||
* name: 'database_query',
|
||||
* attributes: {
|
||||
* 'db.statement': 'SELECT * FROM users',
|
||||
* 'db.duration_ms': 150
|
||||
* }
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
constructor(params: SpanEventParams) {
|
||||
this.name = params.name;
|
||||
this.timestamp = params.timestamp ?? this.getCurrentTimeNano();
|
||||
this.attributes = params.attributes ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a span event from an exception.
|
||||
*
|
||||
* This is a convenience method for creating events that represent exceptions
|
||||
* or errors that occurred during span execution. The event will include
|
||||
* standard exception attributes like message, type, and stack trace.
|
||||
*
|
||||
* @param exception - The exception to create an event from
|
||||
* @returns New SpanEvent instance representing the exception
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* try {
|
||||
* // Some operation that might fail
|
||||
* throw new Error('Database connection failed');
|
||||
* } catch (error) {
|
||||
* const errorEvent = SpanEvent.fromException(error);
|
||||
* span.addEvent(errorEvent);
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
static fromException(exception: Error): SpanEvent {
|
||||
const stackTrace = this.getStackTrace(exception);
|
||||
|
||||
return new SpanEvent({
|
||||
name: 'exception',
|
||||
attributes: {
|
||||
'exception.message': exception.message,
|
||||
'exception.type': exception.name,
|
||||
'exception.stacktrace': stackTrace,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the stack trace from an error object.
|
||||
*
|
||||
* @param error - The error to extract stack trace from
|
||||
* @returns Stack trace as a string, or error representation if stack trace unavailable
|
||||
*/
|
||||
private static getStackTrace(error: Error): string {
|
||||
try {
|
||||
return error.stack ?? String(error);
|
||||
} catch {
|
||||
// Fallback if stack trace extraction fails
|
||||
return String(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this SpanEvent to JSON format
|
||||
* @returns JSON object representation of the span event
|
||||
*/
|
||||
toJson(): Record<string, string | bigint | Record<string, AttributeValue>> {
|
||||
return {
|
||||
name: this.name,
|
||||
timestamp: this.timestamp,
|
||||
attributes: this.attributes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current time in nanoseconds since epoch.
|
||||
*
|
||||
* @returns Current timestamp in nanoseconds
|
||||
*/
|
||||
private getCurrentTimeNano(): bigint {
|
||||
return BigInt(Date.now()) * BigInt(1e6);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { SpanStatus as OTelStatus, SpanStatusCode as OTelSpanStatusCode } from '@opentelemetry/api';
|
||||
|
||||
/**
|
||||
* MLflow Span Status module
|
||||
*
|
||||
* This module provides the MLflow SpanStatusCode enum and SpanStatus class,
|
||||
* matching the Python MLflow implementation.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Enum for status code of a span
|
||||
* Uses the same set of status codes as OTLP SpanStatusCode
|
||||
* https://github.com/open-telemetry/opentelemetry-proto/blob/189b2648d29aa6039aeb2
|
||||
lemetry/proto/trace/v1/trace.proto#L314-L322
|
||||
*/
|
||||
export enum SpanStatusCode {
|
||||
/** Status is unset/unspecified */
|
||||
UNSET = 'STATUS_CODE_UNSET',
|
||||
/** The operation completed successfully */
|
||||
OK = 'STATUS_CODE_OK',
|
||||
/** The operation encountered an error */
|
||||
ERROR = 'STATUS_CODE_ERROR',
|
||||
}
|
||||
|
||||
/**
|
||||
* Status of the span or the trace.
|
||||
*/
|
||||
export class SpanStatus {
|
||||
/**
|
||||
* The status code of the span or the trace.
|
||||
*/
|
||||
readonly statusCode: SpanStatusCode;
|
||||
|
||||
/**
|
||||
* Description of the status. This should be only set when the status is ERROR,
|
||||
* otherwise it will be ignored.
|
||||
*/
|
||||
readonly description: string;
|
||||
|
||||
/**
|
||||
* Create a new SpanStatus instance
|
||||
* @param statusCode The status code - must be one of SpanStatusCode enum values
|
||||
* @param description Optional description, typically used for ERROR status
|
||||
*/
|
||||
constructor(statusCode: SpanStatusCode, description: string = '') {
|
||||
// If user provides a string status code, validate it and convert to enum
|
||||
this.statusCode = statusCode;
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert SpanStatus object to OpenTelemetry status object.
|
||||
*/
|
||||
toOtelStatus(): OTelStatus {
|
||||
let otelStatusCode: OTelSpanStatusCode;
|
||||
|
||||
switch (this.statusCode) {
|
||||
case SpanStatusCode.OK:
|
||||
otelStatusCode = OTelSpanStatusCode.OK;
|
||||
break;
|
||||
case SpanStatusCode.ERROR:
|
||||
otelStatusCode = OTelSpanStatusCode.ERROR;
|
||||
break;
|
||||
case SpanStatusCode.UNSET:
|
||||
default:
|
||||
otelStatusCode = OTelSpanStatusCode.UNSET;
|
||||
break;
|
||||
}
|
||||
|
||||
return {
|
||||
code: otelStatusCode,
|
||||
message: this.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert OpenTelemetry status object to SpanStatus object.
|
||||
*/
|
||||
static fromOtelStatus(otelStatus: OTelStatus): SpanStatus {
|
||||
let statusCode: SpanStatusCode;
|
||||
|
||||
switch (otelStatus.code) {
|
||||
case OTelSpanStatusCode.OK:
|
||||
statusCode = SpanStatusCode.OK;
|
||||
break;
|
||||
case OTelSpanStatusCode.ERROR:
|
||||
statusCode = SpanStatusCode.ERROR;
|
||||
break;
|
||||
case OTelSpanStatusCode.UNSET:
|
||||
default:
|
||||
statusCode = SpanStatusCode.UNSET;
|
||||
break;
|
||||
}
|
||||
|
||||
return new SpanStatus(statusCode, otelStatus.message ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this SpanStatus to JSON format
|
||||
* @returns JSON object representation of the span status
|
||||
*/
|
||||
toJson(): Record<string, string | SpanStatusCode> {
|
||||
return {
|
||||
status_code: this.statusCode,
|
||||
description: this.description,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { SerializedTraceInfo, TraceInfo } from './trace_info';
|
||||
import { SerializedTraceData, TraceData } from './trace_data';
|
||||
|
||||
/**
|
||||
* Represents a complete trace with metadata and span data
|
||||
*/
|
||||
export class Trace {
|
||||
/**
|
||||
* Trace metadata
|
||||
*/
|
||||
info: TraceInfo;
|
||||
|
||||
/**
|
||||
* Trace data containing spans
|
||||
*/
|
||||
data: TraceData;
|
||||
|
||||
/**
|
||||
* Create a new Trace instance
|
||||
* @param info Trace metadata
|
||||
* @param data Trace data containing spans
|
||||
*/
|
||||
constructor(info: TraceInfo, data: TraceData) {
|
||||
this.info = info;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this Trace instance to JSON format
|
||||
* @returns JSON object representation of the Trace
|
||||
*/
|
||||
toJson(): SerializedTrace {
|
||||
return {
|
||||
info: this.info.toJson(),
|
||||
data: this.data.toJson(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Trace instance from JSON data
|
||||
* @param json JSON object containing trace data
|
||||
* @returns Trace instance
|
||||
*/
|
||||
static fromJson(json: SerializedTrace): Trace {
|
||||
const info = TraceInfo.fromJson(json.info);
|
||||
const data = TraceData.fromJson(json.data);
|
||||
return new Trace(info, data);
|
||||
}
|
||||
}
|
||||
|
||||
interface SerializedTrace {
|
||||
info: SerializedTraceInfo;
|
||||
data: SerializedTraceData;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { ISpan, SerializedSpan, Span } from './span';
|
||||
|
||||
/**
|
||||
* Represents the spans and associated data for a trace
|
||||
*/
|
||||
export class TraceData {
|
||||
/**
|
||||
* The spans that make up this trace
|
||||
*/
|
||||
spans: ISpan[];
|
||||
|
||||
/**
|
||||
* Create a new TraceData instance
|
||||
* @param spans List of spans
|
||||
*/
|
||||
constructor(spans: ISpan[] = []) {
|
||||
this.spans = spans;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this TraceData instance to JSON format
|
||||
* @returns JSON object representation of the TraceData
|
||||
*/
|
||||
toJson(): SerializedTraceData {
|
||||
return {
|
||||
spans: this.spans.map((span) => span.toJson()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a TraceData instance from JSON data (following Python implementation)
|
||||
* @param json JSON object containing trace data
|
||||
* @returns TraceData instance
|
||||
*/
|
||||
static fromJson(json: SerializedTraceData): TraceData {
|
||||
const spans: ISpan[] = json.spans.map((spanData) => Span.fromJson(spanData));
|
||||
return new TraceData(spans);
|
||||
}
|
||||
}
|
||||
|
||||
export interface SerializedTraceData {
|
||||
spans: SerializedSpan[];
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
import {
|
||||
deserializeTraceLocation,
|
||||
serializeTraceLocation,
|
||||
type SerializedTraceLocation,
|
||||
type TraceLocation,
|
||||
} from './trace_location';
|
||||
import type { TraceState } from './trace_state';
|
||||
import { TraceMetadataKey } from '../constants';
|
||||
|
||||
/**
|
||||
* Interface for token usage information
|
||||
*/
|
||||
export interface TokenUsage {
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
total_tokens: number;
|
||||
cache_read_input_tokens?: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata about a trace, such as its ID, location, timestamp, etc.
|
||||
*/
|
||||
export class TraceInfo {
|
||||
/**
|
||||
* The primary identifier for the trace
|
||||
*/
|
||||
traceId: string;
|
||||
|
||||
/**
|
||||
* The location where the trace is stored
|
||||
*/
|
||||
traceLocation: TraceLocation;
|
||||
|
||||
/**
|
||||
* Start time of the trace, in milliseconds
|
||||
*/
|
||||
requestTime: number;
|
||||
|
||||
/**
|
||||
* State of the trace
|
||||
*/
|
||||
state: TraceState;
|
||||
|
||||
/**
|
||||
* Request to the model/agent (JSON-encoded, may be truncated)
|
||||
*/
|
||||
requestPreview?: string;
|
||||
|
||||
/**
|
||||
* Response from the model/agent (JSON-encoded, may be truncated)
|
||||
*/
|
||||
responsePreview?: string;
|
||||
|
||||
/**
|
||||
* Client supplied request ID associated with the trace
|
||||
*/
|
||||
clientRequestId?: string;
|
||||
|
||||
/**
|
||||
* Duration of the trace, in milliseconds
|
||||
*/
|
||||
executionDuration?: number;
|
||||
|
||||
/**
|
||||
* Key-value pairs associated with the trace (immutable)
|
||||
*/
|
||||
traceMetadata: Record<string, string>;
|
||||
|
||||
/**
|
||||
* Tags associated with the trace (mutable)
|
||||
*/
|
||||
tags: Record<string, string>;
|
||||
|
||||
/**
|
||||
* List of assessments associated with the trace.
|
||||
* TODO: Assessments are not yet supported in the TypeScript SDK.
|
||||
*/
|
||||
assessments: any[];
|
||||
|
||||
/**
|
||||
* Create a new TraceInfo instance
|
||||
* @param params TraceInfo parameters
|
||||
*/
|
||||
constructor(params: {
|
||||
traceId: string;
|
||||
traceLocation: TraceLocation;
|
||||
requestTime: number;
|
||||
state: TraceState;
|
||||
requestPreview?: string;
|
||||
responsePreview?: string;
|
||||
clientRequestId?: string;
|
||||
executionDuration?: number;
|
||||
traceMetadata?: Record<string, string>;
|
||||
tags?: Record<string, string>;
|
||||
assessments?: any[];
|
||||
}) {
|
||||
this.traceId = params.traceId;
|
||||
this.traceLocation = params.traceLocation;
|
||||
this.requestTime = params.requestTime;
|
||||
this.state = params.state;
|
||||
this.requestPreview = params.requestPreview;
|
||||
this.responsePreview = params.responsePreview;
|
||||
this.clientRequestId = params.clientRequestId;
|
||||
this.executionDuration = params.executionDuration;
|
||||
this.traceMetadata = params.traceMetadata || {};
|
||||
this.tags = params.tags || {};
|
||||
// TODO: Assessments are not yet supported in the TypeScript SDK.
|
||||
this.assessments = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this TraceInfo instance to JSON format
|
||||
* @returns JSON object representation of the TraceInfo
|
||||
*/
|
||||
toJson(): SerializedTraceInfo {
|
||||
return {
|
||||
trace_id: this.traceId,
|
||||
client_request_id: this.clientRequestId,
|
||||
trace_location: serializeTraceLocation(this.traceLocation),
|
||||
request_preview: this.requestPreview,
|
||||
response_preview: this.responsePreview,
|
||||
request_time: new Date(this.requestTime).toISOString(),
|
||||
execution_duration:
|
||||
this.executionDuration != null ? `${this.executionDuration / 1000}s` : undefined,
|
||||
state: this.state,
|
||||
trace_metadata: this.traceMetadata,
|
||||
tags: this.tags,
|
||||
assessments: this.assessments,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregated token usage information for this trace.
|
||||
* Returns null if no token usage data is available.
|
||||
* @returns Token usage object or null
|
||||
*/
|
||||
get tokenUsage(): TokenUsage | null {
|
||||
const tokenUsageJson = this.traceMetadata[TraceMetadataKey.TOKEN_USAGE];
|
||||
|
||||
if (!tokenUsageJson) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const usage = JSON.parse(tokenUsageJson) as TokenUsage;
|
||||
const result: TokenUsage = {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
total_tokens: usage.total_tokens,
|
||||
};
|
||||
if (usage.cache_read_input_tokens != null) {
|
||||
result.cache_read_input_tokens = usage.cache_read_input_tokens;
|
||||
}
|
||||
if (usage.cache_creation_input_tokens != null) {
|
||||
result.cache_creation_input_tokens = usage.cache_creation_input_tokens;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a TraceInfo instance from JSON data
|
||||
* @param json JSON object containing trace info data
|
||||
* @returns TraceInfo instance
|
||||
*/
|
||||
static fromJson(json: SerializedTraceInfo): TraceInfo {
|
||||
return new TraceInfo({
|
||||
traceId: json.trace_id,
|
||||
clientRequestId: json.client_request_id,
|
||||
traceLocation: deserializeTraceLocation(json.trace_location),
|
||||
requestPreview: json.request_preview,
|
||||
responsePreview: json.response_preview,
|
||||
requestTime: json.request_time != null ? new Date(json.request_time).getTime() : Date.now(),
|
||||
executionDuration:
|
||||
json.execution_duration != null
|
||||
? parseFloat(json.execution_duration.replace('s', '')) * 1000
|
||||
: undefined,
|
||||
state: json.state,
|
||||
traceMetadata: json.trace_metadata || {},
|
||||
tags: json.tags || {},
|
||||
assessments: json.assessments || [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface SerializedTraceInfo {
|
||||
trace_id: string;
|
||||
client_request_id?: string;
|
||||
trace_location: SerializedTraceLocation;
|
||||
request_preview?: string;
|
||||
response_preview?: string;
|
||||
// "request_time": "2025-06-15T14:07:41.282Z"
|
||||
request_time: string;
|
||||
execution_duration?: string;
|
||||
state: TraceState;
|
||||
trace_metadata: Record<string, string>;
|
||||
tags: Record<string, string>;
|
||||
// TODO: Define proper type for assessments once supported
|
||||
assessments: any[];
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/**
|
||||
* Types of trace locations
|
||||
*/
|
||||
export enum TraceLocationType {
|
||||
/**
|
||||
* Unspecified trace location type
|
||||
*/
|
||||
TRACE_LOCATION_TYPE_UNSPECIFIED = 'TRACE_LOCATION_TYPE_UNSPECIFIED',
|
||||
|
||||
/**
|
||||
* Trace is stored in an MLflow experiment
|
||||
*/
|
||||
MLFLOW_EXPERIMENT = 'MLFLOW_EXPERIMENT',
|
||||
|
||||
/**
|
||||
* Trace is stored in a Databricks inference table
|
||||
*/
|
||||
INFERENCE_TABLE = 'INFERENCE_TABLE',
|
||||
|
||||
/**
|
||||
* Trace is stored under a Databricks Unity Catalog table prefix.
|
||||
* The user-supplied prefix determines the span/log table names.
|
||||
*/
|
||||
UC_TABLE_PREFIX = 'UC_TABLE_PREFIX',
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface representing an MLflow experiment location
|
||||
*/
|
||||
export interface MlflowExperimentLocation {
|
||||
/**
|
||||
* The ID of the MLflow experiment where the trace is stored
|
||||
*/
|
||||
experimentId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface representing a Databricks inference table location
|
||||
*/
|
||||
export interface InferenceTableLocation {
|
||||
/**
|
||||
* The fully qualified name of the inference table where the trace is stored
|
||||
*/
|
||||
fullTableName: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface representing a Databricks Unity Catalog table-prefix location.
|
||||
* Mirrors Python's `UnityCatalog` location.
|
||||
*/
|
||||
export interface UnityCatalogLocation {
|
||||
catalogName: string;
|
||||
schemaName: string;
|
||||
/** Customer-supplied table prefix; required for trace ID location string. */
|
||||
tablePrefix?: string;
|
||||
/** Backend-populated fully qualified spans table name. */
|
||||
otelSpansTableName?: string;
|
||||
/** Backend-populated fully qualified logs table name. */
|
||||
otelLogsTableName?: string;
|
||||
/** Backend-populated fully qualified annotations table name. */
|
||||
annotationsTableName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface representing the location where the trace is stored
|
||||
*/
|
||||
export interface TraceLocation {
|
||||
/**
|
||||
* The type of the trace location
|
||||
*/
|
||||
type: TraceLocationType;
|
||||
|
||||
/**
|
||||
* The MLflow experiment location
|
||||
* Set this when the location type is MLflow experiment
|
||||
*/
|
||||
mlflowExperiment?: MlflowExperimentLocation;
|
||||
|
||||
/**
|
||||
* The inference table location
|
||||
* Set this when the location type is Databricks Inference table
|
||||
*/
|
||||
inferenceTable?: InferenceTableLocation;
|
||||
|
||||
/**
|
||||
* The Databricks UC table-prefix location. Set when type is UC_TABLE_PREFIX.
|
||||
*/
|
||||
ucTablePrefix?: UnityCatalogLocation;
|
||||
}
|
||||
|
||||
export interface SerializedTraceLocation {
|
||||
type: TraceLocationType;
|
||||
mlflow_experiment?: { experiment_id: string };
|
||||
inference_table?: { full_table_name: string };
|
||||
uc_table_prefix?: {
|
||||
catalog_name: string;
|
||||
schema_name: string;
|
||||
table_prefix?: string;
|
||||
otel_spans_table_name?: string;
|
||||
otel_logs_table_name?: string;
|
||||
annotations_table_name?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeTraceLocation(loc: TraceLocation): SerializedTraceLocation {
|
||||
const out: SerializedTraceLocation = { type: loc.type };
|
||||
if (loc.mlflowExperiment) {
|
||||
out.mlflow_experiment = { experiment_id: loc.mlflowExperiment.experimentId };
|
||||
}
|
||||
if (loc.inferenceTable) {
|
||||
out.inference_table = { full_table_name: loc.inferenceTable.fullTableName };
|
||||
}
|
||||
if (loc.ucTablePrefix) {
|
||||
const uc = loc.ucTablePrefix;
|
||||
out.uc_table_prefix = {
|
||||
catalog_name: uc.catalogName,
|
||||
schema_name: uc.schemaName,
|
||||
...(uc.tablePrefix ? { table_prefix: uc.tablePrefix } : {}),
|
||||
...(uc.otelSpansTableName ? { otel_spans_table_name: uc.otelSpansTableName } : {}),
|
||||
...(uc.otelLogsTableName ? { otel_logs_table_name: uc.otelLogsTableName } : {}),
|
||||
...(uc.annotationsTableName ? { annotations_table_name: uc.annotationsTableName } : {}),
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function deserializeTraceLocation(json: SerializedTraceLocation | undefined): TraceLocation {
|
||||
if (!json?.type) {
|
||||
throw new Error('Invalid trace location: missing type.');
|
||||
}
|
||||
|
||||
return {
|
||||
type: json.type,
|
||||
mlflowExperiment: json.mlflow_experiment
|
||||
? { experimentId: json.mlflow_experiment.experiment_id }
|
||||
: undefined,
|
||||
inferenceTable: json.inference_table
|
||||
? { fullTableName: json.inference_table.full_table_name }
|
||||
: undefined,
|
||||
ucTablePrefix: json.uc_table_prefix
|
||||
? {
|
||||
catalogName: json.uc_table_prefix.catalog_name,
|
||||
schemaName: json.uc_table_prefix.schema_name,
|
||||
tablePrefix: json.uc_table_prefix.table_prefix,
|
||||
otelSpansTableName: json.uc_table_prefix.otel_spans_table_name,
|
||||
otelLogsTableName: json.uc_table_prefix.otel_logs_table_name,
|
||||
annotationsTableName: json.uc_table_prefix.annotations_table_name,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns "catalog.schema.table_prefix" for a UC table-prefix location.
|
||||
* Throws if the prefix is not set; the prefix is required for trace IDs.
|
||||
*/
|
||||
export function ucTablePrefixLocationString(location: UnityCatalogLocation): string {
|
||||
if (!location.tablePrefix) {
|
||||
throw new Error(
|
||||
'Unity Catalog table_prefix is required to build a trace location string. ' +
|
||||
'Provide a tablePrefix when constructing the UnityCatalog destination.',
|
||||
);
|
||||
}
|
||||
return `${location.catalogName}.${location.schemaName}.${location.tablePrefix}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the location string used for V4 trace IDs and UC OTLP routing
|
||||
* (i.e. the `X-Databricks-UC-Table-Name` header value).
|
||||
*/
|
||||
export function getUcLocationString(traceLocation: TraceLocation): string | null {
|
||||
if (traceLocation.type === TraceLocationType.UC_TABLE_PREFIX && traceLocation.ucTablePrefix) {
|
||||
return ucTablePrefixLocationString(traceLocation.ucTablePrefix);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the fully qualified OTel spans table name to use as the
|
||||
* `X-Databricks-UC-Table-Name` header when exporting spans for this
|
||||
* trace via OTLP.
|
||||
*
|
||||
* Falls back to `<catalog>.<schema>.<table_prefix>_otel_spans`, which is the
|
||||
* default spans table name Databricks creates when a UC trace location is
|
||||
* provisioned. Customers with a custom backend-provisioned spans table can
|
||||
* override by setting `ucTablePrefix.otelSpansTableName`.
|
||||
*/
|
||||
export function getOtelSpansTableName(traceLocation: TraceLocation): string | null {
|
||||
if (traceLocation.type === TraceLocationType.UC_TABLE_PREFIX && traceLocation.ucTablePrefix) {
|
||||
const loc = traceLocation.ucTablePrefix;
|
||||
if (loc.otelSpansTableName) {
|
||||
return loc.otelSpansTableName;
|
||||
}
|
||||
if (loc.tablePrefix) {
|
||||
return `${loc.catalogName}.${loc.schemaName}.${loc.tablePrefix}_otel_spans`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a TraceLocation from an experiment ID
|
||||
* @param experimentId The ID of the MLflow experiment
|
||||
*/
|
||||
export function createTraceLocationFromExperimentId(experimentId: string): TraceLocation {
|
||||
return {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: experimentId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a TraceLocation from a Databricks UC table-prefix location.
|
||||
*/
|
||||
export function createTraceLocationFromUcTablePrefix(
|
||||
catalogName: string,
|
||||
schemaName: string,
|
||||
tablePrefix: string,
|
||||
): TraceLocation {
|
||||
return {
|
||||
type: TraceLocationType.UC_TABLE_PREFIX,
|
||||
ucTablePrefix: { catalogName, schemaName, tablePrefix },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* True iff the trace is stored in a Databricks Unity Catalog location.
|
||||
*/
|
||||
export function isUcTraceLocation(traceLocation: TraceLocation): boolean {
|
||||
return traceLocation.type === TraceLocationType.UC_TABLE_PREFIX;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
|
||||
/**
|
||||
* Enum representing the state of a trace
|
||||
*/
|
||||
export enum TraceState {
|
||||
/**
|
||||
* Unspecified trace state
|
||||
*/
|
||||
STATE_UNSPECIFIED = 'STATE_UNSPECIFIED',
|
||||
|
||||
/**
|
||||
* Trace successfully completed
|
||||
*/
|
||||
OK = 'OK',
|
||||
|
||||
/**
|
||||
* Trace encountered an error
|
||||
*/
|
||||
ERROR = 'ERROR',
|
||||
|
||||
/**
|
||||
* Trace is currently in progress
|
||||
*/
|
||||
IN_PROGRESS = 'IN_PROGRESS',
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert OpenTelemetry status code to MLflow TraceState
|
||||
* @param statusCode OpenTelemetry status code
|
||||
*/
|
||||
export function fromOtelStatus(statusCode: SpanStatusCode): TraceState {
|
||||
switch (statusCode) {
|
||||
case SpanStatusCode.OK:
|
||||
return TraceState.OK;
|
||||
case SpanStatusCode.ERROR:
|
||||
return TraceState.ERROR;
|
||||
default:
|
||||
return TraceState.STATE_UNSPECIFIED;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { SpanLogLevel, SpanType } from './constants';
|
||||
|
||||
/**
|
||||
* Span types whose default level is INFO. These represent the user-visible
|
||||
* semantic operations (model calls, tool calls, retrievals, agent turns, etc.)
|
||||
* that should remain visible at the default UI threshold. Everything else --
|
||||
* chain glue, output parsing, internal/custom types -- defaults to DEBUG.
|
||||
*
|
||||
* Mirrors the Python `_INFO_SPAN_TYPES` set in
|
||||
* `mlflow/tracing/utils/default_log_level.py`. Keep the two in sync.
|
||||
*/
|
||||
const INFO_SPAN_TYPES: ReadonlySet<string> = new Set<string>([
|
||||
SpanType.LLM,
|
||||
SpanType.CHAT_MODEL,
|
||||
SpanType.TOOL,
|
||||
SpanType.RETRIEVER,
|
||||
SpanType.AGENT,
|
||||
SpanType.EMBEDDING,
|
||||
]);
|
||||
|
||||
/**
|
||||
* Return the default SpanLogLevel for a span of the given type. Used by the
|
||||
* `LiveSpan` constructor so every span -- autologged or manual -- gets a
|
||||
* sensible level for the trace explorer's verbosity filter without users
|
||||
* annotating every call.
|
||||
*/
|
||||
export function defaultLogLevelForSpanType(spanType: string | undefined | null): SpanLogLevel {
|
||||
if (spanType && INFO_SPAN_TYPES.has(spanType)) {
|
||||
return SpanLogLevel.INFO;
|
||||
}
|
||||
return SpanLogLevel.DEBUG;
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { trace, Tracer } from '@opentelemetry/api';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import { SpanProcessor } from '@opentelemetry/sdk-trace-base';
|
||||
import { MlflowClient } from '../clients';
|
||||
import { MlflowSpanExporter, MlflowSpanProcessor } from '../exporters/mlflow';
|
||||
import {
|
||||
DatabricksUCTableSpanExporter,
|
||||
DatabricksUCTableSpanProcessor,
|
||||
} from '../exporters/uc_table';
|
||||
import { MlflowWalSpanExporter } from '../exporters/wal';
|
||||
import { getConfig, getAuthProvider, type UnityCatalogLocationOptions } from './config';
|
||||
import type { UnityCatalogLocation } from './entities/trace_location';
|
||||
import { asyncExportEnabled } from './utils/env';
|
||||
|
||||
let sdk: NodeSDK | null = null;
|
||||
// Keep a reference to the active span processor for flushing.
|
||||
let processor: SpanProcessor | null = null;
|
||||
|
||||
/**
|
||||
* Initialize the OpenTelemetry SDK and span processor.
|
||||
*
|
||||
* When `config.traceLocation` is provided, wires up the V4 Databricks Unity
|
||||
* Catalog span processor + exporter so that `updateCurrentTrace` tags persist
|
||||
* on UC-backed traces. Otherwise installs the V3 experiment-backed processor.
|
||||
*
|
||||
* TODO: Auto-resolve UC location from the linked Databricks experiment's
|
||||
* `mlflow.experiment.databricksTrace*` tags so customers don't have to pass
|
||||
* `traceLocation` explicitly. This needs `GetExperiment` to run, and Node
|
||||
* has no idiomatic way to do a synchronous HTTP request from `init()`
|
||||
* (the documented `worker_threads` + `Atomics.wait` pattern is for CPU-bound
|
||||
* work, not for converting async I/O into sync). The likely path is a
|
||||
* buffering processor that queues spans until the async fetch resolves;
|
||||
* until that lands, customers configure UC explicitly via `traceLocation`.
|
||||
*/
|
||||
export function initializeSDK(): void {
|
||||
if (sdk) {
|
||||
sdk.shutdown().catch((error) => {
|
||||
console.error('Error shutting down existing SDK:', error);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const config = getConfig();
|
||||
const authProvider = getAuthProvider();
|
||||
|
||||
const client = new MlflowClient({
|
||||
trackingUri: config.trackingUri,
|
||||
authProvider,
|
||||
});
|
||||
|
||||
if (config.traceLocation) {
|
||||
const ucLocation = resolveUcLocation(config.traceLocation);
|
||||
const ucExporter = new DatabricksUCTableSpanExporter(client);
|
||||
processor = new DatabricksUCTableSpanProcessor(ucExporter, ucLocation);
|
||||
} else {
|
||||
if (asyncExportEnabled()) {
|
||||
// WAL path: the hook submits records to a daemon over IPC; the
|
||||
// bundled daemon performs the actual auth setup, fsyncs the
|
||||
// record to `queue.log`, and runs the HTTP upload in a separate
|
||||
// process. The IPC client spawns the daemon on first use, so no
|
||||
// client construction or supervisor wiring is needed here.
|
||||
const mlflowWalExporter = new MlflowWalSpanExporter();
|
||||
processor = new MlflowSpanProcessor(mlflowWalExporter);
|
||||
} else {
|
||||
const exporter = new MlflowSpanExporter(client);
|
||||
processor = new MlflowSpanProcessor(exporter);
|
||||
}
|
||||
}
|
||||
|
||||
sdk = new NodeSDK({ spanProcessors: [processor] });
|
||||
sdk.start();
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize MLflow tracing:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the user-supplied UC location and return it as a
|
||||
* `UnityCatalogLocation`. All three fields are required; the SDK does not
|
||||
* upsert UC trace locations, so we cannot default any of them.
|
||||
*/
|
||||
function resolveUcLocation(options: UnityCatalogLocationOptions): UnityCatalogLocation {
|
||||
if (!options.catalogName || !options.schemaName || !options.tablePrefix) {
|
||||
throw new Error(
|
||||
'traceLocation requires catalogName, schemaName, and tablePrefix. The UC ' +
|
||||
'trace location must already be provisioned in the workspace.',
|
||||
);
|
||||
}
|
||||
return {
|
||||
catalogName: options.catalogName,
|
||||
schemaName: options.schemaName,
|
||||
tablePrefix: options.tablePrefix,
|
||||
};
|
||||
}
|
||||
|
||||
export function getTracer(module_name: string): Tracer {
|
||||
return trace.getTracer(module_name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Force flush all pending trace exports.
|
||||
*
|
||||
* In WAL mode this only awaits on-disk durability of the WAL appends,
|
||||
* not the upstream HTTP upload — that's the daemon's job and the
|
||||
* caller (e.g. the Claude Code Stop hook) is intentionally decoupled
|
||||
* from backend latency.
|
||||
*/
|
||||
export async function flushTraces(): Promise<void> {
|
||||
await processor?.forceFlush();
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import { LiveSpan, Span } from './entities/span';
|
||||
import { TraceInfo } from './entities/trace_info';
|
||||
import { Trace } from './entities/trace';
|
||||
import { TraceData } from './entities/trace_data';
|
||||
import {
|
||||
REQUEST_RESPONSE_PREVIEW_MAX_LENGTH,
|
||||
SpanAttributeKey,
|
||||
TraceMetadataKey,
|
||||
} from './constants';
|
||||
|
||||
/**
|
||||
* Internal representation to keep the state of a trace.
|
||||
* Uses a Map<string, LiveSpan> instead of TraceData to allow access by span_id.
|
||||
*/
|
||||
class _Trace {
|
||||
info: TraceInfo;
|
||||
spanDict: Map<string, LiveSpan>;
|
||||
|
||||
constructor(info: TraceInfo) {
|
||||
this.info = info;
|
||||
this.spanDict = new Map<string, LiveSpan>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the internal trace representation to an MLflow Trace object
|
||||
*/
|
||||
toMlflowTrace(): Trace {
|
||||
// Convert LiveSpan, mutable objects, into immutable Span objects before persisting
|
||||
const traceData = new TraceData([...this.spanDict.values()] as Span[]);
|
||||
|
||||
const root_span = traceData.spans.find((span) => span.parentId == null);
|
||||
if (root_span) {
|
||||
// Only set previews if they haven't been explicitly set by updateCurrentTrace
|
||||
if (!this.info.requestPreview) {
|
||||
this.info.requestPreview = getTruncatedPreview(
|
||||
root_span._span.attributes[SpanAttributeKey.INPUTS] as string,
|
||||
'user',
|
||||
);
|
||||
}
|
||||
if (!this.info.responsePreview) {
|
||||
this.info.responsePreview = getTruncatedPreview(
|
||||
root_span._span.attributes[SpanAttributeKey.OUTPUTS] as string,
|
||||
'assistant',
|
||||
);
|
||||
}
|
||||
|
||||
// TODO: Remove this once the new trace table UI is available that is based on MLflow V3 trace.
|
||||
// Until then, these two metadata are still used to render the "request" and "response" columns.
|
||||
this.info.traceMetadata[TraceMetadataKey.INPUTS] = this.info.requestPreview;
|
||||
this.info.traceMetadata[TraceMetadataKey.OUTPUTS] = this.info.responsePreview;
|
||||
}
|
||||
|
||||
return new Trace(this.info, traceData);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a truncated preview string from span inputs/outputs.
|
||||
* Extracts message text content from known chat formats (OpenAI messages,
|
||||
* choices, Responses API) before truncating, so the preview shows readable
|
||||
* text instead of cut-off JSON.
|
||||
*
|
||||
* Mirrors the Python SDK's `_get_truncated_preview` in
|
||||
* `mlflow/tracing/utils/truncation.py`.
|
||||
*/
|
||||
function getTruncatedPreview(inputsOrOutputs: string, role: string): string {
|
||||
if (!inputsOrOutputs) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const maxLength = REQUEST_RESPONSE_PREVIEW_MAX_LENGTH;
|
||||
let content: string | null = null;
|
||||
|
||||
try {
|
||||
const obj = JSON.parse(inputsOrOutputs);
|
||||
if (obj && typeof obj === 'object') {
|
||||
const messages = tryExtractMessages(obj as Record<string, unknown>);
|
||||
if (messages && messages.length > 0) {
|
||||
const msg = getLastMessageByRole(messages, role);
|
||||
content = getTextContentFromMessage(msg);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Not valid JSON — use as-is
|
||||
}
|
||||
|
||||
const preview = content || inputsOrOutputs;
|
||||
if (preview.length <= maxLength) {
|
||||
return preview;
|
||||
}
|
||||
return preview.slice(0, maxLength - 3) + '...';
|
||||
}
|
||||
|
||||
function tryExtractMessages(obj: Record<string, unknown>): Record<string, unknown>[] | null {
|
||||
// OpenAI ChatCompletion request format: { messages: [...] }
|
||||
if (Array.isArray(obj.messages)) {
|
||||
return obj.messages.filter(isMessage);
|
||||
}
|
||||
|
||||
// OpenAI ChatCompletion response format: { choices: [{ message: {...} }] }
|
||||
if (
|
||||
Array.isArray(obj.choices) &&
|
||||
obj.choices.length > 0 &&
|
||||
typeof obj.choices[0] === 'object' &&
|
||||
obj.choices[0] != null
|
||||
) {
|
||||
const msg = (obj.choices[0] as Record<string, unknown>).message;
|
||||
if (isMessage(msg)) {
|
||||
return [msg];
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI Responses API request format: { input: [...] }
|
||||
if (Array.isArray(obj.input)) {
|
||||
return obj.input.filter(isMessage);
|
||||
}
|
||||
|
||||
// OpenAI Responses API response format: { output: [...] }
|
||||
if (Array.isArray(obj.output)) {
|
||||
return obj.output.filter(isMessage);
|
||||
}
|
||||
|
||||
// ResponsesAgent input: { request: { ... } }
|
||||
if (obj.request && typeof obj.request === 'object') {
|
||||
return tryExtractMessages(obj.request as Record<string, unknown>);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function isMessage(item: unknown): item is Record<string, unknown> {
|
||||
return (
|
||||
item != null &&
|
||||
typeof item === 'object' &&
|
||||
'role' in (item as Record<string, unknown>) &&
|
||||
'content' in (item as Record<string, unknown>)
|
||||
);
|
||||
}
|
||||
|
||||
function getLastMessageByRole(
|
||||
messages: Record<string, unknown>[],
|
||||
role: string,
|
||||
): Record<string, unknown> {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
if (messages[i].role === role) {
|
||||
return messages[i];
|
||||
}
|
||||
}
|
||||
return messages[messages.length - 1];
|
||||
}
|
||||
|
||||
function getTextContentFromMessage(message: Record<string, unknown>): string | null {
|
||||
const content = message.content;
|
||||
if (typeof content === 'string') {
|
||||
return content;
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
for (const part of content) {
|
||||
if (typeof part === 'string') {
|
||||
return part;
|
||||
}
|
||||
if (
|
||||
part &&
|
||||
typeof part === 'object' &&
|
||||
'type' in (part as Record<string, unknown>) &&
|
||||
((part as Record<string, unknown>).type === 'text' ||
|
||||
(part as Record<string, unknown>).type === 'output_text')
|
||||
) {
|
||||
return (part as Record<string, unknown>).text as string;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manage spans and traces created by the tracing system in memory.
|
||||
* This class is implemented as a singleton.
|
||||
*/
|
||||
export class InMemoryTraceManager {
|
||||
private static _instance: InMemoryTraceManager | undefined;
|
||||
|
||||
// In-memory cache to store trace_id -> _Trace mapping
|
||||
// TODO: Add TTL to the trace buffer similarly to Python SDK
|
||||
private _traces: Map<string, _Trace>;
|
||||
// Store mapping between OpenTelemetry trace ID and MLflow trace ID
|
||||
private _otelIdToMlflowTraceId: Map<string, string>;
|
||||
|
||||
// Store the last active trace ID
|
||||
lastActiveTraceId: string | undefined;
|
||||
|
||||
/**
|
||||
* Singleton pattern implementation
|
||||
*/
|
||||
static getInstance(): InMemoryTraceManager {
|
||||
if (InMemoryTraceManager._instance == null) {
|
||||
InMemoryTraceManager._instance = new InMemoryTraceManager();
|
||||
}
|
||||
return InMemoryTraceManager._instance;
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
this._traces = new Map<string, _Trace>();
|
||||
this._otelIdToMlflowTraceId = new Map<string, string>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new trace info object to the in-memory trace registry.
|
||||
* @param otelTraceId The OpenTelemetry trace ID for the new trace
|
||||
* @param traceInfo The trace info object to be stored
|
||||
*/
|
||||
registerTrace(otelTraceId: string, traceInfo: TraceInfo): void {
|
||||
this._traces.set(traceInfo.traceId, new _Trace(traceInfo));
|
||||
this._otelIdToMlflowTraceId.set(otelTraceId, traceInfo.traceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the given span in the in-memory trace data.
|
||||
* @param span The span to be stored
|
||||
*/
|
||||
registerSpan(span: LiveSpan): void {
|
||||
const trace = this._traces.get(span.traceId);
|
||||
if (trace) {
|
||||
trace.spanDict.set(span.spanId, span);
|
||||
} else {
|
||||
console.debug(`Tried to register span ${span.spanId} for trace ${span.traceId}
|
||||
but trace not found. Please make sure to register the trace first.`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the trace for the given trace ID.
|
||||
* Returns the trace object or null if not found.
|
||||
* @param traceId The trace ID to look up
|
||||
*/
|
||||
getTrace(traceId: string): _Trace | null {
|
||||
return this._traces.get(traceId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the MLflow trace ID for the given OpenTelemetry trace ID.
|
||||
* @param otelTraceId The OpenTelemetry trace ID
|
||||
*/
|
||||
getMlflowTraceIdFromOtelId(otelTraceId: string): string | null {
|
||||
return this._otelIdToMlflowTraceId.get(otelTraceId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the span for the given trace ID and span ID.
|
||||
* @param traceId The trace ID
|
||||
* @param spanId The span ID
|
||||
*/
|
||||
getSpan(traceId?: string | null, spanId?: string | null): LiveSpan | null {
|
||||
if (traceId == null || spanId == null) {
|
||||
return null;
|
||||
}
|
||||
return this._traces.get(traceId)?.spanDict.get(spanId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pop trace data for the given OpenTelemetry trace ID and return it as
|
||||
* a ready-to-publish Trace object.
|
||||
* @param otelTraceId The OpenTelemetry trace ID
|
||||
*/
|
||||
popTrace(otelTraceId: string): Trace | null {
|
||||
const mlflowTraceId = this._otelIdToMlflowTraceId.get(otelTraceId);
|
||||
if (!mlflowTraceId) {
|
||||
console.debug(`Tried to pop trace ${otelTraceId} but no trace found.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
this._otelIdToMlflowTraceId.delete(otelTraceId);
|
||||
const trace = this._traces.get(mlflowTraceId);
|
||||
if (trace) {
|
||||
this._traces.delete(mlflowTraceId);
|
||||
return trace.toMlflowTrace();
|
||||
}
|
||||
console.debug(`Tried to pop trace ${otelTraceId} but trace not found.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all the aggregated trace data. This should only be used for testing.
|
||||
*/
|
||||
static reset(): void {
|
||||
if (InMemoryTraceManager._instance) {
|
||||
InMemoryTraceManager._instance._traces.clear();
|
||||
InMemoryTraceManager._instance._otelIdToMlflowTraceId.clear();
|
||||
InMemoryTraceManager._instance = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { isAbsolute } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
/**
|
||||
* Extract the scheme from an artifact URI.
|
||||
*/
|
||||
export function getArtifactUriScheme(uri: string): string {
|
||||
const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):/.exec(uri);
|
||||
if (!match) {
|
||||
return '';
|
||||
}
|
||||
const scheme = match[1].toLowerCase();
|
||||
// A single-letter scheme is a Windows drive designator (e.g. `C:`), not a URI
|
||||
// scheme; treat it as a bare local path.
|
||||
if (scheme.length === 1) {
|
||||
return '';
|
||||
}
|
||||
return scheme;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an artifact URI refers to a location on the local filesystem.
|
||||
*/
|
||||
export function isLocalArtifactUri(uri: string): boolean {
|
||||
const scheme = getArtifactUriScheme(uri);
|
||||
return scheme === '' || scheme === 'file';
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a local artifact URI to an filesystem path.
|
||||
*/
|
||||
export function artifactUriToLocalPath(uri: string): string {
|
||||
if (getArtifactUriScheme(uri) === 'file') {
|
||||
return fileURLToPath(uri);
|
||||
}
|
||||
return uri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a local artifact URI to a filesystem path, requiring it to be absolute.
|
||||
*/
|
||||
export function toAbsoluteLocalPath(uri: string): string {
|
||||
const dir = artifactUriToLocalPath(uri);
|
||||
if (!isAbsolute(dir)) {
|
||||
throw new Error(
|
||||
`Refusing to use a relative local artifact location "${uri}". ` +
|
||||
`Expected an absolute filesystem path or a file:// URI.`,
|
||||
);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Read an env var and require a strictly positive integer.
|
||||
*/
|
||||
export const readPositiveInt = (envName: string, fallback: number): number => {
|
||||
const raw = process.env[envName];
|
||||
if (raw === undefined || raw === '') {
|
||||
return fallback;
|
||||
}
|
||||
// `Number` (vs `Number.parseInt`) refuses trailing garbage and unit
|
||||
// suffixes — e.g. "15000abc", "15s", and "10.9" all become NaN or a
|
||||
// non-integer, so the `isInteger` check below rejects them cleanly.
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
console.warn(
|
||||
`[mlflow] Ignoring invalid ${envName}=${JSON.stringify(raw)}; ` +
|
||||
`expected a positive integer. Falling back to default ${fallback}.`,
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
};
|
||||
|
||||
export const readNonNegativeInt = (envName: string, fallback: number): number => {
|
||||
const raw = process.env[envName];
|
||||
if (raw === undefined || raw === '') {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
console.warn(
|
||||
`[mlflow] Ignoring invalid ${envName}=${JSON.stringify(raw)}; ` +
|
||||
`expected a non-negative integer. Falling back to default ${fallback}.`,
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether to route trace export through an on-disk WAL queue
|
||||
* drained by a background daemon, instead of the synchronous
|
||||
* HTTP exporter.
|
||||
*/
|
||||
export function asyncExportEnabled(): boolean {
|
||||
const raw = process.env.MLFLOW_ENABLE_ASYNC_TRACE_LOGGING;
|
||||
if (raw === undefined || raw === '') {
|
||||
return false;
|
||||
}
|
||||
const lower = raw.toLowerCase();
|
||||
return lower === '1' || lower === 'true';
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
import type { HrTime } from '@opentelemetry/api';
|
||||
import { Span } from '../entities/span';
|
||||
import { SpanAttributeKey } from '../constants';
|
||||
import { TokenUsage } from '../entities/trace_info';
|
||||
|
||||
/**
|
||||
* OpenTelemetry Typescript SDK uses a unique timestamp format `HrTime` to represent
|
||||
* timestamps. This function converts a timestamp in nanoseconds to an `HrTime`
|
||||
* Supports both number and BigInt for large timestamps
|
||||
* Ref: https://github.com/open-telemetry/opentelemetry-js/blob/a9fc600f2bd7dbf9345ec14e4421f1cc034f1f9c/api/src/common/Time.ts#L17-L30C13
|
||||
* @param nanoseconds The timestamp in nanoseconds (number or BigInt)
|
||||
* @returns The timestamp in `HrTime` format
|
||||
*/
|
||||
export function convertNanoSecondsToHrTime(nanoseconds: number | bigint): HrTime {
|
||||
// Handle both number and BigInt inputs
|
||||
if (typeof nanoseconds === 'bigint') {
|
||||
// Use BigInt arithmetic to maintain precision
|
||||
const seconds = Number(nanoseconds / 1_000_000_000n);
|
||||
const nanos = Number(nanoseconds % 1_000_000_000n);
|
||||
return [seconds, nanos] as HrTime;
|
||||
}
|
||||
// For regular numbers, use standard arithmetic
|
||||
return [Math.floor(nanoseconds / 1e9), nanoseconds % 1e9] as HrTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert HrTime to nanoseconds as BigInt
|
||||
* @param hrTime HrTime tuple [seconds, nanoseconds]
|
||||
* @returns BigInt nanoseconds
|
||||
*/
|
||||
export function convertHrTimeToNanoSeconds(hrTime: HrTime): bigint {
|
||||
return BigInt(hrTime[0]) * 1_000_000_000n + BigInt(hrTime[1]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert HrTime to milliseconds
|
||||
* @param hrTime HrTime tuple [seconds, nanoseconds]
|
||||
* @returns Milliseconds
|
||||
*/
|
||||
export function convertHrTimeToMs(hrTime: HrTime): number {
|
||||
return Math.floor(hrTime[0] * 1e3 + hrTime[1] / 1e6);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a hex span ID to base64 format for JSON serialization
|
||||
* Following Python implementation: _encode_span_id_to_byte
|
||||
* @param spanId Hex string span ID (16 chars)
|
||||
* @returns Base64 encoded span ID
|
||||
*/
|
||||
export function encodeSpanIdToBase64(spanId: string): string {
|
||||
// Convert hex string to bytes (8 bytes for span ID)
|
||||
const bytes = new Uint8Array(8);
|
||||
|
||||
// Parse hex string (add padding to 16 chars)
|
||||
const hexStr = spanId.padStart(16, '0');
|
||||
for (let i = 0; i < 8; i++) {
|
||||
bytes[i] = parseInt(hexStr.substring(i * 2, i * 2 + 2), 16);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
return Buffer.from(bytes).toString('base64');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a hex span ID to base64 format for JSON serialization
|
||||
* Following Python implementation: _encode_trace_id_to_byte
|
||||
* @param spanId Hex string span ID (32 chars)
|
||||
* @returns Base64 encoded span ID
|
||||
*/
|
||||
export function encodeTraceIdToBase64(traceId: string): string {
|
||||
// Convert hex string to bytes (16 bytes for trace ID)
|
||||
const bytes = new Uint8Array(16);
|
||||
|
||||
// Parse hex string (add padding to 32 chars)
|
||||
const hexStr = traceId.padStart(32, '0');
|
||||
for (let i = 0; i < 16; i++) {
|
||||
bytes[i] = parseInt(hexStr.substring(i * 2, i * 2 + 2), 16);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
return Buffer.from(bytes).toString('base64');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a base64 span ID back to hex format
|
||||
* Following Python implementation: _decode_id_from_byte
|
||||
* @param base64SpanId Base64 encoded span ID
|
||||
* @returns Hex string span ID
|
||||
*/
|
||||
export function decodeIdFromBase64(base64SpanId: string): string {
|
||||
// Decode from base64
|
||||
const bytes = Buffer.from(base64SpanId, 'base64');
|
||||
|
||||
// Convert to hex string
|
||||
return Array.from(bytes)
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Map function arguments to an object with parameter names as keys
|
||||
* @param func The function to extract parameter names from
|
||||
* @param args The arguments passed to the function
|
||||
* @returns Object mapping parameter names to argument values
|
||||
*/
|
||||
export function mapArgsToObject(func: Function, args: any[]): Record<string, any> {
|
||||
const paramNames = getParameterNames(func);
|
||||
|
||||
// If we can't extract parameter names, return the args as an array
|
||||
if (paramNames.length === 0) {
|
||||
return args.length === 0 ? {} : { args };
|
||||
}
|
||||
|
||||
const result: Record<string, any> = {};
|
||||
|
||||
paramNames.forEach((name, index) => {
|
||||
if (index < args.length) {
|
||||
result[name] = args[index];
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract parameter names from a function using string parsing
|
||||
* @param func The function to extract parameter names from
|
||||
* @returns Array of parameter names
|
||||
*/
|
||||
function getParameterNames(func: Function): string[] {
|
||||
const funcStr = func.toString();
|
||||
|
||||
// Handle arrow functions and regular functions
|
||||
let paramMatch: RegExpMatchArray | null;
|
||||
|
||||
// Try arrow function pattern: (a, b) => or a =>
|
||||
const arrowMatch = funcStr.match(/^[^(]*\(?([^)=]*)\)?\s*=>/);
|
||||
if (arrowMatch) {
|
||||
const params = arrowMatch[1].trim();
|
||||
if (!params) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Handle single parameter without parentheses
|
||||
if (!params.includes(',') && !funcStr.includes('(')) {
|
||||
return [params.trim()];
|
||||
}
|
||||
|
||||
paramMatch = ['', params];
|
||||
} else {
|
||||
// Try regular function pattern: function name(a, b) or (a, b)
|
||||
paramMatch = funcStr.match(/(?:function\s*[^(]*)??\(([^)]*)\)/);
|
||||
}
|
||||
|
||||
if (!paramMatch?.[1]) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Split parameters while handling nested brackets/braces
|
||||
const params = [];
|
||||
let current = '';
|
||||
let depth = 0;
|
||||
|
||||
const paramStr = paramMatch[1];
|
||||
for (let i = 0; i < paramStr.length; i++) {
|
||||
const char = paramStr[i];
|
||||
|
||||
if (char === '{' || char === '[') {
|
||||
depth++;
|
||||
} else if (char === '}' || char === ']') {
|
||||
depth--;
|
||||
} else if (char === ',' && depth === 0) {
|
||||
params.push(current.trim());
|
||||
current = '';
|
||||
continue;
|
||||
}
|
||||
current += char;
|
||||
}
|
||||
if (current.trim()) {
|
||||
params.push(current.trim());
|
||||
}
|
||||
|
||||
return params
|
||||
.map((param) => {
|
||||
let name = param.trim();
|
||||
|
||||
// Skip destructured parameters
|
||||
if (name.includes('{') || name.includes('[')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
name = name.split('=')[0].trim(); // Remove default values: a = 5
|
||||
name = name.split(':')[0].trim(); // Remove type annotations: a: number
|
||||
if (name.startsWith('...')) {
|
||||
return null; // Ignore rest operator: ...args
|
||||
}
|
||||
return name;
|
||||
})
|
||||
.filter((name): name is string => name != null && name !== '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate token usage information from all spans in a trace.
|
||||
*
|
||||
* This function iterates through all spans and extracts token usage from the
|
||||
* SpanAttributeKey.TOKEN_USAGE attribute. It avoids double-counting token usage
|
||||
* when both parent and child spans have usage data (e.g., LangChain ChatOpenAI + OpenAI tracing).
|
||||
*
|
||||
* @param spans - Array of spans to aggregate usage from
|
||||
* @returns Aggregated token usage or null if no usage data exists
|
||||
*/
|
||||
export function aggregateUsageFromSpans(spans: Span[]): TokenUsage | null {
|
||||
const totalUsage: TokenUsage = {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
total_tokens: 0,
|
||||
};
|
||||
let hasUsageData = false;
|
||||
|
||||
// Build parent-children map for DFS traversal (mirrors Python SDK)
|
||||
const spanById = new Map<string, Span>();
|
||||
const childrenMap = new Map<string, Span[]>();
|
||||
const roots: Span[] = [];
|
||||
|
||||
for (const span of spans) {
|
||||
spanById.set(span.spanId, span);
|
||||
if (span.parentId && spanById.has(span.parentId)) {
|
||||
const children = childrenMap.get(span.parentId) || [];
|
||||
children.push(span);
|
||||
childrenMap.set(span.parentId, children);
|
||||
} else {
|
||||
roots.push(span);
|
||||
}
|
||||
}
|
||||
|
||||
function dfs(span: Span, ancestorHasData: boolean): void {
|
||||
const tokenUsageAttr = span.attributes[SpanAttributeKey.TOKEN_USAGE];
|
||||
const spanHasData = tokenUsageAttr != null;
|
||||
|
||||
if (spanHasData && !ancestorHasData) {
|
||||
const tokenUsage = tokenUsageAttr as TokenUsage;
|
||||
totalUsage.input_tokens += tokenUsage.input_tokens || 0;
|
||||
totalUsage.output_tokens += tokenUsage.output_tokens || 0;
|
||||
totalUsage.total_tokens += tokenUsage.total_tokens || 0;
|
||||
// Optional cache keys — only include when present
|
||||
if (tokenUsage.cache_read_input_tokens != null) {
|
||||
totalUsage.cache_read_input_tokens =
|
||||
(totalUsage.cache_read_input_tokens || 0) + tokenUsage.cache_read_input_tokens;
|
||||
}
|
||||
if (tokenUsage.cache_creation_input_tokens != null) {
|
||||
totalUsage.cache_creation_input_tokens =
|
||||
(totalUsage.cache_creation_input_tokens || 0) + tokenUsage.cache_creation_input_tokens;
|
||||
}
|
||||
hasUsageData = true;
|
||||
}
|
||||
|
||||
for (const child of childrenMap.get(span.spanId) || []) {
|
||||
dfs(child, ancestorHasData || spanHasData);
|
||||
}
|
||||
}
|
||||
|
||||
for (const root of roots) {
|
||||
dfs(root, false);
|
||||
}
|
||||
|
||||
return hasUsageData ? totalUsage : null;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
|
||||
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
|
||||
*
|
||||
* The repository contains a critical bug fix for decimal handling, however, it has not been
|
||||
* released to npm yet. This file is a copy of the source code with the bug fix applied.
|
||||
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
|
||||
*
|
||||
* :copyright: Copyright (c) 2013 Andrey Sidorov
|
||||
* :license: The MIT License (MIT)
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
var json_stringify = require('./stringify.js').stringify;
|
||||
var json_parse = require('./parse.js');
|
||||
|
||||
module.exports = function (options) {
|
||||
return {
|
||||
parse: json_parse(options),
|
||||
stringify: json_stringify,
|
||||
};
|
||||
};
|
||||
//create the default method members with no options applied for backwards compatibility
|
||||
module.exports.parse = json_parse();
|
||||
module.exports.stringify = json_stringify;
|
||||
@@ -0,0 +1,455 @@
|
||||
/**
|
||||
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
|
||||
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
|
||||
*
|
||||
* The repository contains a critical bug fix for decimal handling, however, it has not been
|
||||
* released to npm yet. This file is a copy of the source code with the bug fix applied.
|
||||
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
|
||||
*
|
||||
* :copyright: Copyright (c) 2013 Andrey Sidorov
|
||||
* :license: The MIT License (MIT)
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
var BigNumber = null;
|
||||
|
||||
// regexpxs extracted from
|
||||
// (c) BSD-3-Clause
|
||||
// https://github.com/fastify/secure-json-parse/graphs/contributors and https://github.com/hapijs/bourne/graphs/contributors
|
||||
|
||||
const suspectProtoRx =
|
||||
/(?:_|\\u005[Ff])(?:_|\\u005[Ff])(?:p|\\u0070)(?:r|\\u0072)(?:o|\\u006[Ff])(?:t|\\u0074)(?:o|\\u006[Ff])(?:_|\\u005[Ff])(?:_|\\u005[Ff])/;
|
||||
const suspectConstructorRx =
|
||||
/(?:c|\\u0063)(?:o|\\u006[Ff])(?:n|\\u006[Ee])(?:s|\\u0073)(?:t|\\u0074)(?:r|\\u0072)(?:u|\\u0075)(?:c|\\u0063)(?:t|\\u0074)(?:o|\\u006[Ff])(?:r|\\u0072)/;
|
||||
|
||||
/*
|
||||
json_parse.js
|
||||
2012-06-20
|
||||
|
||||
Public Domain.
|
||||
|
||||
NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.
|
||||
|
||||
This file creates a json_parse function.
|
||||
During create you can (optionally) specify some behavioural switches
|
||||
|
||||
require('json-bigint')(options)
|
||||
|
||||
The optional options parameter holds switches that drive certain
|
||||
aspects of the parsing process:
|
||||
* options.strict = true will warn about duplicate-key usage in the json.
|
||||
The default (strict = false) will silently ignore those and overwrite
|
||||
values for keys that are in duplicate use.
|
||||
|
||||
The resulting function follows this signature:
|
||||
json_parse(text, reviver)
|
||||
This method parses a JSON text to produce an object or array.
|
||||
It can throw a SyntaxError exception.
|
||||
|
||||
The optional reviver parameter is a function that can filter and
|
||||
transform the results. It receives each of the keys and values,
|
||||
and its return value is used instead of the original value.
|
||||
If it returns what it received, then the structure is not modified.
|
||||
If it returns undefined then the member is deleted.
|
||||
|
||||
Example:
|
||||
|
||||
// Parse the text. Values that look like ISO date strings will
|
||||
// be converted to Date objects.
|
||||
|
||||
myData = json_parse(text, function (key, value) {
|
||||
var a;
|
||||
if (typeof value === 'string') {
|
||||
a =
|
||||
/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*)?)Z$/.exec(value);
|
||||
if (a) {
|
||||
return new Date(Date.UTC(+a[1], +a[2] - 1, +a[3], +a[4],
|
||||
+a[5], +a[6]));
|
||||
}
|
||||
}
|
||||
return value;
|
||||
});
|
||||
|
||||
This is a reference implementation. You are free to copy, modify, or
|
||||
redistribute.
|
||||
|
||||
This code should be minified before deployment.
|
||||
See http://javascript.crockford.com/jsmin.html
|
||||
|
||||
USE YOUR OWN COPY. IT IS EXTREMELY UNWISE TO LOAD CODE FROM SERVERS YOU DO
|
||||
NOT CONTROL.
|
||||
*/
|
||||
|
||||
/*members "", "\"", "\/", "\\", at, b, call, charAt, f, fromCharCode,
|
||||
hasOwnProperty, message, n, name, prototype, push, r, t, text
|
||||
*/
|
||||
|
||||
var json_parse = function (options) {
|
||||
'use strict';
|
||||
|
||||
// This is a function that can parse a JSON text, producing a JavaScript
|
||||
// data structure. It is a simple, recursive descent parser. It does not use
|
||||
// eval or regular expressions, so it can be used as a model for implementing
|
||||
// a JSON parser in other languages.
|
||||
|
||||
// We are defining the function inside of another function to avoid creating
|
||||
// global variables.
|
||||
|
||||
// Default options one can override by passing options to the parse()
|
||||
var _options = {
|
||||
strict: false, // not being strict means do not generate syntax errors for "duplicate key"
|
||||
storeAsString: false, // toggles whether the values should be stored as BigNumber (default) or a string
|
||||
alwaysParseAsBig: false, // toggles whether all numbers should be Big
|
||||
useNativeBigInt: false, // toggles whether to use native BigInt instead of bignumber.js
|
||||
protoAction: 'error',
|
||||
constructorAction: 'error',
|
||||
};
|
||||
|
||||
// If there are options, then use them to override the default _options
|
||||
if (options !== undefined && options !== null) {
|
||||
if (options.strict === true) {
|
||||
_options.strict = true;
|
||||
}
|
||||
if (options.storeAsString === true) {
|
||||
_options.storeAsString = true;
|
||||
}
|
||||
_options.alwaysParseAsBig =
|
||||
options.alwaysParseAsBig === true ? options.alwaysParseAsBig : false;
|
||||
_options.useNativeBigInt = options.useNativeBigInt === true ? options.useNativeBigInt : false;
|
||||
|
||||
if (typeof options.constructorAction !== 'undefined') {
|
||||
if (
|
||||
options.constructorAction === 'error' ||
|
||||
options.constructorAction === 'ignore' ||
|
||||
options.constructorAction === 'preserve'
|
||||
) {
|
||||
_options.constructorAction = options.constructorAction;
|
||||
} else {
|
||||
throw new Error(
|
||||
`Incorrect value for constructorAction option, must be "error", "ignore" or undefined but passed ${options.constructorAction}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof options.protoAction !== 'undefined') {
|
||||
if (
|
||||
options.protoAction === 'error' ||
|
||||
options.protoAction === 'ignore' ||
|
||||
options.protoAction === 'preserve'
|
||||
) {
|
||||
_options.protoAction = options.protoAction;
|
||||
} else {
|
||||
throw new Error(
|
||||
`Incorrect value for protoAction option, must be "error", "ignore" or undefined but passed ${options.protoAction}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var at, // The index of the current character
|
||||
ch, // The current character
|
||||
escapee = {
|
||||
'"': '"',
|
||||
'\\': '\\',
|
||||
'/': '/',
|
||||
b: '\b',
|
||||
f: '\f',
|
||||
n: '\n',
|
||||
r: '\r',
|
||||
t: '\t',
|
||||
},
|
||||
text,
|
||||
error = function (m) {
|
||||
// Call error when something is wrong.
|
||||
|
||||
throw {
|
||||
name: 'SyntaxError',
|
||||
message: m,
|
||||
at: at,
|
||||
text: text,
|
||||
};
|
||||
},
|
||||
next = function (c) {
|
||||
// If a c parameter is provided, verify that it matches the current character.
|
||||
|
||||
if (c && c !== ch) {
|
||||
error("Expected '" + c + "' instead of '" + ch + "'");
|
||||
}
|
||||
|
||||
// Get the next character. When there are no more characters,
|
||||
// return the empty string.
|
||||
|
||||
ch = text.charAt(at);
|
||||
at += 1;
|
||||
return ch;
|
||||
},
|
||||
number = function () {
|
||||
// Parse a number value.
|
||||
|
||||
var number,
|
||||
string = '';
|
||||
|
||||
if (ch === '-') {
|
||||
string = '-';
|
||||
next('-');
|
||||
}
|
||||
while (ch >= '0' && ch <= '9') {
|
||||
string += ch;
|
||||
next();
|
||||
}
|
||||
if (ch === '.') {
|
||||
string += '.';
|
||||
while (next() && ch >= '0' && ch <= '9') {
|
||||
string += ch;
|
||||
}
|
||||
}
|
||||
if (ch === 'e' || ch === 'E') {
|
||||
string += ch;
|
||||
next();
|
||||
if (ch === '-' || ch === '+') {
|
||||
string += ch;
|
||||
next();
|
||||
}
|
||||
while (ch >= '0' && ch <= '9') {
|
||||
string += ch;
|
||||
next();
|
||||
}
|
||||
}
|
||||
number = +string;
|
||||
if (!isFinite(number)) {
|
||||
error('Bad number');
|
||||
} else {
|
||||
if (BigNumber == null) BigNumber = require('bignumber.js');
|
||||
if (Number.isSafeInteger(number))
|
||||
return !_options.alwaysParseAsBig
|
||||
? number
|
||||
: _options.useNativeBigInt
|
||||
? BigInt(number)
|
||||
: new BigNumber(number);
|
||||
else
|
||||
// Number with fractional part should be treated as number(double) including big integers in scientific notation, i.e 1.79e+308
|
||||
return _options.storeAsString
|
||||
? string
|
||||
: /[\.eE]/.test(string)
|
||||
? number
|
||||
: _options.useNativeBigInt
|
||||
? BigInt(string)
|
||||
: new BigNumber(string);
|
||||
}
|
||||
},
|
||||
string = function () {
|
||||
// Parse a string value.
|
||||
|
||||
var hex,
|
||||
i,
|
||||
string = '',
|
||||
uffff;
|
||||
|
||||
// When parsing for string values, we must look for " and \ characters.
|
||||
|
||||
if (ch === '"') {
|
||||
var startAt = at;
|
||||
while (next()) {
|
||||
if (ch === '"') {
|
||||
if (at - 1 > startAt) string += text.substring(startAt, at - 1);
|
||||
next();
|
||||
return string;
|
||||
}
|
||||
if (ch === '\\') {
|
||||
if (at - 1 > startAt) string += text.substring(startAt, at - 1);
|
||||
next();
|
||||
if (ch === 'u') {
|
||||
uffff = 0;
|
||||
for (i = 0; i < 4; i += 1) {
|
||||
hex = parseInt(next(), 16);
|
||||
if (!isFinite(hex)) {
|
||||
break;
|
||||
}
|
||||
uffff = uffff * 16 + hex;
|
||||
}
|
||||
string += String.fromCharCode(uffff);
|
||||
} else if (typeof escapee[ch] === 'string') {
|
||||
string += escapee[ch];
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
startAt = at;
|
||||
}
|
||||
}
|
||||
}
|
||||
error('Bad string');
|
||||
},
|
||||
white = function () {
|
||||
// Skip whitespace.
|
||||
|
||||
while (ch && ch <= ' ') {
|
||||
next();
|
||||
}
|
||||
},
|
||||
word = function () {
|
||||
// true, false, or null.
|
||||
|
||||
switch (ch) {
|
||||
case 't':
|
||||
next('t');
|
||||
next('r');
|
||||
next('u');
|
||||
next('e');
|
||||
return true;
|
||||
case 'f':
|
||||
next('f');
|
||||
next('a');
|
||||
next('l');
|
||||
next('s');
|
||||
next('e');
|
||||
return false;
|
||||
case 'n':
|
||||
next('n');
|
||||
next('u');
|
||||
next('l');
|
||||
next('l');
|
||||
return null;
|
||||
}
|
||||
error("Unexpected '" + ch + "'");
|
||||
},
|
||||
value, // Place holder for the value function.
|
||||
array = function () {
|
||||
// Parse an array value.
|
||||
|
||||
var array = [];
|
||||
|
||||
if (ch === '[') {
|
||||
next('[');
|
||||
white();
|
||||
if (ch === ']') {
|
||||
next(']');
|
||||
return array; // empty array
|
||||
}
|
||||
while (ch) {
|
||||
array.push(value());
|
||||
white();
|
||||
if (ch === ']') {
|
||||
next(']');
|
||||
return array;
|
||||
}
|
||||
next(',');
|
||||
white();
|
||||
}
|
||||
}
|
||||
error('Bad array');
|
||||
},
|
||||
object = function () {
|
||||
// Parse an object value.
|
||||
|
||||
var key,
|
||||
object = Object.create(null);
|
||||
|
||||
if (ch === '{') {
|
||||
next('{');
|
||||
white();
|
||||
if (ch === '}') {
|
||||
next('}');
|
||||
return object; // empty object
|
||||
}
|
||||
while (ch) {
|
||||
key = string();
|
||||
white();
|
||||
next(':');
|
||||
if (_options.strict === true && Object.hasOwnProperty.call(object, key)) {
|
||||
error('Duplicate key "' + key + '"');
|
||||
}
|
||||
|
||||
if (suspectProtoRx.test(key) === true) {
|
||||
if (_options.protoAction === 'error') {
|
||||
error('Object contains forbidden prototype property');
|
||||
} else if (_options.protoAction === 'ignore') {
|
||||
value();
|
||||
} else {
|
||||
object[key] = value();
|
||||
}
|
||||
} else if (suspectConstructorRx.test(key) === true) {
|
||||
if (_options.constructorAction === 'error') {
|
||||
error('Object contains forbidden constructor property');
|
||||
} else if (_options.constructorAction === 'ignore') {
|
||||
value();
|
||||
} else {
|
||||
object[key] = value();
|
||||
}
|
||||
} else {
|
||||
object[key] = value();
|
||||
}
|
||||
|
||||
white();
|
||||
if (ch === '}') {
|
||||
next('}');
|
||||
return object;
|
||||
}
|
||||
next(',');
|
||||
white();
|
||||
}
|
||||
}
|
||||
error('Bad object');
|
||||
};
|
||||
|
||||
value = function () {
|
||||
// Parse a JSON value. It could be an object, an array, a string, a number,
|
||||
// or a word.
|
||||
|
||||
white();
|
||||
switch (ch) {
|
||||
case '{':
|
||||
return object();
|
||||
case '[':
|
||||
return array();
|
||||
case '"':
|
||||
return string();
|
||||
case '-':
|
||||
return number();
|
||||
default:
|
||||
return ch >= '0' && ch <= '9' ? number() : word();
|
||||
}
|
||||
};
|
||||
|
||||
// Return the json_parse function. It will have access to all of the above
|
||||
// functions and variables.
|
||||
|
||||
return function (source, reviver) {
|
||||
var result;
|
||||
|
||||
text = source + '';
|
||||
at = 0;
|
||||
ch = ' ';
|
||||
result = value();
|
||||
white();
|
||||
if (ch) {
|
||||
error('Syntax error');
|
||||
}
|
||||
|
||||
// If there is a reviver function, we recursively walk the new structure,
|
||||
// passing each name/value pair to the reviver function for possible
|
||||
// transformation, starting with a temporary root object that holds the result
|
||||
// in an empty key. If there is not a reviver function, we simply return the
|
||||
// result.
|
||||
|
||||
return typeof reviver === 'function'
|
||||
? (function walk(holder, key) {
|
||||
var k,
|
||||
v,
|
||||
value = holder[key];
|
||||
if (value && typeof value === 'object') {
|
||||
Object.keys(value).forEach(function (k) {
|
||||
v = walk(value, k);
|
||||
if (v !== undefined) {
|
||||
value[k] = v;
|
||||
} else {
|
||||
delete value[k];
|
||||
}
|
||||
});
|
||||
}
|
||||
return reviver.call(holder, key, value);
|
||||
})({ '': result }, '')
|
||||
: result;
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = json_parse;
|
||||
@@ -0,0 +1,395 @@
|
||||
/**
|
||||
* NOTE: The contents of this file have been inlined from the json-bigint package's source code
|
||||
* https://github.com/sidorares/json-bigint/blob/master/json-bigint.js
|
||||
*
|
||||
* The repository contains a critical bug fix for decimal handling, however, it has not been
|
||||
* released to npm yet. This file is a copy of the source code with the bug fix applied.
|
||||
* https://github.com/sidorares/json-bigint/commit/3530541b016d9041db6c1e7019e6999790bfd857
|
||||
*
|
||||
* :copyright: Copyright (c) 2013 Andrey Sidorov
|
||||
* :license: The MIT License (MIT)
|
||||
*/
|
||||
|
||||
// @ts-nocheck
|
||||
var BigNumber = require('bignumber.js');
|
||||
|
||||
/*
|
||||
json2.js
|
||||
2013-05-26
|
||||
|
||||
Public Domain.
|
||||
|
||||
NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK.
|
||||
|
||||
See http://www.JSON.org/js.html
|
||||
|
||||
|
||||
This code should be minified before deployment.
|
||||
See http://javascript.crockford.com/jsmin.html
|
||||
|
||||
USE YOUR OWN COPY. IT IS EXTREMELY UNWISE TO LOAD CODE FROM SERVERS YOU DO
|
||||
NOT CONTROL.
|
||||
|
||||
|
||||
This file creates a global JSON object containing two methods: stringify
|
||||
and parse.
|
||||
|
||||
JSON.stringify(value, replacer, space)
|
||||
value any JavaScript value, usually an object or array.
|
||||
|
||||
replacer an optional parameter that determines how object
|
||||
values are stringified for objects. It can be a
|
||||
function or an array of strings.
|
||||
|
||||
space an optional parameter that specifies the indentation
|
||||
of nested structures. If it is omitted, the text will
|
||||
be packed without extra whitespace. If it is a number,
|
||||
it will specify the number of spaces to indent at each
|
||||
level. If it is a string (such as '\t' or ' '),
|
||||
it contains the characters used to indent at each level.
|
||||
|
||||
This method produces a JSON text from a JavaScript value.
|
||||
|
||||
When an object value is found, if the object contains a toJSON
|
||||
method, its toJSON method will be called and the result will be
|
||||
stringified. A toJSON method does not serialize: it returns the
|
||||
value represented by the name/value pair that should be serialized,
|
||||
or undefined if nothing should be serialized. The toJSON method
|
||||
will be passed the key associated with the value, and this will be
|
||||
bound to the value
|
||||
|
||||
For example, this would serialize Dates as ISO strings.
|
||||
|
||||
Date.prototype.toJSON = function (key) {
|
||||
function f(n) {
|
||||
// Format integers to have at least two digits.
|
||||
return n < 10 ? '0' + n : n;
|
||||
}
|
||||
|
||||
return this.getUTCFullYear() + '-' +
|
||||
f(this.getUTCMonth() + 1) + '-' +
|
||||
f(this.getUTCDate()) + 'T' +
|
||||
f(this.getUTCHours()) + ':' +
|
||||
f(this.getUTCMinutes()) + ':' +
|
||||
f(this.getUTCSeconds()) + 'Z';
|
||||
};
|
||||
|
||||
You can provide an optional replacer method. It will be passed the
|
||||
key and value of each member, with this bound to the containing
|
||||
object. The value that is returned from your method will be
|
||||
serialized. If your method returns undefined, then the member will
|
||||
be excluded from the serialization.
|
||||
|
||||
If the replacer parameter is an array of strings, then it will be
|
||||
used to select the members to be serialized. It filters the results
|
||||
such that only members with keys listed in the replacer array are
|
||||
stringified.
|
||||
|
||||
Values that do not have JSON representations, such as undefined or
|
||||
functions, will not be serialized. Such values in objects will be
|
||||
dropped; in arrays they will be replaced with null. You can use
|
||||
a replacer function to replace those with JSON values.
|
||||
JSON.stringify(undefined) returns undefined.
|
||||
|
||||
The optional space parameter produces a stringification of the
|
||||
value that is filled with line breaks and indentation to make it
|
||||
easier to read.
|
||||
|
||||
If the space parameter is a non-empty string, then that string will
|
||||
be used for indentation. If the space parameter is a number, then
|
||||
the indentation will be that many spaces.
|
||||
|
||||
Example:
|
||||
|
||||
text = JSON.stringify(['e', {pluribus: 'unum'}]);
|
||||
// text is '["e",{"pluribus":"unum"}]'
|
||||
|
||||
|
||||
text = JSON.stringify(['e', {pluribus: 'unum'}], null, '\t');
|
||||
// text is '[\n\t"e",\n\t{\n\t\t"pluribus": "unum"\n\t}\n]'
|
||||
|
||||
text = JSON.stringify([new Date()], function (key, value) {
|
||||
return this[key] instanceof Date ?
|
||||
'Date(' + this[key] + ')' : value;
|
||||
});
|
||||
// text is '["Date(---current time---)"]'
|
||||
|
||||
|
||||
JSON.parse(text, reviver)
|
||||
This method parses a JSON text to produce an object or array.
|
||||
It can throw a SyntaxError exception.
|
||||
|
||||
The optional reviver parameter is a function that can filter and
|
||||
transform the results. It receives each of the keys and values,
|
||||
and its return value is used instead of the original value.
|
||||
If it returns what it received, then the structure is not modified.
|
||||
If it returns undefined then the member is deleted.
|
||||
|
||||
Example:
|
||||
|
||||
// Parse the text. Values that look like ISO date strings will
|
||||
// be converted to Date objects.
|
||||
|
||||
myData = JSON.parse(text, function (key, value) {
|
||||
var a;
|
||||
if (typeof value === 'string') {
|
||||
a =
|
||||
/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*)?)Z$/.exec(value);
|
||||
if (a) {
|
||||
return new Date(Date.UTC(+a[1], +a[2] - 1, +a[3], +a[4],
|
||||
+a[5], +a[6]));
|
||||
}
|
||||
}
|
||||
return value;
|
||||
});
|
||||
|
||||
myData = JSON.parse('["Date(09/09/2001)"]', function (key, value) {
|
||||
var d;
|
||||
if (typeof value === 'string' &&
|
||||
value.slice(0, 5) === 'Date(' &&
|
||||
value.slice(-1) === ')') {
|
||||
d = new Date(value.slice(5, -1));
|
||||
if (d) {
|
||||
return d;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
});
|
||||
|
||||
|
||||
This is a reference implementation. You are free to copy, modify, or
|
||||
redistribute.
|
||||
*/
|
||||
|
||||
/*jslint evil: true, regexp: true */
|
||||
|
||||
/*members "", "\b", "\t", "\n", "\f", "\r", "\"", JSON, "\\", apply,
|
||||
call, charCodeAt, getUTCDate, getUTCFullYear, getUTCHours,
|
||||
getUTCMinutes, getUTCMonth, getUTCSeconds, hasOwnProperty, join,
|
||||
lastIndex, length, parse, prototype, push, replace, slice, stringify,
|
||||
test, toJSON, toString, valueOf
|
||||
*/
|
||||
|
||||
// Create a JSON object only if one does not already exist. We create the
|
||||
// methods in a closure to avoid creating global variables.
|
||||
|
||||
var JSON = module.exports;
|
||||
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
function f(n) {
|
||||
// Format integers to have at least two digits.
|
||||
return n < 10 ? '0' + n : n;
|
||||
}
|
||||
|
||||
var cx =
|
||||
/[\u0000\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
|
||||
escapable =
|
||||
/[\\\"\x00-\x1f\x7f-\x9f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
|
||||
gap,
|
||||
indent,
|
||||
meta = {
|
||||
// table of character substitutions
|
||||
'\b': '\\b',
|
||||
'\t': '\\t',
|
||||
'\n': '\\n',
|
||||
'\f': '\\f',
|
||||
'\r': '\\r',
|
||||
'"': '\\"',
|
||||
'\\': '\\\\',
|
||||
},
|
||||
rep;
|
||||
|
||||
function quote(string) {
|
||||
// If the string contains no control characters, no quote characters, and no
|
||||
// backslash characters, then we can safely slap some quotes around it.
|
||||
// Otherwise we must also replace the offending characters with safe escape
|
||||
// sequences.
|
||||
|
||||
escapable.lastIndex = 0;
|
||||
return escapable.test(string)
|
||||
? '"' +
|
||||
string.replace(escapable, function (a) {
|
||||
var c = meta[a];
|
||||
return typeof c === 'string'
|
||||
? c
|
||||
: '\\u' + ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
|
||||
}) +
|
||||
'"'
|
||||
: '"' + string + '"';
|
||||
}
|
||||
|
||||
function str(key, holder) {
|
||||
// Produce a string from holder[key].
|
||||
|
||||
var i, // The loop counter.
|
||||
k, // The member key.
|
||||
v, // The member value.
|
||||
length,
|
||||
mind = gap,
|
||||
partial,
|
||||
value = holder[key],
|
||||
isBigNumber = value != null && (value instanceof BigNumber || BigNumber.isBigNumber(value));
|
||||
|
||||
// If the value has a toJSON method, call it to obtain a replacement value.
|
||||
|
||||
if (value && typeof value === 'object' && typeof value.toJSON === 'function') {
|
||||
value = value.toJSON(key);
|
||||
}
|
||||
|
||||
// If we were called with a replacer function, then call the replacer to
|
||||
// obtain a replacement value.
|
||||
|
||||
if (typeof rep === 'function') {
|
||||
value = rep.call(holder, key, value);
|
||||
}
|
||||
|
||||
// What happens next depends on the value's type.
|
||||
|
||||
switch (typeof value) {
|
||||
case 'string':
|
||||
if (isBigNumber) {
|
||||
return value;
|
||||
} else {
|
||||
return quote(value);
|
||||
}
|
||||
|
||||
case 'number':
|
||||
// JSON numbers must be finite. Encode non-finite numbers as null.
|
||||
|
||||
return isFinite(value) ? String(value) : 'null';
|
||||
|
||||
case 'boolean':
|
||||
case 'null':
|
||||
case 'bigint':
|
||||
// If the value is a boolean or null, convert it to a string. Note:
|
||||
// typeof null does not produce 'null'. The case is included here in
|
||||
// the remote chance that this gets fixed someday.
|
||||
|
||||
return String(value);
|
||||
|
||||
// If the type is 'object', we might be dealing with an object or an array or
|
||||
// null.
|
||||
|
||||
case 'object':
|
||||
// Due to a specification blunder in ECMAScript, typeof null is 'object',
|
||||
// so watch out for that case.
|
||||
|
||||
if (!value) {
|
||||
return 'null';
|
||||
}
|
||||
|
||||
// Make an array to hold the partial results of stringifying this object value.
|
||||
|
||||
gap += indent;
|
||||
partial = [];
|
||||
|
||||
// Is the value an array?
|
||||
|
||||
if (Object.prototype.toString.apply(value) === '[object Array]') {
|
||||
// The value is an array. Stringify every element. Use null as a placeholder
|
||||
// for non-JSON values.
|
||||
|
||||
length = value.length;
|
||||
for (i = 0; i < length; i += 1) {
|
||||
partial[i] = str(i, value) || 'null';
|
||||
}
|
||||
|
||||
// Join all of the elements together, separated with commas, and wrap them in
|
||||
// brackets.
|
||||
|
||||
v =
|
||||
partial.length === 0
|
||||
? '[]'
|
||||
: gap
|
||||
? '[\n' + gap + partial.join(',\n' + gap) + '\n' + mind + ']'
|
||||
: '[' + partial.join(',') + ']';
|
||||
gap = mind;
|
||||
return v;
|
||||
}
|
||||
|
||||
// If the replacer is an array, use it to select the members to be stringified.
|
||||
|
||||
if (rep && typeof rep === 'object') {
|
||||
length = rep.length;
|
||||
for (i = 0; i < length; i += 1) {
|
||||
if (typeof rep[i] === 'string') {
|
||||
k = rep[i];
|
||||
v = str(k, value);
|
||||
if (v) {
|
||||
partial.push(quote(k) + (gap ? ': ' : ':') + v);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Otherwise, iterate through all of the keys in the object.
|
||||
|
||||
Object.keys(value).forEach(function (k) {
|
||||
var v = str(k, value);
|
||||
if (v) {
|
||||
partial.push(quote(k) + (gap ? ': ' : ':') + v);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Join all of the member texts together, separated with commas,
|
||||
// and wrap them in braces.
|
||||
|
||||
v =
|
||||
partial.length === 0
|
||||
? '{}'
|
||||
: gap
|
||||
? '{\n' + gap + partial.join(',\n' + gap) + '\n' + mind + '}'
|
||||
: '{' + partial.join(',') + '}';
|
||||
gap = mind;
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
// If the JSON object does not yet have a stringify method, give it one.
|
||||
|
||||
if (typeof JSON.stringify !== 'function') {
|
||||
JSON.stringify = function (value, replacer, space) {
|
||||
// The stringify method takes a value and an optional replacer, and an optional
|
||||
// space parameter, and returns a JSON text. The replacer can be a function
|
||||
// that can replace values, or an array of strings that will select the keys.
|
||||
// A default replacer method can be provided. Use of the space parameter can
|
||||
// produce text that is more easily readable.
|
||||
|
||||
var i;
|
||||
gap = '';
|
||||
indent = '';
|
||||
|
||||
// If the space parameter is a number, make an indent string containing that
|
||||
// many spaces.
|
||||
|
||||
if (typeof space === 'number') {
|
||||
for (i = 0; i < space; i += 1) {
|
||||
indent += ' ';
|
||||
}
|
||||
|
||||
// If the space parameter is a string, it will be used as the indent string.
|
||||
} else if (typeof space === 'string') {
|
||||
indent = space;
|
||||
}
|
||||
|
||||
// If there is a replacer, it must be a function or an array.
|
||||
// Otherwise, throw an error.
|
||||
|
||||
rep = replacer;
|
||||
if (
|
||||
replacer &&
|
||||
typeof replacer !== 'function' &&
|
||||
(typeof replacer !== 'object' || typeof replacer.length !== 'number')
|
||||
) {
|
||||
throw new Error('JSON.stringify');
|
||||
}
|
||||
|
||||
// Make a fake root object containing our value under the key of ''.
|
||||
// Return the result of stringifying the value.
|
||||
|
||||
return str('', { '': value });
|
||||
};
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,52 @@
|
||||
import fastStringify from 'fast-safe-stringify';
|
||||
|
||||
// Configure json-bigint to handle large integers
|
||||
|
||||
type JSON = {
|
||||
parse: (text: string) => any;
|
||||
stringify: (value: any) => string;
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const JSONBigInt = require('./json-bigint/index.js');
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
|
||||
const JSONBig: JSON = JSONBigInt({
|
||||
useNativeBigInt: true,
|
||||
alwaysParseAsBig: false,
|
||||
storeAsString: false,
|
||||
});
|
||||
|
||||
/**
|
||||
* Safely stringify a value to JSON, handling circular references and non-serializable objects.
|
||||
* Uses fast-safe-stringify for optimal performance and reliability.
|
||||
*
|
||||
* @param value The value to stringify
|
||||
* @returns JSON string representation of the value
|
||||
*/
|
||||
export function safeJsonStringify(value: any): string {
|
||||
// MLflow-specific replacer that handles functions, undefined, and errors
|
||||
const mlflowReplacer = (_key: string, val: any): any => {
|
||||
if (typeof val === 'function') {
|
||||
return '[Function]';
|
||||
}
|
||||
if (val === undefined) {
|
||||
return '[Undefined]';
|
||||
}
|
||||
|
||||
if (val instanceof Error) {
|
||||
return {
|
||||
name: val.name,
|
||||
message: val.message,
|
||||
stack: val.stack,
|
||||
};
|
||||
}
|
||||
|
||||
return val;
|
||||
};
|
||||
|
||||
// Use fast-safe-stringify with our MLflow replacer
|
||||
// The circular reference detection is handled by fast-safe-stringify itself
|
||||
return fastStringify(value, mlflowReplacer);
|
||||
}
|
||||
|
||||
export { JSONBig };
|
||||
@@ -0,0 +1,39 @@
|
||||
import { TRACE_ID_PREFIX, TRACE_ID_V4_PREFIX } from '../constants';
|
||||
|
||||
/**
|
||||
* Construct a V3-schema MLflow trace ID from an OTel hex trace ID.
|
||||
* Format: `tr-<hex_trace_id>`.
|
||||
*/
|
||||
export function generateTraceIdV3(otelHexTraceId: string): string {
|
||||
return TRACE_ID_PREFIX + otelHexTraceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a V4-schema MLflow trace ID for the given UC location and
|
||||
* OTel hex trace ID. Format: `trace:/<location>/<hex_trace_id>`.
|
||||
*/
|
||||
export function constructTraceIdV4(location: string, otelHexTraceId: string): string {
|
||||
return `${TRACE_ID_V4_PREFIX}${location}/${otelHexTraceId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an MLflow trace ID. For V4 IDs returns `[location, otelHexTraceId]`;
|
||||
* for V3 IDs returns `[null, raw_id]`. Mirrors Python's `parse_trace_id_v4`.
|
||||
*/
|
||||
export function parseTraceIdV4(traceId: string | null | undefined): [string | null, string | null] {
|
||||
if (!traceId) {
|
||||
return [null, null];
|
||||
}
|
||||
if (traceId.startsWith(TRACE_ID_V4_PREFIX)) {
|
||||
const rest = traceId.slice(TRACE_ID_V4_PREFIX.length);
|
||||
const slash = rest.indexOf('/');
|
||||
if (slash <= 0 || slash === rest.length - 1) {
|
||||
throw new Error(
|
||||
`Invalid trace ID format: ${traceId}. ` +
|
||||
`Expected format: ${TRACE_ID_V4_PREFIX}<location>/<trace_id>`,
|
||||
);
|
||||
}
|
||||
return [rest.slice(0, slash), rest.slice(slash + 1)];
|
||||
}
|
||||
return [null, traceId];
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
import { Trace } from '../core/entities/trace';
|
||||
import { ExportResult } from '@opentelemetry/core';
|
||||
import {
|
||||
Span as OTelSpan,
|
||||
SpanProcessor,
|
||||
ReadableSpan as OTelReadableSpan,
|
||||
SpanExporter,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { Context } from '@opentelemetry/api';
|
||||
import { createAndRegisterMlflowSpan } from '../core/api';
|
||||
import { getConfiguredTraceMetadata, getConfiguredTraceTags } from '../core/context';
|
||||
import { InMemoryTraceManager } from '../core/trace_manager';
|
||||
import { TraceInfo } from '../core/entities/trace_info';
|
||||
import { createTraceLocationFromExperimentId } from '../core/entities/trace_location';
|
||||
import { fromOtelStatus, TraceState } from '../core/entities/trace_state';
|
||||
import {
|
||||
SpanAttributeKey,
|
||||
TRACE_ID_PREFIX,
|
||||
TRACE_SCHEMA_VERSION,
|
||||
TraceMetadataKey,
|
||||
} from '../core/constants';
|
||||
import { convertHrTimeToMs, aggregateUsageFromSpans } from '../core/utils';
|
||||
import { getConfig } from '../core/config';
|
||||
import { MlflowClient } from '../clients';
|
||||
import { executeOnSpanEndHooks, executeOnSpanStartHooks } from './span_processor_hooks';
|
||||
|
||||
/**
|
||||
* Generate a MLflow-compatible trace ID for the given span.
|
||||
* @param span The span to generate the trace ID for
|
||||
*/
|
||||
function generateTraceId(span: OTelSpan): string {
|
||||
// NB: trace Id is already hex string in Typescript OpenTelemetry SDK
|
||||
return TRACE_ID_PREFIX + span.spanContext().traceId;
|
||||
}
|
||||
|
||||
export class MlflowSpanProcessor implements SpanProcessor {
|
||||
private _exporter: SpanExporter;
|
||||
|
||||
constructor(exporter: SpanExporter) {
|
||||
this._exporter = exporter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a {@link Span} is started, if the `span.isRecording()`
|
||||
* returns true.
|
||||
* @param span the Span that just started.
|
||||
*/
|
||||
onStart(span: OTelSpan, _parentContext: Context): void {
|
||||
const otelTraceId = span.spanContext().traceId;
|
||||
|
||||
let traceId: string;
|
||||
const experimentId = getConfig().experimentId;
|
||||
|
||||
if (!span.parentSpanContext?.spanId) {
|
||||
// This is a root span
|
||||
traceId = generateTraceId(span);
|
||||
|
||||
// Build trace metadata, merging context-injected values
|
||||
const traceMetadata: Record<string, string> = {
|
||||
[TraceMetadataKey.SCHEMA_VERSION]: TRACE_SCHEMA_VERSION,
|
||||
};
|
||||
const ctxMetadata = getConfiguredTraceMetadata();
|
||||
if (ctxMetadata) {
|
||||
Object.assign(traceMetadata, ctxMetadata);
|
||||
}
|
||||
|
||||
// Build trace tags, merging context-injected values
|
||||
const tags: Record<string, string> = {};
|
||||
const ctxTags = getConfiguredTraceTags();
|
||||
if (ctxTags) {
|
||||
Object.assign(tags, ctxTags);
|
||||
}
|
||||
|
||||
const trace_info = new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: createTraceLocationFromExperimentId(experimentId),
|
||||
requestTime: convertHrTimeToMs(span.startTime),
|
||||
executionDuration: 0,
|
||||
state: TraceState.IN_PROGRESS,
|
||||
traceMetadata,
|
||||
tags,
|
||||
assessments: [],
|
||||
});
|
||||
InMemoryTraceManager.getInstance().registerTrace(otelTraceId, trace_info);
|
||||
} else {
|
||||
traceId = InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId) || '';
|
||||
|
||||
if (!traceId) {
|
||||
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Set trace ID to the span
|
||||
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
|
||||
|
||||
createAndRegisterMlflowSpan(span);
|
||||
executeOnSpanStartHooks(span);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a {@link ReadableSpan} is ended, if the `span.isRecording()`
|
||||
* returns true.
|
||||
* @param span the Span that just ended.
|
||||
*/
|
||||
onEnd(span: OTelReadableSpan): void {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
|
||||
executeOnSpanEndHooks(span);
|
||||
|
||||
// Only trigger trace export for root span completion
|
||||
if (span.parentSpanContext?.spanId) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Update trace info
|
||||
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
|
||||
if (!traceId) {
|
||||
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const trace = InMemoryTraceManager.getInstance().getTrace(traceId);
|
||||
if (!trace) {
|
||||
console.warn(`No trace found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.updateTraceInfo(trace.info, span);
|
||||
// Aggregate token usage from all spans and add to trace metadata
|
||||
const allSpans = Array.from(trace.spanDict.values());
|
||||
const aggregatedUsage = aggregateUsageFromSpans(allSpans);
|
||||
if (aggregatedUsage) {
|
||||
trace.info.traceMetadata[TraceMetadataKey.TOKEN_USAGE] = JSON.stringify(aggregatedUsage);
|
||||
}
|
||||
|
||||
this._exporter.export([span], (_) => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the trace info with the span end time and status.
|
||||
* @param trace The trace to update
|
||||
* @param span The span to update the trace with
|
||||
*/
|
||||
updateTraceInfo(traceInfo: TraceInfo, span: OTelReadableSpan): void {
|
||||
traceInfo.executionDuration = convertHrTimeToMs(span.endTime) - traceInfo.requestTime;
|
||||
|
||||
let state = fromOtelStatus(span.status.code);
|
||||
// NB: In OpenTelemetry, status code remains UNSET if not explicitly set
|
||||
// by the user. However, there is no way to set the status when using
|
||||
// `trace` function wrapper. Therefore, we just automatically set the status
|
||||
// to OK if it is not ERROR.
|
||||
if (state === TraceState.STATE_UNSPECIFIED) {
|
||||
state = TraceState.OK;
|
||||
}
|
||||
traceInfo.state = state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shuts down the processor. Called when SDK is shut down. This is an
|
||||
* opportunity for processor to do any cleanup required.
|
||||
*/
|
||||
async shutdown() {
|
||||
await this._exporter.shutdown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Forces to export all finished spans
|
||||
*/
|
||||
async forceFlush() {
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
await this._exporter.forceFlush!();
|
||||
}
|
||||
}
|
||||
|
||||
export class MlflowSpanExporter implements SpanExporter {
|
||||
private _client: MlflowClient;
|
||||
private _pendingExports: Record<string, Promise<void>> = {}; // traceId -> export promise
|
||||
|
||||
constructor(client: MlflowClient) {
|
||||
this._client = client;
|
||||
}
|
||||
|
||||
export(spans: OTelReadableSpan[], _resultCallback: (result: ExportResult) => void): void {
|
||||
for (const span of spans) {
|
||||
// Only export root spans
|
||||
if (span.parentSpanContext?.spanId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.popTrace(span.spanContext().traceId);
|
||||
if (!trace) {
|
||||
console.warn(`No trace found for span ${span.name}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Set the last active trace ID
|
||||
traceManager.lastActiveTraceId = trace.info.traceId;
|
||||
|
||||
// Export trace to backend and track the promise
|
||||
const exportPromise = this.exportTraceToBackend(trace).catch((error) => {
|
||||
console.error(`Failed to export trace ${trace.info.traceId}:`, error);
|
||||
});
|
||||
this._pendingExports[trace.info.traceId] = exportPromise;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a complete trace to the MLflow backend
|
||||
* Step 1: Create trace metadata via StartTraceV3 endpoint
|
||||
* Step 2: Upload trace data (spans) via artifact repository pattern
|
||||
*/
|
||||
private async exportTraceToBackend(trace: Trace): Promise<void> {
|
||||
try {
|
||||
// Step 1: Create trace metadata in backend
|
||||
const traceInfo = await this._client.createTrace(trace.info);
|
||||
// Step 2: Upload trace data (spans) to artifact storage
|
||||
await this._client.uploadTraceData(traceInfo, trace.data);
|
||||
} catch (error) {
|
||||
console.error(`Failed to export trace ${trace.info.traceId}:`, error);
|
||||
throw error;
|
||||
} finally {
|
||||
// Remove the promise from the pending exports
|
||||
delete this._pendingExports[trace.info.traceId];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force flush all pending trace exports.
|
||||
* Waits for all async export operations to complete.
|
||||
*/
|
||||
async forceFlush(): Promise<void> {
|
||||
await Promise.all(Object.values(this._pendingExports));
|
||||
this._pendingExports = {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the exporter.
|
||||
* Waits for all pending exports to complete before shutting down.
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
await this.forceFlush();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { Span as OTelSpan, ReadableSpan as OTelReadableSpan } from '@opentelemetry/sdk-trace-base';
|
||||
import { LiveSpan } from '../core/entities/span';
|
||||
import { InMemoryTraceManager } from '../core/trace_manager';
|
||||
|
||||
/**
|
||||
* Hooks to be executed by the span processor.
|
||||
* Primary used for adding custom processing to the span for autologging integrations.
|
||||
*/
|
||||
type OnSpanStartHook = (span: LiveSpan) => void;
|
||||
type OnSpanEndHook = (span: LiveSpan) => void;
|
||||
|
||||
const onSpanStartHooks: Set<OnSpanStartHook> = new Set();
|
||||
const onSpanEndHooks: Set<OnSpanEndHook> = new Set();
|
||||
|
||||
export function registerOnSpanStartHook(hook: OnSpanStartHook): void {
|
||||
onSpanStartHooks.add(hook);
|
||||
}
|
||||
|
||||
export function getOnSpanStartHooks(): OnSpanStartHook[] {
|
||||
return Array.from(onSpanStartHooks);
|
||||
}
|
||||
|
||||
export function registerOnSpanEndHook(hook: OnSpanEndHook): void {
|
||||
onSpanEndHooks.add(hook);
|
||||
}
|
||||
|
||||
export function getOnSpanEndHooks(): OnSpanEndHook[] {
|
||||
return Array.from(onSpanEndHooks);
|
||||
}
|
||||
|
||||
export function executeOnSpanStartHooks(span: OTelSpan): void {
|
||||
// Execute onStart hooks for autologging integrations
|
||||
const hooks = getOnSpanStartHooks();
|
||||
if (hooks.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const mlflowSpan = getMlflowSpan(span);
|
||||
if (!mlflowSpan) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const hook of hooks) {
|
||||
try {
|
||||
hook(mlflowSpan);
|
||||
} catch (error) {
|
||||
console.debug('Error executing onStart hook:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function executeOnSpanEndHooks(span: OTelReadableSpan): void {
|
||||
const hooks = getOnSpanEndHooks();
|
||||
if (hooks.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const mlflowSpan = getMlflowSpan(span);
|
||||
if (!mlflowSpan) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const hook of hooks) {
|
||||
try {
|
||||
hook(mlflowSpan);
|
||||
} catch (error) {
|
||||
console.debug('Error executing onEnd hook:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getMlflowSpan(span: OTelSpan | OTelReadableSpan): LiveSpan | null {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
|
||||
return traceManager.getSpan(traceId, span.spanContext().spanId);
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
|
||||
import { Context } from '@opentelemetry/api';
|
||||
import {
|
||||
Span as OTelSpan,
|
||||
ReadableSpan as OTelReadableSpan,
|
||||
SpanProcessor,
|
||||
SpanExporter,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { createAndRegisterMlflowSpan } from '../core/api';
|
||||
import { MlflowClient } from '../clients';
|
||||
import { SpanAttributeKey, TraceMetadataKey } from '../core/constants';
|
||||
import { getConfiguredTraceMetadata, getConfiguredTraceTags } from '../core/context';
|
||||
import { TraceInfo } from '../core/entities/trace_info';
|
||||
import {
|
||||
TraceLocation,
|
||||
TraceLocationType,
|
||||
UnityCatalogLocation,
|
||||
getOtelSpansTableName,
|
||||
getUcLocationString,
|
||||
} from '../core/entities/trace_location';
|
||||
import { Trace } from '../core/entities/trace';
|
||||
import { fromOtelStatus, TraceState } from '../core/entities/trace_state';
|
||||
import { InMemoryTraceManager } from '../core/trace_manager';
|
||||
import { constructTraceIdV4, parseTraceIdV4 } from '../core/utils/trace_id';
|
||||
import { aggregateUsageFromSpans, convertHrTimeToMs } from '../core/utils';
|
||||
import { executeOnSpanEndHooks, executeOnSpanStartHooks } from './span_processor_hooks';
|
||||
|
||||
/**
|
||||
* Span processor for Databricks Unity Catalog backed traces.
|
||||
*
|
||||
* Mirrors Python's `DatabricksUCTableSpanProcessor`:
|
||||
* - generates V4 trace IDs (`trace:/<location>/<hex>`)
|
||||
* - constructs TraceInfo with the UC trace location, V4 schema version, and
|
||||
* any context-injected metadata/tags so that `updateCurrentTrace({ tags })`
|
||||
* can mutate the in-memory record before export
|
||||
* - delegates to a UC-aware SpanExporter for trace info + OTLP span upload
|
||||
*
|
||||
* Wired up by `init({ trackingUri, experimentId })` when the linked Databricks
|
||||
* experiment carries `mlflow.experiment.databricksTrace*` tags pointing at a UC
|
||||
* destination.
|
||||
*/
|
||||
export class DatabricksUCTableSpanProcessor implements SpanProcessor {
|
||||
private _exporter: SpanExporter;
|
||||
private _location: UnityCatalogLocation;
|
||||
|
||||
constructor(exporter: SpanExporter, location: UnityCatalogLocation) {
|
||||
this._exporter = exporter;
|
||||
this._location = location;
|
||||
}
|
||||
|
||||
onStart(span: OTelSpan, _parentContext: Context): void {
|
||||
const otelTraceId = span.spanContext().traceId;
|
||||
|
||||
if (!span.parentSpanContext?.spanId) {
|
||||
// Root span: build the V4 trace ID and TraceInfo for this trace.
|
||||
const traceLocation: TraceLocation = {
|
||||
type: TraceLocationType.UC_TABLE_PREFIX,
|
||||
ucTablePrefix: { ...this._location },
|
||||
};
|
||||
const locationString = getUcLocationString(traceLocation);
|
||||
if (!locationString) {
|
||||
console.warn(
|
||||
`Unable to derive UC location string for ${JSON.stringify(this._location)}; skipping trace registration.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const traceId = constructTraceIdV4(locationString, otelTraceId);
|
||||
|
||||
const traceMetadata: Record<string, string> = {
|
||||
[TraceMetadataKey.SCHEMA_VERSION]: '4',
|
||||
};
|
||||
const ctxMetadata = getConfiguredTraceMetadata();
|
||||
if (ctxMetadata) {
|
||||
Object.assign(traceMetadata, ctxMetadata);
|
||||
}
|
||||
|
||||
const tags: Record<string, string> = {};
|
||||
const ctxTags = getConfiguredTraceTags();
|
||||
if (ctxTags) {
|
||||
Object.assign(tags, ctxTags);
|
||||
}
|
||||
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId,
|
||||
traceLocation,
|
||||
requestTime: convertHrTimeToMs(span.startTime),
|
||||
executionDuration: 0,
|
||||
state: TraceState.IN_PROGRESS,
|
||||
traceMetadata,
|
||||
tags,
|
||||
assessments: [],
|
||||
});
|
||||
InMemoryTraceManager.getInstance().registerTrace(otelTraceId, traceInfo);
|
||||
|
||||
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
|
||||
} else {
|
||||
const traceId = InMemoryTraceManager.getInstance().getMlflowTraceIdFromOtelId(otelTraceId);
|
||||
if (!traceId) {
|
||||
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
span.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
|
||||
}
|
||||
|
||||
createAndRegisterMlflowSpan(span);
|
||||
executeOnSpanStartHooks(span);
|
||||
}
|
||||
|
||||
onEnd(span: OTelReadableSpan): void {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
executeOnSpanEndHooks(span);
|
||||
|
||||
if (span.parentSpanContext?.spanId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const traceId = traceManager.getMlflowTraceIdFromOtelId(span.spanContext().traceId);
|
||||
if (!traceId) {
|
||||
console.warn(`No trace ID found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const trace = traceManager.getTrace(traceId);
|
||||
if (!trace) {
|
||||
console.warn(`No trace found for span ${span.name}. Skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.updateTraceInfo(trace.info, span);
|
||||
|
||||
const allSpans = Array.from(trace.spanDict.values());
|
||||
const aggregatedUsage = aggregateUsageFromSpans(allSpans);
|
||||
if (aggregatedUsage) {
|
||||
trace.info.traceMetadata[TraceMetadataKey.TOKEN_USAGE] = JSON.stringify(aggregatedUsage);
|
||||
}
|
||||
|
||||
this._exporter.export([span], (_) => {});
|
||||
}
|
||||
|
||||
private updateTraceInfo(traceInfo: TraceInfo, span: OTelReadableSpan): void {
|
||||
traceInfo.executionDuration = convertHrTimeToMs(span.endTime) - traceInfo.requestTime;
|
||||
let state = fromOtelStatus(span.status.code);
|
||||
if (state === TraceState.STATE_UNSPECIFIED) {
|
||||
state = TraceState.OK;
|
||||
}
|
||||
traceInfo.state = state;
|
||||
}
|
||||
|
||||
async shutdown(): Promise<void> {
|
||||
await this._exporter.shutdown();
|
||||
}
|
||||
|
||||
async forceFlush(): Promise<void> {
|
||||
if (typeof this._exporter.forceFlush === 'function') {
|
||||
await this._exporter.forceFlush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exporter for Databricks Unity Catalog backed traces.
|
||||
*
|
||||
* Mirrors Python's `DatabricksUCTableSpanExporter`. Two-step export per trace:
|
||||
* 1. Call the V4 `CreateTraceInfo` REST endpoint with the full `TraceInfo`
|
||||
* (including `tags`, `trace_metadata`, state, durations). This is what
|
||||
* makes `updateCurrentTrace({ tags })` persist as trace-level tags in
|
||||
* `_traces_unified`, `get_trace`, `search_traces`, and the UI.
|
||||
* 2. Upload the OTel spans for this trace to `/api/2.0/otel/v1/traces` with
|
||||
* the `X-Databricks-UC-Table-Name` header pointing to the trace's UC
|
||||
* spans table.
|
||||
*
|
||||
* Errors in either step are logged but do not surface to user code, matching
|
||||
* the non-fatal behavior of `_log_spans` on the Python side.
|
||||
*/
|
||||
export class DatabricksUCTableSpanExporter implements SpanExporter {
|
||||
private _client: MlflowClient;
|
||||
private _pendingExports: Record<string, Promise<void>> = {};
|
||||
private _hasRaisedMissingSpansTableWarning = false;
|
||||
private _hasRaisedSpanExportError = false;
|
||||
|
||||
constructor(client: MlflowClient) {
|
||||
this._client = client;
|
||||
}
|
||||
|
||||
export(spans: OTelReadableSpan[], resultCallback: (result: ExportResult) => void): void {
|
||||
for (const span of spans) {
|
||||
if (span.parentSpanContext?.spanId) {
|
||||
continue;
|
||||
}
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.popTrace(span.spanContext().traceId);
|
||||
if (!trace) {
|
||||
console.warn(`No trace found for span ${span.name}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
traceManager.lastActiveTraceId = trace.info.traceId;
|
||||
|
||||
const exportPromise = this.exportTraceToBackend(trace).catch((error) => {
|
||||
console.error(`Failed to export UC trace ${trace.info.traceId}:`, error);
|
||||
});
|
||||
this._pendingExports[trace.info.traceId] = exportPromise;
|
||||
}
|
||||
// Fire-and-forget: backend errors are logged inside exportTraceToBackend
|
||||
// and intentionally not surfaced (matches Python `_log_spans`). Resolve
|
||||
// the callback synchronously so the SpanExporter contract is honored.
|
||||
resultCallback({ code: ExportResultCode.SUCCESS });
|
||||
}
|
||||
|
||||
private async exportTraceToBackend(trace: Trace): Promise<void> {
|
||||
try {
|
||||
const [location, otelTraceId] = parseTraceIdV4(trace.info.traceId);
|
||||
if (!location || !otelTraceId) {
|
||||
throw new Error(
|
||||
`UC exporter received non-V4 trace ID ${trace.info.traceId}. This indicates the ` +
|
||||
`processor and exporter destinations are out of sync.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Step 1: persist the trace info (tags, metadata, state).
|
||||
const returnedTraceInfo = await this._client.createTraceInfoV4(
|
||||
location,
|
||||
otelTraceId,
|
||||
trace.info,
|
||||
);
|
||||
|
||||
// The backend may populate the spans table name on the returned trace
|
||||
// location. Use that when present; fall back to the location we sent.
|
||||
const spansTable =
|
||||
getOtelSpansTableName(returnedTraceInfo.traceLocation) ??
|
||||
getOtelSpansTableName(trace.info.traceLocation);
|
||||
|
||||
if (!spansTable) {
|
||||
// Without a spans table we can't upload spans, but the trace info
|
||||
// (including tags) is already persisted in step 1.
|
||||
if (!this._hasRaisedMissingSpansTableWarning) {
|
||||
console.warn(
|
||||
`No OTel spans table resolved for UC trace ${trace.info.traceId}; ` +
|
||||
`spans will not be exported. Tags and metadata are still persisted.`,
|
||||
);
|
||||
this._hasRaisedMissingSpansTableWarning = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2: upload spans via OTLP.
|
||||
try {
|
||||
await this._client.exportOtlpSpansToUc(
|
||||
trace.data.spans.map((s) => s._span),
|
||||
spansTable,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!this._hasRaisedSpanExportError) {
|
||||
console.error(`Failed to export UC spans for ${trace.info.traceId}:`, error);
|
||||
this._hasRaisedSpanExportError = true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
delete this._pendingExports[trace.info.traceId];
|
||||
}
|
||||
}
|
||||
|
||||
async forceFlush(): Promise<void> {
|
||||
// Each export self-removes from `_pendingExports` in its `finally` block.
|
||||
// Loop so exports started during the await are still awaited, instead of
|
||||
// resetting the map (which would silently drop new in-flight exports and
|
||||
// make `shutdown()` lose track of them).
|
||||
while (Object.keys(this._pendingExports).length > 0) {
|
||||
await Promise.all(Object.values(this._pendingExports));
|
||||
}
|
||||
}
|
||||
|
||||
async shutdown(): Promise<void> {
|
||||
await this.forceFlush();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Group-commit writer for WAL lines.
|
||||
*/
|
||||
|
||||
import { open, FileHandle } from 'node:fs/promises';
|
||||
import { JSONBig } from '../../core/utils/json';
|
||||
import { getWalPath } from './paths';
|
||||
import { ensureParentDir, queueWriter } from './storage';
|
||||
import { WalLine, WalRecord } from './types';
|
||||
|
||||
interface BatchItem {
|
||||
line: WalLine;
|
||||
resolve: () => void;
|
||||
reject: (err: Error) => void;
|
||||
}
|
||||
|
||||
export class BatchingWriter {
|
||||
private pending: BatchItem[] = [];
|
||||
private flushScheduled = false;
|
||||
|
||||
/**
|
||||
* Enqueue an append for `record`. Resolves once the record (and its
|
||||
* batch-mates) have been fsynced to `queue.log`.
|
||||
*/
|
||||
submit(record: WalRecord): Promise<void> {
|
||||
return this.enqueue({ type: 'append', record });
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a tombstone for `id`. Resolves once the tombstone (and its
|
||||
* batch-mates) have been fsynced to `queue.log`.
|
||||
*/
|
||||
submitTombstone(id: string): Promise<void> {
|
||||
return this.enqueue({ type: 'tombstone', id });
|
||||
}
|
||||
|
||||
private enqueue(line: WalLine): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
this.pending.push({ line, resolve, reject });
|
||||
this.scheduleFlush();
|
||||
});
|
||||
}
|
||||
|
||||
private scheduleFlush(): void {
|
||||
if (this.flushScheduled) {
|
||||
return;
|
||||
}
|
||||
this.flushScheduled = true;
|
||||
// setImmediate fires at the end of the current event-loop tick,
|
||||
// so every `submit` call made within the same tick batches
|
||||
// together.
|
||||
setImmediate(() => {
|
||||
this.flushScheduled = false;
|
||||
if (this.pending.length === 0) {
|
||||
return;
|
||||
}
|
||||
void queueWriter.run(() => this.drainAndFlush());
|
||||
});
|
||||
}
|
||||
|
||||
private async drainAndFlush(): Promise<void> {
|
||||
const batch = this.pending;
|
||||
this.pending = [];
|
||||
if (batch.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const path = getWalPath();
|
||||
try {
|
||||
await ensureParentDir(path);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
for (const item of batch) {
|
||||
item.reject(error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let fh: FileHandle;
|
||||
try {
|
||||
fh = await open(path, 'a');
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
for (const item of batch) {
|
||||
item.reject(error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const buffers = batch.map((item) => Buffer.from(JSONBig.stringify(item.line) + '\n', 'utf8'));
|
||||
await fh.writev(buffers);
|
||||
await fh.sync();
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
for (const item of batch) {
|
||||
item.reject(error);
|
||||
}
|
||||
await fh.close().catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
await fh.close().catch(() => {});
|
||||
for (const item of batch) {
|
||||
item.resolve();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Daemon-side `MlflowClient` cache.
|
||||
*/
|
||||
|
||||
import { createAuthProvider } from '../../auth';
|
||||
import { MlflowClient } from '../../clients/client';
|
||||
|
||||
const cache = new Map<string, MlflowClient>();
|
||||
|
||||
/**
|
||||
* Return the cached `MlflowClient` for `trackingUri`, constructing it
|
||||
* lazily on first access. Subsequent calls with the same URI return the
|
||||
* same instance unless an explicit {@link clearClientForUri} happened
|
||||
* in between (typically driven by the upload loop's auth-error retry).
|
||||
*/
|
||||
export function clientForUri(trackingUri: string): MlflowClient {
|
||||
const cached = cache.get(trackingUri);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
const authProvider = createAuthProvider({ trackingUri });
|
||||
const client = new MlflowClient({ trackingUri, authProvider });
|
||||
cache.set(trackingUri, client);
|
||||
return client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the cached client for `trackingUri` only, leaving other URIs'
|
||||
* entries intact. Called by the upload loop on 401 / 403 so the next
|
||||
* `clientForUri(uri)` call rebuilds the client with freshly-resolved
|
||||
* credentials. No-op if no entry exists for `trackingUri`.
|
||||
*/
|
||||
export function clearClientForUri(trackingUri: string): void {
|
||||
cache.delete(trackingUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop all cached clients. Intended for daemon shutdown paths and
|
||||
* tests; not part of the steady-state batch loop.
|
||||
*/
|
||||
export function clearClientCache(): void {
|
||||
cache.clear();
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Tunable defaults for the WAL exporter and uploader daemon.
|
||||
*/
|
||||
|
||||
import { readNonNegativeInt, readPositiveInt } from '../../core/utils/env';
|
||||
|
||||
/**
|
||||
* Wall-clock budget for retrying a single record before the daemon moves it
|
||||
* to the dead-letter file (`failed.log`), in milliseconds.
|
||||
*/
|
||||
export const RETRY_TIMEOUT_MS: number =
|
||||
readNonNegativeInt('MLFLOW_ASYNC_TRACE_LOGGING_RETRY_TIMEOUT', 500) * 1_000;
|
||||
|
||||
/**
|
||||
* Interval between daemon batch loop iterations, in milliseconds.
|
||||
*/
|
||||
export const BATCH_INTERVAL_MS: number = readPositiveInt(
|
||||
'MLFLOW_ASYNC_TRACE_LOGGING_MAX_INTERVAL_MILLIS',
|
||||
15_000,
|
||||
);
|
||||
|
||||
/**
|
||||
* Threshold for daemon idle shutdown, in milliseconds. After this much
|
||||
* time with the WAL empty and no new records appended, the daemon flushes,
|
||||
* releases its lock, and exits.
|
||||
*/
|
||||
export const IDLE_MS: number = readPositiveInt('MLFLOW_TRACE_DAEMON_IDLE_MS', 120_000);
|
||||
|
||||
/**
|
||||
* Retention window (in UTC days) for the daily-rotated `failed.log.<date>`
|
||||
* and `daemon.log.<date>` files under the WAL directory. Files older than
|
||||
* `today_utc − LOG_RETENTION_DAYS` are unlinked once per daemon startup;
|
||||
* a value of `0` disables the sweep entirely.
|
||||
*/
|
||||
export const LOG_RETENTION_DAYS: number = readNonNegativeInt('MLFLOW_TRACE_LOG_RETENTION_DAYS', 14);
|
||||
@@ -0,0 +1,635 @@
|
||||
/**
|
||||
* The trace-uploader daemon — long-lived background process that drains
|
||||
* the WAL on behalf of all hooks running for this OS user.
|
||||
*/
|
||||
|
||||
import { appendFileSync } from 'node:fs';
|
||||
import { mkdir, readdir, unlink } from 'node:fs/promises';
|
||||
import { createServer, Server, Socket } from 'node:net';
|
||||
import { join } from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { MlflowClient } from '../../clients/client';
|
||||
import { MlflowHttpError } from '../../clients/utils';
|
||||
import { SerializedTraceData, TraceData } from '../../core/entities/trace_data';
|
||||
import { SerializedTraceInfo, TraceInfo } from '../../core/entities/trace_info';
|
||||
import { BatchingWriter } from './batching_writer';
|
||||
import { clearClientCache, clearClientForUri, clientForUri } from './clients';
|
||||
import { BATCH_INTERVAL_MS, IDLE_MS, LOG_RETENTION_DAYS, RETRY_TIMEOUT_MS } from './constants';
|
||||
import { createIpcConnectionHandler } from './ipc';
|
||||
import { getDaemonLogPath, getLockSocketPath, getWalDir } from './paths';
|
||||
import { PidLock, tryAcquirePidLock } from './pid_lock';
|
||||
import { appendDeadLetter, compact, readPending, walSize } from './storage';
|
||||
import { isDaemonAlive } from './supervisor';
|
||||
import { WalRecord } from './types';
|
||||
|
||||
const MAX_BACKOFF_MS = 5 * 60_000;
|
||||
|
||||
const COMPACTION_THRESHOLD_BYTES = 100_000;
|
||||
|
||||
function isErrnoException(err: unknown): err is NodeJS.ErrnoException {
|
||||
return typeof err === 'object' && err != null && 'code' in err;
|
||||
}
|
||||
|
||||
function formatError(err: unknown): string {
|
||||
if (err instanceof Error) {
|
||||
return err.stack ?? err.message;
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a single timestamped line to the daemon log file. Uses sync
|
||||
* I/O so two concurrent log calls cannot interleave;
|
||||
*/
|
||||
export function log(line: string): void {
|
||||
const timestamp = new Date().toISOString();
|
||||
const fullLine = `[${timestamp}] [pid=${process.pid}] ${line}\n`;
|
||||
try {
|
||||
appendFileSync(getDaemonLogPath(), fullLine);
|
||||
} catch {
|
||||
// Failures are swallowed so a broken disk never crashes the daemon mid-batch.
|
||||
}
|
||||
}
|
||||
|
||||
/** Awaits `ms` real milliseconds. Used for the batch loop's pacing. */
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff with a fixed cap.
|
||||
*/
|
||||
export function backoff(attempt: number): number {
|
||||
return Math.min(2_000 * 2 ** (attempt - 1), MAX_BACKOFF_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Default connection handler used when {@link acquireLock} is invoked
|
||||
* without one (e.g. by tests that only care about the lock semantic).
|
||||
*/
|
||||
function defaultConnectionHandler(socket: Socket): void {
|
||||
socket.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of a single `listen()` attempt on the lock socket. Distinguishes
|
||||
* the three outcomes acquireLock has to dispatch on: bound (we won),
|
||||
* EADDRINUSE (someone else holds the path — live daemon or stale file),
|
||||
* or any other error (unexpected).
|
||||
*/
|
||||
type BindResult =
|
||||
| { kind: 'bound'; server: Server }
|
||||
| { kind: 'eaddrinuse' }
|
||||
| { kind: 'error'; err: Error };
|
||||
|
||||
/**
|
||||
* Try to bind a fresh server to `socketPath`. Never throws — every
|
||||
* outcome is encoded in the returned discriminated union so callers can
|
||||
* keep their control flow linear instead of try/catching around a
|
||||
* Promise that resolves three different ways.
|
||||
*/
|
||||
function tryBind(socketPath: string, onConnection: (socket: Socket) => void): Promise<BindResult> {
|
||||
return new Promise<BindResult>((resolve) => {
|
||||
const server = createServer(onConnection);
|
||||
server.once('error', (err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'EADDRINUSE') {
|
||||
resolve({ kind: 'eaddrinuse' });
|
||||
return;
|
||||
}
|
||||
resolve({ kind: 'error', err });
|
||||
});
|
||||
server.listen(socketPath, () => resolve({ kind: 'bound', server }));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle returned by {@link acquireLock} on success. Pairs the bound
|
||||
* server with the optional cross-process PID lock so {@link releaseLock}
|
||||
* can tear both down together. `pidLock` is `null` on Windows because
|
||||
* named pipes already provide kernel-refcounted cross-process
|
||||
* exclusion; the file-based lock would be pure overhead there.
|
||||
*/
|
||||
export interface DaemonLock {
|
||||
server: Server;
|
||||
pidLock: PidLock | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the singleton daemon lock.
|
||||
*
|
||||
* Returns the {@link DaemonLock} on success, or `null` when another
|
||||
* daemon is already running (the caller should `process.exit(0)`
|
||||
* cleanly).
|
||||
*
|
||||
* ## How exclusion is enforced
|
||||
*
|
||||
* On POSIX, exclusion is enforced by the atomic PID lock acquired in
|
||||
* {@link tryAcquirePidLock} *before* the socket bind. The PID lock uses
|
||||
* `link()`, a single kernel-atomic syscall — only one process can win,
|
||||
* the rest see `EEXIST` and concede. With the lock held, no sibling
|
||||
* daemon is past this point at the same time as us, so the subsequent
|
||||
* `bind()` runs uncontested.
|
||||
*
|
||||
* On Windows, named pipes are kernel-refcounted: two daemons cannot
|
||||
* bind the same pipe name simultaneously regardless of the order of
|
||||
* operations. We skip the PID lock there and rely on `listen()`'s
|
||||
* native exclusion — the loser of a concurrent-bind race sees
|
||||
* `EADDRINUSE` from {@link bindUnderPidLock}, which returns `null` so
|
||||
* we concede here the same way the PID-lock-lost path does on POSIX.
|
||||
*/
|
||||
export async function acquireLock(
|
||||
onConnection: (socket: Socket) => void = defaultConnectionHandler,
|
||||
): Promise<DaemonLock | null> {
|
||||
if (await isDaemonAlive()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Cross-process exclusion gate. On POSIX this is the only mechanism
|
||||
// that prevents two daemons from racing through stale-socket
|
||||
// recovery and ending up both bound;
|
||||
const pidLock = process.platform === 'win32' ? null : await tryAcquirePidLock();
|
||||
if (process.platform !== 'win32' && pidLock == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const socketPath = getLockSocketPath();
|
||||
const server = await bindUnderPidLock(socketPath, onConnection);
|
||||
if (server == null) {
|
||||
// Windows-only path: a sibling daemon won the named-pipe race.
|
||||
// No PID lock to release (Windows path; `pidLock` is always
|
||||
// null here). Concede the same way the POSIX PID-lock-lost case
|
||||
// does so the caller can `process.exit(0)` cleanly.
|
||||
return null;
|
||||
}
|
||||
return { server, pidLock };
|
||||
} catch (err) {
|
||||
// Bind path failed after we acquired the PID lock — release it so
|
||||
// the next spawn isn't blocked by our orphaned lockfile.
|
||||
await pidLock?.release();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind the lock socket, with recovery semantics that depend on the
|
||||
* cross-process exclusion guarantee held by the caller.
|
||||
*
|
||||
* Returns:
|
||||
* - the bound `Server` on success.
|
||||
* - `null` on Windows when a sibling daemon won the named-pipe bind
|
||||
* race. POSIX handles the equivalent "sibling won" case via the
|
||||
* PID lock acquisition earlier in {@link acquireLock}, so this
|
||||
* return value only fires on Windows.
|
||||
*
|
||||
* The asymmetric handling reflects platform reality: Unix domain
|
||||
* sockets leave a filesystem entry that survives the owning process,
|
||||
* so POSIX needs an unlink + rebind dance for orphan recovery. Windows
|
||||
* named pipes are kernel-refcounted — when the owning process dies
|
||||
* the pipe disappears from the kernel namespace automatically, so
|
||||
* there is no orphan recovery to do; `EADDRINUSE` there can only mean
|
||||
* a live sibling.
|
||||
*/
|
||||
async function bindUnderPidLock(
|
||||
socketPath: string,
|
||||
onConnection: (socket: Socket) => void,
|
||||
): Promise<Server | null> {
|
||||
const first = await tryBind(socketPath, onConnection);
|
||||
if (first.kind === 'bound') {
|
||||
return first.server;
|
||||
}
|
||||
if (first.kind === 'error') {
|
||||
throw first.err;
|
||||
}
|
||||
|
||||
// EADDRINUSE on the first attempt. The right action depends on
|
||||
// platform:
|
||||
// - Windows: no PID lock is held (named pipes provide their own
|
||||
// kernel-level exclusion), and there is no orphan pipe file to
|
||||
// unlink. EADDRINUSE here means a live sibling daemon won the
|
||||
// race; concede by returning null and let `acquireLock`
|
||||
// propagate that as a clean "another daemon is running" signal.
|
||||
// - POSIX: we hold the PID lock, so no sibling daemon is racing
|
||||
// us. EADDRINUSE must be a leftover socket file from a daemon
|
||||
// that crashed without cleanup. Unlink the orphan and retry.
|
||||
if (process.platform === 'win32') {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
await unlink(socketPath);
|
||||
} catch (err) {
|
||||
if (isErrnoException(err) && err.code !== 'ENOENT') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const second = await tryBind(socketPath, onConnection);
|
||||
if (second.kind === 'bound') {
|
||||
return second.server;
|
||||
}
|
||||
if (second.kind === 'eaddrinuse') {
|
||||
throw new Error(
|
||||
`bind failed with EADDRINUSE on ${socketPath} while holding the PID lock — ` +
|
||||
`another process is squatting on the daemon's socket path`,
|
||||
);
|
||||
}
|
||||
throw second.err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the lock acquired by {@link acquireLock}. Closes the server,
|
||||
* unlinks the POSIX socket file, and (when present) releases the PID
|
||||
* lock so the next daemon's `acquireLock` finds a clean slate.
|
||||
*/
|
||||
export async function releaseLock(server: Server, pidLock: PidLock | null = null): Promise<void> {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
if (process.platform !== 'win32') {
|
||||
try {
|
||||
await unlink(getLockSocketPath());
|
||||
} catch (err) {
|
||||
if (isErrnoException(err) && err.code !== 'ENOENT') {
|
||||
log(`releaseLock: socket unlink failed: ${formatError(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pidLock != null) {
|
||||
try {
|
||||
await pidLock.release();
|
||||
} catch (err) {
|
||||
log(`releaseLock: pid lock release failed: ${formatError(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ROTATED_LOG_REGEX = /^(failed|daemon)\.log\.(\d{4})-(\d{2})-(\d{2})$/;
|
||||
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1_000;
|
||||
|
||||
/**
|
||||
* One-shot retention sweep for the daily-rotated `failed.log.<date>` and
|
||||
* `daemon.log.<date>` files in the WAL dir.
|
||||
*
|
||||
* Called once per daemon startup, after the liveness lock is held and
|
||||
* before the batch loop starts.
|
||||
*
|
||||
* The sweep is best-effort: any failure (missing dir, EPERM on a file the
|
||||
* user hand-edited, malformed filenames) is logged at debug and
|
||||
* swallowed. Retention housekeeping must never abort daemon startup.
|
||||
*/
|
||||
export async function pruneOldLogs(
|
||||
opts: { now?: Date; retentionDays?: number } = {},
|
||||
): Promise<void> {
|
||||
const retentionDays = opts.retentionDays ?? LOG_RETENTION_DAYS;
|
||||
if (retentionDays <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = opts.now ?? new Date();
|
||||
|
||||
const todayUtcMs = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
|
||||
const cutoffMs = todayUtcMs - retentionDays * MS_PER_DAY;
|
||||
|
||||
const walDir = getWalDir();
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await readdir(walDir);
|
||||
} catch (err) {
|
||||
log(`pruneOldLogs: readdir failed for ${walDir}: ${formatError(err)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const name of entries) {
|
||||
const match = ROTATED_LOG_REGEX.exec(name);
|
||||
if (match == null) {
|
||||
continue;
|
||||
}
|
||||
const [, , yyyy, mm, dd] = match;
|
||||
const year = Number(yyyy);
|
||||
const month = Number(mm);
|
||||
const day = Number(dd);
|
||||
const fileDate = new Date(Date.UTC(year, month - 1, day));
|
||||
|
||||
if (
|
||||
fileDate.getUTCFullYear() !== year ||
|
||||
fileDate.getUTCMonth() !== month - 1 ||
|
||||
fileDate.getUTCDate() !== day
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (fileDate.getTime() >= cutoffMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = join(walDir, name);
|
||||
try {
|
||||
await unlink(fullPath);
|
||||
log(`pruneOldLogs: removed ${name} (older than ${retentionDays} day(s))`);
|
||||
} catch (err) {
|
||||
log(`pruneOldLogs: failed to unlink ${name}: ${formatError(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP statuses that {@link uploadOne} treats as "the cached
|
||||
* credentials are stale; rebuild and retry once."
|
||||
*/
|
||||
const AUTH_REFRESH_STATUSES: ReadonlySet<number> = new Set([401, 403]);
|
||||
|
||||
function isAuthRefreshableError(err: unknown): boolean {
|
||||
return err instanceof MlflowHttpError && AUTH_REFRESH_STATUSES.has(err.status);
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform the create-trace + upload-trace-data sequence, then send
|
||||
* the captured OTLP spans to the tracking-store DB.
|
||||
*/
|
||||
async function performUpload(client: MlflowClient, record: WalRecord): Promise<void> {
|
||||
const traceInfo = TraceInfo.fromJson(record.traceInfo as unknown as SerializedTraceInfo);
|
||||
const traceData = TraceData.fromJson(record.traceData as SerializedTraceData);
|
||||
const resolvedInfo = await client.createTrace(traceInfo);
|
||||
await client.uploadTraceData(resolvedInfo, traceData);
|
||||
await sendOtlpSpans(client, record);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ship the record's captured OTLP spans to the tracking store's
|
||||
* span-ingestion endpoint (`exportOtlpSpans` → `/v1/traces`) so they land in the
|
||||
* `SqlSpan` table that powers DB-backed span metrics (e.g. the "Tool calls"
|
||||
* overview).
|
||||
*/
|
||||
async function sendOtlpSpans(client: MlflowClient, record: WalRecord): Promise<void> {
|
||||
if (!record.otlpSpans) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const bytes = Buffer.from(record.otlpSpans, 'base64');
|
||||
await client.exportOtlpSpans(record.experimentId, bytes);
|
||||
} catch (err) {
|
||||
log(
|
||||
`Failed to log OTLP spans for record ${record.id} to ${record.trackingUri}; ` +
|
||||
`spans remain in the JSON artifact only: ${formatError(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single WAL record: deserialize, push to the backend,
|
||||
* tombstone on success or schedule retry / dead-letter on failure.
|
||||
*/
|
||||
export async function uploadOne(
|
||||
record: WalRecord,
|
||||
client: MlflowClient,
|
||||
writer: BatchingWriter,
|
||||
factory: ClientFactory = clientForUri,
|
||||
): Promise<void> {
|
||||
// Anchor the retry-budget window the first time the daemon touches
|
||||
// the record.
|
||||
if (record.firstAttemptAt === undefined) {
|
||||
record.firstAttemptAt = Date.now();
|
||||
}
|
||||
const traceId = (record.traceInfo as { trace_id?: string }).trace_id ?? '<unknown>';
|
||||
|
||||
try {
|
||||
await performUpload(client, record);
|
||||
await writer.submitTombstone(record.id);
|
||||
log(`Uploaded record ${record.id} (trace ${traceId})`);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (!isAuthRefreshableError(err)) {
|
||||
await handleUploadFailure(record, err, writer);
|
||||
return;
|
||||
}
|
||||
|
||||
log(
|
||||
`Auth failure on record ${record.id} (trace ${traceId}) for ${record.trackingUri}; ` +
|
||||
`evicting cached client and retrying once: ${formatError(err)}`,
|
||||
);
|
||||
clearClientForUri(record.trackingUri);
|
||||
|
||||
let refreshed: MlflowClient;
|
||||
try {
|
||||
refreshed = factory(record.trackingUri);
|
||||
} catch (factoryErr) {
|
||||
log(
|
||||
`Failed to rebuild client for ${record.trackingUri} after auth error: ${formatError(factoryErr)}`,
|
||||
);
|
||||
|
||||
await handleUploadFailure(record, factoryErr, writer);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await performUpload(refreshed, record);
|
||||
await writer.submitTombstone(record.id);
|
||||
log(`Uploaded record ${record.id} (trace ${traceId}) after credential refresh`);
|
||||
} catch (retryErr) {
|
||||
// The credential-refresh retry also failed — fall back to the
|
||||
// normal failure machinery (backoff / dead-letter).
|
||||
await handleUploadFailure(record, retryErr, writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Branch the failure path: either dead-letter the record (if the next
|
||||
* retry's backoff would push the total elapsed time past
|
||||
* `RETRY_TIMEOUT_MS`) or re-append it with bumped `attempts` and a
|
||||
* future `nextAttemptAt`. Either way the original row is tombstoned so
|
||||
* the next batch doesn't see it again.
|
||||
*
|
||||
* Retries use a fresh row `id` so per-attempt rows are independently
|
||||
* addressable in the log; the MLflow trace id inside `traceInfo` and
|
||||
* the original `firstAttemptAt` stay stable across attempts.
|
||||
*/
|
||||
async function handleUploadFailure(
|
||||
record: WalRecord,
|
||||
err: unknown,
|
||||
writer: BatchingWriter,
|
||||
): Promise<void> {
|
||||
const nextAttempts = record.attempts + 1;
|
||||
const traceId = (record.traceInfo as { trace_id?: string }).trace_id ?? '<unknown>';
|
||||
const delayMs = backoff(nextAttempts);
|
||||
const anchor = record.firstAttemptAt ?? record.createdAt;
|
||||
const now = Date.now();
|
||||
const elapsedMs = now - anchor;
|
||||
|
||||
if (elapsedMs + delayMs >= RETRY_TIMEOUT_MS) {
|
||||
log(
|
||||
`Dead-lettering record ${record.id} (trace ${traceId}) after ${record.attempts} prior ` +
|
||||
`attempts, elapsed=${elapsedMs}ms; next retry would push past ${RETRY_TIMEOUT_MS}ms ` +
|
||||
`budget: ${formatError(err)}`,
|
||||
);
|
||||
await appendDeadLetter(record);
|
||||
await writer.submitTombstone(record.id);
|
||||
return;
|
||||
}
|
||||
|
||||
const retry: WalRecord = {
|
||||
...record,
|
||||
id: randomUUID(),
|
||||
attempts: nextAttempts,
|
||||
nextAttemptAt: now + delayMs,
|
||||
firstAttemptAt: anchor,
|
||||
};
|
||||
log(
|
||||
`Retry record ${record.id} → ${retry.id} (trace ${traceId}, attempts=${nextAttempts}, ` +
|
||||
`delay=${delayMs}ms, elapsed=${elapsedMs}ms): ${formatError(err)}`,
|
||||
);
|
||||
// Both submissions ride the same `BatchingWriter` tick: one `writev`,
|
||||
// one `fsync`, atomic. Either both lines are durable or neither is.
|
||||
await Promise.all([writer.submit(retry), writer.submitTombstone(record.id)]);
|
||||
}
|
||||
|
||||
export type ClientFactory = (trackingUri: string) => MlflowClient;
|
||||
|
||||
/**
|
||||
* Group `records` by `trackingUri` and upload each group in parallel.
|
||||
*/
|
||||
export async function processBatch(
|
||||
records: WalRecord[],
|
||||
writer: BatchingWriter,
|
||||
factory: ClientFactory = clientForUri,
|
||||
): Promise<void> {
|
||||
if (records.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const groups = new Map<string, WalRecord[]>();
|
||||
for (const record of records) {
|
||||
const list = groups.get(record.trackingUri);
|
||||
if (list === undefined) {
|
||||
groups.set(record.trackingUri, [record]);
|
||||
} else {
|
||||
list.push(record);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
[...groups.entries()].map(async ([trackingUri, group]) => {
|
||||
let client: MlflowClient;
|
||||
try {
|
||||
client = factory(trackingUri);
|
||||
} catch (err) {
|
||||
log(`Failed to build client for ${trackingUri}: ${formatError(err)}`);
|
||||
|
||||
await Promise.all(group.map((record) => handleUploadFailure(record, err, writer)));
|
||||
return;
|
||||
}
|
||||
await Promise.all(group.map((record) => uploadOne(record, client, writer, factory)));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire SIGTERM / SIGINT handlers to abort the supplied controller so
|
||||
* the daemon finishes its current batch and exits cleanly rather than
|
||||
* being torn down mid-write. The abort is consulted at the top of each
|
||||
* {@link runBatchLoop} iteration.
|
||||
*/
|
||||
function installSignalHandlers(controller: AbortController): void {
|
||||
for (const sig of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.on(sig, () => {
|
||||
log(`Received ${sig}; will shut down after current batch.`);
|
||||
controller.abort();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The batch loop. Broken out from {@link main} so the dependencies
|
||||
* (client factory, writer, timings, shutdown signal) can be injected
|
||||
* in tests without standing up the full daemon lifecycle.
|
||||
*
|
||||
* Pass an `AbortSignal` to request a graceful shutdown — the loop
|
||||
* finishes its current iteration and returns. Without a signal the
|
||||
* loop only exits via the `idleMs` timeout path (or by throwing).
|
||||
*/
|
||||
export async function runBatchLoop({
|
||||
factory = clientForUri,
|
||||
writer = new BatchingWriter(),
|
||||
batchIntervalMs = BATCH_INTERVAL_MS,
|
||||
idleMs = IDLE_MS,
|
||||
signal,
|
||||
}: {
|
||||
factory?: ClientFactory;
|
||||
writer?: BatchingWriter;
|
||||
batchIntervalMs?: number;
|
||||
idleMs?: number;
|
||||
signal?: AbortSignal;
|
||||
} = {}): Promise<void> {
|
||||
let lastNonEmpty = Date.now();
|
||||
|
||||
while (signal?.aborted !== true) {
|
||||
try {
|
||||
const pending = await readPending();
|
||||
const now = Date.now();
|
||||
const due = pending.filter((r) => r.nextAttemptAt <= now);
|
||||
|
||||
if (due.length > 0) {
|
||||
lastNonEmpty = now;
|
||||
await processBatch(due, writer, factory);
|
||||
} else if (now - lastNonEmpty >= idleMs) {
|
||||
log('WAL idle threshold reached; shutting down.');
|
||||
return;
|
||||
}
|
||||
|
||||
if ((await walSize()) > COMPACTION_THRESHOLD_BYTES) {
|
||||
await compact().catch((err) => log(`Periodic compact failed: ${formatError(err)}`));
|
||||
}
|
||||
} catch (err) {
|
||||
// Don't let a transient FS / parse error tear down the daemon.
|
||||
log(`Batch iteration error: ${formatError(err)}`);
|
||||
}
|
||||
|
||||
await sleep(batchIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Daemon entry point.
|
||||
*/
|
||||
export async function main(): Promise<void> {
|
||||
await mkdir(getWalDir(), { recursive: true });
|
||||
|
||||
const ipcWriter = new BatchingWriter();
|
||||
const handle = await acquireLock(createIpcConnectionHandler(ipcWriter));
|
||||
if (handle == null) {
|
||||
log('Another daemon is already running; exiting cleanly.');
|
||||
process.exit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
log(`Daemon started; pid=${process.pid}, walDir=${getWalDir()}`);
|
||||
const shutdown = new AbortController();
|
||||
installSignalHandlers(shutdown);
|
||||
|
||||
let exitCode = 0;
|
||||
try {
|
||||
await pruneOldLogs().catch((err) => log(`Retention sweep failed: ${formatError(err)}`));
|
||||
await compact().catch((err) => log(`Startup compact failed: ${formatError(err)}`));
|
||||
await runBatchLoop({ writer: ipcWriter, signal: shutdown.signal });
|
||||
await compact().catch((err) => log(`Final compact failed: ${formatError(err)}`));
|
||||
} catch (err) {
|
||||
log(`Daemon main loop crashed: ${formatError(err)}`);
|
||||
exitCode = 1;
|
||||
} finally {
|
||||
await releaseLock(handle.server, handle.pidLock).catch((err) =>
|
||||
log(`releaseLock failed: ${formatError(err)}`),
|
||||
);
|
||||
clearClientCache();
|
||||
log(`Daemon exited with code ${exitCode}.`);
|
||||
}
|
||||
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
// Run main() only when invoked directly (i.e. `node bundle/daemon.cjs`),
|
||||
// not when this module is imported by tests.
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error('[mlflow][wal-daemon] fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
|
||||
import { ReadableSpan as OTelReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base';
|
||||
import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer';
|
||||
import { getConfig, MLflowTracingConfig } from '../../core/config';
|
||||
import { InMemoryTraceManager } from '../../core/trace_manager';
|
||||
import { ipcRequestByteLength, MAX_REQUEST_BYTES, submitRecord } from './ipc';
|
||||
import { WalRecord } from './types';
|
||||
|
||||
function decodeMlflowSpanAttributes(
|
||||
attributes: OTelReadableSpan['attributes'],
|
||||
): Record<string, unknown> {
|
||||
const decoded: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(attributes)) {
|
||||
if (typeof value === 'string' && value.startsWith('"')) {
|
||||
try {
|
||||
decoded[key] = JSON.parse(value);
|
||||
} catch {
|
||||
decoded[key] = value;
|
||||
}
|
||||
} else {
|
||||
decoded[key] = value;
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a view of `span` whose `attributes` are decoded (see
|
||||
* {@link decodeMlflowSpanAttributes}) while every other property/method is
|
||||
* delegated to the original span untouched.
|
||||
*/
|
||||
function spanWithDecodedAttributes(span: OTelReadableSpan): OTelReadableSpan {
|
||||
const decoded = decodeMlflowSpanAttributes(span.attributes);
|
||||
return new Proxy(span, {
|
||||
get(target, prop) {
|
||||
if (prop === 'attributes') {
|
||||
return decoded;
|
||||
}
|
||||
const value: unknown = Reflect.get(target, prop);
|
||||
return typeof value === 'function' ? (value.bind(target) as unknown) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize live OTel spans into a base64-encoded OTLP
|
||||
* `ExportTraceServiceRequest` protobuf, suitable for storing on a
|
||||
* {@link WalRecord} and later POSTing to the server's OTLP span-ingestion
|
||||
* endpoint
|
||||
*/
|
||||
function serializeSpansToOtlpBase64(spans: OTelReadableSpan[]): string | undefined {
|
||||
const serializable = spans.filter((s) => s?.resource != null && s?.instrumentationScope != null);
|
||||
if (serializable.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const bytes = ProtobufTraceSerializer.serializeRequest(
|
||||
serializable.map(spanWithDecodedAttributes),
|
||||
);
|
||||
if (!bytes || bytes.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return Buffer.from(bytes).toString('base64');
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[mlflow][wal] Failed to serialize spans to OTLP protobuf; ' +
|
||||
'falling back to JSON artifact upload (spans will not appear in DB-backed span metrics).',
|
||||
err,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook-side submit function. Returns once the daemon has fsynced the
|
||||
* record to `queue.log` (ack-after-fsync). Injectable so tests can
|
||||
* swap in a stub without spinning up an actual daemon.
|
||||
*/
|
||||
export type SubmitRecord = (record: WalRecord) => Promise<void>;
|
||||
|
||||
/**
|
||||
* Asynchronous SpanExporter that decouples the user's Stop hook from
|
||||
* backend latency for MlflowSpanProcessor.
|
||||
*
|
||||
* {@link forceFlush} awaits only the IPC submits, *not* the upstream
|
||||
* upload. That gives the Stop hook the exact "trace is durable on this
|
||||
* machine, then return" semantics it needs to bound its runtime.
|
||||
*/
|
||||
export class MlflowWalSpanExporter implements SpanExporter {
|
||||
private _pendingSubmits: Set<Promise<void>> = new Set();
|
||||
private _submit: SubmitRecord;
|
||||
private _shuttingDown = false;
|
||||
|
||||
constructor(opts: { submit?: SubmitRecord } = {}) {
|
||||
this._submit = opts.submit ?? submitRecord;
|
||||
}
|
||||
|
||||
export(spans: OTelReadableSpan[], resultCallback: (result: ExportResult) => void): void {
|
||||
if (this._shuttingDown) {
|
||||
resultCallback({ code: ExportResultCode.FAILED });
|
||||
return;
|
||||
}
|
||||
|
||||
let cfg: MLflowTracingConfig;
|
||||
try {
|
||||
cfg = getConfig();
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[mlflow][wal] Tracing config unavailable; Spans will be lost until config is set.',
|
||||
err,
|
||||
);
|
||||
resultCallback({ code: ExportResultCode.FAILED });
|
||||
return;
|
||||
}
|
||||
|
||||
for (const span of spans) {
|
||||
if (span.parentSpanContext?.spanId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.popTrace(span.spanContext().traceId);
|
||||
if (!trace) {
|
||||
console.warn(`No trace found for span ${span.name}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
traceManager.lastActiveTraceId = trace.info.traceId;
|
||||
|
||||
const otlpSpans = serializeSpansToOtlpBase64(trace.data.spans.map((s) => s._span));
|
||||
|
||||
const record: WalRecord = {
|
||||
id: randomUUID(),
|
||||
trackingUri: cfg.trackingUri,
|
||||
experimentId: cfg.experimentId,
|
||||
traceInfo: trace.info.toJson() as unknown as Record<string, unknown>,
|
||||
traceData: trace.data.toJson(),
|
||||
attempts: 0,
|
||||
nextAttemptAt: 0,
|
||||
createdAt: Date.now(),
|
||||
otlpSpans,
|
||||
};
|
||||
|
||||
const fullSizeBytes = ipcRequestByteLength(record);
|
||||
if (otlpSpans !== undefined && fullSizeBytes > MAX_REQUEST_BYTES) {
|
||||
console.warn(
|
||||
`[mlflow][wal] WAL record for trace ${trace.info.traceId} is ${fullSizeBytes} bytes, exceeding the ` +
|
||||
`${MAX_REQUEST_BYTES}-byte IPC request limit; dropping OTLP spans and attempting JSON artifact upload ` +
|
||||
'(spans will not appear in DB-backed span metrics).',
|
||||
);
|
||||
record.otlpSpans = undefined;
|
||||
}
|
||||
|
||||
const pending = this._submit(record).catch((err) => {
|
||||
console.error(
|
||||
`[mlflow][wal] Failed to submit WAL record for trace ${trace.info.traceId}:`,
|
||||
err,
|
||||
);
|
||||
});
|
||||
this._pendingSubmits.add(pending);
|
||||
|
||||
void pending.finally(() => this._pendingSubmits.delete(pending));
|
||||
}
|
||||
|
||||
resultCallback({ code: ExportResultCode.SUCCESS });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves once every IPC submit that was in flight when this method
|
||||
* was called has been acknowledged (post-fsync). Matches OTel's
|
||||
* "flush currently pending" semantic — concurrent `export()` calls
|
||||
* during the await are not waited on, and the next `forceFlush` will
|
||||
* pick them up. For "drain everything before exit" semantics, use
|
||||
* {@link shutdown}.
|
||||
*/
|
||||
async forceFlush(): Promise<void> {
|
||||
await Promise.all([...this._pendingSubmits]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop accepting new exports and drain every in-flight IPC submit to
|
||||
* convergence before resolving.
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
this._shuttingDown = true;
|
||||
while (this._pendingSubmits.size > 0) {
|
||||
await Promise.all([...this._pendingSubmits]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Public entry point for the WAL exporter package.
|
||||
*/
|
||||
|
||||
export { MlflowWalSpanExporter } from './exporter';
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Hook ↔ daemon IPC over the daemon's lock socket / Named Pipe.
|
||||
*
|
||||
* Client side ({@link submitRecord}):
|
||||
* 1. Connect to {@link getLockSocketPath}.
|
||||
* 2. Write a single newline-terminated JSON request.
|
||||
* 3. Read a single newline-terminated JSON response.
|
||||
* 4. Close.
|
||||
* The promise resolves only after the daemon has fsynced the record's
|
||||
* byte to `queue.log`.
|
||||
*
|
||||
* Server side ({@link createIpcConnectionHandler}):
|
||||
* - Wait for one line of data on the connection.
|
||||
* - Parse it as an {@link IpcRequest}.
|
||||
* - Dispatch the `append` op into the daemon's {@link BatchingWriter}.
|
||||
* - Once the writer's promise settles, write a one-line
|
||||
* {@link IpcResponse} and close.
|
||||
* - Connections that close before sending a line are liveness probes
|
||||
* (used by {@link isDaemonAlive}); we let them go silently.
|
||||
*/
|
||||
|
||||
import { createConnection, Socket } from 'node:net';
|
||||
import { JSONBig } from '../../core/utils/json';
|
||||
import { BatchingWriter } from './batching_writer';
|
||||
import { getLockSocketPath } from './paths';
|
||||
import { IpcRequest, IpcResponse } from './protocol';
|
||||
import { ensureDaemon } from './supervisor';
|
||||
import { WalRecord } from './types';
|
||||
|
||||
/**
|
||||
* Upper bound on a single submit round-trip (connect + write + ACK).
|
||||
*/
|
||||
const SUBMIT_TIMEOUT_MS = 10_000;
|
||||
|
||||
const INITIAL_CONNECT_RETRY_DELAY_MS = 50;
|
||||
const CONNECT_RETRY_DELAYS_MS = Array.from(
|
||||
{ length: 6 },
|
||||
(_, i) => INITIAL_CONNECT_RETRY_DELAY_MS * 2 ** i,
|
||||
);
|
||||
|
||||
const DAEMON_NO_RESPONSE_CODE = 'EDAEMONNORESPONSE';
|
||||
|
||||
export const MAX_REQUEST_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact byte length of the wire payload {@link submitRecord} would send for
|
||||
* `record`, including the `append` envelope and trailing newline. Use this to
|
||||
* compare against {@link MAX_REQUEST_BYTES} before submitting.
|
||||
*/
|
||||
function buildRequestPayload(record: WalRecord): string {
|
||||
return JSONBig.stringify({ op: 'append', record } satisfies IpcRequest) + '\n';
|
||||
}
|
||||
|
||||
export function ipcRequestByteLength(record: WalRecord): number {
|
||||
return Buffer.byteLength(buildRequestPayload(record), 'utf8');
|
||||
}
|
||||
|
||||
function isConnectError(err: unknown): boolean {
|
||||
if (typeof err !== 'object' || err == null) {
|
||||
return false;
|
||||
}
|
||||
const code = (err as { code?: string }).code;
|
||||
return (
|
||||
code === 'ECONNREFUSED' ||
|
||||
code === 'ENOENT' ||
|
||||
code === 'EPIPE' ||
|
||||
code === 'ECONNRESET' ||
|
||||
code === DAEMON_NO_RESPONSE_CODE
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send `record` to the running daemon and wait for the post-fsync ACK.
|
||||
*
|
||||
* Delivery semantics: **at-least-once**.
|
||||
*
|
||||
* A successful resolution means the record was durably fsynced to
|
||||
* `queue.log` at least once. A rejection means the daemon could not be
|
||||
* acknowledged after 7 attempts (1 initial + 6 backoffs from
|
||||
* `CONNECT_RETRY_DELAYS_MS`), but the daemon may still have persisted
|
||||
* the record on one of those attempts before the failure surfaced.
|
||||
*
|
||||
* The internal retry loop retries on five connect-class codes:
|
||||
* `ECONNREFUSED`, `ENOENT`, `EPIPE`, `ECONNRESET`, and
|
||||
* `EDAEMONNORESPONSE`. The first two prove the request never reached a
|
||||
* daemon and are safe; the last three can fire *after* the daemon has
|
||||
* already fsynced (e.g. daemon crashed between fsync and ACK, or the
|
||||
* socket reset post-fsync), so a single logical submit can produce
|
||||
* more than one physical line in `queue.log` and more than one
|
||||
* `createTrace` call to the backend.
|
||||
*
|
||||
*/
|
||||
export async function submitRecord(record: WalRecord): Promise<void> {
|
||||
const payload = buildRequestPayload(record);
|
||||
let lastErr: unknown;
|
||||
for (let i = 0; i <= CONNECT_RETRY_DELAYS_MS.length; i++) {
|
||||
if (i > 0) {
|
||||
await sleep(CONNECT_RETRY_DELAYS_MS[i - 1]);
|
||||
}
|
||||
try {
|
||||
const response = await sendRequest(payload);
|
||||
if (response.ok) {
|
||||
return;
|
||||
}
|
||||
throw new Error(`Daemon rejected record: ${response.error}`);
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (!isConnectError(err)) {
|
||||
throw err;
|
||||
}
|
||||
// Fire-and-forget spawn; the next loop iteration's connect will
|
||||
// observe the bound socket once the daemon finishes binding.
|
||||
ensureDaemon().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
throw lastErr as Error;
|
||||
}
|
||||
|
||||
/**
|
||||
* One IPC round-trip against the daemon's lock socket.
|
||||
*/
|
||||
function sendRequest(payload: string): Promise<IpcResponse> {
|
||||
return new Promise<IpcResponse>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const chunks: Buffer[] = [];
|
||||
let settled = false;
|
||||
|
||||
const finish = (act: () => void): void => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
act();
|
||||
};
|
||||
|
||||
const timer = setTimeout(
|
||||
() => finish(() => reject(new Error(`IPC submit timed out after ${SUBMIT_TIMEOUT_MS}ms`))),
|
||||
SUBMIT_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
socket.once('connect', () => {
|
||||
socket.write(payload);
|
||||
});
|
||||
socket.on('data', (chunk) => chunks.push(chunk));
|
||||
socket.once('end', () => {
|
||||
clearTimeout(timer);
|
||||
const text = Buffer.concat(chunks).toString('utf8').trimEnd();
|
||||
if (text === '') {
|
||||
finish(() =>
|
||||
reject(
|
||||
Object.assign(new Error('Daemon closed connection without responding'), {
|
||||
code: DAEMON_NO_RESPONSE_CODE,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSONBig.parse(text) as IpcResponse;
|
||||
finish(() => resolve(parsed));
|
||||
} catch (err) {
|
||||
finish(() => reject(err as Error));
|
||||
}
|
||||
});
|
||||
socket.once('error', (err) => {
|
||||
clearTimeout(timer);
|
||||
finish(() => reject(err));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-connection state for an in-flight IPC request. One instance per
|
||||
* accepted socket.
|
||||
*/
|
||||
function handleConnection(writer: BatchingWriter, socket: Socket): void {
|
||||
const chunks: Buffer[] = [];
|
||||
let bytesReceived = 0;
|
||||
let dispatched = false;
|
||||
|
||||
socket.on('error', (err: NodeJS.ErrnoException) => {
|
||||
if (err.code !== 'ECONNRESET' && err.code !== 'EPIPE') {
|
||||
console.error(
|
||||
`[mlflow][wal] ipc connection error: code=${err.code ?? 'none'} message=${err.message}`,
|
||||
);
|
||||
}
|
||||
socket.destroy();
|
||||
});
|
||||
|
||||
socket.on('data', (chunk: Buffer) => {
|
||||
if (dispatched) {
|
||||
// Extra bytes after the first line are not part of the protocol
|
||||
// today; ignore them rather than reject
|
||||
return;
|
||||
}
|
||||
bytesReceived += chunk.length;
|
||||
if (bytesReceived > MAX_REQUEST_BYTES) {
|
||||
dispatched = true;
|
||||
socket.pause();
|
||||
sendResponse(socket, {
|
||||
ok: false,
|
||||
error: `request exceeds max size ${MAX_REQUEST_BYTES} bytes without newline terminator`,
|
||||
});
|
||||
const destroyTimer = setTimeout(() => socket.destroy(), 500);
|
||||
destroyTimer.unref();
|
||||
socket.once('close', () => clearTimeout(destroyTimer));
|
||||
return;
|
||||
}
|
||||
// Prior chunks contained no newline (otherwise `dispatched` would
|
||||
// be true and we'd have returned above), so only this chunk needs
|
||||
// scanning. `0x0A` is the byte value of '\n'.
|
||||
const newlineInChunk = chunk.indexOf(0x0a);
|
||||
if (newlineInChunk < 0) {
|
||||
chunks.push(chunk);
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk.subarray(0, newlineInChunk));
|
||||
const line = Buffer.concat(chunks).toString('utf8');
|
||||
dispatched = true;
|
||||
void dispatch(writer, socket, line);
|
||||
});
|
||||
}
|
||||
|
||||
async function dispatch(writer: BatchingWriter, socket: Socket, line: string): Promise<void> {
|
||||
let request: IpcRequest;
|
||||
try {
|
||||
request = JSONBig.parse(line) as IpcRequest;
|
||||
} catch (err) {
|
||||
sendResponse(socket, {
|
||||
ok: false,
|
||||
error: `malformed request: ${(err as Error).message}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.op !== 'append') {
|
||||
sendResponse(socket, {
|
||||
ok: false,
|
||||
error: `unknown op: ${(request as { op: string }).op}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const record = request.record as Partial<WalRecord> | null | undefined;
|
||||
if (
|
||||
typeof record !== 'object' ||
|
||||
record == null ||
|
||||
typeof record.id !== 'string' ||
|
||||
record.id === ''
|
||||
) {
|
||||
sendResponse(socket, {
|
||||
ok: false,
|
||||
error: 'invalid record: must be an object with a non-empty string id',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await writer.submit(request.record);
|
||||
sendResponse(socket, { ok: true });
|
||||
} catch (err) {
|
||||
sendResponse(socket, { ok: false, error: (err as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
function sendResponse(socket: Socket, response: IpcResponse): void {
|
||||
// `socket.end` does not throw synchronously for our inputs (always a
|
||||
// string, never null, never on a not-yet-constructed handle). Any
|
||||
// transport-level failure during the write surfaces asynchronously
|
||||
// via the `'error'` event, which is handled at the top of
|
||||
// handleConnection - no defensive try/catch needed here.
|
||||
socket.end(JSONBig.stringify(response) + '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-connection handler that the daemon installs on its
|
||||
* lock-socket server. Bound to a {@link BatchingWriter} so the
|
||||
* fsync-coalescing state persists across connections.
|
||||
*/
|
||||
export function createIpcConnectionHandler(writer: BatchingWriter): (socket: Socket) => void {
|
||||
return (socket: Socket) => handleConnection(writer, socket);
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Filesystem paths for the MLflow trace WAL (Write-Ahead Log) spool.
|
||||
*
|
||||
* A spool is shared across all Claude Code sessions
|
||||
*
|
||||
* The daemon's singleton lock is scoped per `(user, spool root)` on both
|
||||
* platforms; its exact location depends on the platform and the resolved
|
||||
* spool root's length:
|
||||
* - POSIX, common case: `<spool>/daemon.sock` next to `queue.log`.
|
||||
* - POSIX, long-path fallback: `<tmpdir>/mlflow-<scoped_id>.sock` when the
|
||||
* natural path would exceed `sun_path`.
|
||||
* - Windows: a Named Pipe whose name embeds a hash of the spool root.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { homedir, platform, tmpdir, userInfo } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
|
||||
/**
|
||||
* Maximum allowed length, in bytes, for a UNIX domain socket path.
|
||||
*/
|
||||
const POSIX_SOCKET_PATH_MAX_BYTES = 103;
|
||||
|
||||
/**
|
||||
* Root directory of the per-user spool. Defaults to `~/.mlflow/wal/`.
|
||||
*/
|
||||
export function getWalDir(): string {
|
||||
const override = process.env.MLFLOW_WAL_DIR;
|
||||
const raw =
|
||||
override !== undefined && override !== '' ? override : join(homedir(), '.mlflow', 'wal');
|
||||
return resolve(raw);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append-only Write-Ahead Log of pending trace uploads.
|
||||
*/
|
||||
export function getWalPath(): string {
|
||||
return join(getWalDir(), 'queue.log');
|
||||
}
|
||||
|
||||
function dateSuffix(d: Date = new Date()): string {
|
||||
const yyyy = d.getUTCFullYear().toString().padStart(4, '0');
|
||||
const mm = (d.getUTCMonth() + 1).toString().padStart(2, '0');
|
||||
const dd = d.getUTCDate().toString().padStart(2, '0');
|
||||
return `${yyyy}-${mm}-${dd}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append-only dead-letter file containing records whose retry-budget window
|
||||
* (`MLFLOW_ASYNC_TRACE_LOGGING_RETRY_TIMEOUT`) has elapsed. Records here are
|
||||
* durable and inspectable / replayable by operators.
|
||||
*/
|
||||
export function getDeadLetterPath(now: Date = new Date()): string {
|
||||
return join(getWalDir(), `failed.log.${dateSuffix(now)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Daemon log file with diagnostic lines (retries, DLQ entries, fatal errors).
|
||||
*/
|
||||
export function getDaemonLogPath(now: Date = new Date()): string {
|
||||
return join(getWalDir(), `daemon.log.${dateSuffix(now)}`);
|
||||
}
|
||||
|
||||
export function getPidLockPath(): string {
|
||||
return join(getWalDir(), 'daemon.pid');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable per (user, wal_dir) identifier used to scope the daemon lock
|
||||
*/
|
||||
function getScopedLockId(): string {
|
||||
const sanitizedUser = userInfo().username.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const walDirHash = createHash('sha256').update(getWalDir()).digest('hex').slice(0, 12);
|
||||
return `${sanitizedUser}-${walDirHash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Path used as the daemon's singleton liveness lock.
|
||||
*/
|
||||
export function getLockSocketPath(): string {
|
||||
const scopedId = getScopedLockId();
|
||||
|
||||
if (platform() === 'win32') {
|
||||
return `\\\\?\\pipe\\mlflow-trace-daemon-${scopedId}`;
|
||||
}
|
||||
|
||||
const naturalPath = join(getWalDir(), 'daemon.sock');
|
||||
if (Buffer.byteLength(naturalPath, 'utf8') <= POSIX_SOCKET_PATH_MAX_BYTES) {
|
||||
return naturalPath;
|
||||
}
|
||||
|
||||
// Short basename so the fallback itself stays under sun_path even on
|
||||
// macOS, whose tmpdir is already ~49 chars (`/var/folders/<2>/<30>/T/`).
|
||||
const fallback = join(tmpdir(), `mlflow-${scopedId}.sock`);
|
||||
console.warn(
|
||||
`[mlflow][wal] WAL dir produces a socket path longer than ` +
|
||||
`${POSIX_SOCKET_PATH_MAX_BYTES} bytes (${Buffer.byteLength(naturalPath, 'utf8')}); ` +
|
||||
`using ${fallback} for the daemon lock instead.`,
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Cross-process exclusion lock for the daemon singleton.
|
||||
*
|
||||
* The lock is POSIX-only; on Windows the daemon's named-pipe path is
|
||||
* kernel-refcounted and provides cross-process exclusion natively, so
|
||||
* `acquireLock` skips this module on `process.platform === 'win32'`.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { link, readFile, unlink, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { getPidLockPath, getWalDir } from './paths';
|
||||
|
||||
const MAX_ACQUIRE_ATTEMPTS = 5;
|
||||
|
||||
function isErrnoException(err: unknown): err is NodeJS.ErrnoException {
|
||||
return typeof err === 'object' && err != null && 'code' in err;
|
||||
}
|
||||
|
||||
export interface PidLock {
|
||||
release(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to claim the daemon's cross-process PID lock.
|
||||
*
|
||||
* Returns a {@link PidLock} on success, or `null` when another live
|
||||
* process already holds it. Throws on unrecoverable I/O errors
|
||||
* (`EACCES`, `ENOSPC`, missing WAL dir, etc.) — callers should treat
|
||||
* a throw as "daemon cannot start" rather than "concede to sibling."
|
||||
*/
|
||||
export async function tryAcquirePidLock(): Promise<PidLock | null> {
|
||||
const lockPath = getPidLockPath();
|
||||
const expectedContent = `${process.pid}\n`;
|
||||
|
||||
for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) {
|
||||
const tmpPath = join(getWalDir(), `daemon.pid.${process.pid}.${randomUUID()}.tmp`);
|
||||
|
||||
try {
|
||||
await writeFile(tmpPath, expectedContent, { flag: 'wx' });
|
||||
await link(tmpPath, lockPath);
|
||||
await unlink(tmpPath).catch(() => {});
|
||||
return { release: () => releaseLockFile(lockPath, expectedContent) };
|
||||
} catch (err) {
|
||||
await unlink(tmpPath).catch(() => {});
|
||||
if (!isErrnoException(err) || err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// EEXIST path. Inspect the holder to decide between conceding and
|
||||
// recovering from a stale lockfile left by a crashed daemon.
|
||||
let content: string;
|
||||
try {
|
||||
content = await readFile(lockPath, 'utf8');
|
||||
} catch (err) {
|
||||
if (isErrnoException(err) && err.code === 'ENOENT') {
|
||||
// Holder released between our link attempt and our read; the
|
||||
// path is free again, loop and reattempt the link.
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const holderPid = parsePid(content);
|
||||
if (holderPid != null && isProcessAlive(holderPid)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Stale or malformed. Unlink the lockfile, but only if its content
|
||||
// is still the same bytes we just read - otherwise a successor has
|
||||
// already taken over and we would nuke their fresh lockfile.
|
||||
const fresh = await readFile(lockPath, 'utf8').catch(() => null);
|
||||
if (fresh === content) {
|
||||
await unlink(lockPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`tryAcquirePidLock: gave up after ${MAX_ACQUIRE_ATTEMPTS} contention attempts on ${lockPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the pid recorded in the daemon's lockfile, or `null` if the
|
||||
* file is missing or unparseable. Provided as a diagnostic helper for
|
||||
* callers (e.g. supervisor) that want to log "who owns the lock right
|
||||
* now"; not used by acquisition itself.
|
||||
*/
|
||||
export async function readLockHolderPid(): Promise<number | null> {
|
||||
try {
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
return parsePid(content);
|
||||
} catch (err) {
|
||||
if (isErrnoException(err) && err.code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function parsePid(content: string): number | null {
|
||||
const trimmed = content.trim();
|
||||
if (trimmed === '') {
|
||||
return null;
|
||||
}
|
||||
const n = Number.parseInt(trimmed, 10);
|
||||
// PIDs are positive integers; reject 0 (which `process.kill` would
|
||||
// interpret as "signal every process in our process group") and
|
||||
// negatives (which would target a process group).
|
||||
return Number.isInteger(n) && n > 0 ? n : null;
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (err) {
|
||||
// `EPERM` means the process exists but we lack permission to
|
||||
// signal it (different uid). Treat as alive — the lockfile's
|
||||
// owner is alive enough that we should not unlink their lock.
|
||||
if (isErrnoException(err) && err.code === 'EPERM') {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function releaseLockFile(lockPath: string, expectedContent: string): Promise<void> {
|
||||
const fresh = await readFile(lockPath, 'utf8').catch(() => null);
|
||||
if (fresh === expectedContent) {
|
||||
await unlink(lockPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Wire protocol for the daemon's IPC socket.
|
||||
*
|
||||
* op: 'append' resolves only after the record is durable in
|
||||
* queue.log (the daemon's batching writer fsyncs the batch before
|
||||
* acknowledging).
|
||||
*/
|
||||
|
||||
import { WalRecord } from './types';
|
||||
|
||||
export interface AppendRequest {
|
||||
op: 'append';
|
||||
record: WalRecord;
|
||||
}
|
||||
|
||||
export type IpcRequest = AppendRequest;
|
||||
|
||||
export interface OkResponse {
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export interface ErrResponse {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export type IpcResponse = OkResponse | ErrResponse;
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Append-only JSONL store backing the trace upload WAL.
|
||||
*/
|
||||
|
||||
import { createReadStream, existsSync } from 'node:fs';
|
||||
import { mkdir, open, rename, stat, unlink } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { createInterface } from 'node:readline';
|
||||
import { JSONBig } from '../../core/utils/json';
|
||||
import { getDeadLetterPath, getWalPath } from './paths';
|
||||
import { WalLine, WalRecord } from './types';
|
||||
|
||||
/**
|
||||
* Promise-chained queue that serializes async tasks one at a time.
|
||||
*/
|
||||
class SerialQueue {
|
||||
private tail: Promise<void> = Promise.resolve();
|
||||
|
||||
run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const result = this.tail.catch(() => {}).then(fn);
|
||||
this.tail = result.then(
|
||||
() => {},
|
||||
() => {},
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
export const queueWriter = new SerialQueue();
|
||||
|
||||
const deadLetterWriter = new SerialQueue();
|
||||
|
||||
export async function ensureParentDir(filePath: string): Promise<void> {
|
||||
await mkdir(dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Append one JSONL line to `path`, fsync, and close.
|
||||
*
|
||||
* Encoded as a single `Buffer` so the write goes out in one `O_APPEND`
|
||||
* syscall — line-sized records sit well under `PIPE_BUF`, so concurrent
|
||||
* appenders never interleave their bytes.
|
||||
*/
|
||||
async function appendJsonLine(path: string, line: WalLine): Promise<void> {
|
||||
await ensureParentDir(path);
|
||||
const buf = Buffer.from(JSONBig.stringify(line) + '\n', 'utf8');
|
||||
const fh = await open(path, 'a');
|
||||
try {
|
||||
await fh.write(buf);
|
||||
await fh.sync();
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a pending trace upload to the WAL.
|
||||
*/
|
||||
export function appendRecord(record: WalRecord): Promise<void> {
|
||||
return queueWriter.run(() => appendJsonLine(getWalPath(), { type: 'append', record }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a tombstone for `id` to the WAL. Logically shadows any earlier
|
||||
* `{type:"append"}` line for the same id.
|
||||
*/
|
||||
export function appendTombstone(id: string): Promise<void> {
|
||||
return queueWriter.run(() => appendJsonLine(getWalPath(), { type: 'tombstone', id }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Append `record` to the dead-letter file (`failed.log.<YYYY-MM-DD>`).
|
||||
*/
|
||||
export function appendDeadLetter(record: WalRecord): Promise<void> {
|
||||
return deadLetterWriter.run(() =>
|
||||
appendJsonLine(getDeadLetterPath(), { type: 'append', record }),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay `queue.log` and return the set of records still considered pending.
|
||||
*/
|
||||
export async function readPending(): Promise<WalRecord[]> {
|
||||
const path = getWalPath();
|
||||
if (!existsSync(path)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const alive = new Map<string, WalRecord>();
|
||||
const stream = createReadStream(path, { encoding: 'utf8' });
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
|
||||
for await (const line of rl) {
|
||||
if (line === '') {
|
||||
continue;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
// JSONBig.parse so bigint fields written by `appendJsonLine` are
|
||||
// restored as `bigint` (and not as the lossy `number` JSON.parse
|
||||
// would coerce them to once stringified back through `JSONBig`).
|
||||
parsed = JSONBig.parse(line);
|
||||
} catch {
|
||||
console.debug(`[mlflow][wal] Skipping malformed WAL line: ${line.slice(0, 120)}`);
|
||||
continue;
|
||||
}
|
||||
if (!parsed || typeof parsed !== 'object') {
|
||||
continue;
|
||||
}
|
||||
const obj = parsed as { type?: unknown; record?: unknown; id?: unknown };
|
||||
if (obj.type === 'append') {
|
||||
const record = obj.record as WalRecord | undefined;
|
||||
if (record && typeof record.id === 'string') {
|
||||
alive.set(record.id, record);
|
||||
}
|
||||
} else if (obj.type === 'tombstone' && typeof obj.id === 'string') {
|
||||
alive.delete(obj.id);
|
||||
}
|
||||
}
|
||||
|
||||
return [...alive.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite `queue.log` to contain only currently-live records.
|
||||
*/
|
||||
export function compact(): Promise<void> {
|
||||
return queueWriter.run(async () => {
|
||||
const path = getWalPath();
|
||||
if (!existsSync(path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const liveRecords = await readPending();
|
||||
|
||||
const tmpPath = `${path}.tmp.${process.pid}`;
|
||||
const tmpFh = await open(tmpPath, 'w');
|
||||
try {
|
||||
for (const record of liveRecords) {
|
||||
const buf = Buffer.from(
|
||||
JSONBig.stringify({ type: 'append', record } as WalLine) + '\n',
|
||||
'utf8',
|
||||
);
|
||||
await tmpFh.write(buf);
|
||||
}
|
||||
await tmpFh.sync();
|
||||
await tmpFh.close();
|
||||
await rename(tmpPath, path);
|
||||
} catch (err) {
|
||||
await tmpFh.close().catch(() => {});
|
||||
await unlink(tmpPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Size of `queue.log` in bytes, or 0 if the file does not exist.
|
||||
* Used by the daemon's idle-shutdown heuristic.
|
||||
*/
|
||||
export async function walSize(): Promise<number> {
|
||||
const path = getWalPath();
|
||||
if (!existsSync(path)) {
|
||||
return 0;
|
||||
}
|
||||
return (await stat(path)).size;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Daemon supervisor - handles the "is the uploader daemon alive? if not,
|
||||
* spawn one" check invoked by `MlflowWalSpanExporter` on each WAL append.
|
||||
*/
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createConnection } from 'node:net';
|
||||
import { createRequire } from 'node:module';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { getLockSocketPath } from './paths';
|
||||
|
||||
const PROBE_TIMEOUT_MS = 1000;
|
||||
|
||||
/**
|
||||
* Upper bound on how long {@link ensureDaemon} waits for a freshly
|
||||
* spawned daemon to bind its lock socket.
|
||||
*/
|
||||
const SPAWN_BIND_WAIT_TIMEOUT_MS = 2000;
|
||||
const SPAWN_BIND_WAIT_POLL_MS = 50;
|
||||
|
||||
/**
|
||||
* Probe the daemon's liveness lock with a short timeout.
|
||||
*
|
||||
* Returns:
|
||||
* - `true` if the socket/pipe at `getLockSocketPath()` is currently
|
||||
* accepting connections (i.e. a daemon has bound it).
|
||||
* - `false` on any error (ENOENT — no socket file; ECONNREFUSED — a
|
||||
* stale socket file with no listener) or on probe timeout.
|
||||
*/
|
||||
export function isDaemonAlive(): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const socketPath = getLockSocketPath();
|
||||
const socket = createConnection(socketPath);
|
||||
|
||||
let settled = false;
|
||||
const finish = (alive: boolean): void => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
resolve(alive);
|
||||
};
|
||||
|
||||
const timer = setTimeout(() => finish(false), PROBE_TIMEOUT_MS);
|
||||
|
||||
socket.once('connect', () => {
|
||||
clearTimeout(timer);
|
||||
finish(true);
|
||||
});
|
||||
socket.once('error', () => {
|
||||
clearTimeout(timer);
|
||||
finish(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the absolute path of the daemon entry point.
|
||||
*
|
||||
* Precedence:
|
||||
* 1. `MLFLOW_TRACE_DAEMON_EXECUTABLE` env var — used by tests and
|
||||
* advanced operators who want to override the bundled daemon.
|
||||
* 2. The bundled daemon at `<@mlflow/core install dir>/bundle/daemon.cjs`.
|
||||
*/
|
||||
export function resolveDaemonEntry(): string {
|
||||
const override = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
|
||||
if (override !== undefined && override !== '') {
|
||||
return override;
|
||||
}
|
||||
// DO NOT collapse this into a plain string literal. esbuild
|
||||
// evaluates `require.resolve(literal)` at build time and bakes the
|
||||
// resolved (developer-machine) path directly into the shipped
|
||||
// bundle. On end users' machines that path does not exist, so
|
||||
// `spawn` fails with ENOENT and traces silently stop uploading
|
||||
// while the WAL grows unbounded. The string-concat hides the
|
||||
// specifier from esbuild's AST scanner so the lookup runs at
|
||||
// runtime against the user's own node_modules.
|
||||
//
|
||||
// This defense lives in the supervisor (not in consumers' esbuild
|
||||
// configs) because every plugin-marketplace consumer must bundle
|
||||
// @mlflow/core into their hook to ship as a self-contained file.
|
||||
const pkgJsonModule = '@mlflow' + '/core' + '/package.json';
|
||||
const requireFromHere = createRequire(__filename);
|
||||
const pkgJsonPath = requireFromHere.resolve(pkgJsonModule);
|
||||
return join(dirname(pkgJsonPath), 'bundle', 'daemon.cjs');
|
||||
}
|
||||
|
||||
/**
|
||||
* Fork a detached daemon process. Assumes no daemon is currently bound —
|
||||
* the caller is responsible for guarding with {@link isDaemonAlive}.
|
||||
*/
|
||||
export function spawnDaemon(): void {
|
||||
const entry = resolveDaemonEntry();
|
||||
const child = spawn(process.execPath, [entry], {
|
||||
detached: true,
|
||||
stdio: 'ignore',
|
||||
env: process.env,
|
||||
});
|
||||
|
||||
child.once('error', (err) => {
|
||||
console.error(`[mlflow][wal] daemon spawn failed (entry=${entry}):`, err);
|
||||
});
|
||||
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code != null && code !== 0) {
|
||||
console.error(
|
||||
`[mlflow][wal] daemon exited unexpectedly: entry=${entry} code=${code} signal=${signal ?? 'none'}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
child.unref();
|
||||
}
|
||||
|
||||
let spawnInFlight: Promise<void> | null = null;
|
||||
|
||||
/**
|
||||
* Poll {@link isDaemonAlive} until it returns true or `timeoutMs`
|
||||
* elapses. Does not throw on timeout — the IPC client's own retry
|
||||
* loop will surface any persistent connection failure.
|
||||
*/
|
||||
async function waitForDaemonBind(timeoutMs: number, pollIntervalMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const remainingMs = deadline - Date.now();
|
||||
|
||||
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<boolean | 'timeout'>([
|
||||
isDaemonAlive(),
|
||||
new Promise((resolve) => {
|
||||
timeoutHandle = setTimeout(() => resolve('timeout'), remainingMs);
|
||||
}),
|
||||
]);
|
||||
clearTimeout(timeoutHandle);
|
||||
if (result === true || result === 'timeout') {
|
||||
return;
|
||||
}
|
||||
const sleepMs = Math.min(pollIntervalMs, deadline - Date.now());
|
||||
if (sleepMs > 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, sleepMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a daemon is alive; spawn one if not.
|
||||
*/
|
||||
export function ensureDaemon(): Promise<void> {
|
||||
if (spawnInFlight != null) {
|
||||
return spawnInFlight;
|
||||
}
|
||||
const inFlight = (async () => {
|
||||
try {
|
||||
if (await isDaemonAlive()) {
|
||||
return;
|
||||
}
|
||||
spawnDaemon();
|
||||
await waitForDaemonBind(SPAWN_BIND_WAIT_TIMEOUT_MS, SPAWN_BIND_WAIT_POLL_MS);
|
||||
} catch (err) {
|
||||
console.error('[mlflow][wal] ensureDaemon failed:', err);
|
||||
} finally {
|
||||
spawnInFlight = null;
|
||||
}
|
||||
})();
|
||||
spawnInFlight = inFlight;
|
||||
return inFlight;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Type definitions for the per-user trace upload WAL.
|
||||
*
|
||||
* The WAL is an append-only JSONL file. Each line is one of the variants of
|
||||
* {@link WalLine}: either an append (a complete trace record) or a tombstone
|
||||
* (a marker that shadows a previously-appended record by id).
|
||||
*/
|
||||
|
||||
/**
|
||||
* One queued trace upload, serialized to a single JSONL line.
|
||||
*/
|
||||
export interface WalRecord {
|
||||
id: string;
|
||||
trackingUri: string;
|
||||
experimentId: string;
|
||||
traceInfo: Record<string, unknown>;
|
||||
traceData: unknown;
|
||||
attempts: number;
|
||||
nextAttemptAt: number;
|
||||
createdAt: number;
|
||||
firstAttemptAt?: number;
|
||||
otlpSpans?: string; // base64 encoded ExportTraceServiceRequest protobuf
|
||||
}
|
||||
|
||||
export type WalLine = { type: 'append'; record: WalRecord } | { type: 'tombstone'; id: string };
|
||||
@@ -0,0 +1,44 @@
|
||||
import { init } from './core/config';
|
||||
import {
|
||||
getLastActiveTraceId,
|
||||
getCurrentActiveSpan,
|
||||
updateCurrentTrace,
|
||||
startSpan,
|
||||
trace,
|
||||
withSpan,
|
||||
} from './core/api';
|
||||
import { tracingContext } from './core/context';
|
||||
import { flushTraces } from './core/provider';
|
||||
import { MlflowClient, MlflowHttpError } from './clients';
|
||||
import { InMemoryTraceManager } from './core/trace_manager';
|
||||
import { createAuthProvider } from './auth';
|
||||
|
||||
export {
|
||||
getLastActiveTraceId,
|
||||
getCurrentActiveSpan,
|
||||
updateCurrentTrace,
|
||||
tracingContext,
|
||||
flushTraces,
|
||||
init,
|
||||
startSpan,
|
||||
trace,
|
||||
withSpan,
|
||||
MlflowClient,
|
||||
MlflowHttpError,
|
||||
InMemoryTraceManager,
|
||||
createAuthProvider,
|
||||
};
|
||||
|
||||
// Export entities
|
||||
export * from './core/constants';
|
||||
export type { LiveSpan, Span } from './core/entities/span';
|
||||
export type { Trace } from './core/entities/trace';
|
||||
export type { TraceInfo, TokenUsage } from './core/entities/trace_info';
|
||||
export type { TraceData } from './core/entities/trace_data';
|
||||
export type { TraceLocation, UnityCatalogLocation } from './core/entities/trace_location';
|
||||
export type { SearchTracesOptions, SearchTracesResult } from './clients';
|
||||
export { TraceLocationType } from './core/entities/trace_location';
|
||||
export { SpanStatusCode } from './core/entities/span_status';
|
||||
export type { UpdateCurrentTraceOptions, SpanOptions, TraceOptions } from './core/api';
|
||||
export type { TracingContextOptions } from './core/context';
|
||||
export { registerOnSpanStartHook, registerOnSpanEndHook } from './exporters/span_processor_hooks';
|
||||
@@ -0,0 +1,461 @@
|
||||
import { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import { TraceData } from '../../../src/core/entities/trace_data';
|
||||
import { TraceLocationType } from '../../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../../src/core/entities/trace_state';
|
||||
import { DatabricksArtifactsClient } from '../../../src/clients/artifacts/databricks';
|
||||
import { AuthProvider } from '../../../src/auth';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
|
||||
describe('DatabricksArtifactsClient', () => {
|
||||
let client: DatabricksArtifactsClient;
|
||||
const testHost = 'https://dbc-12345.cloud.databricks.com';
|
||||
const testToken = 'test-token';
|
||||
|
||||
// Create a mock AuthProvider for testing
|
||||
const mockAuthProvider: AuthProvider = {
|
||||
getHost: () => testHost,
|
||||
// eslint-disable-next-line require-await, @typescript-eslint/require-await
|
||||
getHeadersProvider: () => async () => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${testToken}`,
|
||||
}),
|
||||
getDatabricksToken: () => testToken,
|
||||
};
|
||||
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
|
||||
beforeAll(() => {
|
||||
server = setupServer();
|
||||
server.listen();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
client = new DatabricksArtifactsClient({ host: testHost, authProvider: mockAuthProvider });
|
||||
});
|
||||
|
||||
describe('uploadTraceData', () => {
|
||||
it('should get upload credentials and upload to AWS S3 signed URL', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-databricks-123',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock credentials response
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-databricks-123/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AWS_PRESIGNED_URL',
|
||||
signed_uri:
|
||||
'https://s3.amazonaws.com/bucket/traces/tr-databricks-123?signature=xyz',
|
||||
headers: [{ name: 'x-amz-server-side-encryption', value: 'AES256' }],
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Mock successful S3 upload
|
||||
server.use(
|
||||
http.put('https://s3.amazonaws.com/bucket/traces/tr-databricks-123', () => {
|
||||
return HttpResponse.json({}, { status: 200 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should get upload credentials and upload to GCP signed URL', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-gcp-456',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '1' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 2000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock GCP credentials response
|
||||
server.use(
|
||||
http.get(`${testHost}/api/2.0/mlflow/traces/tr-gcp-456/credentials-for-data-upload`, () => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'GCP_SIGNED_URL',
|
||||
signed_uri: 'https://storage.googleapis.com/bucket/traces/tr-gcp-456?signature=abc',
|
||||
},
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
// Mock successful GCP upload
|
||||
server.use(
|
||||
http.put('https://storage.googleapis.com/bucket/traces/tr-gcp-456', () => {
|
||||
return HttpResponse.json({}, { status: 200 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should get upload credentials and upload to Azure Blob Storage', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-azure-789',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '2' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 3000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock Azure credentials response
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-azure-789/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AZURE_SAS_URI',
|
||||
signed_uri: 'https://storage.azure.com/traces/tr-azure-789?sas=token',
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Mock Azure Blob Storage upload
|
||||
server.use(
|
||||
http.put('https://storage.azure.com/traces/tr-azure-789', () => {
|
||||
return new HttpResponse(null, { status: 201 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should handle upload failures', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-fail-upload',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.ERROR,
|
||||
requestTime: 4000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock credentials response
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-fail-upload/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AWS_PRESIGNED_URL',
|
||||
signed_uri: 'https://s3.amazonaws.com/bucket/traces/tr-fail-upload?signature=xyz',
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Mock failed S3 upload
|
||||
server.use(
|
||||
http.put('https://s3.amazonaws.com/bucket/traces/tr-fail-upload', () => {
|
||||
return HttpResponse.json({ error: 'Forbidden' }, { status: 403 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
|
||||
'AWS_PRESIGNED_URL upload failed: 403 Forbidden',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('downloadTraceData', () => {
|
||||
it('should get download credentials and download from AWS S3 signed URL', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-download-123',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '5' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 5000,
|
||||
});
|
||||
|
||||
// Mock credentials response
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-download-123/credentials-for-data-download`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AWS_PRESIGNED_URL',
|
||||
signed_uri: 'https://s3.amazonaws.com/bucket/traces/tr-download-123?signature=xyz',
|
||||
headers: [{ name: 'x-amz-server-side-encryption', value: 'AES256' }],
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const mockTraceData = {
|
||||
spans: [
|
||||
{
|
||||
span_id: 'c3Bhbi1kb3dubG9hZGVk', // base64 encoded
|
||||
trace_id: 'dHItZG93bmxvYWQtMTIz', // base64 encoded
|
||||
name: 'downloaded-span',
|
||||
start_time: '5000000000',
|
||||
end_time: '5100000000',
|
||||
status: { code: 'OK' },
|
||||
attributes: {},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Mock successful S3 download
|
||||
server.use(
|
||||
http.get('https://s3.amazonaws.com/bucket/traces/tr-download-123', () => {
|
||||
return HttpResponse.text(JSON.stringify(mockTraceData));
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await client.downloadTraceData(traceInfo);
|
||||
|
||||
// Verify the result structure
|
||||
|
||||
expect(result).toBeInstanceOf(TraceData);
|
||||
expect(result.spans).toHaveLength(1);
|
||||
expect(result.spans[0].name).toBe('downloaded-span');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('should log and re-throw upload errors', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-error-log',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.ERROR,
|
||||
requestTime: 9000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock network error
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-error-log/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({ error: 'Network error' }, { status: 500 });
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Spy on console.error
|
||||
const consoleSpy = jest.spyOn(console, 'error').mockImplementation();
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow();
|
||||
|
||||
expect(consoleSpy).toHaveBeenCalledWith(
|
||||
'Trace data upload failed for tr-error-log:',
|
||||
expect.any(Error),
|
||||
);
|
||||
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should log and re-throw download errors', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-download-error',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 10000,
|
||||
});
|
||||
|
||||
// Mock network error
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-download-error/credentials-for-data-download`,
|
||||
() => {
|
||||
return HttpResponse.json({ error: 'Connection timeout' }, { status: 500 });
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Spy on console.error
|
||||
const consoleSpy = jest.spyOn(console, 'error').mockImplementation();
|
||||
|
||||
await expect(client.downloadTraceData(traceInfo)).rejects.toThrow();
|
||||
|
||||
expect(consoleSpy).toHaveBeenCalledWith(
|
||||
'Failed to download trace data for tr-download-error:',
|
||||
expect.any(Error),
|
||||
);
|
||||
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Credential Types', () => {
|
||||
const credentialTypes = [
|
||||
{ type: 'AWS_PRESIGNED_URL', url: 'https://s3.amazonaws.com/bucket/file' },
|
||||
{ type: 'GCP_SIGNED_URL', url: 'https://storage.googleapis.com/bucket/file' },
|
||||
];
|
||||
|
||||
credentialTypes.forEach(({ type, url }) => {
|
||||
it(`should handle ${type} credential type`, async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: `tr-${type}`,
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 11000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-${type}/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type,
|
||||
signed_uri: `${url}?signature=test`,
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
server.use(
|
||||
http.put(`${url}`, () => {
|
||||
return HttpResponse.json({}, { status: 200 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('should successfully upload to Azure Blob Storage with AZURE_SAS_URI', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-AZURE_SAS_URI',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 12000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-AZURE_SAS_URI/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AZURE_SAS_URI' as any,
|
||||
signed_uri: 'https://storage.azure.com/file?sas=token',
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Mock Azure Blob Storage upload
|
||||
server.use(
|
||||
http.put('https://storage.azure.com/file', () => {
|
||||
return new HttpResponse(null, { status: 201 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should successfully upload to Azure ADLS Gen2 with AZURE_ADLS_GEN2_SAS_URI', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-AZURE_ADLS_GEN2_SAS_URI',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 12000,
|
||||
});
|
||||
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
server.use(
|
||||
http.get(
|
||||
`${testHost}/api/2.0/mlflow/traces/tr-AZURE_ADLS_GEN2_SAS_URI/credentials-for-data-upload`,
|
||||
() => {
|
||||
return HttpResponse.json({
|
||||
credential_info: {
|
||||
type: 'AZURE_ADLS_GEN2_SAS_URI' as any,
|
||||
signed_uri: 'https://storage.azure.com/file?sas=token',
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Mock Azure ADLS Gen2 operations (create, append, flush)
|
||||
server.use(
|
||||
// Create file
|
||||
http.put('https://storage.azure.com/file', ({ request }) => {
|
||||
const url = new URL(request.url);
|
||||
if (url.searchParams.get('resource') === 'file') {
|
||||
return new HttpResponse(null, { status: 201 });
|
||||
}
|
||||
return new HttpResponse(null, { status: 404 });
|
||||
}),
|
||||
// Append data
|
||||
http.patch('https://storage.azure.com/file', ({ request }) => {
|
||||
const url = new URL(request.url);
|
||||
if (url.searchParams.get('action') === 'append') {
|
||||
return new HttpResponse(null, { status: 202 });
|
||||
}
|
||||
if (url.searchParams.get('action') === 'flush') {
|
||||
return new HttpResponse(null, { status: 200 });
|
||||
}
|
||||
return new HttpResponse(null, { status: 404 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,260 @@
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { MlflowArtifactsClient } from '../../../src/clients/artifacts/mlflow';
|
||||
import { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import { TraceData } from '../../../src/core/entities/trace_data';
|
||||
import { TraceLocationType } from '../../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../../src/core/entities/trace_state';
|
||||
import { AuthProvider } from '../../../src/auth';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
|
||||
describe('MlflowArtifactsClient', () => {
|
||||
let client: MlflowArtifactsClient;
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
const testHost = 'http://localhost:5000';
|
||||
|
||||
// Create a mock AuthProvider for testing
|
||||
const mockAuthProvider: AuthProvider = {
|
||||
getHost: () => testHost,
|
||||
// eslint-disable-next-line require-await, @typescript-eslint/require-await
|
||||
getHeadersProvider: () => async () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
getDatabricksToken: () => undefined,
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
server = setupServer();
|
||||
server.listen();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
client = new MlflowArtifactsClient({ host: testHost, authProvider: mockAuthProvider });
|
||||
});
|
||||
|
||||
describe('uploadTraceData', () => {
|
||||
it('should make PUT request to correct artifact URL', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-abc123',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
tags: {
|
||||
'mlflow.artifactLocation': 'mlflow-artifacts:/0/traces/tr-abc123/artifacts',
|
||||
},
|
||||
});
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
// Mock the artifacts upload endpoint
|
||||
server.use(
|
||||
http.put(
|
||||
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/0/traces/tr-abc123/artifacts/traces.json',
|
||||
() => {
|
||||
return HttpResponse.json({}, { status: 200 });
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await client.uploadTraceData(traceInfo, traceData);
|
||||
|
||||
// Test passes if no errors thrown
|
||||
});
|
||||
|
||||
it('should throw error when artifact location is missing', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-no-artifact',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
tags: {}, // No artifact location
|
||||
});
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
|
||||
'Artifact location not found in trace tags',
|
||||
);
|
||||
|
||||
// Test passes if error is thrown as expected
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadTraceData with local artifact roots', () => {
|
||||
let tmpRoot: string;
|
||||
|
||||
const makeTraceInfo = (artifactLocation: string) =>
|
||||
new TraceInfo({
|
||||
traceId: 'tr-local',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '0' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
tags: {
|
||||
'mlflow.artifactLocation': artifactLocation,
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'mlflow-wal-test-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes traces.json to a bare local path, creating the directory', async () => {
|
||||
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
|
||||
const traceInfo = makeTraceInfo(artifactDir);
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
await client.uploadTraceData(traceInfo, traceData);
|
||||
|
||||
const written = await readFile(join(artifactDir, 'traces.json'), 'utf8');
|
||||
expect(JSON.parse(written)).toEqual({ spans: [] });
|
||||
});
|
||||
|
||||
it('writes traces.json for a file:// artifact location', async () => {
|
||||
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
|
||||
const fileUri = pathToFileURL(artifactDir).href;
|
||||
const traceInfo = makeTraceInfo(fileUri);
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
await client.uploadTraceData(traceInfo, traceData);
|
||||
|
||||
const written = await readFile(join(artifactDir, 'traces.json'), 'utf8');
|
||||
expect(JSON.parse(written)).toEqual({ spans: [] });
|
||||
});
|
||||
|
||||
it('rejects unsupported remote schemes via resolveArtifactUri', async () => {
|
||||
const traceInfo = makeTraceInfo('s3://bucket/0/traces/tr-local/artifacts');
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
|
||||
'Expected mlflow-artifacts:// URI, got s3:',
|
||||
);
|
||||
});
|
||||
|
||||
it('refuses to write to a relative local artifact path', async () => {
|
||||
const traceInfo = makeTraceInfo('relative/0/traces/tr-local/artifacts');
|
||||
const traceData = new TraceData([]);
|
||||
|
||||
await expect(client.uploadTraceData(traceInfo, traceData)).rejects.toThrow(
|
||||
/Refusing to use a relative local artifact location/,
|
||||
);
|
||||
});
|
||||
|
||||
it('round-trips trace data through a bare local path (upload then download)', async () => {
|
||||
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
|
||||
const traceInfo = makeTraceInfo(artifactDir);
|
||||
|
||||
await client.uploadTraceData(traceInfo, new TraceData([]));
|
||||
const result = await client.downloadTraceData(traceInfo);
|
||||
|
||||
expect(result).toBeInstanceOf(TraceData);
|
||||
expect(result.spans).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('reads trace data from a file:// artifact location', async () => {
|
||||
const artifactDir = join(tmpRoot, '0', 'traces', 'tr-local', 'artifacts');
|
||||
const fileUri = pathToFileURL(artifactDir).href;
|
||||
const traceInfo = makeTraceInfo(fileUri);
|
||||
|
||||
await client.uploadTraceData(traceInfo, new TraceData([]));
|
||||
const result = await client.downloadTraceData(traceInfo);
|
||||
|
||||
expect(result).toBeInstanceOf(TraceData);
|
||||
expect(result.spans).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('downloadTraceData', () => {
|
||||
it('should make GET request to correct artifact URL', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-download',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '5' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 4000,
|
||||
tags: {
|
||||
'mlflow.artifactLocation': 'mlflow-artifacts:/5/traces/tr-download/artifacts',
|
||||
},
|
||||
});
|
||||
|
||||
const mockResponse = {
|
||||
spans: [
|
||||
{
|
||||
span_id: 'c3Bhbi1kb3dubG9hZA==',
|
||||
trace_id: 'dHItZG93bmxvYWQ=',
|
||||
name: 'downloaded-span',
|
||||
start_time: '4000000000',
|
||||
end_time: '4100000000',
|
||||
status: { code: 'OK' },
|
||||
attributes: {},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Mock the artifacts download endpoint
|
||||
server.use(
|
||||
http.get(
|
||||
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/5/traces/tr-download/artifacts/traces.json',
|
||||
() => {
|
||||
return HttpResponse.json(mockResponse);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const result = await client.downloadTraceData(traceInfo);
|
||||
|
||||
expect(result).toBeInstanceOf(TraceData);
|
||||
expect(result.spans).toHaveLength(1);
|
||||
expect(result.spans[0].name).toBe('downloaded-span');
|
||||
});
|
||||
|
||||
it('should handle complex artifact paths in URL construction', async () => {
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: 'tr-complex-path',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '42' },
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 5000,
|
||||
tags: {
|
||||
'mlflow.artifactLocation':
|
||||
'mlflow-artifacts:/42/some/nested/path/traces/tr-complex-path/artifacts',
|
||||
},
|
||||
});
|
||||
|
||||
// Mock the complex path artifacts download endpoint
|
||||
server.use(
|
||||
http.get(
|
||||
'http://localhost:5000/api/2.0/mlflow-artifacts/artifacts/42/some/nested/path/traces/tr-complex-path/artifacts/traces.json',
|
||||
() => {
|
||||
return HttpResponse.json({ spans: [] });
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
await client.downloadTraceData(traceInfo);
|
||||
|
||||
// Test passes if no errors thrown
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,367 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import * as mlflow from '../../src';
|
||||
import { MlflowClient, type SearchTracesOptions } from '../../src/clients/client';
|
||||
import type { ArtifactsClient } from '../../src/clients/artifacts';
|
||||
import { Trace } from '../../src/core/entities/trace';
|
||||
import { TraceData } from '../../src/core/entities/trace_data';
|
||||
import { TraceInfo } from '../../src/core/entities/trace_info';
|
||||
import { TraceLocationType } from '../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../src/core/entities/trace_state';
|
||||
import { createAuthProvider } from '../../src/auth';
|
||||
import { TEST_TRACKING_URI } from '../helper';
|
||||
|
||||
describe('MlflowClient', () => {
|
||||
let client: MlflowClient;
|
||||
let experimentId: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
|
||||
// Create a new experiment for each test
|
||||
const experimentName = `test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Clean up: delete the experiment
|
||||
try {
|
||||
await client.deleteExperiment(experimentId);
|
||||
} catch (error) {
|
||||
console.warn('Failed to delete experiment:', error);
|
||||
}
|
||||
});
|
||||
|
||||
describe('createTrace', () => {
|
||||
it('should create a trace', async () => {
|
||||
const traceId = randomUUID();
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: experimentId,
|
||||
},
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
executionDuration: 500,
|
||||
requestPreview: '{"input":"test"}',
|
||||
responsePreview: '{"output":"result"}',
|
||||
clientRequestId: 'client-request-id',
|
||||
traceMetadata: { 'meta-key': 'meta-value' },
|
||||
tags: { 'tag-key': 'tag-value' },
|
||||
assessments: [],
|
||||
});
|
||||
|
||||
const createdTraceInfo = await client.createTrace(traceInfo);
|
||||
|
||||
expect(createdTraceInfo).toBeInstanceOf(TraceInfo);
|
||||
expect(createdTraceInfo.traceId).toBe(traceId);
|
||||
expect(createdTraceInfo.state).toBe(TraceState.OK);
|
||||
expect(createdTraceInfo.requestTime).toBe(1000);
|
||||
expect(createdTraceInfo.executionDuration).toBe(500);
|
||||
expect(createdTraceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(createdTraceInfo.traceLocation.mlflowExperiment?.experimentId).toBe(experimentId);
|
||||
expect(createdTraceInfo.requestPreview).toBe('{"input":"test"}');
|
||||
expect(createdTraceInfo.responsePreview).toBe('{"output":"result"}');
|
||||
expect(createdTraceInfo.clientRequestId).toBe('client-request-id');
|
||||
expect(createdTraceInfo.traceMetadata).toMatchObject({ 'meta-key': 'meta-value' });
|
||||
expect(createdTraceInfo.tags).toEqual({
|
||||
'tag-key': 'tag-value',
|
||||
'mlflow.artifactLocation': expect.any(String),
|
||||
});
|
||||
expect(createdTraceInfo.assessments).toEqual([]);
|
||||
});
|
||||
|
||||
it('should create a trace with error state', async () => {
|
||||
const traceId = randomUUID();
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: experimentId,
|
||||
},
|
||||
},
|
||||
state: TraceState.ERROR,
|
||||
requestTime: 1000,
|
||||
});
|
||||
|
||||
const createdTraceInfo = await client.createTrace(traceInfo);
|
||||
|
||||
expect(createdTraceInfo).toBeInstanceOf(TraceInfo);
|
||||
expect(createdTraceInfo.traceId).toBe(traceId);
|
||||
expect(createdTraceInfo.state).toBe(TraceState.ERROR);
|
||||
expect(createdTraceInfo.requestTime).toBe(1000);
|
||||
expect(createdTraceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(createdTraceInfo.traceLocation.mlflowExperiment?.experimentId).toBe(experimentId);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTraceInfo', () => {
|
||||
it('should retrieve trace info for an existing trace', async () => {
|
||||
const traceId = randomUUID();
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: experimentId,
|
||||
},
|
||||
},
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
});
|
||||
await client.createTrace(traceInfo);
|
||||
|
||||
// Now retrieve it
|
||||
const retrievedTraceInfo = await client.getTraceInfo(traceId);
|
||||
|
||||
expect(retrievedTraceInfo).toBeInstanceOf(TraceInfo);
|
||||
expect(retrievedTraceInfo.traceId).toBe(traceId);
|
||||
expect(retrievedTraceInfo.state).toBe(TraceState.OK);
|
||||
expect(retrievedTraceInfo.requestTime).toBe(1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchTraces', () => {
|
||||
const experimentLocation = () => ({
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: experimentId,
|
||||
},
|
||||
});
|
||||
|
||||
it('should search traces by location and page through results', async () => {
|
||||
const firstTraceId = randomUUID();
|
||||
const secondTraceId = randomUUID();
|
||||
const testFilterTag = randomUUID();
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: firstTraceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
tags: { searchTracesPagination: testFilterTag },
|
||||
}),
|
||||
);
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: secondTraceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 2000,
|
||||
tags: { searchTracesPagination: testFilterTag },
|
||||
}),
|
||||
);
|
||||
|
||||
const firstPage = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
filter: `tags.searchTracesPagination = '${testFilterTag}'`,
|
||||
maxResults: 1,
|
||||
orderBy: ['timestamp_ms DESC'],
|
||||
includeSpans: false,
|
||||
});
|
||||
|
||||
expect(firstPage.traces).toHaveLength(1);
|
||||
expect(firstPage.traces[0]).toBeInstanceOf(Trace);
|
||||
expect(firstPage.traces[0].info).toBeInstanceOf(TraceInfo);
|
||||
expect([firstTraceId, secondTraceId]).toContain(firstPage.traces[0].info.traceId);
|
||||
expect(firstPage.nextPageToken).toBeDefined();
|
||||
|
||||
const secondPage = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
filter: `tags.searchTracesPagination = '${testFilterTag}'`,
|
||||
maxResults: 1,
|
||||
orderBy: ['timestamp_ms DESC'],
|
||||
pageToken: firstPage.nextPageToken,
|
||||
includeSpans: false,
|
||||
});
|
||||
|
||||
const returnedTraceIds = [
|
||||
firstPage.traces[0].info.traceId,
|
||||
...secondPage.traces.map((trace) => trace.info.traceId),
|
||||
];
|
||||
expect(returnedTraceIds).toEqual(expect.arrayContaining([firstTraceId, secondTraceId]));
|
||||
});
|
||||
|
||||
it('should search traces with a filter', async () => {
|
||||
const matchingTraceId = randomUUID();
|
||||
const otherTraceId = randomUUID();
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: matchingTraceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
tags: { searchTracesFilter: 'match' },
|
||||
}),
|
||||
);
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: otherTraceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 2000,
|
||||
tags: { searchTracesFilter: 'skip' },
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
filter: "tags.searchTracesFilter = 'match'",
|
||||
maxResults: 10,
|
||||
includeSpans: false,
|
||||
});
|
||||
|
||||
expect(result.traces.map((trace) => trace.info.traceId)).toContain(matchingTraceId);
|
||||
expect(result.traces.map((trace) => trace.info.traceId)).not.toContain(otherTraceId);
|
||||
});
|
||||
|
||||
it('should fetch span data for each trace by default', async () => {
|
||||
mlflow.init({ trackingUri: TEST_TRACKING_URI, experimentId });
|
||||
const span = mlflow.startSpan({ name: 'search-traces-span' });
|
||||
span.setInputs({ question: 'What is MLflow?' });
|
||||
span.end();
|
||||
await mlflow.flushTraces();
|
||||
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
});
|
||||
|
||||
expect(result.traces).toHaveLength(1);
|
||||
const trace = result.traces[0];
|
||||
expect(trace).toBeInstanceOf(Trace);
|
||||
expect(trace.info.traceId).toBe(span.traceId);
|
||||
expect(trace.data.spans).toHaveLength(1);
|
||||
expect(trace.data.spans[0].name).toBe('search-traces-span');
|
||||
});
|
||||
|
||||
it('should return metadata-only traces with empty span data when includeSpans is false', async () => {
|
||||
const traceId = randomUUID();
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
includeSpans: false,
|
||||
});
|
||||
|
||||
expect(result.traces).toHaveLength(1);
|
||||
expect(result.traces[0].info.traceId).toBe(traceId);
|
||||
expect(result.traces[0].data.spans).toEqual([]);
|
||||
});
|
||||
|
||||
it('should skip traces whose span data cannot be downloaded', async () => {
|
||||
// createTrace registers trace metadata without uploading span data,
|
||||
// so the span-data download fails and the trace is dropped.
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: randomUUID(),
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
});
|
||||
|
||||
expect(result.traces).toEqual([]);
|
||||
});
|
||||
|
||||
it('should warn and drop only the traces whose span data download fails', async () => {
|
||||
const failingTraceId = randomUUID();
|
||||
const okTraceId = randomUUID();
|
||||
for (const traceId of [failingTraceId, okTraceId]) {
|
||||
await client.createTrace(
|
||||
new TraceInfo({
|
||||
traceId: traceId,
|
||||
traceLocation: experimentLocation(),
|
||||
state: TraceState.OK,
|
||||
requestTime: 1000,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const { artifactsClient } = client as unknown as { artifactsClient: ArtifactsClient };
|
||||
const downloadSpy = jest
|
||||
.spyOn(artifactsClient, 'downloadTraceData')
|
||||
.mockImplementation((traceInfo) =>
|
||||
traceInfo.traceId === failingTraceId
|
||||
? Promise.reject(new Error('download failed'))
|
||||
: Promise.resolve(new TraceData([])),
|
||||
);
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
||||
|
||||
try {
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
});
|
||||
|
||||
expect(result.traces.map((trace) => trace.info.traceId)).toEqual([okTraceId]);
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
`Failed to download trace data for trace ${failingTraceId}:`,
|
||||
expect.any(Error),
|
||||
);
|
||||
} finally {
|
||||
downloadSpy.mockRestore();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('should return an empty result when no traces match', async () => {
|
||||
const result = await client.searchTraces({
|
||||
locations: [experimentLocation()],
|
||||
filter: "tags.searchTracesFilter = 'missing'",
|
||||
maxResults: 10,
|
||||
});
|
||||
|
||||
expect(result.traces).toEqual([]);
|
||||
expect(result.nextPageToken).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should reject search without locations', async () => {
|
||||
await expect(client.searchTraces({ locations: [] })).rejects.toThrow(
|
||||
'At least one location must be specified for searching traces.',
|
||||
);
|
||||
});
|
||||
|
||||
it('should reject search when locations is omitted', async () => {
|
||||
// Plain-JS callers can omit locations despite the required type.
|
||||
await expect(client.searchTraces({} as SearchTracesOptions)).rejects.toThrow(
|
||||
'At least one location must be specified for searching traces.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getExperimentByName', () => {
|
||||
it('should retrieve an existing experiment by name', async () => {
|
||||
const experiment = await client.getExperimentByName(
|
||||
`test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`,
|
||||
);
|
||||
|
||||
expect(experiment).toBeNull();
|
||||
|
||||
const createdName = `lookup-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
|
||||
const createdId = await client.createExperiment(createdName);
|
||||
try {
|
||||
const found = await client.getExperimentByName(createdName);
|
||||
expect(found).toEqual({
|
||||
experimentId: createdId,
|
||||
name: createdName,
|
||||
});
|
||||
} finally {
|
||||
await client.deleteExperiment(createdId);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
import { MlflowClient } from '../../src/clients/client';
|
||||
import { createAuthProvider } from '../../src/auth';
|
||||
import { MlflowHttpError } from '../../src/clients/utils';
|
||||
|
||||
/**
|
||||
* Unit tests for {@link MlflowClient.exportOtlpSpans} used for OSS (non-Databricks)
|
||||
* tracking servers. These mock `global.fetch` so they exercise the request shaping
|
||||
* (endpoint, headers, body) and error mapping without needing a live tracking server.
|
||||
*/
|
||||
describe('MlflowClient.exportOtlpSpans', () => {
|
||||
const trackingUri = 'http://localhost:5000';
|
||||
let client: MlflowClient;
|
||||
let fetchMock: jest.Mock;
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
const authProvider = createAuthProvider({ trackingUri });
|
||||
client = new MlflowClient({ trackingUri, authProvider });
|
||||
fetchMock = jest.fn().mockResolvedValue(new Response(null, { status: 200 }));
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it('POSTs OTLP protobuf to the OSS endpoint with the protobuf content type and experiment header', async () => {
|
||||
const bytes = new Uint8Array([0x0a, 0x01, 0x02, 0x03]);
|
||||
|
||||
await client.exportOtlpSpans('exp-123', bytes);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
|
||||
expect(url).toBe('http://localhost:5000/v1/traces');
|
||||
expect(init.method).toBe('POST');
|
||||
const headers = init.headers as Record<string, string>;
|
||||
expect(headers['Content-Type']).toBe('application/x-protobuf');
|
||||
expect(headers['x-mlflow-experiment-id']).toBe('exp-123');
|
||||
expect(init.body).toBe(bytes);
|
||||
});
|
||||
|
||||
it('does not call fetch when there are no span bytes to send', async () => {
|
||||
await client.exportOtlpSpans('exp-123', new Uint8Array(0));
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('surfaces a 501 as an MlflowHttpError so callers can fall back', async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
new Response('REST OTLP span logging is not supported', {
|
||||
status: 501,
|
||||
statusText: 'Not Implemented',
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
client.exportOtlpSpans('exp-123', new Uint8Array([0x0a, 0x01])),
|
||||
).rejects.toBeInstanceOf(MlflowHttpError);
|
||||
});
|
||||
|
||||
it('propagates the 501 status code on the thrown error', async () => {
|
||||
fetchMock.mockResolvedValue(
|
||||
new Response('nope', { status: 501, statusText: 'Not Implemented' }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
client.exportOtlpSpans('exp-123', new Uint8Array([0x0a, 0x01])),
|
||||
).rejects.toMatchObject({
|
||||
status: 501,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,184 @@
|
||||
import { makeRequest, MlflowHttpError } from '../../src/clients/utils';
|
||||
|
||||
describe('makeRequest', () => {
|
||||
const mockHeaderProvider = () => Promise.resolve({ 'Content-Type': 'application/json' });
|
||||
|
||||
beforeEach(() => {
|
||||
// Clear all mocks before each test
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('error handling with response body', () => {
|
||||
it('should include response body in error message when request fails', async () => {
|
||||
const errorResponseBody = JSON.stringify({
|
||||
error_code: 'INVALID_PARAMETER_VALUE',
|
||||
message: 'Invalid experiment ID provided',
|
||||
});
|
||||
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 400,
|
||||
statusText: 'Bad Request',
|
||||
text: jest.fn().mockResolvedValue(errorResponseBody),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'GET',
|
||||
'http://localhost:5000/api/2.0/mlflow/experiments/get',
|
||||
mockHeaderProvider,
|
||||
),
|
||||
).rejects.toThrow(`HTTP 400: Bad Request - ${errorResponseBody}`);
|
||||
|
||||
expect(mockResponse.text).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle empty response body in error', async () => {
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 404,
|
||||
statusText: 'Not Found',
|
||||
text: jest.fn().mockResolvedValue(''),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'GET',
|
||||
'http://localhost:5000/api/2.0/mlflow/experiments/get',
|
||||
mockHeaderProvider,
|
||||
),
|
||||
).rejects.toThrow('HTTP 404: Not Found');
|
||||
|
||||
expect(mockResponse.text).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle error when reading response body fails', async () => {
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 500,
|
||||
statusText: 'Internal Server Error',
|
||||
text: jest.fn().mockRejectedValue(new Error('Failed to read body')),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'GET',
|
||||
'http://localhost:5000/api/2.0/mlflow/experiments/get',
|
||||
mockHeaderProvider,
|
||||
),
|
||||
).rejects.toThrow('HTTP 500: Internal Server Error');
|
||||
|
||||
expect(mockResponse.text).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should include response body with plain text error', async () => {
|
||||
const errorResponseBody = 'Service temporarily unavailable';
|
||||
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 503,
|
||||
statusText: 'Service Unavailable',
|
||||
text: jest.fn().mockResolvedValue(errorResponseBody),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'POST',
|
||||
'http://localhost:5000/api/2.0/mlflow/runs/create',
|
||||
mockHeaderProvider,
|
||||
{},
|
||||
),
|
||||
).rejects.toThrow(`HTTP 503: Service Unavailable - ${errorResponseBody}`);
|
||||
|
||||
expect(mockResponse.text).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw an MlflowHttpError carrying status and error_code', async () => {
|
||||
const errorResponseBody = JSON.stringify({
|
||||
error_code: 'RESOURCE_DOES_NOT_EXIST',
|
||||
message: 'Experiment not found',
|
||||
});
|
||||
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 404,
|
||||
statusText: 'Not Found',
|
||||
text: jest.fn().mockResolvedValue(errorResponseBody),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'GET',
|
||||
'http://localhost:5000/api/2.0/mlflow/experiments/get-by-name',
|
||||
mockHeaderProvider,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: 'MlflowHttpError',
|
||||
status: 404,
|
||||
statusText: 'Not Found',
|
||||
body: errorResponseBody,
|
||||
errorCode: 'RESOURCE_DOES_NOT_EXIST',
|
||||
});
|
||||
});
|
||||
|
||||
it('should leave errorCode undefined when body is not JSON', async () => {
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 500,
|
||||
statusText: 'Internal Server Error',
|
||||
text: jest.fn().mockResolvedValue('upstream timeout'),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
const err = await makeRequest(
|
||||
'GET',
|
||||
'http://localhost:5000/api/2.0/mlflow/experiments/get',
|
||||
mockHeaderProvider,
|
||||
).catch((e: unknown) => e);
|
||||
|
||||
expect(err).toBeInstanceOf(MlflowHttpError);
|
||||
expect((err as MlflowHttpError).status).toBe(500);
|
||||
expect((err as MlflowHttpError).errorCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should truncate large response bodies to prevent memory issues', async () => {
|
||||
// Create a response body larger than 1000 characters
|
||||
const largeResponseBody = 'x'.repeat(1500); // 1500 characters
|
||||
|
||||
const mockResponse = {
|
||||
ok: false,
|
||||
status: 500,
|
||||
statusText: 'Internal Server Error',
|
||||
text: jest.fn().mockResolvedValue(largeResponseBody),
|
||||
};
|
||||
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(mockResponse as any);
|
||||
|
||||
await expect(
|
||||
makeRequest(
|
||||
'POST',
|
||||
'http://localhost:5000/api/2.0/mlflow/runs/create',
|
||||
mockHeaderProvider,
|
||||
{},
|
||||
),
|
||||
).rejects.toThrow(/HTTP 500: Internal Server Error - x+\.\.\. \(truncated\)/);
|
||||
|
||||
expect(mockResponse.text).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,122 @@
|
||||
import { createAuthProvider } from '../../src/auth';
|
||||
|
||||
const ENV_KEYS = [
|
||||
'MLFLOW_TRACKING_TOKEN',
|
||||
'MLFLOW_TRACKING_USERNAME',
|
||||
'MLFLOW_TRACKING_PASSWORD',
|
||||
'MLFLOW_WORKSPACE',
|
||||
] as const;
|
||||
|
||||
describe('createAuthProvider', () => {
|
||||
describe('OSS auth (createOssAuth)', () => {
|
||||
const savedEnv: Record<string, string | undefined> = {};
|
||||
|
||||
beforeEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
savedEnv[key] = process.env[key];
|
||||
delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
if (savedEnv[key] === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = savedEnv[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should include Bearer token header when MLFLOW_TRACKING_TOKEN is set', async () => {
|
||||
process.env.MLFLOW_TRACKING_TOKEN = 'test-token';
|
||||
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['Authorization']).toBe('Bearer test-token');
|
||||
expect(headers['Content-Type']).toBe('application/json');
|
||||
});
|
||||
|
||||
it('should include Basic auth header when username and password are set', async () => {
|
||||
process.env.MLFLOW_TRACKING_USERNAME = 'user';
|
||||
process.env.MLFLOW_TRACKING_PASSWORD = 'pass';
|
||||
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
const expected = `Basic ${Buffer.from('user:pass').toString('base64')}`;
|
||||
expect(headers['Authorization']).toBe(expected);
|
||||
});
|
||||
|
||||
it('should not include Authorization header when no credentials are set', async () => {
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['Authorization']).toBeUndefined();
|
||||
expect(headers['Content-Type']).toBe('application/json');
|
||||
});
|
||||
|
||||
it('should include X-MLFLOW-WORKSPACE header when MLFLOW_WORKSPACE is set', async () => {
|
||||
process.env.MLFLOW_WORKSPACE = 'my-namespace';
|
||||
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['X-MLFLOW-WORKSPACE']).toBe('my-namespace');
|
||||
});
|
||||
|
||||
it('should not include X-MLFLOW-WORKSPACE header when MLFLOW_WORKSPACE is not set', async () => {
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['X-MLFLOW-WORKSPACE']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should include both Authorization and X-MLFLOW-WORKSPACE when both are set', async () => {
|
||||
process.env.MLFLOW_TRACKING_TOKEN = 'test-token';
|
||||
process.env.MLFLOW_WORKSPACE = 'my-namespace';
|
||||
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['Authorization']).toBe('Bearer test-token');
|
||||
expect(headers['X-MLFLOW-WORKSPACE']).toBe('my-namespace');
|
||||
expect(headers['Content-Type']).toBe('application/json');
|
||||
});
|
||||
|
||||
it('should use workspace from options when env var is not set', async () => {
|
||||
const provider = createAuthProvider({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
workspace: 'config-namespace',
|
||||
});
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['X-MLFLOW-WORKSPACE']).toBe('config-namespace');
|
||||
});
|
||||
|
||||
it('should prefer MLFLOW_WORKSPACE env var over options', async () => {
|
||||
process.env.MLFLOW_WORKSPACE = 'env-namespace';
|
||||
|
||||
const provider = createAuthProvider({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
workspace: 'config-namespace',
|
||||
});
|
||||
const headers = await provider.getHeadersProvider()();
|
||||
|
||||
expect(headers['X-MLFLOW-WORKSPACE']).toBe('env-namespace');
|
||||
});
|
||||
|
||||
it('should return the tracking URI as host', () => {
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
|
||||
expect(provider.getHost()).toBe('http://localhost:5000');
|
||||
});
|
||||
|
||||
it('should return undefined for getDatabricksToken', () => {
|
||||
const provider = createAuthProvider({ trackingUri: 'http://localhost:5000' });
|
||||
|
||||
expect(provider.getDatabricksToken()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { init, getConfig } from '../../src/core/config';
|
||||
|
||||
describe('Config', () => {
|
||||
describe('init and getConfig', () => {
|
||||
describe('environment variable resolution', () => {
|
||||
afterEach(() => {
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
});
|
||||
|
||||
it('should read tracking configuration from environment variables when not provided', () => {
|
||||
process.env.MLFLOW_TRACKING_URI = 'http://env-tracking-host:5000';
|
||||
process.env.MLFLOW_EXPERIMENT_ID = 'env-experiment-id';
|
||||
|
||||
init({});
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.trackingUri).toBe('http://env-tracking-host:5000');
|
||||
expect(result.experimentId).toBe('env-experiment-id');
|
||||
expect(result.host).toBe('http://env-tracking-host:5000');
|
||||
});
|
||||
|
||||
it('should throw an error when trackingUri is missing from both config and environment', () => {
|
||||
process.env.MLFLOW_EXPERIMENT_ID = 'env-experiment-id';
|
||||
|
||||
expect(() => init({ experimentId: 'config-experiment-id' })).toThrow(
|
||||
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw an error when experimentId is missing from both config and environment', () => {
|
||||
process.env.MLFLOW_TRACKING_URI = 'http://env-tracking-host:5000';
|
||||
|
||||
expect(() => init({ trackingUri: 'http://explicit-host:5000' })).toThrow(
|
||||
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('should initialize with MLflow tracking server configuration', () => {
|
||||
const config = {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '123456789',
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.trackingUri).toBe('http://localhost:5000');
|
||||
expect(result.experimentId).toBe('123456789');
|
||||
expect(result.host).toBe('http://localhost:5000');
|
||||
});
|
||||
|
||||
it('should throw error if trackingUri is missing', () => {
|
||||
const config = {
|
||||
trackingUri: '',
|
||||
experimentId: '123456789',
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow(
|
||||
'An MLflow Tracking URI is required, please provide the trackingUri option to init, or set the MLFLOW_TRACKING_URI environment variable',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error if experimentId is missing', () => {
|
||||
const config = {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '',
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow(
|
||||
'An MLflow experiment ID is required, please provide the experimentId option to init, or set the MLFLOW_EXPERIMENT_ID environment variable',
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error if trackingUri is not a string', () => {
|
||||
const config = {
|
||||
trackingUri: 123 as any,
|
||||
experimentId: '123456789',
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow('trackingUri must be a string');
|
||||
});
|
||||
|
||||
it('should throw error if experimentId is not a string', () => {
|
||||
const config = {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: 123 as any,
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow('experimentId must be a string');
|
||||
});
|
||||
|
||||
it('should throw error for malformed trackingUri', () => {
|
||||
const config = {
|
||||
trackingUri: 'not-a-valid-uri',
|
||||
experimentId: '123456789',
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow(
|
||||
"Invalid trackingUri: 'not-a-valid-uri'. Must be a valid HTTP or HTTPS URL, or 'databricks' / 'databricks://<profile>'.",
|
||||
);
|
||||
});
|
||||
|
||||
it.skip('should throw error if getConfig is called without init', () => {
|
||||
// Skip this test as it interferes with other tests due to module state
|
||||
expect(() => getConfig()).toThrow(
|
||||
'The MLflow Tracing client is not configured. Please call init() with host and experimentId before using tracing functions.',
|
||||
);
|
||||
});
|
||||
|
||||
describe('Databricks configuration', () => {
|
||||
const tempDir = path.join(os.tmpdir(), 'mlflow-databricks-test-' + Date.now());
|
||||
const configPath = path.join(tempDir, '.databrickscfg');
|
||||
|
||||
beforeEach(() => {
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
delete process.env.DATABRICKS_HOST;
|
||||
delete process.env.DATABRICKS_TOKEN;
|
||||
});
|
||||
|
||||
it('should throw error if host not found anywhere', () => {
|
||||
const config = {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '123456789',
|
||||
databricksConfigPath: '/nonexistent/path/.databrickscfg',
|
||||
};
|
||||
|
||||
expect(() => init(config)).toThrow('Databricks host not found');
|
||||
});
|
||||
|
||||
it('should read host from config file when env var not set', () => {
|
||||
const configContent = `[DEFAULT]
|
||||
host = https://config-workspace.databricks.com
|
||||
token = dapi123456789abcdef`;
|
||||
|
||||
fs.writeFileSync(configPath, configContent);
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '123456789',
|
||||
databricksConfigPath: configPath,
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.host).toBe('https://config-workspace.databricks.com');
|
||||
});
|
||||
|
||||
it('should read host from specific profile in config file', () => {
|
||||
const configContent = `[DEFAULT]
|
||||
host = https://default-workspace.databricks.com
|
||||
token = default-token
|
||||
|
||||
[dev]
|
||||
host = https://dev-workspace.databricks.com
|
||||
token = dev-token`;
|
||||
|
||||
fs.writeFileSync(configPath, configContent);
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks://dev',
|
||||
experimentId: '123456789',
|
||||
databricksConfigPath: configPath,
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.host).toBe('https://dev-workspace.databricks.com');
|
||||
});
|
||||
|
||||
it('should use DATABRICKS_HOST from environment', () => {
|
||||
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
|
||||
process.env.DATABRICKS_TOKEN = 'env-token';
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '123456789',
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.host).toBe('https://env-workspace.databricks.com');
|
||||
expect(result.databricksToken).toBe('env-token');
|
||||
});
|
||||
|
||||
it('should use explicit host/token over environment', () => {
|
||||
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
|
||||
process.env.DATABRICKS_TOKEN = 'env-token';
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '123456789',
|
||||
host: 'https://override-workspace.databricks.com',
|
||||
databricksToken: 'override-token',
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.host).toBe('https://override-workspace.databricks.com');
|
||||
expect(result.databricksToken).toBe('override-token');
|
||||
});
|
||||
|
||||
it('should prefer env var over config file for host', () => {
|
||||
process.env.DATABRICKS_HOST = 'https://env-workspace.databricks.com';
|
||||
|
||||
const configContent = `[DEFAULT]
|
||||
host = https://config-workspace.databricks.com
|
||||
token = dapi123456789abcdef`;
|
||||
|
||||
fs.writeFileSync(configPath, configContent);
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '123456789',
|
||||
databricksConfigPath: configPath,
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
// Env var takes precedence over config file
|
||||
expect(result.host).toBe('https://env-workspace.databricks.com');
|
||||
});
|
||||
|
||||
it('should handle empty profile name in URI', () => {
|
||||
const configContent = `[DEFAULT]
|
||||
host = https://default-workspace.databricks.com
|
||||
token = dapi123456789abcdef`;
|
||||
|
||||
fs.writeFileSync(configPath, configContent);
|
||||
|
||||
const config = {
|
||||
trackingUri: 'databricks://',
|
||||
experimentId: '123456789',
|
||||
databricksConfigPath: configPath,
|
||||
};
|
||||
|
||||
init(config);
|
||||
|
||||
const result = getConfig();
|
||||
expect(result.host).toBe('https://default-workspace.databricks.com');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,418 @@
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
import {
|
||||
createMlflowSpan,
|
||||
Span,
|
||||
NoOpSpan,
|
||||
type LiveSpan,
|
||||
type SerializedSpan,
|
||||
} from '../../../src/core/entities/span';
|
||||
import { SpanEvent } from '../../../src/core/entities/span_event';
|
||||
import { SpanStatus, SpanStatusCode } from '../../../src/core/entities/span_status';
|
||||
import { SpanAttributeKey, SpanType } from '../../../src/core/constants';
|
||||
import { convertHrTimeToNanoSeconds } from '../../../src/core/utils';
|
||||
import { JSONBig } from '../../../src/core/utils/json';
|
||||
|
||||
// Set up a proper tracer provider
|
||||
const provider = new BasicTracerProvider();
|
||||
trace.setGlobalTracerProvider(provider);
|
||||
|
||||
const tracer = trace.getTracer('mlflow-test-tracer', '1.0.0');
|
||||
|
||||
describe('Span', () => {
|
||||
describe('createMlflowSpan', () => {
|
||||
describe('Live Span Creation', () => {
|
||||
it('should create a live span from an active OpenTelemetry span', () => {
|
||||
const traceId = 'tr-12345';
|
||||
|
||||
const span = tracer.startSpan('parent');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.LLM);
|
||||
|
||||
expect(mlflowSpan).toBeInstanceOf(Span);
|
||||
expect(mlflowSpan.traceId).toBe(traceId);
|
||||
expect(mlflowSpan.name).toBe('parent');
|
||||
/* OpenTelemetry's end time default value is 0 when unset */
|
||||
expect(mlflowSpan.startTime[0]).toBeGreaterThan(0); // Check seconds part of HrTime
|
||||
expect(mlflowSpan.endTime?.[1]).toBe(0); // Check nanoseconds part of HrTime
|
||||
expect(mlflowSpan.parentId).toBeNull();
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle span inputs and outputs', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
|
||||
|
||||
mlflowSpan.setInputs({ input: 1 });
|
||||
mlflowSpan.setOutputs(2);
|
||||
|
||||
expect(mlflowSpan.inputs).toEqual({ input: 1 });
|
||||
expect(mlflowSpan.outputs).toBe(2);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle span attributes', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
|
||||
|
||||
mlflowSpan.setAttribute('key', 3);
|
||||
expect(mlflowSpan.getAttribute('key')).toBe(3);
|
||||
|
||||
// Test complex object serialization
|
||||
const complexObject = { nested: { value: 'test' } };
|
||||
mlflowSpan.setAttribute('complex', complexObject);
|
||||
expect(mlflowSpan.getAttribute('complex')).toEqual(complexObject);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle span status', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
|
||||
|
||||
mlflowSpan.setStatus(SpanStatusCode.OK);
|
||||
expect(mlflowSpan.status).toBeInstanceOf(SpanStatus);
|
||||
expect(mlflowSpan.status?.statusCode).toBe(SpanStatusCode.OK);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle span events', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
|
||||
|
||||
const event = new SpanEvent({
|
||||
name: 'test_event',
|
||||
timestamp: 99999n,
|
||||
attributes: { foo: 'bar' },
|
||||
});
|
||||
|
||||
mlflowSpan.addEvent(event);
|
||||
|
||||
const events = mlflowSpan.events;
|
||||
expect(events.length).toBeGreaterThan(0);
|
||||
|
||||
// Find our test event
|
||||
const testEvent = events.find((e) => e.name === 'test_event');
|
||||
expect(testEvent).toBeDefined();
|
||||
expect(testEvent?.attributes).toEqual({ foo: 'bar' });
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Completed Span Creation', () => {
|
||||
it('should create a completed span from an active OpenTelemetry span', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('parent');
|
||||
span.setAttribute(SpanAttributeKey.TRACE_ID, JSONBig.stringify(traceId));
|
||||
span.setAttribute(SpanAttributeKey.INPUTS, '{"input": 1}');
|
||||
span.setAttribute(SpanAttributeKey.OUTPUTS, '2');
|
||||
span.setAttribute('custom_attr', 'custom_value');
|
||||
span.end();
|
||||
|
||||
const mlflowSpan = createMlflowSpan(span, traceId);
|
||||
|
||||
expect(mlflowSpan).toBeInstanceOf(Span);
|
||||
expect(mlflowSpan.traceId).toBe(traceId);
|
||||
expect(mlflowSpan.name).toBe('parent');
|
||||
expect(convertHrTimeToNanoSeconds(mlflowSpan.startTime)).toBeGreaterThan(0n);
|
||||
expect(mlflowSpan.endTime).toBeDefined();
|
||||
if (mlflowSpan.endTime) {
|
||||
expect(convertHrTimeToNanoSeconds(mlflowSpan.endTime)).toBeGreaterThan(0n);
|
||||
}
|
||||
expect(mlflowSpan.parentId).toBeNull();
|
||||
expect(mlflowSpan.inputs).toEqual({ input: 1 });
|
||||
expect(mlflowSpan.outputs).toBe(2);
|
||||
expect(mlflowSpan.getAttribute('custom_attr')).toBe('custom_value');
|
||||
|
||||
// Setter should not be defined for completed span
|
||||
expect('setInputs' in mlflowSpan).toBe(false);
|
||||
expect('setOutputs' in mlflowSpan).toBe(false);
|
||||
expect('setAttribute' in mlflowSpan).toBe(false);
|
||||
expect('addEvent' in mlflowSpan).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('No-Op Span Creation', () => {
|
||||
it('should create a no-op span from null input', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = createMlflowSpan(null, traceId);
|
||||
|
||||
expect(span).toBeInstanceOf(NoOpSpan);
|
||||
expect(span.traceId).toBe('no-op-span-trace-id');
|
||||
expect(span.spanId).toBe('');
|
||||
expect(span.name).toBe('');
|
||||
expect(span.startTime).toEqual([0, 0]); // HrTime format [seconds, nanoseconds]
|
||||
expect(span.endTime).toBeNull();
|
||||
expect(span.parentId).toBeNull();
|
||||
expect(span.inputs).toBeNull();
|
||||
expect(span.outputs).toBeNull();
|
||||
});
|
||||
|
||||
it('should create a no-op span from undefined input', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = createMlflowSpan(undefined, traceId);
|
||||
|
||||
expect(span).toBeInstanceOf(NoOpSpan);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Exception Safety', () => {
|
||||
it('should handle exceptions in span.end() gracefully', () => {
|
||||
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test-span');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId) as LiveSpan;
|
||||
|
||||
// Mock the underlying OTel span to throw on end()
|
||||
// eslint-disable-next-line @typescript-eslint/unbound-method
|
||||
const originalEnd = span.end;
|
||||
span.end = jest.fn(() => {
|
||||
throw new Error('OTel span.end() failed');
|
||||
});
|
||||
|
||||
expect(() => mlflowSpan.end()).not.toThrow();
|
||||
|
||||
// Restore original end method
|
||||
span.end = originalEnd;
|
||||
} finally {
|
||||
span.end();
|
||||
consoleErrorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Circular Reference Handling', () => {
|
||||
it('should handle circular references in span attributes', () => {
|
||||
const traceId = 'tr-circular';
|
||||
const span = tracer.startSpan('circular-test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.UNKNOWN) as LiveSpan;
|
||||
|
||||
// Create an object with circular reference
|
||||
const circularObj: any = { name: 'test' };
|
||||
circularObj.self = circularObj;
|
||||
|
||||
// This should not throw
|
||||
expect(() => {
|
||||
mlflowSpan.setAttribute('circular', circularObj);
|
||||
}).not.toThrow();
|
||||
|
||||
// Verify the circular reference is replaced with placeholder
|
||||
const attrValue = mlflowSpan.getAttribute('circular');
|
||||
expect(attrValue.name).toBe('test');
|
||||
expect(attrValue.self).toBe('[Circular]');
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle non-serializable objects', () => {
|
||||
const traceId = 'tr-non-serializable';
|
||||
const span = tracer.startSpan('non-serializable-test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.UNKNOWN) as LiveSpan;
|
||||
|
||||
// Test various non-serializable objects
|
||||
const testData = {
|
||||
func: () => console.log('test'),
|
||||
undefinedVal: undefined,
|
||||
error: new Error('Test error'),
|
||||
date: new Date('2024-01-01'),
|
||||
regex: /test/g,
|
||||
map: new Map([['key', 'value']]),
|
||||
set: new Set([1, 2, 3]),
|
||||
};
|
||||
|
||||
// This should not throw
|
||||
expect(() => {
|
||||
mlflowSpan.setAttribute('nonSerializable', testData);
|
||||
}).not.toThrow();
|
||||
|
||||
const attrValue = mlflowSpan.getAttribute('nonSerializable');
|
||||
expect(attrValue.func).toBe('[Function]');
|
||||
expect(attrValue.undefinedVal).toBe('[Undefined]');
|
||||
expect(attrValue.error.name).toBe('Error');
|
||||
expect(attrValue.error.message).toBe('Test error');
|
||||
expect(attrValue.date).toBeDefined(); // Date should be serialized normally
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('JSON Serialization', () => {
|
||||
it('should produce correct JSON format for toJson()', () => {
|
||||
const traceId = 'tr-12345';
|
||||
const span = tracer.startSpan('test-span');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.LLM) as LiveSpan;
|
||||
|
||||
// Set inputs, outputs, and attributes to match expected format
|
||||
mlflowSpan.setInputs({ x: 1 });
|
||||
mlflowSpan.setOutputs(2);
|
||||
mlflowSpan.setAttribute('mlflow.spanFunctionName', 'f');
|
||||
mlflowSpan.setStatus(SpanStatusCode.OK);
|
||||
|
||||
// Add an event
|
||||
const event = new SpanEvent({
|
||||
name: 'test_event',
|
||||
timestamp: 1500000000000n,
|
||||
attributes: { eventAttr: 'value' },
|
||||
});
|
||||
mlflowSpan.addEvent(event);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
|
||||
// Get the completed span
|
||||
const completedSpan = createMlflowSpan(span, traceId);
|
||||
const json = completedSpan.toJson();
|
||||
|
||||
// Validate JSON structure matches expected format
|
||||
expect(json).toHaveProperty('span_id');
|
||||
expect(json).toHaveProperty('name', 'test-span');
|
||||
expect(json).toHaveProperty('start_time_unix_nano');
|
||||
expect(json).toHaveProperty('end_time_unix_nano');
|
||||
expect(json).toHaveProperty('attributes');
|
||||
expect(json).toHaveProperty('status');
|
||||
expect(json).toHaveProperty('events');
|
||||
|
||||
// Validate that attributes exist and have correct values
|
||||
expect(json.attributes['mlflow.spanInputs']).toEqual({ x: 1 });
|
||||
expect(json.attributes['mlflow.spanOutputs']).toBe(2);
|
||||
expect(json.attributes['mlflow.spanType']).toBe('LLM');
|
||||
expect(json.attributes['mlflow.traceRequestId']).toBe(traceId);
|
||||
|
||||
// Validate status format
|
||||
expect(json.status).toHaveProperty('code');
|
||||
expect(json.status.code).toBe('STATUS_CODE_OK');
|
||||
|
||||
// Validate timestamps are bigint for precision
|
||||
expect(typeof json.start_time_unix_nano).toBe('bigint');
|
||||
expect(typeof json.end_time_unix_nano).toBe('bigint');
|
||||
|
||||
// Validate events structure
|
||||
expect(Array.isArray(json.events)).toBe(true);
|
||||
if (json.events.length > 0) {
|
||||
const event = json.events[0];
|
||||
expect(event).toHaveProperty('name');
|
||||
expect(event).toHaveProperty('time_unix_nano');
|
||||
expect(event).toHaveProperty('attributes');
|
||||
}
|
||||
});
|
||||
|
||||
it('should match expected JSON format from real MLflow data', () => {
|
||||
// Create a span that matches the real MLflow data structure
|
||||
const expectedSpanData = {
|
||||
trace_id: 'rZo9DIws+6d2tejICXD4gw==',
|
||||
span_id: 'DOD2qjZ6ZrU=',
|
||||
trace_state: '',
|
||||
parent_span_id: '',
|
||||
name: 'python',
|
||||
start_time_unix_nano: 1749996461282772491n,
|
||||
end_time_unix_nano: 1749996461365717111n,
|
||||
attributes: {
|
||||
'mlflow.spanOutputs': '2',
|
||||
'mlflow.spanType': '"LLM"',
|
||||
'mlflow.spanInputs': '{"x": 1}',
|
||||
'mlflow.traceRequestId': '"tr-ad9a3d0c8c2cfba776b5e8c80970f883"',
|
||||
'mlflow.spanFunctionName': '"f"',
|
||||
},
|
||||
status: {
|
||||
message: '',
|
||||
code: 'STATUS_CODE_OK',
|
||||
},
|
||||
events: [],
|
||||
};
|
||||
|
||||
// Convert to JSON string and parse with json-bigint to get proper bigints
|
||||
const jsonString = JSONBig.stringify(expectedSpanData);
|
||||
const parsedData = JSONBig.parse(jsonString) as SerializedSpan;
|
||||
|
||||
// Test fromJson can handle this format
|
||||
const span = Span.fromJson(parsedData);
|
||||
|
||||
// Validate key properties
|
||||
expect(span.traceId).toBe('tr-ad9a3d0c8c2cfba776b5e8c80970f883');
|
||||
expect(span.spanId).toBe('0ce0f6aa367a66b5');
|
||||
expect(span.name).toBe('python');
|
||||
expect(span.spanType).toBe(SpanType.LLM);
|
||||
expect(convertHrTimeToNanoSeconds(span.startTime)).toBe(1749996461282772491n);
|
||||
expect(convertHrTimeToNanoSeconds(span.endTime!)).toBe(1749996461365717111n);
|
||||
expect(span.parentId).toBeNull();
|
||||
expect(span.status.statusCode).toBe(SpanStatusCode.OK);
|
||||
expect(span.status.description).toBe('');
|
||||
expect(span.inputs).toStrictEqual({ x: 1 });
|
||||
expect(span.outputs).toBe(2);
|
||||
expect(span.events).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle round-trip JSON serialization correctly', () => {
|
||||
const traceId = 'tr-test-round-trip';
|
||||
const span = tracer.startSpan('round-trip-test');
|
||||
|
||||
try {
|
||||
const mlflowSpan = createMlflowSpan(span, traceId, SpanType.CHAIN) as LiveSpan;
|
||||
|
||||
// Set comprehensive test data
|
||||
mlflowSpan.setInputs({ input: 'test', number: 42 });
|
||||
mlflowSpan.setOutputs({ result: 'success', count: 100 });
|
||||
mlflowSpan.setAttribute('custom.attribute', 'custom_value');
|
||||
mlflowSpan.setAttribute('numeric.attribute', 123);
|
||||
mlflowSpan.setStatus(SpanStatusCode.OK);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
|
||||
// Round-trip test: span -> JSON -> span -> JSON
|
||||
const originalSpan = createMlflowSpan(span, traceId);
|
||||
const originalJson = originalSpan.toJson();
|
||||
|
||||
// Create new span from JSON
|
||||
const reconstructedSpan = Span.fromJson(originalJson);
|
||||
|
||||
// Key properties should match
|
||||
expect(reconstructedSpan.name).toBe(originalSpan.name);
|
||||
expect(reconstructedSpan.parentId).toBe(originalSpan.parentId);
|
||||
expect(reconstructedSpan.startTime).toEqual(originalSpan.startTime);
|
||||
expect(reconstructedSpan.endTime).toEqual(originalSpan.endTime);
|
||||
|
||||
// Attributes should be preserved
|
||||
expect(reconstructedSpan.attributes).toEqual(originalSpan.attributes);
|
||||
|
||||
// Status should be preserved
|
||||
expect(reconstructedSpan.status.statusCode).toBe(originalSpan.status.statusCode);
|
||||
|
||||
// Events should be preserved
|
||||
expect(reconstructedSpan.events).toEqual(originalSpan.events);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import { SpanEvent } from '../../../src/core/entities/span_event';
|
||||
|
||||
describe('SpanEvent', () => {
|
||||
describe('constructor', () => {
|
||||
it('should create a span event with all parameters', () => {
|
||||
const timestamp = BigInt(Date.now()) * 1_000_000n; // nanoseconds
|
||||
const attributes = {
|
||||
key1: 'value1',
|
||||
key2: 42,
|
||||
key3: true,
|
||||
key4: ['a', 'b', 'c'],
|
||||
};
|
||||
|
||||
const event = new SpanEvent({
|
||||
name: 'test_event',
|
||||
timestamp,
|
||||
attributes,
|
||||
});
|
||||
|
||||
expect(event.name).toBe('test_event');
|
||||
expect(event.timestamp).toBe(timestamp);
|
||||
expect(event.attributes).toEqual(attributes);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fromException', () => {
|
||||
it('should create a span event from a basic error', () => {
|
||||
const error = new Error('Test error message');
|
||||
const event = SpanEvent.fromException(error);
|
||||
|
||||
expect(event.name).toBe('exception');
|
||||
expect(event.attributes['exception.message']).toBe('Test error message');
|
||||
expect(event.attributes['exception.type']).toBe('Error');
|
||||
expect(event.attributes['exception.stacktrace']).toContain('Test error message');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toJson round-trip serialization', () => {
|
||||
it('should serialize and maintain all properties', () => {
|
||||
const originalEvent = new SpanEvent({
|
||||
name: 'test_event',
|
||||
timestamp: 1234567890000n,
|
||||
attributes: {
|
||||
string_attr: 'test_value',
|
||||
number_attr: 42,
|
||||
boolean_attr: true,
|
||||
string_array: ['a', 'b', 'c'],
|
||||
number_array: [1, 2, 3],
|
||||
boolean_array: [true, false, true],
|
||||
},
|
||||
});
|
||||
|
||||
const json = originalEvent.toJson();
|
||||
|
||||
// Verify JSON structure
|
||||
expect(json).toEqual({
|
||||
name: 'test_event',
|
||||
timestamp: 1234567890000n,
|
||||
attributes: {
|
||||
string_attr: 'test_value',
|
||||
number_attr: 42,
|
||||
boolean_attr: true,
|
||||
string_array: ['a', 'b', 'c'],
|
||||
number_array: [1, 2, 3],
|
||||
boolean_array: [true, false, true],
|
||||
},
|
||||
});
|
||||
|
||||
// Create new event from JSON data
|
||||
const recreatedEvent = new SpanEvent({
|
||||
name: json.name as string,
|
||||
timestamp: json.timestamp as bigint,
|
||||
attributes: json.attributes as Record<string, any>,
|
||||
});
|
||||
|
||||
// Verify round-trip preservation
|
||||
expect(recreatedEvent.name).toBe(originalEvent.name);
|
||||
expect(recreatedEvent.timestamp).toBe(originalEvent.timestamp);
|
||||
expect(recreatedEvent.attributes).toEqual(originalEvent.attributes);
|
||||
expect(recreatedEvent.toJson()).toEqual(originalEvent.toJson());
|
||||
});
|
||||
|
||||
it('should handle events with minimal properties', () => {
|
||||
const originalEvent = new SpanEvent({
|
||||
name: 'minimal_event',
|
||||
});
|
||||
|
||||
const json = originalEvent.toJson();
|
||||
|
||||
expect(json.name).toBe('minimal_event');
|
||||
expect(json.timestamp).toBeGreaterThan(0);
|
||||
expect(json.attributes).toEqual({});
|
||||
|
||||
// Recreate and verify
|
||||
const recreatedEvent = new SpanEvent({
|
||||
name: json.name as string,
|
||||
timestamp: json.timestamp as bigint,
|
||||
attributes: json.attributes as Record<string, any>,
|
||||
});
|
||||
|
||||
expect(recreatedEvent.name).toBe(originalEvent.name);
|
||||
expect(recreatedEvent.timestamp).toBe(originalEvent.timestamp);
|
||||
expect(recreatedEvent.attributes).toEqual(originalEvent.attributes);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { SpanStatusCode as OTelSpanStatusCode } from '@opentelemetry/api';
|
||||
import { SpanStatus, SpanStatusCode } from '../../../src/core/entities/span_status';
|
||||
|
||||
describe('SpanStatus', () => {
|
||||
describe('initialization', () => {
|
||||
// Test both enum and string initialization (parameterized test equivalent)
|
||||
const testCases = [
|
||||
{ input: 'STATUS_CODE_OK', expected: SpanStatusCode.OK },
|
||||
{ input: 'STATUS_CODE_ERROR', expected: SpanStatusCode.ERROR },
|
||||
{ input: 'STATUS_CODE_UNSET', expected: SpanStatusCode.UNSET },
|
||||
];
|
||||
|
||||
testCases.forEach(({ input, expected }) => {
|
||||
it(`should initialize with status code ${input}`, () => {
|
||||
const spanStatus = new SpanStatus(input as SpanStatusCode, 'test');
|
||||
expect(spanStatus.statusCode).toBe(expected);
|
||||
expect(spanStatus.description).toBe('test');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenTelemetry status conversion', () => {
|
||||
const conversionTestCases = [
|
||||
{
|
||||
mlflowStatus: SpanStatusCode.OK,
|
||||
otelStatus: OTelSpanStatusCode.OK,
|
||||
},
|
||||
{
|
||||
mlflowStatus: SpanStatusCode.ERROR,
|
||||
otelStatus: OTelSpanStatusCode.ERROR,
|
||||
},
|
||||
{
|
||||
mlflowStatus: SpanStatusCode.UNSET,
|
||||
otelStatus: OTelSpanStatusCode.UNSET,
|
||||
},
|
||||
];
|
||||
|
||||
conversionTestCases.forEach(({ mlflowStatus, otelStatus }) => {
|
||||
it(`should convert ${mlflowStatus} to OpenTelemetry status correctly`, () => {
|
||||
const spanStatus = new SpanStatus(mlflowStatus);
|
||||
const otelStatusResult = spanStatus.toOtelStatus();
|
||||
|
||||
expect(otelStatusResult.code).toBe(otelStatus);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('toJson round-trip serialization', () => {
|
||||
it('should serialize and recreate status with all properties', () => {
|
||||
const originalStatus = new SpanStatus(SpanStatusCode.ERROR, 'Something went wrong');
|
||||
|
||||
const json = originalStatus.toJson();
|
||||
|
||||
// Verify JSON structure
|
||||
expect(json).toEqual({
|
||||
status_code: SpanStatusCode.ERROR,
|
||||
description: 'Something went wrong',
|
||||
});
|
||||
|
||||
// Create new status from JSON data
|
||||
const recreatedStatus = new SpanStatus(json.status_code as SpanStatusCode, json.description);
|
||||
|
||||
// Verify round-trip preservation
|
||||
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
|
||||
expect(recreatedStatus.description).toBe(originalStatus.description);
|
||||
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
|
||||
});
|
||||
|
||||
it('should handle status with different status codes', () => {
|
||||
const testCases = [
|
||||
{ code: SpanStatusCode.OK, description: 'All good' },
|
||||
{ code: SpanStatusCode.ERROR, description: 'Failed operation' },
|
||||
{ code: SpanStatusCode.UNSET, description: '' },
|
||||
];
|
||||
|
||||
testCases.forEach(({ code, description }) => {
|
||||
const originalStatus = new SpanStatus(code, description);
|
||||
const json = originalStatus.toJson();
|
||||
const recreatedStatus = new SpanStatus(
|
||||
json.status_code as SpanStatusCode,
|
||||
json.description,
|
||||
);
|
||||
|
||||
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
|
||||
expect(recreatedStatus.description).toBe(originalStatus.description);
|
||||
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle status with minimal properties', () => {
|
||||
const originalStatus = new SpanStatus(SpanStatusCode.OK);
|
||||
|
||||
const json = originalStatus.toJson();
|
||||
|
||||
expect(json).toEqual({
|
||||
status_code: SpanStatusCode.OK,
|
||||
description: '',
|
||||
});
|
||||
|
||||
const recreatedStatus = new SpanStatus(json.status_code as SpanStatusCode, json.description);
|
||||
|
||||
expect(recreatedStatus.statusCode).toBe(originalStatus.statusCode);
|
||||
expect(recreatedStatus.description).toBe(originalStatus.description);
|
||||
expect(recreatedStatus.toJson()).toEqual(originalStatus.toJson());
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
import { Trace } from '../../../src/core/entities/trace';
|
||||
import { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import { TraceData } from '../../../src/core/entities/trace_data';
|
||||
import { TraceState } from '../../../src/core/entities/trace_state';
|
||||
import { createTraceLocationFromExperimentId } from '../../../src/core/entities/trace_location';
|
||||
import { createMlflowSpan } from '../../../src/core/entities/span';
|
||||
|
||||
// Set up a proper tracer provider
|
||||
const provider = new BasicTracerProvider();
|
||||
trace.setGlobalTracerProvider(provider);
|
||||
|
||||
const tracer = trace.getTracer('mlflow-test-tracer', '1.0.0');
|
||||
|
||||
describe('Trace', () => {
|
||||
function createMockTraceInfo(): TraceInfo {
|
||||
return new TraceInfo({
|
||||
traceId: 'tr-12345',
|
||||
clientRequestId: 'client-request-id',
|
||||
traceLocation: createTraceLocationFromExperimentId('exp-123'),
|
||||
requestTime: Date.now(),
|
||||
state: TraceState.OK,
|
||||
executionDuration: 1000,
|
||||
requestPreview: '{"prompt": "Hello"}',
|
||||
responsePreview: '{"response": "Hi there"}',
|
||||
traceMetadata: { key: 'value' },
|
||||
tags: { env: 'test' },
|
||||
assessments: [],
|
||||
});
|
||||
}
|
||||
|
||||
function createMockTraceData(): TraceData {
|
||||
const span = tracer.startSpan('parent');
|
||||
const mlflowSpan = createMlflowSpan(span, 'tr-12345');
|
||||
span.end();
|
||||
return new TraceData([mlflowSpan]);
|
||||
}
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should create a Trace with info and data', () => {
|
||||
const traceInfo = createMockTraceInfo();
|
||||
const traceData = createMockTraceData();
|
||||
|
||||
const trace = new Trace(traceInfo, traceData);
|
||||
|
||||
expect(trace.info).toBe(traceInfo);
|
||||
expect(trace.data).toBe(traceData);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toJson/fromJson round-trip serialization', () => {
|
||||
it('should serialize and deserialize a complete trace correctly', () => {
|
||||
const originalTraceInfo = createMockTraceInfo();
|
||||
const originalTraceData = createMockTraceData();
|
||||
const originalTrace = new Trace(originalTraceInfo, originalTraceData);
|
||||
|
||||
const json = originalTrace.toJson();
|
||||
|
||||
// Verify JSON structure
|
||||
expect(json).toHaveProperty('info');
|
||||
expect(json).toHaveProperty('data');
|
||||
expect(json.info).toMatchObject({
|
||||
trace_id: 'tr-12345',
|
||||
request_time: expect.any(String),
|
||||
state: TraceState.OK,
|
||||
});
|
||||
expect(json.data).toMatchObject({
|
||||
spans: [
|
||||
{
|
||||
name: 'parent',
|
||||
trace_id: expect.any(String),
|
||||
span_id: expect.any(String),
|
||||
parent_span_id: '',
|
||||
start_time_unix_nano: expect.any(BigInt),
|
||||
end_time_unix_nano: expect.any(BigInt),
|
||||
attributes: {
|
||||
'mlflow.spanType': 'UNKNOWN',
|
||||
},
|
||||
status: { code: 'STATUS_CODE_UNSET' },
|
||||
events: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Round-trip test
|
||||
const recreatedTrace = Trace.fromJson(json);
|
||||
|
||||
expect(recreatedTrace).toBeInstanceOf(Trace);
|
||||
expect(recreatedTrace.info).toBeInstanceOf(TraceInfo);
|
||||
expect(recreatedTrace.data).toBeInstanceOf(TraceData);
|
||||
|
||||
// Verify that key properties are preserved
|
||||
expect(recreatedTrace.info.traceId).toBe(originalTrace.info.traceId);
|
||||
expect(recreatedTrace.info.state).toBe(originalTrace.info.state);
|
||||
expect(recreatedTrace.data.spans.length).toEqual(originalTrace.data.spans.length);
|
||||
|
||||
// Verify round-trip JSON serialization matches
|
||||
expect(recreatedTrace.toJson()).toEqual(originalTrace.toJson());
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import { TraceLocationType } from '../../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../../src/core/entities/trace_state';
|
||||
|
||||
describe('TraceInfo', () => {
|
||||
const createTestTraceInfo = () => {
|
||||
return new TraceInfo({
|
||||
traceId: 'test-trace-id',
|
||||
traceLocation: {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: {
|
||||
experimentId: 'test-experiment-id',
|
||||
},
|
||||
},
|
||||
requestTime: 1000,
|
||||
state: TraceState.OK,
|
||||
requestPreview: '{"input":"test"}',
|
||||
responsePreview: '{"output":"result"}',
|
||||
clientRequestId: 'client-request-id',
|
||||
executionDuration: 500,
|
||||
traceMetadata: { 'meta-key': 'meta-value' },
|
||||
tags: { 'tag-key': 'tag-value' },
|
||||
assessments: [],
|
||||
});
|
||||
};
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should create a TraceInfo instance with all properties', () => {
|
||||
const traceInfo = createTestTraceInfo();
|
||||
|
||||
expect(traceInfo.traceId).toBe('test-trace-id');
|
||||
expect(traceInfo.traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(traceInfo.traceLocation.mlflowExperiment?.experimentId).toBe('test-experiment-id');
|
||||
expect(traceInfo.requestTime).toBe(1000);
|
||||
expect(traceInfo.state).toBe(TraceState.OK);
|
||||
expect(traceInfo.requestPreview).toBe('{"input":"test"}');
|
||||
expect(traceInfo.responsePreview).toBe('{"output":"result"}');
|
||||
expect(traceInfo.clientRequestId).toBe('client-request-id');
|
||||
expect(traceInfo.executionDuration).toBe(500);
|
||||
expect(traceInfo.traceMetadata['meta-key']).toBe('meta-value');
|
||||
expect(traceInfo.tags['tag-key']).toBe('tag-value');
|
||||
expect(traceInfo.assessments).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
import {
|
||||
TraceLocation,
|
||||
TraceLocationType,
|
||||
createTraceLocationFromExperimentId,
|
||||
deserializeTraceLocation,
|
||||
} from '../../../src/core/entities/trace_location';
|
||||
|
||||
describe('TraceLocation', () => {
|
||||
describe('constructor and basic functionality', () => {
|
||||
it('should create a TraceLocation with MLflow experiment', () => {
|
||||
const traceLocation: TraceLocation = {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
mlflowExperiment: { experimentId: '123' },
|
||||
};
|
||||
|
||||
expect(traceLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(traceLocation.mlflowExperiment?.experimentId).toBe('123');
|
||||
expect(traceLocation.inferenceTable).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should create a TraceLocation with inference table', () => {
|
||||
const traceLocation: TraceLocation = {
|
||||
type: TraceLocationType.INFERENCE_TABLE,
|
||||
inferenceTable: { fullTableName: 'a.b.c' },
|
||||
};
|
||||
|
||||
expect(traceLocation.type).toBe(TraceLocationType.INFERENCE_TABLE);
|
||||
expect(traceLocation.inferenceTable?.fullTableName).toBe('a.b.c');
|
||||
expect(traceLocation.mlflowExperiment).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('validation', () => {
|
||||
it('should validate that only one location type can be provided', () => {
|
||||
// This test validates the conceptual constraint that only one location should be provided
|
||||
// In TypeScript, this is more of a usage pattern validation
|
||||
const invalidLocation: TraceLocation = {
|
||||
type: TraceLocationType.TRACE_LOCATION_TYPE_UNSPECIFIED,
|
||||
mlflowExperiment: { experimentId: '123' },
|
||||
inferenceTable: { fullTableName: 'a.b.c' },
|
||||
};
|
||||
|
||||
// Both are defined, which violates the constraint
|
||||
expect(invalidLocation.mlflowExperiment).toBeDefined();
|
||||
expect(invalidLocation.inferenceTable).toBeDefined();
|
||||
// In a real implementation, this would throw an error during validation
|
||||
});
|
||||
|
||||
it('should validate type matches MLflow experiment location', () => {
|
||||
// This represents a mismatch: INFERENCE_TABLE type with mlflowExperiment data
|
||||
const mismatchedLocation: TraceLocation = {
|
||||
type: TraceLocationType.INFERENCE_TABLE,
|
||||
mlflowExperiment: { experimentId: '123' },
|
||||
};
|
||||
|
||||
expect(mismatchedLocation.type).toBe(TraceLocationType.INFERENCE_TABLE);
|
||||
expect(mismatchedLocation.mlflowExperiment).toBeDefined();
|
||||
expect(mismatchedLocation.inferenceTable).toBeUndefined();
|
||||
// In a real implementation, this would be caught by validation
|
||||
});
|
||||
|
||||
it('should validate type matches inference table location', () => {
|
||||
// This represents a mismatch: MLFLOW_EXPERIMENT type with inferenceTable data
|
||||
const mismatchedLocation: TraceLocation = {
|
||||
type: TraceLocationType.MLFLOW_EXPERIMENT,
|
||||
inferenceTable: { fullTableName: 'a.b.c' },
|
||||
};
|
||||
|
||||
expect(mismatchedLocation.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(mismatchedLocation.inferenceTable).toBeDefined();
|
||||
expect(mismatchedLocation.mlflowExperiment).toBeUndefined();
|
||||
// In a real implementation, this would be caught by validation
|
||||
});
|
||||
});
|
||||
|
||||
describe('createTraceLocationFromExperimentId', () => {
|
||||
it('should create a TraceLocation with MLflow experiment', () => {
|
||||
const experimentId = 'experiment123';
|
||||
const location = createTraceLocationFromExperimentId(experimentId);
|
||||
|
||||
expect(location.type).toBe(TraceLocationType.MLFLOW_EXPERIMENT);
|
||||
expect(location.mlflowExperiment).toBeDefined();
|
||||
expect(location.mlflowExperiment?.experimentId).toBe(experimentId);
|
||||
expect(location.inferenceTable).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('deserializeTraceLocation', () => {
|
||||
it('should reject missing trace location data', () => {
|
||||
expect(() => deserializeTraceLocation(undefined)).toThrow(
|
||||
'Invalid trace location: missing type.',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import { TraceState, fromOtelStatus } from '../../../src/core/entities/trace_state';
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
|
||||
describe('TraceState', () => {
|
||||
describe('enum values', () => {
|
||||
it('should have the correct enum values', () => {
|
||||
expect(TraceState.STATE_UNSPECIFIED).toBe('STATE_UNSPECIFIED');
|
||||
expect(TraceState.OK).toBe('OK');
|
||||
expect(TraceState.ERROR).toBe('ERROR');
|
||||
expect(TraceState.IN_PROGRESS).toBe('IN_PROGRESS');
|
||||
});
|
||||
});
|
||||
|
||||
describe('fromOtelStatus', () => {
|
||||
it('should convert OpenTelemetry OK status to TraceState.OK', () => {
|
||||
const result = fromOtelStatus(SpanStatusCode.OK);
|
||||
expect(result).toBe(TraceState.OK);
|
||||
});
|
||||
|
||||
it('should convert OpenTelemetry ERROR status to TraceState.ERROR', () => {
|
||||
const result = fromOtelStatus(SpanStatusCode.ERROR);
|
||||
expect(result).toBe(TraceState.ERROR);
|
||||
});
|
||||
|
||||
it('should convert OpenTelemetry UNSET status to TraceState.STATE_UNSPECIFIED', () => {
|
||||
const result = fromOtelStatus(SpanStatusCode.UNSET);
|
||||
expect(result).toBe(TraceState.STATE_UNSPECIFIED);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
import { trace as otelTrace } from '@opentelemetry/api';
|
||||
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
|
||||
import { SpanAttributeKey, SpanLogLevel, SpanType, toSpanLogLevel } from '../../src/core/constants';
|
||||
import { defaultLogLevelForSpanType } from '../../src/core/log_level';
|
||||
import { createMlflowSpan, type LiveSpan } from '../../src/core/entities/span';
|
||||
import { SpanEvent } from '../../src/core/entities/span_event';
|
||||
|
||||
const provider = new BasicTracerProvider();
|
||||
otelTrace.setGlobalTracerProvider(provider);
|
||||
const tracer = otelTrace.getTracer('mlflow-log-level-test', '1.0.0');
|
||||
|
||||
const newLiveSpan = (spanType: SpanType = SpanType.UNKNOWN): LiveSpan => {
|
||||
const otelSpan = tracer.startSpan('test');
|
||||
return createMlflowSpan(otelSpan, 'tr-test', spanType) as LiveSpan;
|
||||
};
|
||||
|
||||
describe('toSpanLogLevel', () => {
|
||||
it.each<[SpanLogLevel | string, SpanLogLevel]>([
|
||||
[SpanLogLevel.DEBUG, SpanLogLevel.DEBUG],
|
||||
[SpanLogLevel.CRITICAL, SpanLogLevel.CRITICAL],
|
||||
['DEBUG', SpanLogLevel.DEBUG],
|
||||
['info', SpanLogLevel.INFO],
|
||||
['Warning', SpanLogLevel.WARNING],
|
||||
[' ERROR ', SpanLogLevel.ERROR], // whitespace tolerated
|
||||
])('accepts %p and yields %p', (input, expected) => {
|
||||
expect(toSpanLogLevel(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each(['NOPE', 'TRACE', 'FATAL', 'INFOO', '', 'WARN', 'warn'])(
|
||||
'rejects invalid string %p',
|
||||
(value) => {
|
||||
// "WARN" is no longer accepted; only the full names work.
|
||||
expect(() => toSpanLogLevel(value)).toThrow(/Invalid SpanLogLevel/);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([0, 7, 100, -1])('rejects unknown number %p (defensive runtime check)', (value) => {
|
||||
// Raw numbers aren't part of the public type (`SpanLogLevel | string`),
|
||||
// but `SpanLogLevel` is a numeric enum so members arrive as primitive
|
||||
// numbers at runtime. The runtime branch validates them defensively;
|
||||
// anything that isn't a known enum value throws.
|
||||
expect(() => toSpanLogLevel(value as unknown as SpanLogLevel)).toThrow(/Invalid SpanLogLevel/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LiveSpan default applied at end()', () => {
|
||||
it.each<[SpanType, SpanLogLevel]>([
|
||||
[SpanType.LLM, SpanLogLevel.INFO],
|
||||
[SpanType.CHAT_MODEL, SpanLogLevel.INFO],
|
||||
[SpanType.TOOL, SpanLogLevel.INFO],
|
||||
[SpanType.RETRIEVER, SpanLogLevel.INFO],
|
||||
[SpanType.AGENT, SpanLogLevel.INFO],
|
||||
[SpanType.EMBEDDING, SpanLogLevel.INFO],
|
||||
[SpanType.CHAIN, SpanLogLevel.DEBUG],
|
||||
[SpanType.PARSER, SpanLogLevel.DEBUG],
|
||||
[SpanType.RERANKER, SpanLogLevel.DEBUG],
|
||||
[SpanType.MEMORY, SpanLogLevel.DEBUG],
|
||||
[SpanType.UNKNOWN, SpanLogLevel.DEBUG],
|
||||
])('resolves default %p for span type %p', (spanType, expected) => {
|
||||
const span = newLiveSpan(spanType);
|
||||
// Mid-span: log level isn't resolved yet.
|
||||
expect(span.logLevel).toBeNull();
|
||||
span.end();
|
||||
// Post-end: type-derived default is applied.
|
||||
expect(span.logLevel).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LiveSpan log-level accessors', () => {
|
||||
it.each<[SpanLogLevel | string, SpanLogLevel]>([
|
||||
[SpanLogLevel.WARNING, SpanLogLevel.WARNING],
|
||||
[SpanLogLevel.ERROR, SpanLogLevel.ERROR],
|
||||
['info', SpanLogLevel.INFO],
|
||||
['CRITICAL', SpanLogLevel.CRITICAL],
|
||||
])('setLogLevel(%p) round-trips to %p', (input, expected) => {
|
||||
const span = newLiveSpan();
|
||||
span.setLogLevel(input);
|
||||
expect(span.logLevel).toBe(expected);
|
||||
// Stored as the canonical int under the reserved attribute key for portability.
|
||||
expect(span.getAttribute(SpanAttributeKey.LOG_LEVEL)).toBe(expected as number);
|
||||
span.end();
|
||||
});
|
||||
|
||||
it('setLogLevel rejects invalid input', () => {
|
||||
const span = newLiveSpan();
|
||||
expect(() => span.setLogLevel('NOPE')).toThrow(/Invalid SpanLogLevel/);
|
||||
span.end();
|
||||
});
|
||||
});
|
||||
|
||||
describe('exception event bumps log level to ERROR', () => {
|
||||
it('bumps an unset span to ERROR via addEvent (DEBUG-defaulting type)', () => {
|
||||
const span = newLiveSpan(SpanType.PARSER);
|
||||
expect(span.logLevel).toBeNull();
|
||||
span.addEvent(
|
||||
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
|
||||
);
|
||||
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
|
||||
span.end();
|
||||
// Stays at ERROR after end(); the resolution branch only fires when unset.
|
||||
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
|
||||
});
|
||||
|
||||
it('bumps an unset span to ERROR via addEvent (INFO-defaulting type)', () => {
|
||||
const span = newLiveSpan(SpanType.CHAT_MODEL);
|
||||
expect(span.logLevel).toBeNull();
|
||||
span.addEvent(
|
||||
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
|
||||
);
|
||||
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
|
||||
span.end();
|
||||
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
|
||||
});
|
||||
|
||||
it('preserves user-set CRITICAL', () => {
|
||||
const span = newLiveSpan(SpanType.PARSER);
|
||||
span.setLogLevel(SpanLogLevel.CRITICAL);
|
||||
span.addEvent(
|
||||
new SpanEvent({ name: 'exception', attributes: { 'exception.message': 'boom' } }),
|
||||
);
|
||||
expect(span.logLevel).toBe(SpanLogLevel.CRITICAL);
|
||||
span.end();
|
||||
});
|
||||
|
||||
it('bumps via recordException (which bypasses our addEvent wrapper)', () => {
|
||||
const span = newLiveSpan(SpanType.PARSER);
|
||||
span.recordException(new Error('boom'));
|
||||
expect(span.logLevel).toBe(SpanLogLevel.ERROR);
|
||||
span.end();
|
||||
});
|
||||
|
||||
it('non-exception events do not bump the level', () => {
|
||||
const span = newLiveSpan(SpanType.CHAT_MODEL);
|
||||
span.addEvent(new SpanEvent({ name: 'my_event', attributes: { k: 'v' } }));
|
||||
// Plain event must not pre-resolve the level; resolution waits until end().
|
||||
expect(span.logLevel).toBeNull();
|
||||
span.end();
|
||||
expect(span.logLevel).toBe(SpanLogLevel.INFO);
|
||||
});
|
||||
});
|
||||
|
||||
describe('defaultLogLevelForSpanType', () => {
|
||||
it.each<[string, SpanLogLevel]>([
|
||||
// INFO set: user-visible semantic operations.
|
||||
[SpanType.LLM, SpanLogLevel.INFO],
|
||||
[SpanType.CHAT_MODEL, SpanLogLevel.INFO],
|
||||
[SpanType.AGENT, SpanLogLevel.INFO],
|
||||
[SpanType.TOOL, SpanLogLevel.INFO],
|
||||
[SpanType.RETRIEVER, SpanLogLevel.INFO],
|
||||
[SpanType.EMBEDDING, SpanLogLevel.INFO],
|
||||
// DEBUG set: internal/glue work and unclassified types.
|
||||
[SpanType.CHAIN, SpanLogLevel.DEBUG],
|
||||
[SpanType.PARSER, SpanLogLevel.DEBUG],
|
||||
[SpanType.RERANKER, SpanLogLevel.DEBUG],
|
||||
[SpanType.MEMORY, SpanLogLevel.DEBUG],
|
||||
[SpanType.UNKNOWN, SpanLogLevel.DEBUG],
|
||||
['MY_CUSTOM_TYPE', SpanLogLevel.DEBUG], // custom types fall through to DEBUG
|
||||
])('maps %p to %p', (spanType, expected) => {
|
||||
expect(defaultLogLevelForSpanType(spanType)).toBe(expected);
|
||||
});
|
||||
|
||||
it('maps undefined/null to DEBUG', () => {
|
||||
expect(defaultLogLevelForSpanType(undefined)).toBe(SpanLogLevel.DEBUG);
|
||||
expect(defaultLogLevelForSpanType(null)).toBe(SpanLogLevel.DEBUG);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
|
||||
// Mock the OTLP proto exporter (Jest can't load its dynamic http imports).
|
||||
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
|
||||
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
|
||||
OTLPTraceExporter: jest
|
||||
.fn()
|
||||
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
|
||||
exporterCtors.push(cfg);
|
||||
return {
|
||||
export: (_spans: unknown[], cb: (r: { code: number }) => void) => cb({ code: 0 }),
|
||||
shutdown: () => Promise.resolve(),
|
||||
forceFlush: () => Promise.resolve(),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
import { init } from '../../src/core/config';
|
||||
import { flushTraces } from '../../src/core/provider';
|
||||
import { withSpan } from '../../src/core/api';
|
||||
|
||||
const testHost = 'https://dbc-12345.cloud.databricks.com';
|
||||
|
||||
// `init()` is documented as call-once-per-process: NodeSDK's background
|
||||
// shutdown can race with a subsequent start and clobber the global tracer
|
||||
// provider. This file therefore only exercises a single `init()` call.
|
||||
// UC processor + exporter behavior is covered in detail by the unit tests
|
||||
// in `tests/exporters/uc_table*.test.ts`; this file just verifies that
|
||||
// `init()` wires up the UC processor when `traceLocation` is provided.
|
||||
describe('init() with traceLocation wires the UC span processor', () => {
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
const v4TraceInfoCalls: { url: string; body: any }[] = [];
|
||||
|
||||
beforeAll(() => {
|
||||
process.env.DATABRICKS_HOST = testHost;
|
||||
process.env.DATABRICKS_TOKEN = 'test-token';
|
||||
server = setupServer(
|
||||
http.post(
|
||||
`${testHost}/api/4.0/mlflow/traces/:location/:otelTraceId/info`,
|
||||
async ({ request }) => {
|
||||
const body = (await request.json()) as any;
|
||||
v4TraceInfoCalls.push({ url: request.url, body });
|
||||
return HttpResponse.json({
|
||||
trace_id: body.trace_id,
|
||||
trace_location: body.trace_location,
|
||||
request_time: body.request_time,
|
||||
execution_duration: body.execution_duration,
|
||||
state: body.state,
|
||||
trace_metadata: body.trace_metadata,
|
||||
tags: body.tags,
|
||||
assessments: [],
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
server.listen();
|
||||
|
||||
init({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '4118495900667593',
|
||||
traceLocation: { catalogName: 'cat', schemaName: 'sch', tablePrefix: 'agent' },
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
delete process.env.DATABRICKS_HOST;
|
||||
delete process.env.DATABRICKS_TOKEN;
|
||||
});
|
||||
|
||||
it('routes spans through the V4 endpoint with the configured UC location', async () => {
|
||||
void withSpan(() => {}, { name: 'root' });
|
||||
await flushTraces();
|
||||
|
||||
expect(v4TraceInfoCalls).toHaveLength(1);
|
||||
expect(v4TraceInfoCalls[0].body.trace_id).toMatch(/^trace:\/cat\.sch\.agent\//);
|
||||
expect(v4TraceInfoCalls[0].body.trace_location.uc_table_prefix).toEqual({
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'agent',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { InMemoryTraceManager } from '../../src/core/trace_manager';
|
||||
import { Trace } from '../../src/core/entities/trace';
|
||||
import { TraceInfo } from '../../src/core/entities/trace_info';
|
||||
import { createTraceLocationFromExperimentId } from '../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../src/core/entities/trace_state';
|
||||
import { createTestSpan } from '../helper';
|
||||
import { Span } from '../../src/core/entities/span';
|
||||
import { TraceMetadataKey } from '../../src/core/constants';
|
||||
|
||||
/**
|
||||
* Helper function to create a test TraceInfo object
|
||||
*/
|
||||
function createTestTraceInfo(traceId: string): TraceInfo {
|
||||
return new TraceInfo({
|
||||
traceId,
|
||||
traceLocation: createTraceLocationFromExperimentId('1'),
|
||||
requestTime: Date.now(),
|
||||
state: TraceState.IN_PROGRESS,
|
||||
requestPreview: undefined,
|
||||
responsePreview: undefined,
|
||||
traceMetadata: {},
|
||||
tags: {},
|
||||
});
|
||||
}
|
||||
|
||||
describe('InMemoryTraceManager', () => {
|
||||
beforeEach(() => {
|
||||
// Reset the singleton instance before each test
|
||||
InMemoryTraceManager.reset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Clean up after each test
|
||||
InMemoryTraceManager.reset();
|
||||
});
|
||||
|
||||
it('should return the same instance when called multiple times', () => {
|
||||
const obj1 = InMemoryTraceManager.getInstance();
|
||||
const obj2 = InMemoryTraceManager.getInstance();
|
||||
expect(obj1).toBe(obj2);
|
||||
});
|
||||
|
||||
it('should add spans and pop traces correctly', () => {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
|
||||
// Add a new trace info
|
||||
const traceId = 'tr-1';
|
||||
const otelTraceId = '12345';
|
||||
traceManager.registerTrace(otelTraceId, createTestTraceInfo(traceId));
|
||||
|
||||
// Add a span for a new trace
|
||||
const span11 = createTestSpan('test', traceId, 'span11');
|
||||
traceManager.registerSpan(span11);
|
||||
|
||||
expect(traceManager.getTrace(traceId)).toBeTruthy();
|
||||
const trace1 = traceManager.getTrace(traceId);
|
||||
expect(trace1?.info.traceId).toBe(traceId);
|
||||
expect(trace1?.spanDict.size).toBe(1);
|
||||
|
||||
// Add more spans to the same trace
|
||||
const span111 = createTestSpan('test', traceId, 'span111');
|
||||
const span112 = createTestSpan('test', traceId, 'span112');
|
||||
traceManager.registerSpan(span111);
|
||||
traceManager.registerSpan(span112);
|
||||
expect(trace1?.spanDict.size).toBe(3);
|
||||
|
||||
// Pop the trace data
|
||||
const poppedTrace1 = traceManager.popTrace(otelTraceId);
|
||||
expect(poppedTrace1).toBeInstanceOf(Trace);
|
||||
expect(poppedTrace1?.info.traceId).toBe(traceId);
|
||||
expect(poppedTrace1?.data.spans.length).toBe(3);
|
||||
expect(traceManager.getTrace(traceId)).toBeNull();
|
||||
expect(poppedTrace1?.data.spans[0]).toBeInstanceOf(Span);
|
||||
});
|
||||
|
||||
it('should truncate the request/response preview if it exceeds the max length', () => {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const traceId = 'tr-1';
|
||||
const otelTraceId = '12345';
|
||||
traceManager.registerTrace(otelTraceId, createTestTraceInfo(traceId));
|
||||
|
||||
const span = createTestSpan('test', traceId, 'span11');
|
||||
span.setInputs('a'.repeat(5000));
|
||||
traceManager.registerSpan(span);
|
||||
|
||||
const trace = traceManager.popTrace(otelTraceId);
|
||||
expect(trace?.info.requestPreview).toHaveLength(1000);
|
||||
expect(trace?.info.traceMetadata[TraceMetadataKey.INPUTS]).toHaveLength(1000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import {
|
||||
TraceLocationType,
|
||||
createTraceLocationFromUcTablePrefix,
|
||||
getOtelSpansTableName,
|
||||
getUcLocationString,
|
||||
isUcTraceLocation,
|
||||
ucTablePrefixLocationString,
|
||||
} from '../../src/core/entities/trace_location';
|
||||
import {
|
||||
constructTraceIdV4,
|
||||
generateTraceIdV3,
|
||||
parseTraceIdV4,
|
||||
} from '../../src/core/utils/trace_id';
|
||||
import {
|
||||
DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG,
|
||||
DATABRICKS_TRACE_DESTINATION_PATH_TAG,
|
||||
DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG,
|
||||
DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG,
|
||||
ucLocationFromExperimentTags,
|
||||
} from '../../src/core/destination';
|
||||
import { TraceInfo } from '../../src/core/entities/trace_info';
|
||||
import { TraceState } from '../../src/core/entities/trace_state';
|
||||
|
||||
describe('Trace ID v4 helpers', () => {
|
||||
it('parses a v4 trace ID into location and otel trace ID', () => {
|
||||
const [location, otelId] = parseTraceIdV4(
|
||||
'trace:/cat.sch.tbl/abcdef1234567890abcdef1234567890',
|
||||
);
|
||||
expect(location).toBe('cat.sch.tbl');
|
||||
expect(otelId).toBe('abcdef1234567890abcdef1234567890');
|
||||
});
|
||||
|
||||
it('returns [null, raw] for v3 trace IDs', () => {
|
||||
const [location, otelId] = parseTraceIdV4('tr-abcdef1234567890');
|
||||
expect(location).toBeNull();
|
||||
expect(otelId).toBe('tr-abcdef1234567890');
|
||||
});
|
||||
|
||||
it('throws on a malformed v4 trace ID', () => {
|
||||
expect(() => parseTraceIdV4('trace:/onlyone')).toThrow(/Invalid trace ID format/);
|
||||
expect(() => parseTraceIdV4('trace://abc')).toThrow(/Invalid trace ID format/);
|
||||
});
|
||||
|
||||
it('constructs v4 and v3 IDs in their canonical formats', () => {
|
||||
expect(constructTraceIdV4('cat.sch.tbl', 'deadbeef')).toBe('trace:/cat.sch.tbl/deadbeef');
|
||||
expect(generateTraceIdV3('deadbeef')).toBe('tr-deadbeef');
|
||||
});
|
||||
});
|
||||
|
||||
describe('UC trace location helpers', () => {
|
||||
it('builds a UC table-prefix TraceLocation and returns its location string', () => {
|
||||
const loc = createTraceLocationFromUcTablePrefix('cat', 'sch', 'agent');
|
||||
expect(loc.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
|
||||
expect(isUcTraceLocation(loc)).toBe(true);
|
||||
expect(getUcLocationString(loc)).toBe('cat.sch.agent');
|
||||
expect(ucTablePrefixLocationString(loc.ucTablePrefix!)).toBe('cat.sch.agent');
|
||||
// Default spans table convention: `<prefix>_otel_spans` under the same
|
||||
// catalog/schema. Customers can override by setting otelSpansTableName.
|
||||
expect(getOtelSpansTableName(loc)).toBe('cat.sch.agent_otel_spans');
|
||||
});
|
||||
|
||||
it('respects an explicit backend-populated spans table name', () => {
|
||||
const loc = createTraceLocationFromUcTablePrefix('cat', 'sch', 'agent');
|
||||
loc.ucTablePrefix!.otelSpansTableName = 'cat.sch.custom_spans';
|
||||
expect(getOtelSpansTableName(loc)).toBe('cat.sch.custom_spans');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ucLocationFromExperimentTags', () => {
|
||||
it('returns null when the experiment has no Databricks trace tags', () => {
|
||||
expect(ucLocationFromExperimentTags({})).toBeNull();
|
||||
expect(ucLocationFromExperimentTags({ unrelated: 'x' })).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the destination path is not three-segment', () => {
|
||||
expect(
|
||||
ucLocationFromExperimentTags({
|
||||
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch',
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when any path segment is empty', () => {
|
||||
expect(
|
||||
ucLocationFromExperimentTags({
|
||||
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch.',
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
ucLocationFromExperimentTags({
|
||||
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: '.sch.prefix',
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('parses a UC table-prefix location and copies the spans / logs / annotations tables', () => {
|
||||
const loc = ucLocationFromExperimentTags({
|
||||
[DATABRICKS_TRACE_DESTINATION_PATH_TAG]: 'cat.sch.prefix',
|
||||
[DATABRICKS_TRACE_SPAN_STORAGE_TABLE_TAG]: 'cat.sch.prefix_otel_spans',
|
||||
[DATABRICKS_TRACE_LOG_STORAGE_TABLE_TAG]: 'cat.sch.prefix_otel_logs',
|
||||
[DATABRICKS_TRACE_ANNOTATIONS_TABLE_TAG]: 'cat.sch.prefix_annotations',
|
||||
});
|
||||
expect(loc).not.toBeNull();
|
||||
expect(loc).toEqual({
|
||||
catalogName: 'cat',
|
||||
schemaName: 'sch',
|
||||
tablePrefix: 'prefix',
|
||||
otelSpansTableName: 'cat.sch.prefix_otel_spans',
|
||||
otelLogsTableName: 'cat.sch.prefix_otel_logs',
|
||||
annotationsTableName: 'cat.sch.prefix_annotations',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('TraceInfo serialization with UC locations', () => {
|
||||
it('serializes and deserializes a UC table-prefix TraceLocation', () => {
|
||||
const info = new TraceInfo({
|
||||
traceId: 'trace:/cat.sch.tbl/abc',
|
||||
traceLocation: createTraceLocationFromUcTablePrefix('cat', 'sch', 'tbl'),
|
||||
requestTime: 1700000000000,
|
||||
state: TraceState.OK,
|
||||
tags: { user_id: 'u1', family_id: 'f1' },
|
||||
traceMetadata: {},
|
||||
});
|
||||
const json = info.toJson();
|
||||
expect(json.trace_location.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
|
||||
expect(json.trace_location.uc_table_prefix).toEqual({
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'tbl',
|
||||
});
|
||||
expect(json.tags).toEqual({ user_id: 'u1', family_id: 'f1' });
|
||||
|
||||
const roundTripped = TraceInfo.fromJson(json);
|
||||
expect(roundTripped.traceLocation.type).toBe(TraceLocationType.UC_TABLE_PREFIX);
|
||||
expect(roundTripped.traceLocation.ucTablePrefix?.catalogName).toBe('cat');
|
||||
expect(roundTripped.traceLocation.ucTablePrefix?.schemaName).toBe('sch');
|
||||
expect(roundTripped.traceLocation.ucTablePrefix?.tablePrefix).toBe('tbl');
|
||||
expect(roundTripped.tags).toEqual({ user_id: 'u1', family_id: 'f1' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import {
|
||||
artifactUriToLocalPath,
|
||||
getArtifactUriScheme,
|
||||
isLocalArtifactUri,
|
||||
toAbsoluteLocalPath,
|
||||
} from '../../../src/core/utils/artifact_uri';
|
||||
|
||||
describe('artifact_uri helpers', () => {
|
||||
describe('getArtifactUriScheme', () => {
|
||||
it('returns the scheme for URIs that have one', () => {
|
||||
expect(getArtifactUriScheme('mlflow-artifacts:/0/traces/tr-1/artifacts')).toBe(
|
||||
'mlflow-artifacts',
|
||||
);
|
||||
expect(getArtifactUriScheme('file:///tmp/a/b')).toBe('file');
|
||||
expect(getArtifactUriScheme('s3://bucket/key')).toBe('s3');
|
||||
expect(getArtifactUriScheme('S3://bucket/key')).toBe('s3');
|
||||
});
|
||||
|
||||
it('returns an empty string for bare filesystem paths', () => {
|
||||
expect(getArtifactUriScheme('/tmp/mlflow-artifacts/0/traces/tr-1/artifacts')).toBe('');
|
||||
expect(getArtifactUriScheme('relative/path')).toBe('');
|
||||
});
|
||||
|
||||
it('treats a single-letter (Windows drive) scheme as a bare path', () => {
|
||||
expect(getArtifactUriScheme('C:\\tmp\\artifacts')).toBe('');
|
||||
expect(getArtifactUriScheme('d:/tmp/artifacts')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isLocalArtifactUri', () => {
|
||||
it('treats bare paths and file:// URIs as local', () => {
|
||||
expect(isLocalArtifactUri('/tmp/mlflow-artifacts/0')).toBe(true);
|
||||
expect(isLocalArtifactUri('file:///tmp/mlflow-artifacts/0')).toBe(true);
|
||||
expect(isLocalArtifactUri('C:\\tmp\\artifacts')).toBe(true);
|
||||
});
|
||||
|
||||
it('treats mlflow-artifacts and remote schemes as non-local', () => {
|
||||
expect(isLocalArtifactUri('mlflow-artifacts:/0/traces/tr-1/artifacts')).toBe(false);
|
||||
expect(isLocalArtifactUri('s3://bucket/key')).toBe(false);
|
||||
expect(isLocalArtifactUri('gs://bucket/key')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('artifactUriToLocalPath', () => {
|
||||
it('returns bare paths unchanged', () => {
|
||||
expect(artifactUriToLocalPath('/tmp/mlflow-artifacts/0/traces/tr-1/artifacts')).toBe(
|
||||
'/tmp/mlflow-artifacts/0/traces/tr-1/artifacts',
|
||||
);
|
||||
});
|
||||
|
||||
it('decodes file:// URIs to filesystem paths', () => {
|
||||
expect(artifactUriToLocalPath('file:///tmp/mlflow-artifacts/0')).toBe(
|
||||
'/tmp/mlflow-artifacts/0',
|
||||
);
|
||||
// Percent-encoded spaces are decoded.
|
||||
expect(artifactUriToLocalPath('file:///tmp/with%20space/0')).toBe('/tmp/with space/0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toAbsoluteLocalPath', () => {
|
||||
it('returns absolute bare paths and file:// URIs unchanged', () => {
|
||||
expect(toAbsoluteLocalPath('/tmp/mlflow-artifacts/0')).toBe('/tmp/mlflow-artifacts/0');
|
||||
expect(toAbsoluteLocalPath('file:///tmp/mlflow-artifacts/0')).toBe('/tmp/mlflow-artifacts/0');
|
||||
});
|
||||
|
||||
it('throws for relative paths', () => {
|
||||
expect(() => toAbsoluteLocalPath('relative/path')).toThrow(
|
||||
/Refusing to use a relative local artifact location/,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,404 @@
|
||||
import type { HrTime } from '@opentelemetry/api';
|
||||
import {
|
||||
convertNanoSecondsToHrTime,
|
||||
convertHrTimeToNanoSeconds,
|
||||
encodeSpanIdToBase64,
|
||||
encodeTraceIdToBase64,
|
||||
decodeIdFromBase64,
|
||||
mapArgsToObject,
|
||||
} from '../../../src/core/utils';
|
||||
describe('utils', () => {
|
||||
describe('convertNanoSecondsToHrTime', () => {
|
||||
// Using table-driven tests with test.each for time conversion
|
||||
test.each([
|
||||
{
|
||||
description: 'small nanosecond values',
|
||||
input: 123456789,
|
||||
expected: [0, 123456789],
|
||||
},
|
||||
{
|
||||
description: 'values exactly at 1 second',
|
||||
input: 1_000_000_000,
|
||||
expected: [1, 0],
|
||||
},
|
||||
{
|
||||
description: 'zero',
|
||||
input: 0,
|
||||
expected: [0, 0],
|
||||
},
|
||||
])('should convert $description correctly', ({ input, expected }) => {
|
||||
const result = convertNanoSecondsToHrTime(input);
|
||||
expect(result).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should convert large nanosecond values correctly', () => {
|
||||
// Note: JavaScript loses precision with very large numbers
|
||||
// The computation seconds * 1e9 + nanosInSecond loses precision
|
||||
const seconds = 1234567890;
|
||||
const nanosInSecond = 123456789;
|
||||
const nanos = seconds * 1e9 + nanosInSecond;
|
||||
const result = convertNanoSecondsToHrTime(nanos);
|
||||
expect(result[0]).toBe(seconds);
|
||||
// Due to precision loss in the calculation above
|
||||
expect(result[1]).toBe(123456768);
|
||||
});
|
||||
|
||||
it('should handle maximum safe integer', () => {
|
||||
const maxSafeInt = Number.MAX_SAFE_INTEGER;
|
||||
const result = convertNanoSecondsToHrTime(maxSafeInt);
|
||||
expect(result[0]).toBe(Math.floor(maxSafeInt / 1e9));
|
||||
expect(result[1]).toBe(maxSafeInt % 1e9);
|
||||
});
|
||||
});
|
||||
|
||||
describe('convertHrTimeToNanoSeconds', () => {
|
||||
it('should convert HrTime with zero seconds correctly', () => {
|
||||
const hrTime: HrTime = [0, 123456789];
|
||||
const result = convertHrTimeToNanoSeconds(hrTime);
|
||||
expect(result).toBe(123456789n);
|
||||
});
|
||||
|
||||
it('should convert HrTime with seconds correctly', () => {
|
||||
const hrTime: HrTime = [5, 500000000];
|
||||
const result = convertHrTimeToNanoSeconds(hrTime);
|
||||
expect(result).toBe(5_500_000_000n);
|
||||
});
|
||||
|
||||
it('should handle zero HrTime', () => {
|
||||
const hrTime: HrTime = [0, 0];
|
||||
const result = convertHrTimeToNanoSeconds(hrTime);
|
||||
expect(result).toBe(0n);
|
||||
});
|
||||
|
||||
it('should handle large HrTime values', () => {
|
||||
const hrTime: HrTime = [1234567890, 123456789];
|
||||
const result = convertHrTimeToNanoSeconds(hrTime);
|
||||
const expected = 1234567890n * 1_000_000_000n + 123456789n;
|
||||
expect(result).toBe(expected);
|
||||
});
|
||||
|
||||
it('should be reversible with convertNanoSecondsToHrTime', () => {
|
||||
const testValues = [0, 123456789, 1_000_000_000, 5_500_000_000];
|
||||
|
||||
testValues.forEach((nanos) => {
|
||||
const hrTime = convertNanoSecondsToHrTime(nanos);
|
||||
const result = convertHrTimeToNanoSeconds(hrTime);
|
||||
expect(result).toBe(BigInt(nanos));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('encodeSpanIdToBase64', () => {
|
||||
// Using array syntax for test.each (alternative to object syntax)
|
||||
test.each([
|
||||
['standard 16-character hex span ID', '0123456789abcdef', 'ASNFZ4mrze8='],
|
||||
['short span IDs with zero padding', 'abc', 'AAAAAAAACrw='],
|
||||
['all zeros', '0000000000000000', 'AAAAAAAAAAA='],
|
||||
['all F characters', 'ffffffffffffffff', '//////////8='],
|
||||
])('should encode %s', (description, spanId, expected) => {
|
||||
const result = encodeSpanIdToBase64(spanId);
|
||||
expect(result).toBe(expected);
|
||||
});
|
||||
|
||||
it('should handle mixed case hex strings', () => {
|
||||
const spanId = 'AbCdEf1234567890';
|
||||
const result = encodeSpanIdToBase64(spanId);
|
||||
// Should work the same as lowercase
|
||||
const lowercase = encodeSpanIdToBase64('abcdef1234567890');
|
||||
expect(result).toBe(lowercase);
|
||||
});
|
||||
});
|
||||
|
||||
describe('encodeTraceIdToBase64', () => {
|
||||
it('should encode a standard 32-character hex trace ID', () => {
|
||||
const traceId = '0123456789abcdef0123456789abcdef';
|
||||
const result = encodeTraceIdToBase64(traceId);
|
||||
expect(result).toBe('ASNFZ4mrze8BI0VniavN7w==');
|
||||
});
|
||||
|
||||
it('should pad short trace IDs with zeros', () => {
|
||||
const traceId = 'abc';
|
||||
const result = encodeTraceIdToBase64(traceId);
|
||||
// Should be padded to '00000000000000000000000000000abc'
|
||||
expect(result).toBe('AAAAAAAAAAAAAAAAAAAKvA==');
|
||||
});
|
||||
|
||||
it('should handle all zeros', () => {
|
||||
const traceId = '00000000000000000000000000000000';
|
||||
const result = encodeTraceIdToBase64(traceId);
|
||||
expect(result).toBe('AAAAAAAAAAAAAAAAAAAAAA==');
|
||||
});
|
||||
|
||||
it('should handle all F characters', () => {
|
||||
const traceId = 'ffffffffffffffffffffffffffffffff';
|
||||
const result = encodeTraceIdToBase64(traceId);
|
||||
expect(result).toBe('/////////////////////w==');
|
||||
});
|
||||
|
||||
it('should handle mixed case hex strings', () => {
|
||||
const traceId = 'AbCdEf1234567890AbCdEf1234567890';
|
||||
const result = encodeTraceIdToBase64(traceId);
|
||||
// Should work the same as lowercase
|
||||
const lowercase = encodeTraceIdToBase64('abcdef1234567890abcdef1234567890');
|
||||
expect(result).toBe(lowercase);
|
||||
});
|
||||
});
|
||||
|
||||
describe('decodeIdFromBase64', () => {
|
||||
it('should decode a base64 encoded span ID back to hex', () => {
|
||||
const base64Id = 'ASNFZ4mrze8=';
|
||||
const result = decodeIdFromBase64(base64Id);
|
||||
expect(result).toBe('0123456789abcdef');
|
||||
});
|
||||
|
||||
it('should decode a base64 encoded trace ID back to hex', () => {
|
||||
const base64Id = 'ASNFZ4mrze8BI0VniavN7w==';
|
||||
const result = decodeIdFromBase64(base64Id);
|
||||
expect(result).toBe('0123456789abcdef0123456789abcdef');
|
||||
});
|
||||
|
||||
it('should handle all zeros', () => {
|
||||
const spanBase64 = 'AAAAAAAAAAA=';
|
||||
const traceBase64 = 'AAAAAAAAAAAAAAAAAAAAAA==';
|
||||
|
||||
expect(decodeIdFromBase64(spanBase64)).toBe('0000000000000000');
|
||||
expect(decodeIdFromBase64(traceBase64)).toBe('00000000000000000000000000000000');
|
||||
});
|
||||
|
||||
it('should handle all F values', () => {
|
||||
const spanBase64 = '//////////8=';
|
||||
const traceBase64 = '/////////////////////w==';
|
||||
|
||||
expect(decodeIdFromBase64(spanBase64)).toBe('ffffffffffffffff');
|
||||
expect(decodeIdFromBase64(traceBase64)).toBe('ffffffffffffffffffffffffffffffff');
|
||||
});
|
||||
|
||||
it('should be reversible with encodeSpanIdToBase64', () => {
|
||||
const testSpanIds = [
|
||||
'0123456789abcdef',
|
||||
'0000000000000000',
|
||||
'ffffffffffffffff',
|
||||
'deadbeef12345678',
|
||||
];
|
||||
|
||||
testSpanIds.forEach((spanId) => {
|
||||
const encoded = encodeSpanIdToBase64(spanId);
|
||||
const decoded = decodeIdFromBase64(encoded);
|
||||
expect(decoded).toBe(spanId);
|
||||
});
|
||||
});
|
||||
|
||||
it('should be reversible with encodeTraceIdToBase64', () => {
|
||||
const testTraceIds = [
|
||||
'0123456789abcdef0123456789abcdef',
|
||||
'00000000000000000000000000000000',
|
||||
'ffffffffffffffffffffffffffffffff',
|
||||
'deadbeef12345678deadbeef12345678',
|
||||
];
|
||||
|
||||
testTraceIds.forEach((traceId) => {
|
||||
const encoded = encodeTraceIdToBase64(traceId);
|
||||
const decoded = decodeIdFromBase64(encoded);
|
||||
expect(decoded).toBe(traceId);
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle empty base64 string', () => {
|
||||
const result = decodeIdFromBase64('');
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should handle single byte base64', () => {
|
||||
// Base64 for single byte 0xFF
|
||||
const result = decodeIdFromBase64('/w==');
|
||||
expect(result).toBe('ff');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge cases and integration', () => {
|
||||
it('should handle conversion chain for span IDs', () => {
|
||||
// Test that we can go from hex -> base64 -> hex without loss
|
||||
const originalHex = 'a1b2c3d4e5f67890';
|
||||
const base64 = encodeSpanIdToBase64(originalHex);
|
||||
const decodedHex = decodeIdFromBase64(base64);
|
||||
|
||||
expect(decodedHex).toBe(originalHex);
|
||||
});
|
||||
|
||||
it('should handle conversion chain for trace IDs', () => {
|
||||
// Test that we can go from hex -> base64 -> hex without loss
|
||||
const originalHex = 'a1b2c3d4e5f67890a1b2c3d4e5f67890';
|
||||
const base64 = encodeTraceIdToBase64(originalHex);
|
||||
const decodedHex = decodeIdFromBase64(base64);
|
||||
|
||||
expect(decodedHex).toBe(originalHex);
|
||||
});
|
||||
|
||||
it('should handle odd-length hex strings for span ID', () => {
|
||||
const oddHex = '12345'; // 5 characters
|
||||
const base64 = encodeSpanIdToBase64(oddHex);
|
||||
const decoded = decodeIdFromBase64(base64);
|
||||
|
||||
// Should be padded to '0000000000012345'
|
||||
expect(decoded).toBe('0000000000012345');
|
||||
});
|
||||
|
||||
it('should handle odd-length hex strings for trace ID', () => {
|
||||
const oddHex = '12345'; // 5 characters
|
||||
const base64 = encodeTraceIdToBase64(oddHex);
|
||||
const decoded = decodeIdFromBase64(base64);
|
||||
|
||||
// Should be padded to '00000000000000000000000000012345'
|
||||
expect(decoded).toBe('00000000000000000000000000012345');
|
||||
});
|
||||
|
||||
it('should handle very long hex strings by truncation', () => {
|
||||
// Span ID should only use first 16 chars
|
||||
const longHex = '0123456789abcdef0123456789abcdef0123456789abcdef';
|
||||
const spanBase64 = encodeSpanIdToBase64(longHex);
|
||||
const decodedSpan = decodeIdFromBase64(spanBase64);
|
||||
|
||||
// Should only encode first 16 chars
|
||||
expect(decodedSpan).toBe('0123456789abcdef');
|
||||
});
|
||||
|
||||
it('should produce consistent results across multiple calls', () => {
|
||||
const spanId = 'deadbeef12345678';
|
||||
const traceId = 'deadbeef12345678deadbeef12345678';
|
||||
|
||||
// Multiple calls should produce identical results
|
||||
const spanBase64_1 = encodeSpanIdToBase64(spanId);
|
||||
const spanBase64_2 = encodeSpanIdToBase64(spanId);
|
||||
expect(spanBase64_1).toBe(spanBase64_2);
|
||||
|
||||
const traceBase64_1 = encodeTraceIdToBase64(traceId);
|
||||
const traceBase64_2 = encodeTraceIdToBase64(traceId);
|
||||
expect(traceBase64_1).toBe(traceBase64_2);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('mapArgsToObject', () => {
|
||||
// Basic argument mapping test cases
|
||||
test.each([
|
||||
{
|
||||
description: 'regular functions',
|
||||
func: function add(a: number, b: number) {
|
||||
return a + b;
|
||||
},
|
||||
args: [5, 10],
|
||||
expected: { a: 5, b: 10 },
|
||||
},
|
||||
{
|
||||
description: 'arrow functions',
|
||||
func: (x: number, y: number) => x * y,
|
||||
args: [3, 4],
|
||||
expected: { x: 3, y: 4 },
|
||||
},
|
||||
{
|
||||
description: 'single parameter functions',
|
||||
func: (value: number) => value * 2,
|
||||
args: [7],
|
||||
expected: { value: 7 },
|
||||
},
|
||||
{
|
||||
description: 'functions with no parameters',
|
||||
func: () => 42,
|
||||
args: [],
|
||||
expected: {},
|
||||
},
|
||||
{
|
||||
description: 'functions with default parameters',
|
||||
func: function withDefaults(name: string, greeting: string = 'Hello') {
|
||||
return greeting + ' ' + name;
|
||||
},
|
||||
args: ['World'],
|
||||
expected: { name: 'World' },
|
||||
},
|
||||
{
|
||||
description: 'functions with type annotations',
|
||||
func: function typed(id: number, name: string, active: boolean) {
|
||||
return { id, name, active };
|
||||
},
|
||||
args: [123, 'John', true],
|
||||
expected: { id: 123, name: 'John', active: true },
|
||||
},
|
||||
{
|
||||
description: 'anonymous functions',
|
||||
func: function (first: string, second: number) {
|
||||
return first + second;
|
||||
},
|
||||
args: ['hello', 42],
|
||||
expected: { first: 'hello', second: 42 },
|
||||
},
|
||||
{
|
||||
description: 'fewer arguments than parameters',
|
||||
func: function threeParams(a: number, b: number, c: number) {
|
||||
return a + b + c;
|
||||
},
|
||||
args: [1, 2],
|
||||
expected: { a: 1, b: 2 },
|
||||
},
|
||||
{
|
||||
description: 'more arguments than parameters',
|
||||
func: function twoParams(a: number, b: number) {
|
||||
return a + b;
|
||||
},
|
||||
args: [1, 2, 3, 4],
|
||||
expected: { a: 1, b: 2 },
|
||||
},
|
||||
{
|
||||
description: 'complex argument types (objects, arrays)',
|
||||
func: function complex(obj: object, arr: any[], str: string) {
|
||||
return { obj, arr, str };
|
||||
},
|
||||
args: [{ key: 'value' }, [1, 2, 3], 'test'],
|
||||
expected: { obj: { key: 'value' }, arr: [1, 2, 3], str: 'test' },
|
||||
},
|
||||
{
|
||||
description: 'null and undefined arguments',
|
||||
func: function nullable(a: any, b: any, c: any) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return a + b + c;
|
||||
},
|
||||
args: [null, undefined, 'value'],
|
||||
expected: { a: null, b: undefined, c: 'value' },
|
||||
},
|
||||
{
|
||||
description: 'functions with destructured parameters gracefully',
|
||||
func: function withDestructuring({ prop }: any, normal: string) {
|
||||
return prop + normal;
|
||||
},
|
||||
args: [{ prop: 'value' }, 'normal'],
|
||||
expected: { normal: { prop: 'value' } },
|
||||
},
|
||||
{
|
||||
description: 'fallback to args array when parameter extraction fails',
|
||||
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
||||
func: new Function('return arguments[0] + arguments[1];'),
|
||||
args: [1, 2],
|
||||
expected: { args: [1, 2] },
|
||||
},
|
||||
{
|
||||
description: 'empty object for no parameters and no arguments',
|
||||
func: () => {},
|
||||
args: [],
|
||||
expected: {},
|
||||
},
|
||||
{
|
||||
description: 'edge case with only whitespace parameters',
|
||||
func: () => {},
|
||||
args: [],
|
||||
expected: {},
|
||||
},
|
||||
{
|
||||
description: 'functions with object destructuring parameters (JS/TS kwarg-only pattern)',
|
||||
func: ({ a, b }: { a: number; b: number }) => a + b,
|
||||
args: [{ a: 5, b: 10 }],
|
||||
expected: { args: [{ a: 5, b: 10 }] },
|
||||
},
|
||||
])('should handle $description', ({ func, args, expected }) => {
|
||||
const result = mapArgsToObject(func, args);
|
||||
expect(result).toEqual(expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
import { safeJsonStringify } from '../../../src/core/utils/json';
|
||||
|
||||
describe('safeJsonStringify', () => {
|
||||
it('should stringify simple values correctly', () => {
|
||||
expect(safeJsonStringify('hello')).toBe('"hello"');
|
||||
expect(safeJsonStringify(123)).toBe('123');
|
||||
expect(safeJsonStringify(true)).toBe('true');
|
||||
expect(safeJsonStringify(null)).toBe('null');
|
||||
expect(safeJsonStringify({ a: 1, b: 'test' })).toBe('{"a":1,"b":"test"}');
|
||||
expect(safeJsonStringify([1, 2, 3])).toBe('[1,2,3]');
|
||||
});
|
||||
|
||||
it('should handle circular references', () => {
|
||||
const obj: any = { name: 'test' };
|
||||
obj.self = obj;
|
||||
|
||||
const result = safeJsonStringify(obj);
|
||||
const parsed = JSON.parse(result);
|
||||
|
||||
expect(parsed.name).toBe('test');
|
||||
expect(parsed.self).toBe('[Circular]');
|
||||
});
|
||||
|
||||
it('should handle functions', () => {
|
||||
const obj = {
|
||||
name: 'test',
|
||||
fn: () => console.log('hello'),
|
||||
method: function () {
|
||||
return 42;
|
||||
},
|
||||
};
|
||||
|
||||
const result = safeJsonStringify(obj);
|
||||
const parsed = JSON.parse(result);
|
||||
|
||||
expect(parsed.name).toBe('test');
|
||||
expect(parsed.fn).toBe('[Function]');
|
||||
expect(parsed.method).toBe('[Function]');
|
||||
});
|
||||
|
||||
it('should handle undefined values', () => {
|
||||
const obj = {
|
||||
name: 'test',
|
||||
value: undefined,
|
||||
nested: { prop: undefined },
|
||||
};
|
||||
|
||||
const result = safeJsonStringify(obj);
|
||||
const parsed = JSON.parse(result);
|
||||
|
||||
expect(parsed.name).toBe('test');
|
||||
expect(parsed.value).toBe('[Undefined]');
|
||||
expect(parsed.nested.prop).toBe('[Undefined]');
|
||||
});
|
||||
|
||||
it('should handle Error objects', () => {
|
||||
const error = new Error('Test error');
|
||||
const obj = {
|
||||
status: 'failed',
|
||||
error,
|
||||
};
|
||||
|
||||
const result = safeJsonStringify(obj);
|
||||
const parsed = JSON.parse(result);
|
||||
|
||||
expect(parsed.status).toBe('failed');
|
||||
expect(parsed.error.name).toBe('Error');
|
||||
expect(parsed.error.message).toBe('Test error');
|
||||
expect(parsed.error.stack).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { ROOT_CONTEXT, SpanKind } from '@opentelemetry/api';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
ReadableSpan as OTelReadableSpan,
|
||||
Span as OTelSpan,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
|
||||
// Mock the OTLP proto exporter. The real implementation does dynamic
|
||||
// `import('http')` calls that Jest needs --experimental-vm-modules to run.
|
||||
// We assert on constructor args (url + UC table header) and assume the
|
||||
// upstream OTel library serializes correctly.
|
||||
const mockExport = jest.fn();
|
||||
const mockShutdown = jest.fn().mockResolvedValue(undefined);
|
||||
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
|
||||
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
|
||||
OTLPTraceExporter: jest
|
||||
.fn()
|
||||
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
|
||||
exporterCtors.push(cfg);
|
||||
return {
|
||||
export: (spans: unknown[], cb: (r: { code: number }) => void) => {
|
||||
mockExport(spans);
|
||||
cb({ code: 0 });
|
||||
},
|
||||
shutdown: mockShutdown,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
import { AuthProvider } from '../../src/auth';
|
||||
import { MlflowClient } from '../../src/clients/client';
|
||||
import {
|
||||
TraceLocationType,
|
||||
createTraceLocationFromUcTablePrefix,
|
||||
} from '../../src/core/entities/trace_location';
|
||||
import { TraceInfo } from '../../src/core/entities/trace_info';
|
||||
import { TraceState } from '../../src/core/entities/trace_state';
|
||||
import { DATABRICKS_UC_TABLE_HEADER, TraceMetadataKey } from '../../src/core/constants';
|
||||
|
||||
const testHost = 'https://dbc-12345.cloud.databricks.com';
|
||||
const testToken = 'test-token';
|
||||
|
||||
const mockAuthProvider: AuthProvider = {
|
||||
getHost: () => testHost,
|
||||
getHeadersProvider: () => () =>
|
||||
Promise.resolve({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${testToken}`,
|
||||
}),
|
||||
getDatabricksToken: () => testToken,
|
||||
};
|
||||
|
||||
describe('MlflowClient UC methods', () => {
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
let client: MlflowClient;
|
||||
|
||||
beforeAll(() => {
|
||||
server = setupServer();
|
||||
server.listen();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
client = new MlflowClient({ trackingUri: 'databricks', authProvider: mockAuthProvider });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
it('createTraceInfoV4 POSTs trace_info to the V4 endpoint with tags preserved', async () => {
|
||||
const location = 'cat.sch.tbl';
|
||||
const otelTraceId = 'abcdef1234567890abcdef1234567890';
|
||||
const traceInfo = new TraceInfo({
|
||||
traceId: `trace:/${location}/${otelTraceId}`,
|
||||
traceLocation: createTraceLocationFromUcTablePrefix('cat', 'sch', 'tbl'),
|
||||
requestTime: 1_700_000_000_000,
|
||||
executionDuration: 1500,
|
||||
state: TraceState.OK,
|
||||
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
|
||||
traceMetadata: { [TraceMetadataKey.SCHEMA_VERSION]: '4' },
|
||||
});
|
||||
|
||||
let capturedBody: unknown = null;
|
||||
let capturedUrl: string | null = null;
|
||||
server.use(
|
||||
http.post(
|
||||
`${testHost}/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`,
|
||||
async ({ request }) => {
|
||||
capturedUrl = request.url;
|
||||
capturedBody = await request.json();
|
||||
// Echo back a TraceInfo with the backend populating the spans table.
|
||||
return HttpResponse.json({
|
||||
trace_id: traceInfo.traceId,
|
||||
trace_location: {
|
||||
type: TraceLocationType.UC_TABLE_PREFIX,
|
||||
uc_table_prefix: {
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'tbl',
|
||||
otel_spans_table_name: 'cat.sch.tbl_otel_spans',
|
||||
},
|
||||
},
|
||||
request_time: new Date(traceInfo.requestTime).toISOString(),
|
||||
execution_duration: '1.5s',
|
||||
state: TraceState.OK,
|
||||
trace_metadata: traceInfo.traceMetadata,
|
||||
tags: traceInfo.tags,
|
||||
assessments: [],
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const returned = await client.createTraceInfoV4(location, otelTraceId, traceInfo);
|
||||
|
||||
expect(capturedUrl).toContain(
|
||||
`/api/4.0/mlflow/traces/${encodeURIComponent(location)}/${otelTraceId}/info`,
|
||||
);
|
||||
// Body is the TraceInfo JSON directly (Databricks RPC convention).
|
||||
expect(capturedBody).toMatchObject({
|
||||
trace_id: traceInfo.traceId,
|
||||
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
|
||||
trace_metadata: { [TraceMetadataKey.SCHEMA_VERSION]: '4' },
|
||||
trace_location: {
|
||||
type: TraceLocationType.UC_TABLE_PREFIX,
|
||||
uc_table_prefix: {
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'tbl',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(returned.traceLocation.ucTablePrefix?.otelSpansTableName).toBe('cat.sch.tbl_otel_spans');
|
||||
});
|
||||
|
||||
it('exportOtlpSpansToUc constructs an OTLP proto exporter with the UC table header', async () => {
|
||||
exporterCtors.length = 0;
|
||||
mockExport.mockClear();
|
||||
|
||||
const provider = new BasicTracerProvider();
|
||||
const tracer = provider.getTracer('uc-test');
|
||||
const span = tracer.startSpan('root', { kind: SpanKind.INTERNAL }, ROOT_CONTEXT) as OTelSpan;
|
||||
span.setAttribute('user.id', 'u1');
|
||||
span.end();
|
||||
|
||||
await client.exportOtlpSpansToUc(
|
||||
[span as unknown as OTelReadableSpan],
|
||||
'cat.sch.tbl_otel_spans',
|
||||
);
|
||||
|
||||
expect(exporterCtors).toHaveLength(1);
|
||||
expect(exporterCtors[0].url).toBe(`${testHost}/api/2.0/otel/v1/traces`);
|
||||
expect(exporterCtors[0].headers?.[DATABRICKS_UC_TABLE_HEADER]).toBe('cat.sch.tbl_otel_spans');
|
||||
expect(exporterCtors[0].headers?.['Authorization']).toBe(`Bearer ${testToken}`);
|
||||
expect(mockExport).toHaveBeenCalledTimes(1);
|
||||
expect((mockExport.mock.calls[0][0] as unknown[])[0]).toBe(span);
|
||||
});
|
||||
|
||||
it('exportOtlpSpansToUc short-circuits with no spans', async () => {
|
||||
// No mock registered: if it tried to hit the network the call would throw.
|
||||
await expect(client.exportOtlpSpansToUc([], 'cat.sch.tbl_otel_spans')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { context, SpanKind, ROOT_CONTEXT, trace as otelTrace } from '@opentelemetry/api';
|
||||
import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
ReadableSpan as OTelReadableSpan,
|
||||
Span as OTelSpan,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
|
||||
// Mock the OTLP proto exporter (Jest can't load its dynamic http imports).
|
||||
const exporterCtors: { url?: string; headers?: Record<string, string> }[] = [];
|
||||
jest.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
|
||||
OTLPTraceExporter: jest
|
||||
.fn()
|
||||
.mockImplementation((cfg: { url?: string; headers?: Record<string, string> }) => {
|
||||
exporterCtors.push(cfg);
|
||||
return {
|
||||
export: (_spans: unknown[], cb: (r: { code: number }) => void) => cb({ code: 0 }),
|
||||
shutdown: () => Promise.resolve(),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
import { AuthProvider } from '../../src/auth';
|
||||
import { MlflowClient } from '../../src/clients/client';
|
||||
import {
|
||||
DatabricksUCTableSpanExporter,
|
||||
DatabricksUCTableSpanProcessor,
|
||||
} from '../../src/exporters/uc_table';
|
||||
import { updateCurrentTrace } from '../../src/core/api';
|
||||
import { TraceMetadataKey, DATABRICKS_UC_TABLE_HEADER } from '../../src/core/constants';
|
||||
import { InMemoryTraceManager } from '../../src/core/trace_manager';
|
||||
import { TraceLocationType, isUcTraceLocation } from '../../src/core/entities/trace_location';
|
||||
|
||||
const testHost = 'https://dbc-12345.cloud.databricks.com';
|
||||
|
||||
const mockAuthProvider: AuthProvider = {
|
||||
getHost: () => testHost,
|
||||
getHeadersProvider: () => () =>
|
||||
Promise.resolve({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer test',
|
||||
}),
|
||||
getDatabricksToken: () => 'test',
|
||||
};
|
||||
|
||||
function makeOtelRootSpan(): OTelSpan {
|
||||
// Use a real tracer to get a real Span object whose context() works.
|
||||
const provider = new BasicTracerProvider();
|
||||
const tracer = provider.getTracer('uc-test');
|
||||
const span = tracer.startSpan('root', { kind: SpanKind.INTERNAL }, ROOT_CONTEXT) as OTelSpan;
|
||||
return span;
|
||||
}
|
||||
|
||||
describe('DatabricksUCTableSpanProcessor + Exporter end-to-end', () => {
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
let traceInfoCalls: { url: string; body: any }[];
|
||||
let contextManager: AsyncLocalStorageContextManager;
|
||||
|
||||
beforeAll(() => {
|
||||
// Default NoopContextManager makes context.with() a no-op, so
|
||||
// updateCurrentTrace can't locate the active span. Wire a real manager.
|
||||
contextManager = new AsyncLocalStorageContextManager();
|
||||
contextManager.enable();
|
||||
context.setGlobalContextManager(contextManager);
|
||||
server = setupServer();
|
||||
server.listen();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
contextManager.disable();
|
||||
context.disable();
|
||||
server.close();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
traceInfoCalls = [];
|
||||
exporterCtors.length = 0;
|
||||
server.resetHandlers();
|
||||
server.use(
|
||||
http.post(
|
||||
`${testHost}/api/4.0/mlflow/traces/:location/:otelTraceId/info`,
|
||||
async ({ request, params }) => {
|
||||
const body = (await request.json()) as any;
|
||||
traceInfoCalls.push({ url: request.url, body });
|
||||
return HttpResponse.json({
|
||||
trace_id: body.trace_id,
|
||||
trace_location: {
|
||||
type: TraceLocationType.UC_TABLE_PREFIX,
|
||||
uc_table_prefix: {
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'tbl',
|
||||
otel_spans_table_name: 'cat.sch.tbl_otel_spans',
|
||||
},
|
||||
},
|
||||
request_time: body.request_time,
|
||||
execution_duration: body.execution_duration,
|
||||
state: body.state,
|
||||
trace_metadata: body.trace_metadata,
|
||||
tags: body.tags,
|
||||
assessments: [],
|
||||
});
|
||||
void params;
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('persists tags set via updateCurrentTrace through the V4 endpoint and uploads spans via OTLP', async () => {
|
||||
const client = new MlflowClient({
|
||||
trackingUri: 'databricks',
|
||||
authProvider: mockAuthProvider,
|
||||
});
|
||||
const location = { catalogName: 'cat', schemaName: 'sch', tablePrefix: 'tbl' };
|
||||
const exporter = new DatabricksUCTableSpanExporter(client);
|
||||
const processor = new DatabricksUCTableSpanProcessor(exporter, location);
|
||||
|
||||
const span = makeOtelRootSpan();
|
||||
processor.onStart(span, context.active());
|
||||
|
||||
const otelTraceId = span.spanContext().traceId;
|
||||
const mgr = InMemoryTraceManager.getInstance();
|
||||
const mlflowTraceId = mgr.getMlflowTraceIdFromOtelId(otelTraceId)!;
|
||||
expect(mlflowTraceId.startsWith('trace:/cat.sch.tbl/')).toBe(true);
|
||||
|
||||
// Drive the public `updateCurrentTrace` API with the span as the OTel
|
||||
// active context — the UC processor must be the one mutating the in-memory
|
||||
// TraceInfo so the tags reach the V4 endpoint.
|
||||
context.with(otelTrace.setSpan(context.active(), span), () => {
|
||||
updateCurrentTrace({
|
||||
tags: { user_id: 'u1', family_id: 'f1', conversation_id: 'c1' },
|
||||
});
|
||||
});
|
||||
|
||||
const trace = mgr.getTrace(mlflowTraceId)!;
|
||||
expect(isUcTraceLocation(trace.info.traceLocation)).toBe(true);
|
||||
expect(trace.info.traceMetadata[TraceMetadataKey.SCHEMA_VERSION]).toBe('4');
|
||||
expect(trace.info.tags).toMatchObject({
|
||||
user_id: 'u1',
|
||||
family_id: 'f1',
|
||||
conversation_id: 'c1',
|
||||
});
|
||||
|
||||
span.end();
|
||||
processor.onEnd(span as unknown as OTelReadableSpan);
|
||||
await processor.forceFlush();
|
||||
|
||||
expect(traceInfoCalls).toHaveLength(1);
|
||||
// The Databricks RPC convention sends the TraceInfo JSON directly as the
|
||||
// body (not wrapped in `{ trace_info: ... }`), so the body IS the trace info.
|
||||
const posted = traceInfoCalls[0].body;
|
||||
expect(posted.trace_id).toBe(mlflowTraceId);
|
||||
expect(posted.tags).toEqual({ user_id: 'u1', family_id: 'f1', conversation_id: 'c1' });
|
||||
expect(posted.trace_metadata[TraceMetadataKey.SCHEMA_VERSION]).toBe('4');
|
||||
expect(posted.trace_location.uc_table_prefix).toEqual({
|
||||
catalog_name: 'cat',
|
||||
schema_name: 'sch',
|
||||
table_prefix: 'tbl',
|
||||
});
|
||||
|
||||
expect(exporterCtors).toHaveLength(1);
|
||||
expect(exporterCtors[0].url).toBe(`${testHost}/api/2.0/otel/v1/traces`);
|
||||
expect(exporterCtors[0].headers?.[DATABRICKS_UC_TABLE_HEADER]).toBe('cat.sch.tbl_otel_spans');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
import * as fsPromises from 'node:fs/promises';
|
||||
import { mkdtemp, readFile, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
|
||||
import { getWalPath } from '../../../src/exporters/wal/paths';
|
||||
import { appendTombstone, readPending } from '../../../src/exporters/wal/storage';
|
||||
import type { WalRecord } from '../../../src/exporters/wal/types';
|
||||
|
||||
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
|
||||
return {
|
||||
id: `wal-${idSuffix}`,
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
traceInfo: { trace_id: `t-${idSuffix}` },
|
||||
traceData: { spans: [] },
|
||||
attempts: 0,
|
||||
nextAttemptAt: 0,
|
||||
createdAt: Date.now(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function spyOnFileHandleSync(walDir: string): Promise<jest.SpyInstance> {
|
||||
let spy: jest.SpyInstance | undefined;
|
||||
try {
|
||||
const probeFh = await fsPromises.open(join(walDir, '.probe'), 'w');
|
||||
const fdProto = Object.getPrototypeOf(probeFh) as { sync?: () => Promise<void> };
|
||||
await probeFh.close();
|
||||
expect(typeof fdProto.sync).toBe('function');
|
||||
spy = jest.spyOn(fdProto as { sync: () => Promise<void> }, 'sync');
|
||||
return spy;
|
||||
} catch (err) {
|
||||
// `spy` is undefined for any throw before the install, so the
|
||||
// optional chain is a no-op; if the install succeeded and a later
|
||||
// line throws, this restores the prototype before propagating.
|
||||
spy?.mockRestore();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
describe('wal/batching_writer', () => {
|
||||
let walDir: string;
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-batch-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('persists a single submitted record after the ack', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
const record = makeRecord('s1');
|
||||
await writer.submit(record);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(JSON.parse(lines[0])).toEqual({ type: 'append', record });
|
||||
});
|
||||
|
||||
it('coalesces concurrent submits into a single fsync (group commit)', async () => {
|
||||
const syncSpy = await spyOnFileHandleSync(walDir);
|
||||
try {
|
||||
const writer = new BatchingWriter();
|
||||
const records = Array.from({ length: 25 }, (_, i) =>
|
||||
makeRecord(i.toString().padStart(2, '0')),
|
||||
);
|
||||
|
||||
const syncsBefore = syncSpy.mock.calls.length;
|
||||
// Fire all submits in the same tick before awaiting any of them.
|
||||
await Promise.all(records.map((r) => writer.submit(r)));
|
||||
const syncsAfter = syncSpy.mock.calls.length;
|
||||
|
||||
// Allow at most one extra fsync above the one we expect, in case
|
||||
// a stray tick splits the batch in two; the regression we care
|
||||
// about is the "N fsyncs for N submits" shape.
|
||||
expect(syncsAfter - syncsBefore).toBeLessThanOrEqual(2);
|
||||
expect(syncsAfter - syncsBefore).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(records.length);
|
||||
const ids = lines.map((l) => (JSON.parse(l) as { record: WalRecord }).record.id).sort();
|
||||
expect(ids).toEqual(records.map((r) => r.id).sort());
|
||||
} finally {
|
||||
syncSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('resolves submit() only after the byte is durable', async () => {
|
||||
// The post-submit file read must observe the record — i.e. the
|
||||
// ack-after-fsync invariant. If submit resolved before write+sync
|
||||
// completed this read would race and occasionally come up empty.
|
||||
const writer = new BatchingWriter();
|
||||
const record = makeRecord('durable');
|
||||
await writer.submit(record);
|
||||
|
||||
const contents = await readFile(getWalPath(), 'utf8');
|
||||
expect(contents).toContain(`"id":"${record.id}"`);
|
||||
});
|
||||
|
||||
it('produces well-formed lines when batched submits interleave with direct writers', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
const r1 = makeRecord('mixed-1');
|
||||
const r2 = makeRecord('mixed-2');
|
||||
|
||||
const submitR1 = writer.submit(r1);
|
||||
const tombstone = appendTombstone(r1.id);
|
||||
const submitR2 = writer.submit(r2);
|
||||
await Promise.all([submitR1, tombstone, submitR2]);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
// Two appends + one tombstone, none truncated, all valid JSON.
|
||||
expect(lines).toHaveLength(3);
|
||||
for (const line of lines) {
|
||||
expect(() => {
|
||||
JSON.parse(line);
|
||||
}).not.toThrow();
|
||||
}
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending.map((r) => r.id)).toContain(r2.id);
|
||||
});
|
||||
|
||||
it('persists a single submitted tombstone after the ack', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
const record = makeRecord('t-single');
|
||||
await writer.submit(record);
|
||||
await writer.submitTombstone(record.id);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(2);
|
||||
expect(JSON.parse(lines[1])).toEqual({ type: 'tombstone', id: record.id });
|
||||
|
||||
// Tombstone has shadowed the append: nothing should remain pending.
|
||||
const pending = await readPending();
|
||||
expect(pending.find((r) => r.id === record.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resolves submitTombstone() only after the byte is durable', async () => {
|
||||
// Symmetric to the submit() ack-after-fsync test: the post-ack file
|
||||
// read must observe the tombstone, otherwise tombstones could be
|
||||
// ack'd before being durable and a daemon crash mid-batch would
|
||||
// resurrect already-uploaded records on the next pass.
|
||||
const writer = new BatchingWriter();
|
||||
const id = 'wal-t-durable';
|
||||
await writer.submitTombstone(id);
|
||||
|
||||
const contents = await readFile(getWalPath(), 'utf8');
|
||||
expect(contents).toContain(`"id":"${id}"`);
|
||||
expect(contents).toContain('"type":"tombstone"');
|
||||
});
|
||||
|
||||
it('coalesces appends and tombstones into a single fsync when submitted in the same tick', async () => {
|
||||
// The whole point of routing tombstones through the BatchingWriter:
|
||||
// a tombstone storm from the upload loop should batch with concurrent
|
||||
// hook submits instead of paying its own fsync per call.
|
||||
const syncSpy = await spyOnFileHandleSync(walDir);
|
||||
try {
|
||||
const writer = new BatchingWriter();
|
||||
const records = Array.from({ length: 10 }, (_, i) => makeRecord(`mix-a-${i}`));
|
||||
const tombstoneIds = Array.from({ length: 10 }, (_, i) => `mix-t-${i}`);
|
||||
|
||||
const syncsBefore = syncSpy.mock.calls.length;
|
||||
await Promise.all([
|
||||
...records.map((r) => writer.submit(r)),
|
||||
...tombstoneIds.map((id) => writer.submitTombstone(id)),
|
||||
]);
|
||||
const syncsAfter = syncSpy.mock.calls.length;
|
||||
|
||||
// Same shape guarantee as the append-only group-commit test: a
|
||||
// stray tick may split the batch into two, but never N fsyncs for
|
||||
// N submits.
|
||||
expect(syncsAfter - syncsBefore).toBeLessThanOrEqual(2);
|
||||
expect(syncsAfter - syncsBefore).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(records.length + tombstoneIds.length);
|
||||
|
||||
const tombstoneLines = lines.filter((l) => l.includes('"type":"tombstone"'));
|
||||
const appendLines = lines.filter((l) => l.includes('"type":"append"'));
|
||||
expect(tombstoneLines).toHaveLength(tombstoneIds.length);
|
||||
expect(appendLines).toHaveLength(records.length);
|
||||
} finally {
|
||||
syncSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import {
|
||||
clearClientCache,
|
||||
clearClientForUri,
|
||||
clientForUri,
|
||||
} from '../../../src/exporters/wal/clients';
|
||||
import { MlflowClient } from '../../../src/clients/client';
|
||||
|
||||
describe('wal/clients', () => {
|
||||
beforeEach(() => {
|
||||
clearClientCache();
|
||||
});
|
||||
|
||||
it('returns the same MlflowClient instance for repeat calls with the same URI', () => {
|
||||
const a = clientForUri('http://localhost:5000');
|
||||
const b = clientForUri('http://localhost:5000');
|
||||
expect(a).toBeInstanceOf(MlflowClient);
|
||||
expect(b).toBe(a);
|
||||
});
|
||||
|
||||
it('returns distinct MlflowClient instances for different URIs', () => {
|
||||
const local = clientForUri('http://localhost:5000');
|
||||
const remote = clientForUri('http://other-host:5000');
|
||||
expect(local).not.toBe(remote);
|
||||
expect(local.getHost()).toBe('http://localhost:5000');
|
||||
expect(remote.getHost()).toBe('http://other-host:5000');
|
||||
});
|
||||
|
||||
it('rebuilds the client after the cache is cleared', () => {
|
||||
const before = clientForUri('http://localhost:5000');
|
||||
clearClientCache();
|
||||
const after = clientForUri('http://localhost:5000');
|
||||
expect(after).not.toBe(before);
|
||||
expect(after).toBeInstanceOf(MlflowClient);
|
||||
});
|
||||
|
||||
it('isolates state for different tracking URIs after a clear', () => {
|
||||
const a1 = clientForUri('http://a:5000');
|
||||
const b1 = clientForUri('http://b:5000');
|
||||
clearClientCache();
|
||||
const a2 = clientForUri('http://a:5000');
|
||||
const b2 = clientForUri('http://b:5000');
|
||||
expect(a2).not.toBe(a1);
|
||||
expect(b2).not.toBe(b1);
|
||||
expect(a2).not.toBe(b2);
|
||||
});
|
||||
|
||||
describe('clearClientForUri', () => {
|
||||
it('evicts only the targeted URI and rebuilds it on next access', () => {
|
||||
// Targeted eviction is the upload loop's auth-error retry path:
|
||||
// we want the *specific* stale client gone, not every other URI's
|
||||
// working cache entry blown away as collateral damage.
|
||||
const a = clientForUri('http://a:5000');
|
||||
const b = clientForUri('http://b:5000');
|
||||
|
||||
clearClientForUri('http://a:5000');
|
||||
|
||||
const aAfter = clientForUri('http://a:5000');
|
||||
const bAfter = clientForUri('http://b:5000');
|
||||
|
||||
expect(aAfter).not.toBe(a);
|
||||
expect(bAfter).toBe(b);
|
||||
});
|
||||
|
||||
it('is a silent no-op for a URI that was never cached', () => {
|
||||
// The upload loop calls clearClientForUri unconditionally before
|
||||
// re-resolving via the factory; it must not throw if the cache
|
||||
// entry was already evicted (e.g. by a prior auth-retry in a
|
||||
// sibling group, or by clearClientCache() during shutdown).
|
||||
expect(() => clearClientForUri('http://never-cached:5000')).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,825 @@
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
|
||||
import {
|
||||
acquireLock,
|
||||
backoff,
|
||||
log,
|
||||
processBatch,
|
||||
pruneOldLogs,
|
||||
releaseLock,
|
||||
runBatchLoop,
|
||||
uploadOne,
|
||||
} from '../../../src/exporters/wal/daemon';
|
||||
import {
|
||||
getDaemonLogPath,
|
||||
getDeadLetterPath,
|
||||
getLockSocketPath,
|
||||
getPidLockPath,
|
||||
getWalPath,
|
||||
} from '../../../src/exporters/wal/paths';
|
||||
import { appendRecord, readPending } from '../../../src/exporters/wal/storage';
|
||||
import type { WalRecord } from '../../../src/exporters/wal/types';
|
||||
import type { MlflowClient } from '../../../src/clients/client';
|
||||
import { MlflowHttpError } from '../../../src/clients/utils';
|
||||
import type { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import type { TraceData } from '../../../src/core/entities/trace_data';
|
||||
|
||||
/**
|
||||
* Spawn a trivial child, wait for it to exit, and return its now-freed
|
||||
* pid. Used so the integration tests can seed a stale PID lockfile
|
||||
* with a pid that is guaranteed to fail `process.kill(pid, 0)` with
|
||||
* ESRCH and trigger the stale-recovery branch in `tryAcquirePidLock`.
|
||||
*/
|
||||
function spawnAndReapPid(): Promise<number> {
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['-e', 'process.exit(0)']);
|
||||
const pid = child.pid;
|
||||
if (pid == null) {
|
||||
reject(new Error('child spawn produced no pid'));
|
||||
return;
|
||||
}
|
||||
child.once('exit', () => resolve(pid));
|
||||
child.once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function makeRecord(overrides: Partial<WalRecord> = {}): WalRecord {
|
||||
// Fields shaped so TraceInfo.fromJson / TraceData.fromJson succeed:
|
||||
// execution_duration is the "Ns" string form expected by the parser
|
||||
// (a raw number throws because of `.replace`), spans is an empty array.
|
||||
const base: WalRecord = {
|
||||
id: 'row-1',
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
traceInfo: {
|
||||
trace_id: 'tr-1',
|
||||
trace_location: { type: 'MLFLOW_EXPERIMENT', mlflow_experiment: { experiment_id: '0' } },
|
||||
request_time: 0,
|
||||
execution_duration: '0s',
|
||||
state: 'OK',
|
||||
request_preview: '',
|
||||
response_preview: '',
|
||||
client_request_id: '',
|
||||
tags: {},
|
||||
trace_metadata: {},
|
||||
assessments: [],
|
||||
},
|
||||
traceData: { spans: [] },
|
||||
attempts: 0,
|
||||
nextAttemptAt: 0,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
return { ...base, ...overrides };
|
||||
}
|
||||
|
||||
function makeClient(opts: {
|
||||
createTrace?: jest.Mock;
|
||||
uploadTraceData?: jest.Mock;
|
||||
exportOtlpSpans?: jest.Mock;
|
||||
}): MlflowClient {
|
||||
return {
|
||||
createTrace:
|
||||
opts.createTrace ?? jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info)),
|
||||
uploadTraceData:
|
||||
opts.uploadTraceData ??
|
||||
jest.fn().mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve()),
|
||||
exportOtlpSpans:
|
||||
opts.exportOtlpSpans ??
|
||||
jest
|
||||
.fn()
|
||||
.mockImplementation((_experimentId: string, _bytes: Uint8Array) => Promise.resolve()),
|
||||
getHost: () => 'http://localhost:5000',
|
||||
} as unknown as MlflowClient;
|
||||
}
|
||||
|
||||
describe('wal/daemon', () => {
|
||||
let walDir: string;
|
||||
// Capture the developer's pre-test value so we restore (not just unset)
|
||||
// in afterEach. Mirrors the pattern in paths.test.ts so running
|
||||
// `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override for later
|
||||
// tests in the same worker.
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-daemon-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('backoff', () => {
|
||||
it('doubles on each attempt until the cap', () => {
|
||||
expect(backoff(1)).toBe(2_000);
|
||||
expect(backoff(2)).toBe(4_000);
|
||||
expect(backoff(3)).toBe(8_000);
|
||||
expect(backoff(8)).toBe(256_000);
|
||||
});
|
||||
|
||||
it('caps at 5 minutes', () => {
|
||||
expect(backoff(9)).toBe(300_000);
|
||||
expect(backoff(20)).toBe(300_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('log', () => {
|
||||
it('appends a timestamped line to the daemon log file', () => {
|
||||
log('hello');
|
||||
log('world');
|
||||
const path = getDaemonLogPath();
|
||||
// daemon.log is daily-rotated; assert the suffix is present so a
|
||||
// regression to a single unbounded file is caught here too.
|
||||
expect(path).toMatch(/[/\\]daemon\.log\.\d{4}-\d{2}-\d{2}$/);
|
||||
const contents = readFileSync(path, 'utf8');
|
||||
const lines = contents.trim().split('\n');
|
||||
expect(lines).toHaveLength(2);
|
||||
expect(lines[0]).toMatch(/^\[\d{4}-\d{2}-\d{2}T.+Z\] \[pid=\d+\] hello$/);
|
||||
expect(lines[1]).toMatch(/world$/);
|
||||
});
|
||||
|
||||
it('swallows write errors silently', () => {
|
||||
// Point the log path at a directory that doesn't exist by deleting
|
||||
// the WAL dir mid-test. appendFileSync will throw internally; log()
|
||||
// must not propagate.
|
||||
process.env.MLFLOW_WAL_DIR = join(walDir, 'nope', 'deeper');
|
||||
expect(() => log('ignored')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('acquireLock / releaseLock', () => {
|
||||
it('binds the lock socket and releases cleanly', async () => {
|
||||
const handle = await acquireLock();
|
||||
expect(handle).not.toBeNull();
|
||||
expect(existsSync(getLockSocketPath())).toBe(true);
|
||||
await releaseLock(handle!.server, handle!.pidLock);
|
||||
expect(existsSync(getLockSocketPath())).toBe(false);
|
||||
});
|
||||
|
||||
it('returns null when another daemon already holds the lock', async () => {
|
||||
const first = await acquireLock();
|
||||
expect(first).not.toBeNull();
|
||||
try {
|
||||
const second = await acquireLock();
|
||||
expect(second).toBeNull();
|
||||
} finally {
|
||||
await releaseLock(first!.server, first!.pidLock);
|
||||
}
|
||||
});
|
||||
|
||||
it('unlinks a stale POSIX socket file before binding', async () => {
|
||||
if (process.platform === 'win32') {
|
||||
return; // Named pipes have no stale-file concept.
|
||||
}
|
||||
// Simulate a previously-crashed daemon by writing a regular file
|
||||
// at the socket path. acquireLock should unlink it on its way to
|
||||
// a fresh bind.
|
||||
await writeFile(getLockSocketPath(), 'stale');
|
||||
const handle = await acquireLock();
|
||||
try {
|
||||
expect(handle).not.toBeNull();
|
||||
} finally {
|
||||
await releaseLock(handle!.server, handle!.pidLock);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// POSIX-only PID-lock integration. On Windows the named pipe is
|
||||
// kernel-refcounted and acquireLock returns `pidLock: null`, so
|
||||
// there's nothing for these tests to assert. Use a describe-level
|
||||
// skip rather than per-`it` guards so the suite reports honestly on
|
||||
// CI ("skipped" instead of "passed in 0ms").
|
||||
const describePosix = process.platform === 'win32' ? describe.skip : describe;
|
||||
|
||||
describePosix('acquireLock / releaseLock — PID lock integration', () => {
|
||||
it('creates the pid lockfile with our pid and removes it on release', async () => {
|
||||
const handle = await acquireLock();
|
||||
try {
|
||||
expect(handle).not.toBeNull();
|
||||
expect(handle!.pidLock).not.toBeNull();
|
||||
// Lockfile must exist alongside the socket and record our pid
|
||||
// exactly — content-stamped release relies on the exact match.
|
||||
expect(existsSync(getPidLockPath())).toBe(true);
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await releaseLock(handle!.server, handle!.pidLock);
|
||||
}
|
||||
expect(existsSync(getPidLockPath())).toBe(false);
|
||||
});
|
||||
|
||||
it('recovers from a stale pid lockfile left behind by a crashed daemon', async () => {
|
||||
// A previously-crashed daemon's pid file with a now-dead pid.
|
||||
// tryAcquirePidLock's liveness probe (`process.kill(pid, 0)`)
|
||||
// should return ESRCH, the content-stamped unlink should clear
|
||||
// the file, and acquireLock should proceed to bind normally.
|
||||
const deadPid = await spawnAndReapPid();
|
||||
await writeFile(getPidLockPath(), `${deadPid}\n`);
|
||||
const handle = await acquireLock();
|
||||
try {
|
||||
expect(handle).not.toBeNull();
|
||||
expect(handle!.pidLock).not.toBeNull();
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await releaseLock(handle!.server, handle!.pidLock);
|
||||
}
|
||||
});
|
||||
|
||||
it('concedes when an alive process already holds the pid lockfile', async () => {
|
||||
await writeFile(getPidLockPath(), `${process.pid}\n`);
|
||||
const handle = await acquireLock();
|
||||
expect(handle).toBeNull();
|
||||
// Lockfile must remain untouched when we concede.
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
// And the socket must not have been bound.
|
||||
expect(existsSync(getLockSocketPath())).toBe(false);
|
||||
});
|
||||
|
||||
it('serializes concurrent acquireLock calls to a single winner', async () => {
|
||||
// Both calls run inside the same Node event loop, but the
|
||||
// `link()` syscall inside tryAcquirePidLock still goes through
|
||||
// libuv's fs threadpool and the kernel's inode lock — so exactly
|
||||
// one wins the atomic create, and the other reads our (alive)
|
||||
// pid and concedes via the live-holder branch.
|
||||
const [first, second] = await Promise.all([acquireLock(), acquireLock()]);
|
||||
try {
|
||||
const acquired = [first, second].filter((h) => h != null);
|
||||
const conceded = [first, second].filter((h) => h == null);
|
||||
expect(acquired).toHaveLength(1);
|
||||
expect(conceded).toHaveLength(1);
|
||||
// The winner owns the socket; the loser never bound anything.
|
||||
expect(existsSync(getLockSocketPath())).toBe(true);
|
||||
// The winner's pid is recorded in the lockfile.
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
if (first != null) {
|
||||
await releaseLock(first.server, first.pidLock);
|
||||
}
|
||||
if (second != null) {
|
||||
await releaseLock(second.server, second.pidLock);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('releases the pid lockfile when both socket and pid lock are unlinked together', async () => {
|
||||
// Verifies the teardown order is symmetric to acquisition:
|
||||
// releaseLock must remove the socket file AND the pid lockfile
|
||||
// so a subsequent acquireLock starts from a fully-clean slate.
|
||||
const handle = await acquireLock();
|
||||
expect(handle).not.toBeNull();
|
||||
expect(existsSync(getLockSocketPath())).toBe(true);
|
||||
expect(existsSync(getPidLockPath())).toBe(true);
|
||||
await releaseLock(handle!.server, handle!.pidLock);
|
||||
expect(existsSync(getLockSocketPath())).toBe(false);
|
||||
expect(existsSync(getPidLockPath())).toBe(false);
|
||||
});
|
||||
|
||||
it('allows a fresh acquire after a clean release', async () => {
|
||||
// End-to-end smoke for the acquire → release → reacquire cycle.
|
||||
// Catches regressions where releaseLock leaves stale state that
|
||||
// the next acquireLock cannot recover from.
|
||||
const first = await acquireLock();
|
||||
expect(first).not.toBeNull();
|
||||
await releaseLock(first!.server, first!.pidLock);
|
||||
const second = await acquireLock();
|
||||
try {
|
||||
expect(second).not.toBeNull();
|
||||
} finally {
|
||||
await releaseLock(second!.server, second!.pidLock);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadOne', () => {
|
||||
it('uploads the trace and tombstones the WAL row on success', async () => {
|
||||
const createTrace = jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info));
|
||||
const uploadTraceData = jest
|
||||
.fn()
|
||||
.mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve());
|
||||
const client = makeClient({ createTrace, uploadTraceData });
|
||||
|
||||
const record = makeRecord({ id: 'row-success' });
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
expect(createTrace).toHaveBeenCalledTimes(1);
|
||||
expect(uploadTraceData).toHaveBeenCalledTimes(1);
|
||||
|
||||
// After tombstone the record should not be in the live set.
|
||||
const pending = await readPending();
|
||||
expect(pending.find((r) => r.id === 'row-success')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('logs captured OTLP spans to the DB when record.otlpSpans is present', async () => {
|
||||
const exportOtlpSpans = jest
|
||||
.fn()
|
||||
.mockImplementation((_experimentId: string, _bytes: Uint8Array) => Promise.resolve());
|
||||
const client = makeClient({ exportOtlpSpans });
|
||||
|
||||
const otlpBytes = Buffer.from([0x0a, 0x01, 0x02, 0x03]);
|
||||
const record = makeRecord({
|
||||
id: 'row-otlp',
|
||||
experimentId: 'exp-7',
|
||||
otlpSpans: otlpBytes.toString('base64'),
|
||||
});
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
expect(exportOtlpSpans).toHaveBeenCalledTimes(1);
|
||||
const [experimentId, bytes] = exportOtlpSpans.mock.calls[0] as [string, Uint8Array];
|
||||
expect(experimentId).toBe('exp-7');
|
||||
expect(Buffer.from(bytes).equals(otlpBytes)).toBe(true);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending.find((r) => r.id === 'row-otlp')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips span logging when record.otlpSpans is absent', async () => {
|
||||
const exportOtlpSpans = jest.fn();
|
||||
const client = makeClient({ exportOtlpSpans });
|
||||
|
||||
const record = makeRecord({ id: 'row-no-otlp' });
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
expect(exportOtlpSpans).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('tombstones the record even when span logging fails (non-fatal)', async () => {
|
||||
const exportOtlpSpans = jest.fn().mockRejectedValue(new Error('501 not supported'));
|
||||
const createTrace = jest.fn().mockImplementation((info: TraceInfo) => Promise.resolve(info));
|
||||
const uploadTraceData = jest
|
||||
.fn()
|
||||
.mockImplementation((_info: TraceInfo, _data: TraceData) => Promise.resolve());
|
||||
const client = makeClient({ createTrace, uploadTraceData, exportOtlpSpans });
|
||||
|
||||
const record = makeRecord({
|
||||
id: 'row-otlp-fail',
|
||||
otlpSpans: Buffer.from([0x0a, 0x01]).toString('base64'),
|
||||
});
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
expect(createTrace).toHaveBeenCalledTimes(1);
|
||||
expect(uploadTraceData).toHaveBeenCalledTimes(1);
|
||||
expect(exportOtlpSpans).toHaveBeenCalledTimes(1);
|
||||
|
||||
// The trace info + artifact already uploaded, so a span-log failure must
|
||||
// not retry or dead-letter: the row is tombstoned and nothing re-appended.
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('re-appends a fresh row with bumped attempts on transient failure', async () => {
|
||||
const createTrace = jest.fn().mockRejectedValue(new Error('5xx'));
|
||||
const client = makeClient({ createTrace });
|
||||
|
||||
const record = makeRecord({ id: 'row-retry', attempts: 0 });
|
||||
const before = Date.now();
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
const after = Date.now();
|
||||
|
||||
const pending = await readPending();
|
||||
const retry = pending.find((r) => r.id !== 'row-retry');
|
||||
expect(pending.find((r) => r.id === 'row-retry')).toBeUndefined();
|
||||
expect(retry).toBeDefined();
|
||||
expect(retry!.attempts).toBe(1);
|
||||
// backoff(1) = 2_000 ms; the new schedule must be in [before+2s, after+2s].
|
||||
expect(retry!.nextAttemptAt).toBeGreaterThanOrEqual(before + 2_000);
|
||||
expect(retry!.nextAttemptAt).toBeLessThanOrEqual(after + 2_000);
|
||||
// firstAttemptAt is anchored on the first daemon touch and pinned
|
||||
// onto the retry row so the wall-clock retry budget is measured
|
||||
// from a stable point even across multiple retries / restarts.
|
||||
expect(retry!.firstAttemptAt).toBeGreaterThanOrEqual(before);
|
||||
expect(retry!.firstAttemptAt).toBeLessThanOrEqual(after);
|
||||
});
|
||||
|
||||
it('preserves firstAttemptAt across retries (anchor survives daemon respawn)', async () => {
|
||||
const createTrace = jest.fn().mockRejectedValue(new Error('5xx'));
|
||||
const client = makeClient({ createTrace });
|
||||
|
||||
// A record a previous daemon already tried twice. A new daemon
|
||||
// picking it up must preserve the original anchor, not reset it
|
||||
// — otherwise a record could indefinitely outlive the retry
|
||||
// budget by piggybacking on each daemon respawn's fresh clock.
|
||||
const originalAnchor = Date.now() - 50_000;
|
||||
const record = makeRecord({
|
||||
id: 'row-restart',
|
||||
attempts: 2,
|
||||
firstAttemptAt: originalAnchor,
|
||||
});
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
const pending = await readPending();
|
||||
const retry = pending.find((r) => r.id !== 'row-restart')!;
|
||||
expect(retry.firstAttemptAt).toBe(originalAnchor);
|
||||
expect(retry.attempts).toBe(3);
|
||||
});
|
||||
|
||||
it('dead-letters the record when elapsed retry time exceeds RETRY_TIMEOUT_MS', async () => {
|
||||
const createTrace = jest.fn().mockRejectedValue(new Error('still broken'));
|
||||
const client = makeClient({ createTrace });
|
||||
|
||||
// Backdate firstAttemptAt by more than the default 500s budget,
|
||||
// so the next backoff (~2s) plus the elapsed wall clock blows
|
||||
// through the deadline and the record dead-letters immediately
|
||||
// regardless of the attempts counter.
|
||||
const longAgo = Date.now() - 600_000;
|
||||
const record = makeRecord({
|
||||
id: 'row-dlq',
|
||||
attempts: 3,
|
||||
firstAttemptAt: longAgo,
|
||||
});
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(0);
|
||||
|
||||
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
|
||||
const failedLines = failedContents.trim().split('\n');
|
||||
expect(failedLines).toHaveLength(1);
|
||||
const parsed = JSON.parse(failedLines[0]) as {
|
||||
type: 'append';
|
||||
record: WalRecord;
|
||||
};
|
||||
expect(parsed.type).toBe('append');
|
||||
expect(parsed.record.id).toBe('row-dlq');
|
||||
});
|
||||
|
||||
// The credential-refresh-on-401/403 retry path. The cached
|
||||
// MlflowClient holds an AuthProvider snapshot from daemon start; a
|
||||
// token rotation must surface as a transient failure that the daemon
|
||||
// recovers from automatically rather than dead-letters forever.
|
||||
describe('auth-refresh retry', () => {
|
||||
it.each([
|
||||
[401, 'Unauthorized'],
|
||||
[403, 'Forbidden'],
|
||||
] as const)(
|
||||
'evicts the cached client and retries once after a %i, succeeding the second time',
|
||||
async (status, statusText) => {
|
||||
const staleCreateTrace = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new MlflowHttpError(status, statusText, ''));
|
||||
const freshCreateTrace = jest
|
||||
.fn()
|
||||
.mockImplementation((info: TraceInfo) => Promise.resolve(info));
|
||||
const freshUploadTraceData = jest.fn().mockResolvedValue(undefined);
|
||||
const stale = makeClient({ createTrace: staleCreateTrace });
|
||||
const fresh = makeClient({
|
||||
createTrace: freshCreateTrace,
|
||||
uploadTraceData: freshUploadTraceData,
|
||||
});
|
||||
const factory = jest.fn(() => fresh);
|
||||
|
||||
const record = makeRecord({ id: `row-${status}-retry` });
|
||||
await uploadOne(record, stale, new BatchingWriter(), factory);
|
||||
|
||||
// First attempt used the stale client and got the auth error;
|
||||
// the auth-refresh branch then asked the factory for a fresh
|
||||
// client (exactly once, for this URI) and the retry pushed
|
||||
// through successfully.
|
||||
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledWith(record.trackingUri);
|
||||
expect(freshCreateTrace).toHaveBeenCalledTimes(1);
|
||||
expect(freshUploadTraceData).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Successful retry: tombstoned, no live row, no dead-letter.
|
||||
const pending = await readPending();
|
||||
expect(pending.find((r) => r.id === record.id)).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it('falls through to handleUploadFailure when both attempts hit 401', async () => {
|
||||
const staleCreateTrace = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
|
||||
const freshCreateTrace = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
|
||||
const stale = makeClient({ createTrace: staleCreateTrace });
|
||||
const fresh = makeClient({ createTrace: freshCreateTrace });
|
||||
const factory = jest.fn(() => fresh);
|
||||
|
||||
// Backdate firstAttemptAt past the 500s budget so the post-retry
|
||||
// failure dead-letters immediately rather than re-appending yet
|
||||
// another retry row we'd have to sit on.
|
||||
const longAgo = Date.now() - 600_000;
|
||||
const record = makeRecord({
|
||||
id: 'row-401-persistent',
|
||||
firstAttemptAt: longAgo,
|
||||
});
|
||||
await uploadOne(record, stale, new BatchingWriter(), factory);
|
||||
|
||||
// Both attempts ran; factory was consulted exactly once for the
|
||||
// refresh; the second failure routed to dead-letter (not a
|
||||
// perpetual invalidate-retry loop).
|
||||
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
|
||||
expect(freshCreateTrace).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(0);
|
||||
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
|
||||
expect(failedContents).toContain('"id":"row-401-persistent"');
|
||||
});
|
||||
|
||||
it('dead-letters with the factory error when refresh-time factory throws', async () => {
|
||||
const staleCreateTrace = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new MlflowHttpError(401, 'Unauthorized', ''));
|
||||
const stale = makeClient({ createTrace: staleCreateTrace });
|
||||
const factory = jest.fn(() => {
|
||||
throw new Error('credential provider blew up');
|
||||
});
|
||||
|
||||
const longAgo = Date.now() - 600_000;
|
||||
const record = makeRecord({
|
||||
id: 'row-refresh-factory-throws',
|
||||
firstAttemptAt: longAgo,
|
||||
});
|
||||
await uploadOne(record, stale, new BatchingWriter(), factory);
|
||||
|
||||
// Stale client was tried once and got the auth error; the
|
||||
// refresh asked the factory once and that throw aborted the
|
||||
// retry before any fresh client call could happen.
|
||||
expect(staleCreateTrace).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledWith(record.trackingUri);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(0);
|
||||
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
|
||||
expect(failedContents).toContain('"id":"row-refresh-factory-throws"');
|
||||
});
|
||||
|
||||
it('does not refresh the client on non-auth HTTP errors', async () => {
|
||||
const createTrace = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new MlflowHttpError(500, 'Internal Server Error', ''));
|
||||
const client = makeClient({ createTrace });
|
||||
const factory = jest.fn();
|
||||
|
||||
const record = makeRecord({ id: 'row-5xx' });
|
||||
await uploadOne(record, client, new BatchingWriter(), factory);
|
||||
|
||||
// 5xx is not a credential problem — single attempt, no factory
|
||||
// consultation, regular retry row appended by handleUploadFailure.
|
||||
expect(createTrace).toHaveBeenCalledTimes(1);
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
|
||||
const pending = await readPending();
|
||||
const retry = pending.find((r) => r.id !== 'row-5xx');
|
||||
expect(pending.find((r) => r.id === 'row-5xx')).toBeUndefined();
|
||||
expect(retry).toBeDefined();
|
||||
expect(retry!.attempts).toBe(1);
|
||||
});
|
||||
|
||||
it('does not refresh the client on transport-level errors', async () => {
|
||||
// A non-MlflowHttpError (e.g. fetch DNS / ECONNRESET / abort
|
||||
// wrapped by makeRequest into `new Error('API request failed: ...')`)
|
||||
// must not be confused with an auth failure — the `instanceof`
|
||||
// gate in isAuthRefreshableError is what protects us here.
|
||||
const createTrace = jest.fn().mockRejectedValue(new Error('ECONNRESET'));
|
||||
const client = makeClient({ createTrace });
|
||||
const factory = jest.fn();
|
||||
|
||||
const record = makeRecord({ id: 'row-transport' });
|
||||
await uploadOne(record, client, new BatchingWriter(), factory);
|
||||
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
const pending = await readPending();
|
||||
const retry = pending.find((r) => r.id !== 'row-transport');
|
||||
expect(pending.find((r) => r.id === 'row-transport')).toBeUndefined();
|
||||
expect(retry).toBeDefined();
|
||||
expect(retry!.attempts).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('processBatch', () => {
|
||||
it('groups records by trackingUri and calls the factory once per URI', async () => {
|
||||
const clientA = makeClient({});
|
||||
const clientB = makeClient({});
|
||||
const factory = jest.fn((uri: string) => (uri.endsWith(':5001') ? clientB : clientA));
|
||||
|
||||
const records = [
|
||||
makeRecord({ id: 'r1', trackingUri: 'http://localhost:5000' }),
|
||||
makeRecord({ id: 'r2', trackingUri: 'http://localhost:5000' }),
|
||||
makeRecord({ id: 'r3', trackingUri: 'http://localhost:5001' }),
|
||||
];
|
||||
|
||||
await processBatch(records, new BatchingWriter(), factory);
|
||||
|
||||
expect(factory).toHaveBeenCalledTimes(2);
|
||||
expect(factory).toHaveBeenCalledWith('http://localhost:5000');
|
||||
expect(factory).toHaveBeenCalledWith('http://localhost:5001');
|
||||
const pending = await readPending();
|
||||
// All three should have been tombstoned by their respective uploads.
|
||||
expect(pending).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('no-ops on an empty input', async () => {
|
||||
const factory = jest.fn();
|
||||
await processBatch([], new BatchingWriter(), factory);
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dead-letters records when their client factory throws', async () => {
|
||||
const factory = jest.fn(() => {
|
||||
throw new Error('auth provider blew up');
|
||||
});
|
||||
|
||||
// Backdate firstAttemptAt past the default 500s retry budget so
|
||||
// the synthetic factory failure dead-letters immediately instead
|
||||
// of just bumping the retry counter.
|
||||
const longAgo = Date.now() - 600_000;
|
||||
const records = [makeRecord({ id: 'r-bad', firstAttemptAt: longAgo })];
|
||||
await processBatch(records, new BatchingWriter(), factory);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(0);
|
||||
|
||||
const failedContents = await readFile(getDeadLetterPath(), 'utf8');
|
||||
expect(failedContents).toContain('"id":"r-bad"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('queue.log layout after retry', () => {
|
||||
it('atomically batches the retry append + original tombstone in one fsync', async () => {
|
||||
const createTrace = jest.fn().mockRejectedValueOnce(new Error('first attempt fails'));
|
||||
const client = makeClient({ createTrace });
|
||||
|
||||
const record = makeRecord({ id: 'row-order', attempts: 0 });
|
||||
await uploadOne(record, client, new BatchingWriter());
|
||||
|
||||
const contents = await readFile(getWalPath(), 'utf8');
|
||||
const lines = contents.trim().split('\n');
|
||||
// `handleUploadFailure`'s retry branch enqueues both the new
|
||||
// retry row and the tombstone for the original id on the same
|
||||
// `BatchingWriter` tick, so `drainAndFlush` packs them into a
|
||||
// single `writev` + `fsync`. The on-disk order matches the
|
||||
// enqueue order (append first, then tombstone) and the pair is
|
||||
// atomic — either both lines are durable or neither is, so a
|
||||
// crash between them is structurally impossible.
|
||||
expect(lines).toHaveLength(2);
|
||||
const first = JSON.parse(lines[0]) as { type: string; record?: WalRecord };
|
||||
const second = JSON.parse(lines[1]) as { type: string; id?: string };
|
||||
expect(first.type).toBe('append');
|
||||
expect(first.record!.attempts).toBe(1);
|
||||
expect(second.type).toBe('tombstone');
|
||||
expect(second.id).toBe('row-order');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pruneOldLogs', () => {
|
||||
// All retention tests pin `now` to a fixed UTC date so the cutoff
|
||||
// arithmetic stays stable regardless of when CI happens to run.
|
||||
const NOW = new Date('2026-05-20T12:00:00.000Z');
|
||||
|
||||
async function seed(names: string[]): Promise<void> {
|
||||
for (const name of names) {
|
||||
await writeFile(join(walDir, name), 'x');
|
||||
}
|
||||
}
|
||||
|
||||
function existsInWal(name: string): boolean {
|
||||
return existsSync(join(walDir, name));
|
||||
}
|
||||
|
||||
// Per-file existence checks are used throughout this block instead of
|
||||
// full directory equality because `pruneOldLogs` itself calls `log()`
|
||||
// when it unlinks a file, which writes to today's `daemon.log.<date>`
|
||||
// and would show up in any whole-listing comparison. We assert the
|
||||
// narrower property the function actually owns: which seeded files
|
||||
// survive vs disappear.
|
||||
|
||||
it('removes files older than the retention window and keeps newer ones', async () => {
|
||||
// retentionDays=14 with NOW=2026-05-20 → keep dates >= 2026-05-06.
|
||||
const kept = ['failed.log.2026-05-19', 'failed.log.2026-05-06', 'daemon.log.2026-05-19'];
|
||||
const removed = ['failed.log.2026-05-05', 'failed.log.2026-04-01', 'daemon.log.2026-05-05'];
|
||||
await seed([...kept, ...removed]);
|
||||
|
||||
await pruneOldLogs({ now: NOW, retentionDays: 14 });
|
||||
|
||||
for (const name of kept) {
|
||||
expect(existsInWal(name)).toBe(true);
|
||||
}
|
||||
for (const name of removed) {
|
||||
expect(existsInWal(name)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('is a no-op when retention is 0 (disabled)', async () => {
|
||||
const seeded = ['failed.log.2020-01-01', 'daemon.log.2020-01-01'];
|
||||
await seed(seeded);
|
||||
|
||||
await pruneOldLogs({ now: NOW, retentionDays: 0 });
|
||||
|
||||
for (const name of seeded) {
|
||||
expect(existsInWal(name)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores files that do not match the rotated-log pattern', async () => {
|
||||
// Including `queue.log`, the lock socket, files without a date
|
||||
// suffix, malformed dates, and unrelated artifacts the operator
|
||||
// may have dropped into the WAL dir.
|
||||
const seeded = [
|
||||
'queue.log',
|
||||
'failed.log',
|
||||
'daemon.log',
|
||||
'failed.log.bogus',
|
||||
'failed.log.2026-13-99',
|
||||
'failed.log.2026-02-30',
|
||||
'README.md',
|
||||
'failed.log.2026-05-19.bak',
|
||||
];
|
||||
await seed(seeded);
|
||||
|
||||
await pruneOldLogs({ now: NOW, retentionDays: 14 });
|
||||
|
||||
// Every seeded entry must still be there — the regex must not
|
||||
// strip-and-reparse, and Date.UTC's overflow-tolerance must not
|
||||
// leak into the cutoff comparison.
|
||||
for (const name of seeded) {
|
||||
expect(existsInWal(name)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves the active queue.log even when the WAL dir is full of expired logs', async () => {
|
||||
await seed(['queue.log', 'failed.log.2020-01-01', 'daemon.log.2020-01-01']);
|
||||
|
||||
await pruneOldLogs({ now: NOW, retentionDays: 14 });
|
||||
|
||||
expect(existsInWal('queue.log')).toBe(true);
|
||||
expect(existsInWal('failed.log.2020-01-01')).toBe(false);
|
||||
expect(existsInWal('daemon.log.2020-01-01')).toBe(false);
|
||||
});
|
||||
|
||||
it('does not throw when the WAL directory is missing', async () => {
|
||||
// Simulate "daemon started, then someone rm -rf'd the dir before
|
||||
// the sweep ran". The sweep must log-and-continue, never throw.
|
||||
process.env.MLFLOW_WAL_DIR = join(walDir, 'does-not-exist');
|
||||
await expect(pruneOldLogs({ now: NOW, retentionDays: 14 })).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runBatchLoop idle semantics', () => {
|
||||
it('shuts down via idleMs when the WAL is empty', async () => {
|
||||
// Baseline: no records seeded → due.length === 0 every
|
||||
// iteration → idle-shutdown branch fires on the first iteration
|
||||
// after `idleMs` elapses.
|
||||
const factory = jest.fn();
|
||||
const start = Date.now();
|
||||
await runBatchLoop({
|
||||
factory,
|
||||
writer: new BatchingWriter(),
|
||||
batchIntervalMs: 10,
|
||||
idleMs: 50,
|
||||
});
|
||||
const elapsed = Date.now() - start;
|
||||
expect(elapsed).toBeGreaterThanOrEqual(50);
|
||||
// Generous upper bound — guards against an accidental infinite
|
||||
// loop while tolerating CI scheduler jitter.
|
||||
expect(elapsed).toBeLessThan(2_000);
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shuts down via idleMs when all pending records are future-dated', async () => {
|
||||
// Regression test for the bug where `pending.length > 0` short-
|
||||
// circuited the idle-shutdown check, pinning the daemon alive
|
||||
// until each retry matured even though nothing was actually
|
||||
// being worked on. The fix gates the idle check on
|
||||
// `due.length === 0`, so a WAL populated only by future-dated
|
||||
// retries is treated as not-actively-working and the loop
|
||||
// winds down through the normal `idleMs` path.
|
||||
await appendRecord(makeRecord({ id: 'row-future', nextAttemptAt: Date.now() + 60_000 }));
|
||||
const factory = jest.fn();
|
||||
const start = Date.now();
|
||||
await runBatchLoop({
|
||||
factory,
|
||||
writer: new BatchingWriter(),
|
||||
batchIntervalMs: 10,
|
||||
idleMs: 50,
|
||||
});
|
||||
const elapsed = Date.now() - start;
|
||||
expect(elapsed).toBeGreaterThanOrEqual(50);
|
||||
expect(elapsed).toBeLessThan(2_000);
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
// The record is still durably parked in the WAL: the supervisor's
|
||||
// next respawn (driven by a fresh IPC submission) will pick it
|
||||
// up and either retry or dead-letter against the original
|
||||
// `firstAttemptAt` budget — at-least-once delivery preserved.
|
||||
expect(await readPending()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,621 @@
|
||||
import { mkdtemp, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { ROOT_CONTEXT, SpanKind } from '@opentelemetry/api';
|
||||
import { ExportResult, ExportResultCode } from '@opentelemetry/core';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
ReadableSpan as OTelReadableSpan,
|
||||
Span as OTelSpan,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer';
|
||||
import { MlflowWalSpanExporter } from '../../../src/exporters/wal/exporter';
|
||||
import * as ipc from '../../../src/exporters/wal/ipc';
|
||||
import { init, resetConfig } from '../../../src/core/config';
|
||||
import { InMemoryTraceManager } from '../../../src/core/trace_manager';
|
||||
import { Trace } from '../../../src/core/entities/trace';
|
||||
import { Span as MlflowSpan, NoOpSpan } from '../../../src/core/entities/span';
|
||||
import { TraceInfo } from '../../../src/core/entities/trace_info';
|
||||
import { TraceData } from '../../../src/core/entities/trace_data';
|
||||
import { createTraceLocationFromExperimentId } from '../../../src/core/entities/trace_location';
|
||||
import { TraceState } from '../../../src/core/entities/trace_state';
|
||||
import { SpanAttributeKey } from '../../../src/core/constants';
|
||||
import type { WalRecord } from '../../../src/exporters/wal/types';
|
||||
|
||||
function makeTrace(traceId: string): Trace {
|
||||
const info = new TraceInfo({
|
||||
traceId,
|
||||
traceLocation: createTraceLocationFromExperimentId('exp-test'),
|
||||
requestTime: 1_700_000_000_000,
|
||||
state: TraceState.OK,
|
||||
executionDuration: 42,
|
||||
traceMetadata: { 'mlflow.trace.schemaVersion': '3' },
|
||||
tags: {},
|
||||
assessments: [],
|
||||
});
|
||||
const data = new TraceData([]);
|
||||
return new Trace(info, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a trace whose `data.spans` carries real, ended OTel spans (wrapped in
|
||||
* MLflow `Span`s) so the exporter's OTLP serialization runs end-to-end
|
||||
*/
|
||||
function makeTraceWithSpans(traceId: string, attributes: Record<string, string> = {}): Trace {
|
||||
const info = new TraceInfo({
|
||||
traceId,
|
||||
traceLocation: createTraceLocationFromExperimentId('exp-test'),
|
||||
requestTime: 1_700_000_000_000,
|
||||
state: TraceState.OK,
|
||||
executionDuration: 42,
|
||||
traceMetadata: { 'mlflow.trace.schemaVersion': '3' },
|
||||
tags: {},
|
||||
assessments: [],
|
||||
});
|
||||
|
||||
const provider = new BasicTracerProvider();
|
||||
const tracer = provider.getTracer('wal-exporter-test');
|
||||
const otelSpan = tracer.startSpan(
|
||||
'tool-call',
|
||||
{ kind: SpanKind.INTERNAL },
|
||||
ROOT_CONTEXT,
|
||||
) as OTelSpan;
|
||||
otelSpan.setAttribute(SpanAttributeKey.TRACE_ID, JSON.stringify(traceId));
|
||||
// Attribute values are stored JSON-encoded on the OTel span, mirroring the
|
||||
// SDK's `safeJsonStringify` behavior (see core/entities/span.ts).
|
||||
for (const [key, value] of Object.entries(attributes)) {
|
||||
otelSpan.setAttribute(key, value);
|
||||
}
|
||||
otelSpan.end();
|
||||
|
||||
return new Trace(info, new TraceData([new MlflowSpan(otelSpan)]));
|
||||
}
|
||||
|
||||
function makeSpan(opts: { traceId: string; rootSpan: boolean; name?: string }): OTelReadableSpan {
|
||||
return {
|
||||
name: opts.name ?? (opts.rootSpan ? 'root' : 'child'),
|
||||
spanContext: () => ({ traceId: opts.traceId, spanId: 'span-id' }),
|
||||
parentSpanContext: opts.rootSpan ? undefined : { spanId: 'parent-span-id' },
|
||||
} as unknown as OTelReadableSpan;
|
||||
}
|
||||
|
||||
function captureExportResult(): {
|
||||
callback: (r: ExportResult) => void;
|
||||
promise: Promise<ExportResult>;
|
||||
} {
|
||||
let resolveFn: (r: ExportResult) => void = () => {};
|
||||
const promise = new Promise<ExportResult>((resolve) => {
|
||||
resolveFn = resolve;
|
||||
});
|
||||
return { callback: resolveFn, promise };
|
||||
}
|
||||
|
||||
/**
|
||||
* In OTLP protobuf wire format, each attribute's key and value strings are
|
||||
* emitted close together. Require `expectedValue` to appear after `attributeKey`
|
||||
* so a bare substring scan cannot false-positive on unrelated fields.
|
||||
*/
|
||||
function expectOtlpAttributeValue(
|
||||
decoded: Buffer,
|
||||
attributeKey: string,
|
||||
expectedValue: string,
|
||||
): void {
|
||||
const keyOffset = decoded.indexOf(attributeKey);
|
||||
expect(keyOffset).toBeGreaterThanOrEqual(0);
|
||||
const valueOffset = decoded.indexOf(expectedValue, keyOffset + attributeKey.length);
|
||||
expect(valueOffset).toBeGreaterThan(keyOffset);
|
||||
}
|
||||
|
||||
function expectOtlpAttributeValueAbsent(
|
||||
decoded: Buffer,
|
||||
attributeKey: string,
|
||||
unexpectedValue: string,
|
||||
): void {
|
||||
const keyOffset = decoded.indexOf(attributeKey);
|
||||
expect(keyOffset).toBeGreaterThanOrEqual(0);
|
||||
const valueOffset = decoded.indexOf(unexpectedValue, keyOffset + attributeKey.length);
|
||||
expect(valueOffset).toBe(-1);
|
||||
}
|
||||
|
||||
describe('wal/exporter', () => {
|
||||
let walDir: string;
|
||||
let popTraceSpy: jest.SpyInstance;
|
||||
// Capture the developer's pre-test value so we restore (not just unset)
|
||||
// in afterEach. Mirrors the pattern in paths.test.ts so running
|
||||
// `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override for later
|
||||
// tests in the same worker.
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-exporter-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
|
||||
init({ trackingUri: 'http://localhost:5000', experimentId: 'exp-test' });
|
||||
|
||||
popTraceSpy = jest.spyOn(InMemoryTraceManager.getInstance(), 'popTrace');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
popTraceSpy.mockRestore();
|
||||
InMemoryTraceManager.reset();
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('submits one WAL record per root span via the injected submitter', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-abc'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
expect(submit).toHaveBeenCalledTimes(1);
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(record.trackingUri).toBe('http://localhost:5000');
|
||||
expect(record.experimentId).toBe('exp-test');
|
||||
expect((record.traceInfo as { trace_id: string }).trace_id).toBe('tr-abc');
|
||||
expect(typeof record.id).toBe('string');
|
||||
expect(record.id.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('skips non-root spans without invoking the submitter', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
const childSpan = makeSpan({ traceId: 'otel-1', rootSpan: false });
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([childSpan], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
expect(submit).not.toHaveBeenCalled();
|
||||
expect(popTraceSpy).not.toHaveBeenCalled();
|
||||
|
||||
await exporter.forceFlush();
|
||||
});
|
||||
|
||||
it('warns and skips when popTrace returns null', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(null);
|
||||
const span = makeSpan({ traceId: 'otel-missing', rootSpan: true, name: 'orphan' });
|
||||
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
expect(submit).not.toHaveBeenCalled();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('No trace found for span orphan'),
|
||||
);
|
||||
|
||||
await exporter.forceFlush();
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('handles a batch with mixed root and child spans', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValueOnce(makeTrace('tr-1')).mockReturnValueOnce(makeTrace('tr-2'));
|
||||
|
||||
const spans = [
|
||||
makeSpan({ traceId: 'otel-1', rootSpan: true, name: 'root-1' }),
|
||||
makeSpan({ traceId: 'otel-1', rootSpan: false, name: 'child' }),
|
||||
makeSpan({ traceId: 'otel-2', rootSpan: true, name: 'root-2' }),
|
||||
];
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export(spans, callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
expect(submit).toHaveBeenCalledTimes(2);
|
||||
expect(popTraceSpy).toHaveBeenCalledTimes(2);
|
||||
|
||||
await exporter.forceFlush();
|
||||
|
||||
const traceIds = submit.mock.calls
|
||||
.map(([record]) => (record.traceInfo as { trace_id: string }).trace_id)
|
||||
.sort();
|
||||
expect(traceIds).toEqual(['tr-1', 'tr-2']);
|
||||
});
|
||||
|
||||
it('forceFlush resolves only once every in-flight submit has acked', async () => {
|
||||
let release: (() => void) | undefined;
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockImplementation(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
}),
|
||||
);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-flush'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
let flushResolved = false;
|
||||
const flush = exporter.forceFlush().then(() => {
|
||||
flushResolved = true;
|
||||
});
|
||||
// The submit is parked on `release`; until we resolve it, forceFlush
|
||||
// must remain pending. Yield a few microtasks to give any premature
|
||||
// resolution a chance to surface.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(flushResolved).toBe(false);
|
||||
|
||||
release?.();
|
||||
await flush;
|
||||
expect(flushResolved).toBe(true);
|
||||
});
|
||||
|
||||
it('shutdown awaits in-flight submits before resolving', async () => {
|
||||
let release: (() => void) | undefined;
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockImplementation(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
}),
|
||||
);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-shutdown'));
|
||||
const { callback } = captureExportResult();
|
||||
exporter.export([makeSpan({ traceId: 'otel-1', rootSpan: true })], callback);
|
||||
|
||||
let shutdownResolved = false;
|
||||
const shutdownPromise = exporter.shutdown().then(() => {
|
||||
shutdownResolved = true;
|
||||
});
|
||||
|
||||
// The submit is parked; shutdown must remain pending until we
|
||||
// release it. Yield a few microtasks so any premature resolution
|
||||
// surfaces here rather than silently passing.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(shutdownResolved).toBe(false);
|
||||
|
||||
release?.();
|
||||
await shutdownPromise;
|
||||
expect(shutdownResolved).toBe(true);
|
||||
});
|
||||
|
||||
it('shutdown rejects subsequent export calls with FAILED per OTel spec', async () => {
|
||||
// OTel SpanExporter.Shutdown spec: "After the call to Shutdown
|
||||
// subsequent calls to Export are not allowed and SHOULD return
|
||||
// a failure." Locks in the post-shutdown contract.
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
await exporter.shutdown();
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-post-shutdown'));
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([makeSpan({ traceId: 'otel-post', rootSpan: true })], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.FAILED);
|
||||
expect(submit).not.toHaveBeenCalled();
|
||||
expect(popTraceSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shutdown awaits all in-flight submits, not just the first to resolve', async () => {
|
||||
// Both `export()` calls run synchronously to completion before
|
||||
// `shutdown()` is invoked, so both submit promises are already
|
||||
// in `_pendingSubmits` when the drain loop takes its first
|
||||
// snapshot. (The `_shuttingDown` flag in `export()` makes the
|
||||
// "submit lands after the flag is set" scenario structurally
|
||||
// impossible in single-threaded JS — once the flag flips, every
|
||||
// export call short-circuits with FAILED before touching the
|
||||
// Set.
|
||||
let releaseFirst: (() => void) | undefined;
|
||||
let releaseSecond: (() => void) | undefined;
|
||||
const submit = jest
|
||||
.fn<Promise<void>, [WalRecord]>()
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
}),
|
||||
)
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseSecond = resolve;
|
||||
}),
|
||||
);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy
|
||||
.mockReturnValueOnce(makeTrace('tr-first'))
|
||||
.mockReturnValueOnce(makeTrace('tr-second'));
|
||||
|
||||
// Two submits enter `_pendingSubmits` synchronously, parked on
|
||||
// their respective `release*` resolvers.
|
||||
const cb1 = captureExportResult();
|
||||
exporter.export([makeSpan({ traceId: 'otel-1', rootSpan: true })], cb1.callback);
|
||||
const cb2 = captureExportResult();
|
||||
exporter.export([makeSpan({ traceId: 'otel-2', rootSpan: true })], cb2.callback);
|
||||
|
||||
let shutdownResolved = false;
|
||||
const shutdownPromise = exporter.shutdown().then(() => {
|
||||
shutdownResolved = true;
|
||||
});
|
||||
|
||||
// Release submit #1 only. #2 is still parked, so `Promise.all`
|
||||
// inside the drain loop must remain pending — proving shutdown
|
||||
// awaits both, not just one.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
releaseFirst?.();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(shutdownResolved).toBe(false);
|
||||
|
||||
releaseSecond?.();
|
||||
await shutdownPromise;
|
||||
expect(shutdownResolved).toBe(true);
|
||||
expect(submit).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('returns FAILED and warns when getConfig() throws (init not called)', async () => {
|
||||
resetConfig();
|
||||
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-no-config'));
|
||||
const span = makeSpan({ traceId: 'otel-no-config', rootSpan: true });
|
||||
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
const result = await promise;
|
||||
expect(result.code).toBe(ExportResultCode.FAILED);
|
||||
expect(submit).not.toHaveBeenCalled();
|
||||
expect(popTraceSpy).not.toHaveBeenCalled();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Tracing config unavailable'),
|
||||
expect.any(Error),
|
||||
);
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
// Restore config defensively so a downstream test that forgets
|
||||
// to re-init doesn't observe the cleared global state.
|
||||
init({ trackingUri: 'http://localhost:5000', experimentId: 'exp-test' });
|
||||
}
|
||||
});
|
||||
|
||||
it('logs and swallows submit failures so OTel never observes them', async () => {
|
||||
const submit = jest
|
||||
.fn<Promise<void>, [WalRecord]>()
|
||||
.mockRejectedValue(new Error('daemon unreachable'));
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-err'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
try {
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
// OTel sees SUCCESS regardless of submit outcome — the exporter
|
||||
// owns durability semantics and must not bubble per-record
|
||||
// failures into the OTel result callback (which would surface as
|
||||
// an SDK-level error users can't act on).
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
|
||||
await exporter.forceFlush();
|
||||
expect(submit).toHaveBeenCalledTimes(1);
|
||||
expect(errSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Failed to submit WAL record for trace tr-err'),
|
||||
expect.any(Error),
|
||||
);
|
||||
} finally {
|
||||
errSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('captures OTLP protobuf bytes on the record when the trace has spans', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-with-spans'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(typeof record.otlpSpans).toBe('string');
|
||||
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
|
||||
|
||||
expect(decoded[0]).toBe(0x0a);
|
||||
// And the span itself must be encoded in the payload, not just an empty
|
||||
// envelope: its name is embedded as a UTF-8 string in the protobuf.
|
||||
expect(decoded.includes(Buffer.from('tool-call'))).toBe(true);
|
||||
// The JSON trace data is retained alongside the OTLP bytes (artifact fallback).
|
||||
expect((record.traceData as { spans: unknown[] }).spans).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('omits otlpSpans when the trace has no spans', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTrace('tr-no-spans'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(record.otlpSpans).toBeUndefined();
|
||||
});
|
||||
|
||||
it('filters malformed spans (missing resource/scope) and still serializes the valid ones', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
const trace = makeTraceWithSpans('tr-mixed');
|
||||
trace.data.spans.push(new NoOpSpan());
|
||||
popTraceSpy.mockReturnValue(trace);
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(typeof record.otlpSpans).toBe('string');
|
||||
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
|
||||
expect(decoded[0]).toBe(0x0a);
|
||||
expect(decoded.includes(Buffer.from('tool-call'))).toBe(true);
|
||||
});
|
||||
|
||||
it('warns and submits without otlpSpans when OTLP serialization throws', async () => {
|
||||
const serializeSpy = jest
|
||||
.spyOn(ProtobufTraceSerializer, 'serializeRequest')
|
||||
.mockImplementation(() => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-serialize-fail'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
try {
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
// The record is still submitted (artifact fallback), just without spans.
|
||||
expect(submit).toHaveBeenCalledTimes(1);
|
||||
expect(submit.mock.calls[0][0].otlpSpans).toBeUndefined();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Failed to serialize spans to OTLP protobuf'),
|
||||
expect.any(Error),
|
||||
);
|
||||
} finally {
|
||||
serializeSpy.mockRestore();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('drops otlpSpans but still submits when the record would exceed the IPC size budget', async () => {
|
||||
const sizeSpy = jest
|
||||
.spyOn(ipc, 'ipcRequestByteLength')
|
||||
.mockReturnValue(ipc.MAX_REQUEST_BYTES + 1);
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(makeTraceWithSpans('tr-too-big'));
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
try {
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
// The trace is not dropped: it is still submitted, just without the OTLP
|
||||
// blob, so the artifact fallback (traceData) keeps it alive.
|
||||
expect(submit).toHaveBeenCalledTimes(1);
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(record.otlpSpans).toBeUndefined();
|
||||
expect((record.traceData as { spans: unknown[] }).spans).toHaveLength(1);
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('exceeding the'));
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('dropping OTLP spans'));
|
||||
} finally {
|
||||
sizeSpy.mockRestore();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('serializes decoded attribute values (no double JSON-encoding)', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
popTraceSpy.mockReturnValue(
|
||||
makeTraceWithSpans('tr-decode', {
|
||||
[SpanAttributeKey.SPAN_TYPE]: JSON.stringify('TOOL'),
|
||||
tool_name: JSON.stringify('WebSearch'),
|
||||
}),
|
||||
);
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
const record = submit.mock.calls[0][0];
|
||||
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
|
||||
|
||||
expectOtlpAttributeValue(decoded, SpanAttributeKey.SPAN_TYPE, 'TOOL');
|
||||
expectOtlpAttributeValue(decoded, 'tool_name', 'WebSearch');
|
||||
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.SPAN_TYPE, '"TOOL"');
|
||||
expectOtlpAttributeValueAbsent(decoded, 'tool_name', '"WebSearch"');
|
||||
});
|
||||
|
||||
it('keeps JSON object attributes as single-encoded strings in OTLP output', async () => {
|
||||
const submit = jest.fn<Promise<void>, [WalRecord]>().mockResolvedValue(undefined);
|
||||
const exporter = new MlflowWalSpanExporter({ submit });
|
||||
|
||||
const inputsJson = JSON.stringify({ prompt: 'hello' });
|
||||
const outputsJson = JSON.stringify({ response: 'world' });
|
||||
popTraceSpy.mockReturnValue(
|
||||
makeTraceWithSpans('tr-obj-attrs', {
|
||||
[SpanAttributeKey.SPAN_TYPE]: JSON.stringify('TOOL'),
|
||||
[SpanAttributeKey.INPUTS]: inputsJson,
|
||||
[SpanAttributeKey.OUTPUTS]: outputsJson,
|
||||
}),
|
||||
);
|
||||
const span = makeSpan({ traceId: 'otel-1', rootSpan: true });
|
||||
|
||||
const { callback, promise } = captureExportResult();
|
||||
exporter.export([span], callback);
|
||||
|
||||
expect((await promise).code).toBe(ExportResultCode.SUCCESS);
|
||||
await exporter.forceFlush();
|
||||
|
||||
const record = submit.mock.calls[0][0];
|
||||
expect(record.otlpSpans).toBeDefined();
|
||||
const decoded = Buffer.from(record.otlpSpans as string, 'base64');
|
||||
|
||||
expectOtlpAttributeValue(decoded, SpanAttributeKey.INPUTS, inputsJson);
|
||||
expectOtlpAttributeValue(decoded, SpanAttributeKey.OUTPUTS, outputsJson);
|
||||
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.INPUTS, JSON.stringify(inputsJson));
|
||||
expectOtlpAttributeValueAbsent(decoded, SpanAttributeKey.OUTPUTS, JSON.stringify(outputsJson));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,430 @@
|
||||
import { mkdtemp, readFile, rm } from 'fs/promises';
|
||||
import { createServer, Server } from 'net';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { BatchingWriter } from '../../../src/exporters/wal/batching_writer';
|
||||
import {
|
||||
createIpcConnectionHandler,
|
||||
ipcRequestByteLength,
|
||||
MAX_REQUEST_BYTES,
|
||||
submitRecord,
|
||||
} from '../../../src/exporters/wal/ipc';
|
||||
import { getLockSocketPath, getWalPath } from '../../../src/exporters/wal/paths';
|
||||
import type { IpcRequest, IpcResponse } from '../../../src/exporters/wal/protocol';
|
||||
import type { WalRecord } from '../../../src/exporters/wal/types';
|
||||
import { JSONBig } from '../../../src/core/utils/json';
|
||||
|
||||
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
|
||||
return {
|
||||
id: `wal-${idSuffix}`,
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
traceInfo: { trace_id: `t-${idSuffix}` },
|
||||
traceData: { spans: [] },
|
||||
attempts: 0,
|
||||
nextAttemptAt: 0,
|
||||
createdAt: Date.now(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function closeServer(server: Server): Promise<void> {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
|
||||
describe('wal/ipc submitRecord', () => {
|
||||
let walDir: string;
|
||||
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
const originalDaemonExe = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-ipc-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = join(walDir, 'noop-daemon.cjs');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
if (originalDaemonExe === undefined) {
|
||||
delete process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
|
||||
} else {
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = originalDaemonExe;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('round-trips an append request: request → ok ACK', async () => {
|
||||
const requestChunks: Buffer[] = [];
|
||||
const server = createServer((socket) => {
|
||||
let responded = false;
|
||||
socket.on('data', (chunk: Buffer) => {
|
||||
requestChunks.push(chunk);
|
||||
if (responded) {
|
||||
return;
|
||||
}
|
||||
const buffered = Buffer.concat(requestChunks).toString('utf8');
|
||||
if (!buffered.includes('\n')) {
|
||||
return;
|
||||
}
|
||||
responded = true;
|
||||
const response: IpcResponse = { ok: true };
|
||||
socket.end(JSON.stringify(response) + '\n');
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
|
||||
|
||||
try {
|
||||
const record = makeRecord('a');
|
||||
await expect(submitRecord(record)).resolves.toBeUndefined();
|
||||
const request = Buffer.concat(requestChunks).toString('utf8').trim();
|
||||
const parsed = JSON.parse(request) as IpcRequest;
|
||||
expect(parsed.op).toBe('append');
|
||||
expect(parsed.record.id).toBe(record.id);
|
||||
} finally {
|
||||
await closeServer(server);
|
||||
}
|
||||
});
|
||||
|
||||
it('throws when the daemon responds with an error', async () => {
|
||||
const server = createServer((socket) => {
|
||||
socket.on('data', () => {
|
||||
const response: IpcResponse = { ok: false, error: 'disk full' };
|
||||
socket.end(JSON.stringify(response) + '\n');
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
|
||||
|
||||
try {
|
||||
await expect(submitRecord(makeRecord('b'))).rejects.toThrow(/disk full/);
|
||||
} finally {
|
||||
await closeServer(server);
|
||||
}
|
||||
});
|
||||
|
||||
it('throws after exhausting retries when no daemon is listening', async () => {
|
||||
// No server bound; submitRecord should retry, fail to reach
|
||||
// anyone, and surface a connect error to the caller. The full
|
||||
// CONNECT_RETRY_DELAYS_MS sleeps (~3.15s) still elapse - we
|
||||
// don't currently inject those in tests - but pointing
|
||||
// MLFLOW_WAL_DIR at a tmpdir guarantees each connect attempt
|
||||
// fails fast with ENOENT instead of stacking on top of the
|
||||
// (much longer) per-attempt kernel connect timeout that would
|
||||
// fire against an unreachable bound socket. The 10s test
|
||||
// timeout accommodates the configured backoff with margin; if
|
||||
// retry latency ever becomes a CI bottleneck the fix is to make
|
||||
// CONNECT_RETRY_DELAYS_MS injectable rather than to chase it
|
||||
// here.
|
||||
const r = makeRecord('c');
|
||||
await expect(submitRecord(r)).rejects.toMatchObject({
|
||||
// The contract we care about is that retry exhaustion surfaces
|
||||
// a transport-level error code to the caller. The exact value
|
||||
// varies by platform (POSIX UNIX-domain sockets vs. Windows
|
||||
// Named Pipes) and by Node/libuv version, so pinning to
|
||||
// ECONNREFUSED|ENOENT produced unnecessary Windows-CI
|
||||
// fragility; matching any non-empty errno-style string is what
|
||||
// this assertion actually wants to express.
|
||||
code: expect.stringMatching(/^\S+$/),
|
||||
});
|
||||
}, 10_000);
|
||||
});
|
||||
|
||||
describe('wal/ipc createIpcConnectionHandler', () => {
|
||||
let walDir: string;
|
||||
let server: Server | null = null;
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-ipc-handler-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (server) {
|
||||
await closeServer(server);
|
||||
server = null;
|
||||
}
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function startHandlerServer(writer: BatchingWriter): Promise<Server> {
|
||||
const s = createServer(createIpcConnectionHandler(writer));
|
||||
await new Promise<void>((resolve) => s.listen(getLockSocketPath(), resolve));
|
||||
return s;
|
||||
}
|
||||
|
||||
it('persists the submitted record and acks with ok=true', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
const record = makeRecord('h1');
|
||||
await submitRecord(record);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(1);
|
||||
const parsed = JSON.parse(lines[0]) as { type: string; record: WalRecord };
|
||||
expect(parsed.type).toBe('append');
|
||||
expect(parsed.record.id).toBe(record.id);
|
||||
});
|
||||
|
||||
it('responds with ok=false for an unknown op', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
const result = await new Promise<string>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const chunks: Buffer[] = [];
|
||||
socket.on('connect', () => {
|
||||
socket.write(JSON.stringify({ op: 'frobnicate', record: makeRecord('x') }) + '\n');
|
||||
});
|
||||
socket.on('data', (chunk) => chunks.push(chunk));
|
||||
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
|
||||
socket.on('error', reject);
|
||||
});
|
||||
const parsed = JSON.parse(result) as IpcResponse;
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (!parsed.ok) {
|
||||
expect(parsed.error).toMatch(/unknown op/);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects requests whose record is missing or has no string id', async () => {
|
||||
// Exercises the structural check in dispatch() that stops bad
|
||||
// payloads from being durably persisted to queue.log and then
|
||||
// poisoning every subsequent batch loop iteration that tries to
|
||||
// upload them. Walks each rejection branch (non-object, null,
|
||||
// missing id, non-string id, empty string id) through one IPC
|
||||
// round-trip per case so the WAL is never written.
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
const send = (record: unknown): Promise<IpcResponse> =>
|
||||
new Promise<IpcResponse>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const chunks: Buffer[] = [];
|
||||
socket.on('connect', () => {
|
||||
socket.write(JSON.stringify({ op: 'append', record }) + '\n');
|
||||
});
|
||||
socket.on('data', (chunk) => chunks.push(chunk));
|
||||
socket.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8').trim()) as IpcResponse);
|
||||
} catch (err) {
|
||||
reject(err as Error);
|
||||
}
|
||||
});
|
||||
socket.on('error', reject);
|
||||
});
|
||||
|
||||
const badCases: unknown[] = [null, undefined, 42, 'not an object', {}, { id: 123 }, { id: '' }];
|
||||
for (const bad of badCases) {
|
||||
const response = await send(bad);
|
||||
expect(response.ok).toBe(false);
|
||||
if (!response.ok) {
|
||||
expect(response.error).toMatch(/invalid record/);
|
||||
}
|
||||
}
|
||||
|
||||
// queue.log must never have been written: no successful submits
|
||||
// and no partial lines from rejected ones.
|
||||
await expect(readFile(getWalPath(), 'utf8')).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it('rejects oversized requests without exhausting memory', async () => {
|
||||
// Stream bytes well past the cap without ever sending a newline. The
|
||||
// handler should respond with ok=false, pause its read side so the kernel
|
||||
// buffer stops growing, and half-close the write side so we observe a clean
|
||||
// `'end'` here.
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
// One MiB per chunk; send just enough chunks to overshoot the cap by 1 MiB,
|
||||
// derived from MAX_REQUEST_BYTES so the test tracks the constant.
|
||||
const oneMibBytes = 1024 * 1024;
|
||||
const chunkCount = Math.ceil(MAX_REQUEST_BYTES / oneMibBytes) + 1;
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
const result = await new Promise<string>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const chunks: Buffer[] = [];
|
||||
socket.on('connect', () => {
|
||||
// No trailing newline. We send chunks back-to-back; once the daemon's
|
||||
// cumulative buffer crosses the cap it must short-circuit and respond,
|
||||
// after which it ignores any further bytes.
|
||||
const oneMib = Buffer.alloc(oneMibBytes, 0x41);
|
||||
const pump = (remaining: number): void => {
|
||||
if (remaining === 0 || socket.destroyed) {
|
||||
return;
|
||||
}
|
||||
if (!socket.write(oneMib)) {
|
||||
socket.once('drain', () => pump(remaining - 1));
|
||||
return;
|
||||
}
|
||||
setImmediate(() => pump(remaining - 1));
|
||||
};
|
||||
pump(chunkCount);
|
||||
});
|
||||
socket.on('data', (chunk) => chunks.push(chunk));
|
||||
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
|
||||
socket.on('error', (err: NodeJS.ErrnoException) => {
|
||||
// EPIPE can surface from the pump's writes once the daemon
|
||||
// pauses reading and our kernel send buffer eventually fills.
|
||||
// That's expected; the contract we care about is that the
|
||||
// daemon replied before that backpressure hit.
|
||||
if (err.code === 'EPIPE') {
|
||||
return;
|
||||
}
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
const parsed = JSON.parse(result) as IpcResponse;
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (!parsed.ok) {
|
||||
expect(parsed.error).toMatch(/exceeds max size/);
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it('measures the cap in wire bytes, not UTF-16 code units', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
// Each '中' is 3 UTF-8 bytes but a single UTF-16 code unit, so a cap
|
||||
// measured in code units would count this chunk at a third of its wire
|
||||
// size. Send enough chunks to overshoot the (byte-measured) cap; a
|
||||
// code-unit measurement would stay well under it and never trip.
|
||||
const cjkChunk = Buffer.from('中'.repeat(349_525), 'utf8');
|
||||
const chunkCount = Math.ceil(MAX_REQUEST_BYTES / cjkChunk.length) + 1;
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
const result = await new Promise<string>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const chunks: Buffer[] = [];
|
||||
socket.on('connect', () => {
|
||||
const pump = (remaining: number): void => {
|
||||
if (remaining === 0 || socket.destroyed) {
|
||||
return;
|
||||
}
|
||||
if (!socket.write(cjkChunk)) {
|
||||
socket.once('drain', () => pump(remaining - 1));
|
||||
return;
|
||||
}
|
||||
setImmediate(() => pump(remaining - 1));
|
||||
};
|
||||
pump(chunkCount);
|
||||
});
|
||||
socket.on('data', (chunk) => chunks.push(chunk));
|
||||
socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
|
||||
socket.on('error', (err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'EPIPE') {
|
||||
return;
|
||||
}
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
const parsed = JSON.parse(result) as IpcResponse;
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (!parsed.ok) {
|
||||
expect(parsed.error).toMatch(/exceeds max size/);
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it('handles a multi-byte UTF-8 codepoint split across chunk boundaries', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
const record = makeRecord('with-中');
|
||||
const payload = Buffer.from(JSON.stringify({ op: 'append', record }) + '\n', 'utf8');
|
||||
// First high-bit byte in the payload is the `0xE4` of `中` (the
|
||||
// rest of the JSON is ASCII), so split immediately after it to
|
||||
// straddle the codepoint.
|
||||
const cjkStart = payload.indexOf(0xe4);
|
||||
expect(cjkStart).toBeGreaterThan(0);
|
||||
const chunkA = payload.subarray(0, cjkStart + 1);
|
||||
const chunkB = payload.subarray(cjkStart + 1);
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
const response = await new Promise<IpcResponse>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
const respChunks: Buffer[] = [];
|
||||
socket.on('connect', () => {
|
||||
socket.write(chunkA, () => {
|
||||
// setTimeout (rather than setImmediate) gives the kernel a
|
||||
// tick to deliver chunk A as its own `'data'` event before
|
||||
// chunk B arrives, otherwise the writes may coalesce and
|
||||
// the test won't actually exercise the split-codepoint
|
||||
// path it's meant to guard.
|
||||
setTimeout(() => socket.write(chunkB), 10);
|
||||
});
|
||||
});
|
||||
socket.on('data', (chunk) => respChunks.push(chunk));
|
||||
socket.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(respChunks).toString('utf8').trim()) as IpcResponse);
|
||||
} catch (err) {
|
||||
reject(err as Error);
|
||||
}
|
||||
});
|
||||
socket.on('error', reject);
|
||||
});
|
||||
expect(response.ok).toBe(true);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(1);
|
||||
const persisted = JSON.parse(lines[0]) as { record: WalRecord };
|
||||
expect(persisted.record.id).toBe('wal-with-中');
|
||||
});
|
||||
|
||||
it('tolerates probe connections that close without sending data', async () => {
|
||||
const writer = new BatchingWriter();
|
||||
server = await startHandlerServer(writer);
|
||||
|
||||
const { createConnection } = await import('node:net');
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = createConnection(getLockSocketPath());
|
||||
socket.on('connect', () => {
|
||||
socket.destroy();
|
||||
resolve();
|
||||
});
|
||||
socket.on('error', reject);
|
||||
});
|
||||
|
||||
// A subsequent real submit must still succeed: the probe-and-tear
|
||||
// pattern must not have left the handler in a bad state.
|
||||
await submitRecord(makeRecord('after-probe'));
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wal/ipc ipcRequestByteLength', () => {
|
||||
it('measures the real wire size of a record, including BigInt fields', () => {
|
||||
const record = makeRecord('measure', {
|
||||
traceData: { spans: [{ startTimeUnixNano: 1_700_000_000_000_000_000n }] },
|
||||
});
|
||||
|
||||
const expected = Buffer.byteLength(JSONBig.stringify({ op: 'append', record }) + '\n', 'utf8');
|
||||
expect(ipcRequestByteLength(record)).toBe(expected);
|
||||
});
|
||||
|
||||
it('returns a value above the cap for a genuinely oversized otlpSpans payload', () => {
|
||||
// A baseline record sits comfortably under the cap...
|
||||
expect(ipcRequestByteLength(makeRecord('small'))).toBeLessThan(MAX_REQUEST_BYTES);
|
||||
|
||||
// ...while a record carrying an over-cap otlpSpans blob measures over it.
|
||||
const oversized = makeRecord('oversized', {
|
||||
otlpSpans: 'A'.repeat(MAX_REQUEST_BYTES + 1),
|
||||
});
|
||||
expect(ipcRequestByteLength(oversized)).toBeGreaterThan(MAX_REQUEST_BYTES);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
import { mkdtemp, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join, resolve } from 'path';
|
||||
import {
|
||||
getDaemonLogPath,
|
||||
getDeadLetterPath,
|
||||
getLockSocketPath,
|
||||
getWalDir,
|
||||
getWalPath,
|
||||
} from '../../../src/exporters/wal/paths';
|
||||
|
||||
describe('wal/paths', () => {
|
||||
const originalEnv = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalEnv === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalEnv;
|
||||
}
|
||||
});
|
||||
|
||||
describe('getWalDir', () => {
|
||||
it('honors MLFLOW_WAL_DIR and returns an absolute, resolved path', () => {
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/foo/';
|
||||
expect(getWalDir()).toBe(resolve('/tmp/foo'));
|
||||
});
|
||||
|
||||
it('treats empty MLFLOW_WAL_DIR as unset', () => {
|
||||
process.env.MLFLOW_WAL_DIR = '';
|
||||
expect(getWalDir()).toMatch(/[/\\]\.mlflow[/\\]wal$/);
|
||||
});
|
||||
|
||||
it('produces the same value for /tmp/foo and /tmp/foo/', () => {
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/foo';
|
||||
const a = getWalDir();
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/foo/';
|
||||
const b = getWalDir();
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getLockSocketPath', () => {
|
||||
let scratchDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
scratchDir = await mkdtemp(join(tmpdir(), 'mlflow-paths-'));
|
||||
process.env.MLFLOW_WAL_DIR = scratchDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(scratchDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const isWindows = process.platform === 'win32';
|
||||
|
||||
(isWindows ? it.skip : it)('POSIX: returns <wal_dir>/daemon.sock for short WAL dirs', () => {
|
||||
const lock = getLockSocketPath();
|
||||
expect(lock).toBe(join(scratchDir, 'daemon.sock'));
|
||||
});
|
||||
|
||||
(isWindows ? it.skip : it)(
|
||||
'POSIX: falls back to tmpdir-based path when wal_dir would exceed sun_path',
|
||||
() => {
|
||||
// Construct a long WAL dir that pushes <wal_dir>/daemon.sock past 103
|
||||
// bytes. We do not need this directory to exist; getLockSocketPath
|
||||
// only inspects path lengths.
|
||||
const longWalDir = '/tmp/' + 'x'.repeat(150);
|
||||
process.env.MLFLOW_WAL_DIR = longWalDir;
|
||||
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const lock = getLockSocketPath();
|
||||
expect(lock.startsWith(tmpdir())).toBe(true);
|
||||
expect(lock).toMatch(/[/\\]mlflow-[A-Za-z0-9._-]+-[0-9a-f]{12}\.sock$/);
|
||||
expect(Buffer.byteLength(lock, 'utf8')).toBeLessThanOrEqual(103);
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
expect(warnSpy.mock.calls[0]?.[0]).toContain('longer than 103 bytes');
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
(isWindows ? it.skip : it)(
|
||||
'POSIX: fallback identifier is stable for the same wal_dir and differs for distinct wal_dirs',
|
||||
() => {
|
||||
const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/' + 'a'.repeat(150);
|
||||
const lockA1 = getLockSocketPath();
|
||||
const lockA2 = getLockSocketPath();
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/' + 'b'.repeat(150);
|
||||
const lockB = getLockSocketPath();
|
||||
expect(lockA1).toBe(lockA2);
|
||||
expect(lockA1).not.toBe(lockB);
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
(isWindows ? it : it.skip)(
|
||||
'Windows: returns a Named Pipe path scoped by user + wal_dir hash',
|
||||
() => {
|
||||
const lock = getLockSocketPath();
|
||||
expect(lock).toMatch(/^\\\\\?\\pipe\\mlflow-trace-daemon-[A-Za-z0-9._-]+-[0-9a-f]{12}$/);
|
||||
},
|
||||
);
|
||||
|
||||
(isWindows ? it : it.skip)('Windows: pipe name differs for distinct wal_dirs', () => {
|
||||
process.env.MLFLOW_WAL_DIR = 'C:\\tmp\\walA';
|
||||
const lockA = getLockSocketPath();
|
||||
process.env.MLFLOW_WAL_DIR = 'C:\\tmp\\walB';
|
||||
const lockB = getLockSocketPath();
|
||||
expect(lockA).not.toBe(lockB);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getWalPath', () => {
|
||||
it('lives inside the resolved WAL dir', () => {
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/foo';
|
||||
expect(getWalPath()).toBe(join(resolve('/tmp/foo'), 'queue.log'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('daily rotation of failed.log / daemon.log', () => {
|
||||
beforeEach(() => {
|
||||
process.env.MLFLOW_WAL_DIR = '/tmp/wal-rotation';
|
||||
});
|
||||
|
||||
it('suffixes failed.log with the UTC date of the supplied Date', () => {
|
||||
const d = new Date('2026-05-20T12:34:56.000Z');
|
||||
expect(getDeadLetterPath(d)).toBe(
|
||||
join(resolve('/tmp/wal-rotation'), 'failed.log.2026-05-20'),
|
||||
);
|
||||
});
|
||||
|
||||
it('suffixes daemon.log with the UTC date of the supplied Date', () => {
|
||||
const d = new Date('2026-05-20T12:34:56.000Z');
|
||||
expect(getDaemonLogPath(d)).toBe(join(resolve('/tmp/wal-rotation'), 'daemon.log.2026-05-20'));
|
||||
});
|
||||
|
||||
it('rolls failed.log to the next day across midnight UTC', () => {
|
||||
const lastSecondOfDay = new Date('2026-05-20T23:59:59.999Z');
|
||||
const firstInstantOfNextDay = new Date('2026-05-21T00:00:00.000Z');
|
||||
expect(getDeadLetterPath(lastSecondOfDay)).toMatch(/failed\.log\.2026-05-20$/);
|
||||
expect(getDeadLetterPath(firstInstantOfNextDay)).toMatch(/failed\.log\.2026-05-21$/);
|
||||
});
|
||||
|
||||
it('rolls daemon.log to the next day across midnight UTC', () => {
|
||||
const lastSecondOfDay = new Date('2026-05-20T23:59:59.999Z');
|
||||
const firstInstantOfNextDay = new Date('2026-05-21T00:00:00.000Z');
|
||||
expect(getDaemonLogPath(lastSecondOfDay)).toMatch(/daemon\.log\.2026-05-20$/);
|
||||
expect(getDaemonLogPath(firstInstantOfNextDay)).toMatch(/daemon\.log\.2026-05-21$/);
|
||||
});
|
||||
|
||||
it('uses UTC date, not local date, regardless of the process timezone', () => {
|
||||
// 2026-05-20T18:00:00 UTC = 2026-05-21T02:00:00 in UTC+8 (e.g. Singapore).
|
||||
// The suffix must reflect UTC, not the host TZ.
|
||||
const d = new Date('2026-05-20T18:00:00.000Z');
|
||||
expect(getDaemonLogPath(d)).toMatch(/daemon\.log\.2026-05-20$/);
|
||||
expect(getDeadLetterPath(d)).toMatch(/failed\.log\.2026-05-20$/);
|
||||
});
|
||||
|
||||
it('zero-pads month and day', () => {
|
||||
const earlyDay = new Date('2026-01-03T12:00:00.000Z');
|
||||
expect(getDaemonLogPath(earlyDay)).toMatch(/daemon\.log\.2026-01-03$/);
|
||||
});
|
||||
|
||||
it('defaults to "now" when called without an argument', () => {
|
||||
expect(getDaemonLogPath()).toMatch(/daemon\.log\.\d{4}-\d{2}-\d{2}$/);
|
||||
expect(getDeadLetterPath()).toMatch(/failed\.log\.\d{4}-\d{2}-\d{2}$/);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,223 @@
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { readLockHolderPid, tryAcquirePidLock } from '../../../src/exporters/wal/pid_lock';
|
||||
import { getPidLockPath } from '../../../src/exporters/wal/paths';
|
||||
|
||||
// The lock is a POSIX-only mechanism — Windows daemons rely on
|
||||
// kernel-refcounted named pipes for cross-process exclusion, so the
|
||||
// entire suite is a no-op there. Wrap with a describe variant that
|
||||
// skips on Windows to keep the test report honest about coverage.
|
||||
const describePosix = process.platform === 'win32' ? describe.skip : describe;
|
||||
|
||||
/**
|
||||
* Spawn a trivial child, wait for it to exit, and return its now-freed
|
||||
* pid. Used to obtain a pid that is guaranteed to fail
|
||||
* `process.kill(pid, 0)` with ESRCH so the stale-recovery path can be
|
||||
* exercised deterministically. The PID could in principle be recycled
|
||||
* by the OS between the child exiting and our liveness check, but in a
|
||||
* single-digit-millisecond test window the probability is negligible
|
||||
* — and even if it happened, the test would falsely fail rather than
|
||||
* silently pass, which is the safe failure mode.
|
||||
*/
|
||||
function spawnAndReapPid(): Promise<number> {
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ['-e', 'process.exit(0)']);
|
||||
const pid = child.pid;
|
||||
if (pid == null) {
|
||||
reject(new Error('child spawn produced no pid'));
|
||||
return;
|
||||
}
|
||||
child.once('exit', () => resolve(pid));
|
||||
child.once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
describePosix('wal/pid_lock', () => {
|
||||
let walDir: string;
|
||||
// Mirror daemon.test.ts: capture the developer's pre-test value so
|
||||
// running `MLFLOW_WAL_DIR=/some/dir jest` doesn't lose the override
|
||||
// for later tests in the same worker.
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-pidlock-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('tryAcquirePidLock', () => {
|
||||
it('claims the lock and writes our pid when no lockfile exists', async () => {
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('returns null when an alive process already holds the lock', async () => {
|
||||
// Seed the lockfile with our own pid — guaranteed alive — so the
|
||||
// liveness probe inside tryAcquirePidLock has to detect us and
|
||||
// concede rather than steal the lock from ourselves.
|
||||
await writeFile(getPidLockPath(), `${process.pid}\n`);
|
||||
const lock = await tryAcquirePidLock();
|
||||
expect(lock).toBeNull();
|
||||
// Lockfile must remain untouched after a conceding acquisition.
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
});
|
||||
|
||||
it('recovers from a stale lockfile whose pid is dead', async () => {
|
||||
const deadPid = await spawnAndReapPid();
|
||||
await writeFile(getPidLockPath(), `${deadPid}\n`);
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers from a malformed lockfile (non-numeric content)', async () => {
|
||||
await writeFile(getPidLockPath(), 'this is not a pid\n');
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers from an empty lockfile', async () => {
|
||||
await writeFile(getPidLockPath(), '');
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a lockfile that records pid 0 (process-group signal target)', async () => {
|
||||
// pid 0 would make process.kill(0, 0) signal our entire process
|
||||
// group; parsePid must screen it out before the liveness probe
|
||||
// ever runs. Recovery should treat it as malformed and reclaim.
|
||||
await writeFile(getPidLockPath(), '0\n');
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe(`${process.pid}\n`);
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a lockfile that records a negative pid (process-group signal target)', async () => {
|
||||
await writeFile(getPidLockPath(), '-12345\n');
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
expect(lock).not.toBeNull();
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('serializes concurrent acquirers in the same event loop to a single winner', async () => {
|
||||
// Both acquirers run in the same Node process so they see the
|
||||
// same `process.pid`. The link() atomicity is what arbitrates;
|
||||
// the loser observes EEXIST, reads our own (alive) pid, and
|
||||
// concedes via the live-holder branch.
|
||||
const [first, second] = await Promise.all([tryAcquirePidLock(), tryAcquirePidLock()]);
|
||||
try {
|
||||
const acquired = [first, second].filter((l) => l != null);
|
||||
const conceded = [first, second].filter((l) => l == null);
|
||||
expect(acquired).toHaveLength(1);
|
||||
expect(conceded).toHaveLength(1);
|
||||
} finally {
|
||||
await first?.release();
|
||||
await second?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('cleans up the temp file on a successful acquisition', async () => {
|
||||
const lock = await tryAcquirePidLock();
|
||||
try {
|
||||
// No `daemon.pid.<pid>.<uuid>.tmp` should survive a successful
|
||||
// acquire — the implementation unlinks it after the link()
|
||||
// succeeds so the WAL dir doesn't accumulate per-spawn garbage.
|
||||
const { readdir } = await import('fs/promises');
|
||||
const entries = await readdir(walDir);
|
||||
const tmps = entries.filter((e) => e.startsWith('daemon.pid.') && e.endsWith('.tmp'));
|
||||
expect(tmps).toEqual([]);
|
||||
} finally {
|
||||
await lock?.release();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('PidLock.release', () => {
|
||||
it('unlinks the lockfile when content still matches our pid', async () => {
|
||||
const lock = await tryAcquirePidLock();
|
||||
expect(lock).not.toBeNull();
|
||||
await lock!.release();
|
||||
expect(existsSync(getPidLockPath())).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves the lockfile intact when content has been hijacked', async () => {
|
||||
const lock = await tryAcquirePidLock();
|
||||
expect(lock).not.toBeNull();
|
||||
// Simulate an external rewrite (operator intervention, or a
|
||||
// successor that erroneously stole the path while we were still
|
||||
// running). release() must not blind-unlink the file in this
|
||||
// case — that would erase the successor's claim.
|
||||
await writeFile(getPidLockPath(), '99999\n');
|
||||
await lock!.release();
|
||||
expect(existsSync(getPidLockPath())).toBe(true);
|
||||
const content = await readFile(getPidLockPath(), 'utf8');
|
||||
expect(content).toBe('99999\n');
|
||||
});
|
||||
|
||||
it('is idempotent', async () => {
|
||||
const lock = await tryAcquirePidLock();
|
||||
expect(lock).not.toBeNull();
|
||||
await lock!.release();
|
||||
// Second release: lockfile already gone, must be a no-op.
|
||||
await expect(lock!.release()).resolves.toBeUndefined();
|
||||
expect(existsSync(getPidLockPath())).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readLockHolderPid', () => {
|
||||
it('returns the recorded pid when a lockfile exists', async () => {
|
||||
await writeFile(getPidLockPath(), '4242\n');
|
||||
await expect(readLockHolderPid()).resolves.toBe(4242);
|
||||
});
|
||||
|
||||
it('returns null when no lockfile exists', async () => {
|
||||
await expect(readLockHolderPid()).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the lockfile is malformed', async () => {
|
||||
await writeFile(getPidLockPath(), 'garbage\n');
|
||||
await expect(readLockHolderPid()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
// Mock node:fs/promises with a passthrough so individual tests can flip a
|
||||
// single method (e.g. `rename`) to fail/inject without breaking the
|
||||
// dozens of real fs operations the rest of the suite needs. Hoisted by
|
||||
// ts-jest above the storage.ts import so storage's
|
||||
// `import { ... } from 'node:fs/promises'` resolves to the mocked bindings.
|
||||
|
||||
jest.mock('node:fs/promises', () => {
|
||||
const actual = jest.requireActual<typeof import('node:fs/promises')>('node:fs/promises');
|
||||
return {
|
||||
...actual,
|
||||
rename: jest.fn(actual.rename),
|
||||
};
|
||||
});
|
||||
|
||||
import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import * as fsPromises from 'node:fs/promises';
|
||||
import {
|
||||
appendDeadLetter,
|
||||
appendRecord,
|
||||
appendTombstone,
|
||||
compact,
|
||||
readPending,
|
||||
walSize,
|
||||
} from '../../../src/exporters/wal/storage';
|
||||
import { getDeadLetterPath, getWalPath } from '../../../src/exporters/wal/paths';
|
||||
import type { WalRecord } from '../../../src/exporters/wal/types';
|
||||
|
||||
const renameMock = fsPromises.rename as jest.MockedFunction<typeof fsPromises.rename>;
|
||||
|
||||
function makeRecord(idSuffix: string, overrides: Partial<WalRecord> = {}): WalRecord {
|
||||
return {
|
||||
id: `wal-${idSuffix}`,
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
traceInfo: { trace_id: `t-${idSuffix}` },
|
||||
traceData: { spans: [] },
|
||||
attempts: 0,
|
||||
nextAttemptAt: 0,
|
||||
createdAt: Date.now(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('wal/storage', () => {
|
||||
let walDir: string;
|
||||
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-wal-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns an empty array when no WAL file exists yet', async () => {
|
||||
expect(await readPending()).toEqual([]);
|
||||
expect(await walSize()).toBe(0);
|
||||
});
|
||||
|
||||
it('appends a record and reads it back', async () => {
|
||||
const record = makeRecord('a');
|
||||
await appendRecord(record);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(1);
|
||||
expect(pending[0]).toEqual(record);
|
||||
expect(await walSize()).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('round-trips bigint fields (regression for SerializedSpan timestamps)', async () => {
|
||||
const startNs = 1_750_000_000_123_456_789n;
|
||||
const record = makeRecord('bigint', {
|
||||
traceData: {
|
||||
spans: [
|
||||
{
|
||||
trace_id: 't-bigint',
|
||||
start_time_unix_nano: startNs,
|
||||
end_time_unix_nano: startNs + 5_000_000n,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await appendRecord(record);
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(1);
|
||||
const restored = (pending[0].traceData as { spans: Array<Record<string, unknown>> }).spans[0];
|
||||
expect(restored.start_time_unix_nano).toBe(startNs);
|
||||
expect(restored.end_time_unix_nano).toBe(startNs + 5_000_000n);
|
||||
});
|
||||
|
||||
it('hides a record once it has been tombstoned', async () => {
|
||||
const r1 = makeRecord('a');
|
||||
const r2 = makeRecord('b');
|
||||
await appendRecord(r1);
|
||||
await appendRecord(r2);
|
||||
await appendTombstone(r1.id);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending.map((r) => r.id)).toEqual(['wal-b']);
|
||||
});
|
||||
|
||||
it('treats a tombstone before its corresponding append as a no-op', async () => {
|
||||
// Records use fresh ids per attempt, so a tombstone never legitimately
|
||||
// precedes its append; if it does (e.g. corrupt log), the later append
|
||||
// must still win.
|
||||
const r = makeRecord('a');
|
||||
await appendTombstone(r.id);
|
||||
await appendRecord(r);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(1);
|
||||
expect(pending[0]?.id).toBe(r.id);
|
||||
});
|
||||
|
||||
it('accumulates entries in the dead-letter file without touching queue.log', async () => {
|
||||
const live = makeRecord('live');
|
||||
await appendRecord(live);
|
||||
|
||||
const dead1 = makeRecord('dead-1', { attempts: 10 });
|
||||
const dead2 = makeRecord('dead-2', { attempts: 10 });
|
||||
await appendDeadLetter(dead1);
|
||||
await appendDeadLetter(dead2);
|
||||
|
||||
const dlqPath = getDeadLetterPath();
|
||||
// failed.log is daily-rotated; the resolved path must include a
|
||||
// YYYY-MM-DD suffix so a regression to a single unbounded file is
|
||||
// caught in CI.
|
||||
expect(dlqPath).toMatch(/[/\\]failed\.log\.\d{4}-\d{2}-\d{2}$/);
|
||||
const dlqContents = await readFile(dlqPath, 'utf8');
|
||||
const dlqLines = dlqContents.split('\n').filter((l) => l.length > 0);
|
||||
expect(dlqLines).toHaveLength(2);
|
||||
expect(JSON.parse(dlqLines[0])).toEqual({ type: 'append', record: dead1 });
|
||||
expect(JSON.parse(dlqLines[1])).toEqual({ type: 'append', record: dead2 });
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toEqual([live]);
|
||||
});
|
||||
|
||||
it('preserves live records and shrinks the file when compacting', async () => {
|
||||
const r1 = makeRecord('a');
|
||||
const r2 = makeRecord('b');
|
||||
const r3 = makeRecord('c');
|
||||
await appendRecord(r1);
|
||||
await appendRecord(r2);
|
||||
await appendRecord(r3);
|
||||
await appendTombstone(r2.id);
|
||||
|
||||
const sizeBefore = await walSize();
|
||||
await compact();
|
||||
const sizeAfter = await walSize();
|
||||
|
||||
expect(sizeAfter).toBeLessThan(sizeBefore);
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending.map((r) => r.id).sort()).toEqual(['wal-a', 'wal-c']);
|
||||
|
||||
const lines = (await readFile(getWalPath(), 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(2);
|
||||
for (const line of lines) {
|
||||
expect(JSON.parse(line).type).toBe('append');
|
||||
}
|
||||
});
|
||||
|
||||
it('is a no-op when compacting a non-existent WAL', async () => {
|
||||
await expect(compact()).resolves.toBeUndefined();
|
||||
await expect(stat(getWalPath())).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it('cleans up the tmp file when rename throws after a successful write', async () => {
|
||||
// Regression: an earlier shape had `close()` and `rename()` outside
|
||||
// the try/catch, so a flaky FS at the rename step would orphan one
|
||||
// `queue.log.tmp.<pid>` per failed compaction. The fix moved both
|
||||
// into the try so the catch's `unlink(tmpPath)` always runs on
|
||||
// failure regardless of which step blew up.
|
||||
await appendRecord(makeRecord('a'));
|
||||
|
||||
renameMock.mockRejectedValueOnce(
|
||||
Object.assign(new Error('EBUSY: simulated rename failure'), { code: 'EBUSY' }),
|
||||
);
|
||||
|
||||
await expect(compact()).rejects.toThrow(/simulated rename failure/);
|
||||
|
||||
// After the failed compaction, the WAL itself is unchanged and no
|
||||
// `.tmp.<pid>` orphan remains in the spool dir.
|
||||
const entries = await readdir(walDir);
|
||||
expect(entries.some((e) => /\.tmp\.\d+$/.test(e))).toBe(false);
|
||||
const pending = await readPending();
|
||||
expect(pending.map((r) => r.id)).toEqual(['wal-a']);
|
||||
});
|
||||
|
||||
it('skips malformed lines but keeps surrounding records', async () => {
|
||||
const r1 = makeRecord('a');
|
||||
const r2 = makeRecord('b');
|
||||
await appendRecord(r1);
|
||||
await writeFile(getWalPath(), '{not valid json}\n', { flag: 'a' });
|
||||
await appendRecord(r2);
|
||||
|
||||
const debugSpy = jest.spyOn(console, 'debug').mockImplementation(() => {});
|
||||
try {
|
||||
const pending = await readPending();
|
||||
expect(pending.map((r) => r.id).sort()).toEqual(['wal-a', 'wal-b']);
|
||||
expect(debugSpy).toHaveBeenCalled();
|
||||
} finally {
|
||||
debugSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('survives 50 concurrent appends with no torn or lost lines', async () => {
|
||||
const records = Array.from({ length: 50 }, (_, i) => makeRecord(i.toString().padStart(2, '0')));
|
||||
await Promise.all(records.map((r) => appendRecord(r)));
|
||||
|
||||
const contents = await readFile(getWalPath(), 'utf8');
|
||||
const lines = contents.split('\n').filter((l) => l.length > 0);
|
||||
expect(lines).toHaveLength(50);
|
||||
for (const line of lines) {
|
||||
expect(() => {
|
||||
JSON.parse(line);
|
||||
}).not.toThrow();
|
||||
}
|
||||
|
||||
const pending = await readPending();
|
||||
expect(pending).toHaveLength(50);
|
||||
const ids = new Set(pending.map((r) => r.id));
|
||||
for (const r of records) {
|
||||
expect(ids.has(r.id)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('records nonzero walSize after an append', async () => {
|
||||
expect(await walSize()).toBe(0);
|
||||
await appendRecord(makeRecord('a'));
|
||||
const size = await walSize();
|
||||
expect(size).toBeGreaterThan(0);
|
||||
|
||||
const fsSize = (await stat(getWalPath())).size;
|
||||
expect(size).toBe(fsSize);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,294 @@
|
||||
import * as childProcess from 'node:child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdtemp, readFile, rm, unlink, writeFile } from 'fs/promises';
|
||||
import { createServer, Server } from 'net';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
ensureDaemon,
|
||||
isDaemonAlive,
|
||||
resolveDaemonEntry,
|
||||
} from '../../../src/exporters/wal/supervisor';
|
||||
import { getLockSocketPath } from '../../../src/exporters/wal/paths';
|
||||
|
||||
jest.mock('node:child_process', () => {
|
||||
const actual = jest.requireActual<typeof import('node:child_process')>('node:child_process');
|
||||
return {
|
||||
...actual,
|
||||
spawn: jest.fn((...args: Parameters<typeof actual.spawn>) => actual.spawn(...args)),
|
||||
};
|
||||
});
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitFor(
|
||||
predicate: () => boolean | Promise<boolean>,
|
||||
timeoutMs = 2000,
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
for (;;) {
|
||||
if (await predicate()) {
|
||||
return;
|
||||
}
|
||||
if (Date.now() - start > timeoutMs) {
|
||||
throw new Error(`waitFor: predicate did not become true within ${timeoutMs}ms`);
|
||||
}
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
describe('wal/supervisor', () => {
|
||||
let walDir: string;
|
||||
|
||||
const originalWalDir = process.env.MLFLOW_WAL_DIR;
|
||||
const originalDaemonExecutable = process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
|
||||
|
||||
beforeEach(async () => {
|
||||
walDir = await mkdtemp(join(tmpdir(), 'mlflow-supervisor-'));
|
||||
process.env.MLFLOW_WAL_DIR = walDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalWalDir === undefined) {
|
||||
delete process.env.MLFLOW_WAL_DIR;
|
||||
} else {
|
||||
process.env.MLFLOW_WAL_DIR = originalWalDir;
|
||||
}
|
||||
if (originalDaemonExecutable === undefined) {
|
||||
delete process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE;
|
||||
} else {
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = originalDaemonExecutable;
|
||||
}
|
||||
await rm(walDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('isDaemonAlive', () => {
|
||||
it('returns false when no socket is bound', async () => {
|
||||
expect(await isDaemonAlive()).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true when a server is bound to the lock socket', async () => {
|
||||
const server: Server = createServer((s) => s.end());
|
||||
await new Promise<void>((resolve) => server.listen(getLockSocketPath(), resolve));
|
||||
try {
|
||||
expect(await isDaemonAlive()).toBe(true);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveDaemonEntry', () => {
|
||||
it('returns the value of MLFLOW_TRACE_DAEMON_EXECUTABLE when set', () => {
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = '/custom/path/to/daemon.cjs';
|
||||
expect(resolveDaemonEntry()).toBe('/custom/path/to/daemon.cjs');
|
||||
});
|
||||
|
||||
it('treats an empty MLFLOW_TRACE_DAEMON_EXECUTABLE as unset', () => {
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = '';
|
||||
// With the env var empty, the resolver falls through to package
|
||||
// resolution. We can't predict the absolute path, but it must end
|
||||
// in `bundle/daemon.cjs` per the documented contract.
|
||||
const resolved = resolveDaemonEntry();
|
||||
expect(resolved.endsWith(join('bundle', 'daemon.cjs'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureDaemon', () => {
|
||||
let stubScript: string;
|
||||
let counterFile: string;
|
||||
let pidFile: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
stubScript = join(walDir, 'stub-daemon.js');
|
||||
counterFile = join(walDir, 'spawns.log');
|
||||
pidFile = join(walDir, 'pids.log');
|
||||
|
||||
// Minimal CJS script: record that we spawned, bind the lock
|
||||
// socket, and self-terminate after a short window so a failing
|
||||
// test cannot leak indefinitely. fs writes happen *before* the
|
||||
// listen() call so the counter / pid record is durable even if
|
||||
// the parent has already moved on by the time we bind.
|
||||
const lockPath = getLockSocketPath();
|
||||
const stubSource = [
|
||||
"const net = require('net');",
|
||||
"const fs = require('fs');",
|
||||
`fs.appendFileSync(${JSON.stringify(counterFile)}, '+');`,
|
||||
`fs.appendFileSync(${JSON.stringify(pidFile)}, process.pid + '\\n');`,
|
||||
'const server = net.createServer((s) => s.end());',
|
||||
`server.listen(${JSON.stringify(lockPath)}, () => {`,
|
||||
' setTimeout(() => process.exit(0), 3000);',
|
||||
'});',
|
||||
].join('\n');
|
||||
await writeFile(stubScript, stubSource);
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = stubScript;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Kill any stub daemons that haven't self-destructed yet so
|
||||
// subsequent tests don't see stale processes / sockets.
|
||||
if (existsSync(pidFile)) {
|
||||
const pids = (await readFile(pidFile, 'utf8')).split('\n').filter((l) => l.length > 0);
|
||||
for (const pid of pids) {
|
||||
try {
|
||||
process.kill(Number.parseInt(pid, 10), 'SIGKILL');
|
||||
} catch {
|
||||
// Already exited.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (process.platform !== 'win32') {
|
||||
try {
|
||||
await unlink(getLockSocketPath());
|
||||
} catch {
|
||||
// ENOENT (no stale file from this test) or any other error —
|
||||
// best-effort cleanup; the next test's beforeEach reseeds
|
||||
// everything regardless.
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('spawns a daemon when none is alive', async () => {
|
||||
await ensureDaemon();
|
||||
await waitFor(() => existsSync(counterFile));
|
||||
expect(await readFile(counterFile, 'utf8')).toBe('+');
|
||||
});
|
||||
|
||||
it('does not spawn a second daemon when one is already alive', async () => {
|
||||
await ensureDaemon();
|
||||
await waitFor(() => existsSync(counterFile));
|
||||
// Wait until the first stub has actually bound the socket; only
|
||||
// after that will the second probe see "alive".
|
||||
await waitFor(() => isDaemonAlive(), 3000);
|
||||
|
||||
await ensureDaemon();
|
||||
// Give any (errant) second spawn time to write to the counter.
|
||||
await sleep(200);
|
||||
|
||||
expect(await readFile(counterFile, 'utf8')).toBe('+');
|
||||
});
|
||||
|
||||
it('swallows broken-entry failures and surfaces a diagnostic to stderr', async () => {
|
||||
// Broken-install failures surface asynchronously, *after*
|
||||
// `ensureDaemon` has already resolved (the spawn itself is
|
||||
// fire-and-forget by design). The listener in `spawnDaemon` must:
|
||||
// 1. let `ensureDaemon` resolve cleanly (no upstream crash);
|
||||
// 2. log a breadcrumb mentioning the broken entry so a broken
|
||||
// install (missing `bundle/daemon.cjs`, bad
|
||||
// `MLFLOW_TRACE_DAEMON_EXECUTABLE`) doesn't degrade into
|
||||
// ~3.85 s of opaque ECONNREFUSED retries in the WAL exporter
|
||||
// with zero indication of cause.
|
||||
//
|
||||
// Two child events can carry the diagnostic and the listener
|
||||
// hooks both: `'error'` fires when libuv can't start the binary
|
||||
// at all (we can't easily trigger this here because `spawnDaemon`
|
||||
// runs `spawn(process.execPath, [entry], ...)` and the node
|
||||
// binary always exists), and `'exit'` fires with a non-zero code
|
||||
// when the binary starts but the entry script fails to load —
|
||||
// which is what pointing `MLFLOW_TRACE_DAEMON_EXECUTABLE` at a
|
||||
// nonexistent `.js` file actually triggers. The test accepts
|
||||
// either: both equally serve the reviewer-visible "broken
|
||||
// install" purpose.
|
||||
const missing = join(walDir, 'does-not-exist.js');
|
||||
process.env.MLFLOW_TRACE_DAEMON_EXECUTABLE = missing;
|
||||
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
try {
|
||||
await expect(ensureDaemon()).resolves.toBeUndefined();
|
||||
// Generous timeout: node cold-start + script-resolution failure
|
||||
// + exit can comfortably exceed 1 s on a busy CI box.
|
||||
await waitFor(
|
||||
() =>
|
||||
errSpy.mock.calls.some(
|
||||
([msg]) =>
|
||||
typeof msg === 'string' &&
|
||||
msg.includes(missing) &&
|
||||
/daemon (spawn failed|exited unexpectedly)/.test(msg),
|
||||
),
|
||||
5000,
|
||||
);
|
||||
const match = errSpy.mock.calls.find(
|
||||
([msg]) =>
|
||||
typeof msg === 'string' &&
|
||||
msg.includes(missing) &&
|
||||
/daemon (spawn failed|exited unexpectedly)/.test(msg),
|
||||
);
|
||||
expect(match).toBeDefined();
|
||||
expect(match![0]).toContain(missing);
|
||||
} finally {
|
||||
errSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('catches synchronous spawn failures via the outer ensureDaemon catch', async () => {
|
||||
const spawnMock = childProcess.spawn as jest.MockedFunction<typeof childProcess.spawn>;
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
throw new Error('synthetic synchronous spawn failure');
|
||||
});
|
||||
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
try {
|
||||
// The outer catch must convert the synchronous throw into a
|
||||
// clean resolution + a diagnostic log line. Anything else
|
||||
// (rejection, unhandled crash, no log) is a regression in the
|
||||
// defense-in-depth path.
|
||||
await expect(ensureDaemon()).resolves.toBeUndefined();
|
||||
expect(spawnMock).toHaveBeenCalled();
|
||||
expect(
|
||||
errSpy.mock.calls.some(
|
||||
([msg]) => typeof msg === 'string' && msg.includes('ensureDaemon failed'),
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
errSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('coalesces concurrent calls into a single spawn', async () => {
|
||||
// The realistic trigger is multiple `submitRecord` calls in one
|
||||
// hook all hitting ECONNREFUSED during a daemon cold-start. Each
|
||||
// would otherwise spawn its own child; the in-flight latch should
|
||||
// dedupe them down to a single `spawn()` invocation.
|
||||
const concurrent = 5;
|
||||
await Promise.all(Array.from({ length: concurrent }, () => ensureDaemon()));
|
||||
await waitFor(() => existsSync(counterFile));
|
||||
// Give any (errant) extra spawns time to write to the counter.
|
||||
await sleep(200);
|
||||
|
||||
const counterContents = await readFile(counterFile, 'utf8');
|
||||
expect(counterContents).toBe('+');
|
||||
});
|
||||
|
||||
it('dedupes a second ensureDaemon call during the spawn -> bind window', async () => {
|
||||
const bindDelayMs = 300;
|
||||
const lockPath = getLockSocketPath();
|
||||
const slowBindStub = [
|
||||
"const net = require('net');",
|
||||
"const fs = require('fs');",
|
||||
`fs.appendFileSync(${JSON.stringify(counterFile)}, '+');`,
|
||||
`fs.appendFileSync(${JSON.stringify(pidFile)}, process.pid + '\\n');`,
|
||||
'const server = net.createServer((s) => s.end());',
|
||||
// Delay listen() to widen the spawn→bind window the fix is meant to cover.
|
||||
`setTimeout(() => {`,
|
||||
` server.listen(${JSON.stringify(lockPath)}, () => {`,
|
||||
` setTimeout(() => process.exit(0), 3000);`,
|
||||
` });`,
|
||||
`}, ${bindDelayMs});`,
|
||||
].join('\n');
|
||||
await writeFile(stubScript, slowBindStub);
|
||||
|
||||
const spawnMock = childProcess.spawn as jest.MockedFunction<typeof childProcess.spawn>;
|
||||
spawnMock.mockClear();
|
||||
|
||||
const first = ensureDaemon();
|
||||
// Land the second call mid-bind-window: after spawnDaemon() has
|
||||
// returned (~ms) but well before the stub's 300ms listen() fires.
|
||||
await sleep(100);
|
||||
const second = ensureDaemon();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(await readFile(counterFile, 'utf8')).toBe('+');
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { LiveSpan } from '../src/core/entities/span';
|
||||
import { SpanType } from '../src/core/constants';
|
||||
|
||||
/**
|
||||
* Port and tracking URI for the local MLflow server used for testing.
|
||||
* If the server is not running, jest.global-setup.ts will start it.
|
||||
*/
|
||||
export const TEST_PORT = 5000;
|
||||
export const TEST_TRACKING_URI = `http://localhost:${TEST_PORT}`;
|
||||
|
||||
/**
|
||||
* Mock OpenTelemetry span class for testing
|
||||
*/
|
||||
export class MockOtelSpan {
|
||||
name: string;
|
||||
attributes: Record<string, any>;
|
||||
spanId: string;
|
||||
traceId: string;
|
||||
|
||||
constructor(
|
||||
name: string = 'test-span',
|
||||
spanId: string = 'test-span-id',
|
||||
traceId: string = 'test-trace-id',
|
||||
) {
|
||||
this.name = name;
|
||||
this.spanId = spanId;
|
||||
this.traceId = traceId;
|
||||
this.attributes = {};
|
||||
}
|
||||
|
||||
getAttribute(key: string): any {
|
||||
return this.attributes[key];
|
||||
}
|
||||
|
||||
setAttribute(key: string, value: any): void {
|
||||
this.attributes[key] = value;
|
||||
}
|
||||
|
||||
spanContext() {
|
||||
return {
|
||||
spanId: this.spanId,
|
||||
traceId: this.traceId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock OpenTelemetry span with the given parameters
|
||||
*/
|
||||
export function createMockOtelSpan(
|
||||
name: string = 'test-span',
|
||||
spanId: string = 'test-span-id',
|
||||
traceId: string = 'test-trace-id',
|
||||
): MockOtelSpan {
|
||||
return new MockOtelSpan(name, spanId, traceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test LiveSpan with mock OpenTelemetry span
|
||||
*/
|
||||
export function createTestSpan(
|
||||
name: string = 'test-span',
|
||||
traceId: string = 'test-trace-id',
|
||||
spanId: string = 'test-span-id',
|
||||
spanType: SpanType = SpanType.UNKNOWN,
|
||||
): LiveSpan {
|
||||
const mockOtelSpan = createMockOtelSpan(name, spanId, traceId);
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
return new LiveSpan(mockOtelSpan as any, traceId, spanType);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"allowJs": true,
|
||||
"checkJs": true,
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
# MLflow Typescript SDK - Anthropic
|
||||
|
||||
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with Anthropic to automatically trace your Claude API calls.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
|
||||
| [@mlflow/anthropic](./) | [](https://www.npmjs.com/package/@mlflow/anthropic) | Auto-instrumentation integration for Anthropic. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/anthropic
|
||||
```
|
||||
|
||||
The package includes the [`@mlflow/core`](https://github.com/mlflow/mlflow/tree/main/libs/typescript) package and `@anthropic-ai/sdk` package as peer dependencies. Depending on your package manager, you may need to install these two packages separately.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server if you don't have one already:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
Create a trace for Anthropic Claude:
|
||||
|
||||
```typescript
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { tracedAnthropic } from '@mlflow/anthropic';
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
|
||||
const client = tracedAnthropic(anthropic);
|
||||
|
||||
const response = await client.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude' }],
|
||||
});
|
||||
```
|
||||
|
||||
View traces in MLflow UI:
|
||||
|
||||

|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests', '<rootDir>/src'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
|
||||
},
|
||||
globalSetup: '<rootDir>/../../jest.global-server-setup.ts',
|
||||
globalTeardown: '<rootDir>/../../jest.global-server-teardown.ts',
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"name": "@mlflow/anthropic",
|
||||
"version": "0.3.0",
|
||||
"description": "Anthropic integration package for MLflow Tracing",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"opentelemetry",
|
||||
"llm",
|
||||
"anthropic",
|
||||
"claude",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "jest",
|
||||
"lint": "eslint . --ext .ts --max-warnings 0",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@mlflow/core": "^0.3.0",
|
||||
"@anthropic-ai/sdk": "^0.71.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"jest": "^29.6.2",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"files": [
|
||||
"dist/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* Main tracedAnthropic wrapper function for MLflow tracing integration
|
||||
*/
|
||||
|
||||
import {
|
||||
startSpan,
|
||||
getCurrentActiveSpan,
|
||||
SpanAttributeKey,
|
||||
SpanType,
|
||||
SpanStatusCode,
|
||||
TokenUsage,
|
||||
LiveSpan,
|
||||
withSpan,
|
||||
} from '@mlflow/core';
|
||||
|
||||
const SUPPORTED_MODULES = ['Messages'];
|
||||
const SUPPORTED_METHODS = ['create', 'stream'];
|
||||
|
||||
/**
|
||||
* Create a traced version of Anthropic client with MLflow tracing
|
||||
* @param anthropicClient - The Anthropic client instance to trace
|
||||
* @returns Traced Anthropic client with tracing capabilities
|
||||
*/
|
||||
export function tracedAnthropic<T = any>(anthropicClient: T): T {
|
||||
const tracedClient = new Proxy(anthropicClient as any, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
const moduleName = (target as object).constructor?.name;
|
||||
|
||||
if (typeof original === 'function') {
|
||||
if (shouldTraceMethod(moduleName, String(prop))) {
|
||||
const methodName = String(prop);
|
||||
if (methodName === 'stream') {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapStreamWithTracing(original as Function, moduleName) as T;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapWithTracing(original as Function, moduleName) as T;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return (original as Function).bind(target) as T;
|
||||
}
|
||||
|
||||
if (
|
||||
original &&
|
||||
!Array.isArray(original) &&
|
||||
!(original instanceof Date) &&
|
||||
typeof original === 'object'
|
||||
) {
|
||||
return tracedAnthropic(original) as T;
|
||||
}
|
||||
|
||||
return original as T;
|
||||
},
|
||||
});
|
||||
|
||||
return tracedClient as T;
|
||||
}
|
||||
|
||||
function shouldTraceMethod(moduleName: string | undefined, methodName: string): boolean {
|
||||
if (!moduleName) {
|
||||
return false;
|
||||
}
|
||||
return SUPPORTED_MODULES.includes(moduleName) && SUPPORTED_METHODS.includes(methodName);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapWithTracing(fn: Function, moduleName: string): Function {
|
||||
const spanType = getSpanType(moduleName);
|
||||
const name = moduleName;
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// Skip tracing for create() calls with stream: true
|
||||
// The Anthropic SDK's stream() method internally calls create() with stream: true,
|
||||
// and we don't want to create a duplicate trace - the streaming wrapper handles it
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (args[0]?.stream === true) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(args[0]);
|
||||
|
||||
const result = await fn.apply(this, args);
|
||||
|
||||
span.setOutputs(result);
|
||||
|
||||
try {
|
||||
const usage = extractTokenUsage(result);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return result;
|
||||
},
|
||||
{ name, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
function getSpanType(moduleName: string): SpanType | undefined {
|
||||
switch (moduleName) {
|
||||
case 'Messages':
|
||||
return SpanType.LLM;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapStreamWithTracing(fn: Function, moduleName: string): Function {
|
||||
const spanType = getSpanType(moduleName);
|
||||
const name = moduleName;
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// Create the stream (synchronous call)
|
||||
const stream = fn.apply(this, args);
|
||||
|
||||
// Return a proxy that wraps the stream with tracing
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return wrapMessageStream(stream, args[0], name, spanType);
|
||||
};
|
||||
}
|
||||
|
||||
function wrapMessageStream(stream: any, inputs: any, name: string, spanType: SpanType): any {
|
||||
// Use a flag that is set synchronously on first access to prevent duplicate spans.
|
||||
// It is claimed either when the async iterator getter is invoked or when the wrapped
|
||||
// finalMessage function is called, before any asynchronous work begins, so only one
|
||||
// of these access paths will perform tracing for a given stream instance.
|
||||
let tracingClaimed = false;
|
||||
|
||||
return new Proxy(stream, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
|
||||
// Wrap finalMessage() to add tracing
|
||||
if (prop === 'finalMessage') {
|
||||
return async function () {
|
||||
if (tracingClaimed) {
|
||||
// Already traced via async iteration, just return the message
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
return await target.finalMessage();
|
||||
}
|
||||
tracingClaimed = true;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(inputs);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
const message = await target.finalMessage();
|
||||
|
||||
span.setOutputs(message);
|
||||
|
||||
try {
|
||||
const usage = extractTokenUsage(message);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return message;
|
||||
},
|
||||
{ name, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
// Wrap async iterator for `for await (const event of stream)` pattern
|
||||
if (prop === Symbol.asyncIterator) {
|
||||
return function () {
|
||||
if (tracingClaimed) {
|
||||
// In practice, MessageStreams are typically consumed once and iterating again would
|
||||
// yield no events, so this may not be a real issue but for completeness we return the
|
||||
// unwrapped iterator in this case.
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call
|
||||
return target[Symbol.asyncIterator]();
|
||||
}
|
||||
tracingClaimed = true;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument
|
||||
return wrapAsyncIterator(target[Symbol.asyncIterator](), target, inputs, name, spanType);
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof original === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
return original.bind(target);
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return original;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function* wrapAsyncIterator(
|
||||
iterator: AsyncIterator<any>,
|
||||
stream: any,
|
||||
inputs: any,
|
||||
name: string,
|
||||
spanType: SpanType,
|
||||
): AsyncGenerator<any> {
|
||||
// Use startSpan for manual lifecycle management since withSpan doesn't support async generators
|
||||
const parentSpan = getCurrentActiveSpan();
|
||||
const span = startSpan({ name, spanType, parent: parentSpan ?? undefined });
|
||||
span.setInputs(inputs);
|
||||
|
||||
let iterationError: Error | undefined;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await iterator.next();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
yield value;
|
||||
}
|
||||
} catch (error) {
|
||||
iterationError = error as Error;
|
||||
throw error;
|
||||
} finally {
|
||||
if (iterationError) {
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
span.setStatus(SpanStatusCode.ERROR, iterationError.message);
|
||||
span.end();
|
||||
} else {
|
||||
// After iteration completes (or early termination via break/return),
|
||||
// get the final message for outputs and token usage
|
||||
try {
|
||||
// Prefer a proxy-aware finalMessage if available on the iterator, and fall back to the stream.
|
||||
let finalMessage: unknown;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
|
||||
const iteratorAny = iterator as any;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (iteratorAny && typeof iteratorAny.finalMessage === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
finalMessage = await iteratorAny.finalMessage();
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
} else if (stream && typeof stream.finalMessage === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
finalMessage = await stream.finalMessage();
|
||||
}
|
||||
|
||||
if (finalMessage !== undefined) {
|
||||
span.setOutputs(finalMessage);
|
||||
|
||||
const usage = extractTokenUsage(finalMessage);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Stream may have completed without finalMessage available
|
||||
console.debug('Could not get final message from stream', e);
|
||||
span.setAttribute('mlflow.tracing.token_usage_capture_failed', true);
|
||||
span.setAttribute(
|
||||
'mlflow.tracing.token_usage_capture_error',
|
||||
e instanceof Error ? e.message : String(e),
|
||||
);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractTokenUsage(response: any): TokenUsage | undefined {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const usage = response?.usage;
|
||||
if (!usage) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const inputTokens = usage.input_tokens;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const outputTokens = usage.output_tokens;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const totalTokens = usage.total_tokens ?? (inputTokens ?? 0) + (outputTokens ?? 0);
|
||||
|
||||
const result: TokenUsage = {
|
||||
input_tokens: inputTokens ?? 0,
|
||||
output_tokens: outputTokens ?? 0,
|
||||
total_tokens: totalTokens ?? 0,
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (usage.cache_read_input_tokens != null) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
result.cache_read_input_tokens = usage.cache_read_input_tokens;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (usage.cache_creation_input_tokens != null) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
result.cache_creation_input_tokens = usage.cache_creation_input_tokens;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
/**
|
||||
* Tests for MLflow Anthropic integration with MSW mock server
|
||||
*/
|
||||
|
||||
import * as mlflow from '@mlflow/core';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { tracedAnthropic } from '../src';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { anthropicMockHandlers, createStreamingErrorHandler } from './mockAnthropicServer';
|
||||
import { createAuthProvider } from '@mlflow/core/src/auth';
|
||||
|
||||
const TEST_TRACKING_URI = 'http://localhost:5000';
|
||||
|
||||
describe('tracedAnthropic', () => {
|
||||
let experimentId: string;
|
||||
let client: mlflow.MlflowClient;
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = setupServer(...anthropicMockHandlers);
|
||||
server.listen();
|
||||
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new mlflow.MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
|
||||
const experimentName = `anthropic-test-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: TEST_TRACKING_URI,
|
||||
experimentId,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
server.close();
|
||||
await client.deleteExperiment(experimentId);
|
||||
});
|
||||
|
||||
const getLastActiveTrace = async (): Promise<mlflow.Trace> => {
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
const trace = await client.getTrace(traceId!);
|
||||
return trace;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should trace messages.create()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude!' }],
|
||||
});
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const tokenUsage = trace.info.tokenUsage;
|
||||
expect(tokenUsage).toBeDefined();
|
||||
expect(typeof tokenUsage?.input_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.output_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.total_tokens).toBe('number');
|
||||
expect(tokenUsage?.cache_read_input_tokens).toBe(64);
|
||||
expect(tokenUsage?.cache_creation_input_tokens).toBe(32);
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.logLevel).toBe(mlflow.SpanLogLevel.INFO);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude!' }],
|
||||
});
|
||||
expect(span.outputs).toEqual(result);
|
||||
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should handle Anthropic errors properly', async () => {
|
||||
server.use(
|
||||
http.post('https://api.anthropic.com/v1/messages', () =>
|
||||
HttpResponse.json(
|
||||
{
|
||||
error: {
|
||||
type: 'rate_limit',
|
||||
message: 'Rate limit exceeded',
|
||||
},
|
||||
},
|
||||
{ status: 429 },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// Disable retries to prevent the SDK from retrying on errors.
|
||||
// SDK 0.50+ calls CancelReadableStream on error responses before retrying,
|
||||
// which hangs indefinitely with MSW mock responses.
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
await expect(
|
||||
wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'This request should fail.' }],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'This request should fail.' }],
|
||||
});
|
||||
expect(span.outputs).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should trace Anthropic request wrapped in a parent span', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const response = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello from parent span.' }],
|
||||
});
|
||||
return response.content[0];
|
||||
},
|
||||
{
|
||||
name: 'predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello from parent span.',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.data.spans.length).toBe(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('predict');
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
|
||||
const childSpan = trace.data.spans[1];
|
||||
expect(childSpan.name).toBe('Messages');
|
||||
expect(childSpan.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(childSpan.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello from parent span.' }],
|
||||
});
|
||||
expect(childSpan.outputs).toBeDefined();
|
||||
|
||||
const messageFormat = childSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should trace messages.stream() with async iteration', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const events: any[] = [];
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming!' }],
|
||||
});
|
||||
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
expect(events.length).toBeGreaterThan(0);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toMatchObject({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming!' }],
|
||||
});
|
||||
|
||||
// Verify outputs contain actual message content, not a stream object
|
||||
expect(span.outputs).toBeDefined();
|
||||
expect(span.outputs).toHaveProperty('id');
|
||||
expect(span.outputs).toHaveProperty('type', 'message');
|
||||
expect(span.outputs).toHaveProperty('content');
|
||||
expect(span.outputs.content[0]).toHaveProperty('type', 'text');
|
||||
|
||||
// Verify token usage is captured for streaming
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should trace messages.stream() with finalMessage()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
|
||||
});
|
||||
|
||||
const finalMessage = await stream.finalMessage();
|
||||
|
||||
expect(finalMessage).toBeDefined();
|
||||
expect(finalMessage.content).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
|
||||
});
|
||||
expect(span.outputs).toEqual(finalMessage);
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should handle errors during streaming async iteration', async () => {
|
||||
server.use(createStreamingErrorHandler());
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming error!' }],
|
||||
});
|
||||
|
||||
await expect(async () => {
|
||||
for await (const _event of stream) {
|
||||
// consume events until error
|
||||
}
|
||||
}).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toMatchObject({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming error!' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle errors during streaming finalMessage()', async () => {
|
||||
server.use(createStreamingErrorHandler());
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
|
||||
});
|
||||
|
||||
await expect(stream.finalMessage()).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('should create only one span when async iteration is followed by finalMessage()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello dedup!' }],
|
||||
});
|
||||
|
||||
// First consume via async iteration (claims tracing)
|
||||
for await (const _event of stream) {
|
||||
// consume all events
|
||||
}
|
||||
|
||||
// Then call finalMessage() on the same stream — should NOT create a second span
|
||||
const finalMessage = await stream.finalMessage();
|
||||
expect(finalMessage).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
// Only one LLM span should exist (no duplicate from finalMessage)
|
||||
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
|
||||
expect(llmSpans.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should not trace messages.create() when stream: true is passed directly', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
// Calling create() with stream: true is skipped by the tracing wrapper
|
||||
// because the SDK's stream() method internally calls create({ stream: true }),
|
||||
// and tracing is handled by the stream wrapper instead.
|
||||
// Users should use messages.stream() for traced streaming.
|
||||
const response = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello direct stream!' }],
|
||||
stream: true,
|
||||
});
|
||||
|
||||
// The SDK returns a Stream object when stream: true
|
||||
expect(response).toBeDefined();
|
||||
|
||||
// Consume the stream to completion
|
||||
for await (const _event of response as any) {
|
||||
// consume
|
||||
}
|
||||
|
||||
// No traced span should have been created by the create() wrapper
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
// The last active trace should be from a previous test, not this call,
|
||||
// OR if it is from this call, it should not have a Messages LLM span
|
||||
// created by wrapWithTracing (since create with stream:true is skipped).
|
||||
if (traceId) {
|
||||
const trace = await client.getTrace(traceId);
|
||||
const matchingSpans = trace.data.spans.filter(
|
||||
(s) =>
|
||||
s.spanType === mlflow.SpanType.LLM &&
|
||||
s.inputs?.messages?.[0]?.content === 'Hello direct stream!',
|
||||
);
|
||||
expect(matchingSpans.length).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should trace streaming request wrapped in a parent span', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello streaming parent!' }],
|
||||
});
|
||||
|
||||
const message = await stream.finalMessage();
|
||||
return message.content[0];
|
||||
},
|
||||
{
|
||||
name: 'streaming-predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello streaming parent!',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
// At least 2 spans: parent chain span and child LLM span
|
||||
expect(trace.data.spans.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('streaming-predict');
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
|
||||
// Find child spans with LLM type
|
||||
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
|
||||
expect(llmSpans.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const llmSpan = llmSpans[0];
|
||||
expect(llmSpan.outputs).toBeDefined();
|
||||
|
||||
const messageFormat = llmSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* MSW-based Anthropic API mock server for testing
|
||||
*/
|
||||
|
||||
import { http, HttpResponse } from 'msw';
|
||||
|
||||
interface MessagesCreateRequest {
|
||||
model: string;
|
||||
messages: Array<{
|
||||
role: string;
|
||||
content: string | Array<{ type: string; text: string }>;
|
||||
}>;
|
||||
max_tokens?: number;
|
||||
system?: string | Array<{ type: string; text: string }>;
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
function createMessageResponse(request: MessagesCreateRequest) {
|
||||
return {
|
||||
id: `msg_${Math.random().toString(36).slice(2)}`,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: request.model,
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: 128,
|
||||
output_tokens: 256,
|
||||
total_tokens: 384,
|
||||
cache_read_input_tokens: 64,
|
||||
cache_creation_input_tokens: 32,
|
||||
},
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'This is a mocked Anthropic response.',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function createStreamingEvents(request: MessagesCreateRequest): string[] {
|
||||
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
|
||||
const responseText = 'This is a mocked streaming response.';
|
||||
|
||||
return [
|
||||
`event: message_start\ndata: ${JSON.stringify({
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id: messageId,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: request.model,
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: 128,
|
||||
output_tokens: 0,
|
||||
},
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_start\ndata: ${JSON.stringify({
|
||||
type: 'content_block_start',
|
||||
index: 0,
|
||||
content_block: {
|
||||
type: 'text',
|
||||
text: '',
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_delta\ndata: ${JSON.stringify({
|
||||
type: 'content_block_delta',
|
||||
index: 0,
|
||||
delta: {
|
||||
type: 'text_delta',
|
||||
text: responseText,
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_stop\ndata: ${JSON.stringify({
|
||||
type: 'content_block_stop',
|
||||
index: 0,
|
||||
})}\n\n`,
|
||||
`event: message_delta\ndata: ${JSON.stringify({
|
||||
type: 'message_delta',
|
||||
delta: {
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
},
|
||||
usage: {
|
||||
output_tokens: 256,
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: message_stop\ndata: ${JSON.stringify({
|
||||
type: 'message_stop',
|
||||
})}\n\n`,
|
||||
];
|
||||
}
|
||||
|
||||
export function createStreamingErrorHandler() {
|
||||
return http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
|
||||
const body = (await request.json()) as MessagesCreateRequest;
|
||||
|
||||
if (body.stream) {
|
||||
// Send a partial stream that ends with an error event
|
||||
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
|
||||
const events = [
|
||||
`event: message_start\ndata: ${JSON.stringify({
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id: messageId,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: body.model,
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 128, output_tokens: 0 },
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: error\ndata: ${JSON.stringify({
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'overloaded_error',
|
||||
message: 'Overloaded',
|
||||
},
|
||||
})}\n\n`,
|
||||
];
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
for (const event of events) {
|
||||
controller.enqueue(encoder.encode(event));
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new HttpResponse(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return HttpResponse.json(
|
||||
{ error: { type: 'overloaded_error', message: 'Overloaded' } },
|
||||
{ status: 529 },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export const anthropicMockHandlers = [
|
||||
http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
|
||||
const body = (await request.json()) as MessagesCreateRequest;
|
||||
|
||||
// Check if streaming is requested
|
||||
if (body.stream) {
|
||||
const events = createStreamingEvents(body);
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
for (const event of events) {
|
||||
controller.enqueue(encoder.encode(event));
|
||||
// Small delay to simulate streaming
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new HttpResponse(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return HttpResponse.json(createMessageResponse(body));
|
||||
}),
|
||||
];
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user