chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:44:08 +08:00
commit 983960e2dd
1244 changed files with 281996 additions and 0 deletions
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Business & Finance",
"description": "Tools for business and finance.",
"icon_name": "money-dollar-circle-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Calendar & Scheduling",
"description": "Tools for calendars and scheduling.",
"icon_name": "calendar-2-line",
"context_files": [],
"tools": []
}
@@ -0,0 +1,3 @@
from .src.python.opencode_tool import OpenCodeTool
__all__ = ["OpenCodeTool"]
@@ -0,0 +1,4 @@
{
"OPENCODE_OPENROUTER_API_KEY": null,
"OPENCODE_OPENROUTER_MODEL": "openrouter/openai/gpt-5.2-codex"
}
@@ -0,0 +1 @@
export { default } from './opencode-tool'
@@ -0,0 +1,68 @@
{{SYSTEM_PROMPT_SECTION}}
{{REPO_SNAPSHOT}}
{{TOOLKIT_INFO}}
# Leon Skill Creation (Concise)
You are generating a Leon skill in **{{LANGUAGE}}**.
## Core Rules
- Use the **{{BRIDGE}}** bridge for all source files.
- Skills live directly under `skills/` (no subfolders).
- All source files use `{{FILE_EXTENSION}}`.
- Validate JSON files against `schemas/skill-schemas/*`.
- Write all required files to disk under the chosen `skills/<name>_skill` folder.
## Required Structure
```
skills/skill_name/
skill.json
locales/en.json
src/
settings.sample.json
settings.json
actions/
widgets/ (optional)
```
## skill.json Rules
- `actions` required, `flow` optional.
- If `flow` exists, only the first action receives user parameters.
- Use `"skill_name:action_name"` for cross-skill flow steps.
- Set `author.name` to `Leon` unless explicitly specified.
## Settings Files
- `src/settings.sample.json` and `src/settings.json` must both exist and start identical.
- Use `{}` if no settings.
## Toolkits (Plan First)
- Choose relevant toolkits from above **before** writing code.
- Use existing tools instead of duplicating functionality.
## leon.answer Basics
{{LEON_ANSWER_BASIC_EXAMPLE}}
## Passing Data Between Actions
{{CONTEXT_DATA_EXAMPLE}}
## Settings Usage
{{SETTINGS_USAGE_EXAMPLE}}
## Widget Rules
- Do not use `Card` as the parent component. The `WidgetWrapper` is already applied by default.
- For icons, use only the icon name without the `ri-` prefix and `-line` suffix. The system automatically completes them to `ri-{icon-name}-line`. For example, use `snow` instead of `ri-snow-line`.
## Action Parameters
{{ACTION_PARAMS_EXAMPLE}}
{{REFERENCE_FILES_SECTION}}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,68 @@
{{SYSTEM_PROMPT_SECTION}}
{{REPO_SNAPSHOT}}
{{TOOLKIT_INFO}}
# Leon Skill Creation (Concise)
You are generating a Leon skill in **{{LANGUAGE}}**.
## Core Rules
- Use the **{{BRIDGE}}** bridge for all source files.
- Skills live directly under `skills/` (no subfolders).
- All source files use `{{FILE_EXTENSION}}`.
- Validate JSON files against `schemas/skill-schemas/*`.
- Write all required files to disk under the chosen `skills/<name>_skill` folder.
## Required Structure
```
skills/skill_name/
skill.json
locales/en.json
src/
settings.sample.json
settings.json
actions/
widgets/ (optional)
```
## skill.json Rules
- `actions` required, `flow` optional.
- If `flow` exists, only the first action receives user parameters.
- Use `"skill_name:action_name"` for cross-skill flow steps.
- Set `author.name` to `Leon` unless explicitly specified.
## Settings Files
- `src/settings.sample.json` and `src/settings.json` must both exist and start identical.
- Use `{}` if no settings.
## Toolkits (Plan First)
- Choose relevant toolkits from above **before** writing code.
- Use existing tools instead of duplicating functionality.
## leon.answer Basics
{{LEON_ANSWER_BASIC_EXAMPLE}}
## Passing Data Between Actions
{{CONTEXT_DATA_EXAMPLE}}
## Settings Usage
{{SETTINGS_USAGE_EXAMPLE}}
## Widget Rules
- Do not use `Card` as the parent component. The `WidgetWrapper` is already applied by default.
- For icons, use only the icon name without the `ri-` prefix and `-line` suffix. The system automatically completes them to `ri-{icon-name}-line`. For example, use `snow` instead of `ri-snow-line`.
## Action Parameters
{{ACTION_PARAMS_EXAMPLE}}
{{REFERENCE_FILES_SECTION}}
File diff suppressed because it is too large Load Diff
+111
View File
@@ -0,0 +1,111 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "opencode",
"toolkit_id": "coding_development",
"name": "OpenCode",
"description": "An AI-powered coding agent tool that generates skills using multiple LLM providers (Cerebras, MiniMax, Anthropic, OpenAI, Gemini).",
"icon_name": "code-box-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"binaries": {
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-linux-x86_64.tar.gz",
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-linux-aarch64.tar.gz",
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-macosx-x86_64.zip",
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-macosx-arm64.zip",
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-win-amd64.zip"
},
"functions": {
"configureProvider": {
"description": "Configure a provider with an API key and optional model.",
"parameters": {
"type": "object",
"properties": {
"provider": {
"type": "string"
},
"apiKey": {
"type": "string"
},
"model": {
"type": "string"
}
},
"required": [
"provider",
"apiKey"
]
}
},
"getConfiguredProviders": {
"description": "List the providers currently configured with API keys.",
"parameters": {
"type": "object",
"properties": {}
}
},
"getAvailableProviders": {
"description": "List providers supported by OpenCode.",
"parameters": {
"type": "object",
"properties": {}
}
},
"getDefaultModel": {
"description": "Get the default model name for a provider.",
"parameters": {
"type": "object",
"properties": {
"provider": {
"type": "string"
}
},
"required": [
"provider"
]
}
},
"generateSkill": {
"description": "Generate a new skill using OpenCode CLI with an agentic loop.",
"parameters": {
"type": "object",
"properties": {
"description": {
"type": "string"
},
"provider": {
"type": "string"
},
"model": {
"type": "string"
},
"api_key": {
"type": "string"
},
"target_path": {
"type": "string"
},
"context_files": {
"type": "array",
"items": {
"type": "string"
}
},
"system_prompt": {
"type": "string"
},
"bridge": {
"type": "string"
}
},
"required": [
"description",
"provider",
"target_path"
]
}
}
}
}
+15
View File
@@ -0,0 +1,15 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Coding & Development",
"description": "Tools for code generation, development, and automation.",
"icon_name": "code-s-slash-line",
"context_files": [
"ARCHITECTURE.md",
"WORKSPACE_INTELLIGENCE.md",
"LEON_RUNTIME.md",
"HOME.md"
],
"tools": [
"opencode"
]
}
+3
View File
@@ -0,0 +1,3 @@
from .src.python.cerebras_tool import CerebrasTool
__all__ = ["CerebrasTool"]
@@ -0,0 +1,4 @@
{
"CEREBRAS_API_KEY": null,
"CEREBRAS_MODEL": "zai-glm-4.7"
}
@@ -0,0 +1,352 @@
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network, NetworkError } from '@sdk/network'
// Hardcoded default settings for Cerebras tool
const CEREBRAS_API_KEY: string | null = null
const CEREBRAS_MODEL = 'zai-glm-4.7'
const DEFAULT_SETTINGS: Record<string, unknown> = {
CEREBRAS_API_KEY,
CEREBRAS_MODEL
}
const REQUIRED_SETTINGS = ['CEREBRAS_API_KEY']
interface ChatMessage {
role: string
content: string
}
interface ChatCompletionOptions {
messages: ChatMessage[]
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
use_structured_output?: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema?: Record<string, any>
}
interface CompletionOptions {
prompt: string
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
use_structured_output?: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema?: Record<string, any>
}
interface StructuredCompletionOptions {
prompt: string
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema: Record<string, any>
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
}
interface ApiResponse {
success: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data?: any
model_used?: string
error?: string
status_code?: number
}
export default class CerebrasTool extends Tool {
private static readonly TOOLKIT = 'communication'
private readonly config: ReturnType<typeof ToolkitConfig.load>
private api_key: string | null
private model: string
private readonly network: Network
// Popular Cerebras-hosted models (override with full model IDs if needed)
private readonly popular_models = {
'zai-glm-4.7': 'zai-glm-4.7',
'qwen-3-235b-a22b-instruct-2507': 'qwen-3-235b-a22b-instruct-2507',
'qwen-3-32b': 'qwen-3-32b'
}
constructor(apiKey?: string) {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(CerebrasTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
CerebrasTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
// Priority: skill-provided apiKey > toolkit settings > hardcoded default
this.api_key =
apiKey ||
(this.settings['CEREBRAS_API_KEY'] as string) ||
CEREBRAS_API_KEY
// Load model from toolkit settings or hardcoded default
this.model = (this.settings['CEREBRAS_MODEL'] as string) || CEREBRAS_MODEL
this.network = new Network({ baseURL: 'https://api.cerebras.ai/v1' })
}
get toolName(): string {
return 'cerebras'
}
get toolkit(): string {
return CerebrasTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Set the Cerebras API key
*/
setApiKey(apiKey: string): void {
this.api_key = apiKey
}
/**
* Get list of popular available models
*/
getAvailableModels(): string[] {
return Object.keys(this.popular_models)
}
/**
* Convert friendly model name to Cerebras model ID
*/
getModelId(modelName: string): string {
return (
this.popular_models[modelName as keyof typeof this.popular_models] ||
modelName
)
}
/**
* Send a chat completion request to Cerebras
*/
async chatCompletion(options: ChatCompletionOptions): Promise<ApiResponse> {
const {
messages,
model,
temperature = 0.7,
max_tokens,
system_prompt,
use_structured_output = false,
json_schema
} = options
if (!this.api_key) {
return {
success: false,
error: 'Cerebras API key not configured'
}
}
// Use default model if none provided
const finalModel = model || this.model
const modelId = this.getModelId(finalModel)
const requestMessages = []
if (system_prompt) {
requestMessages.push({ role: 'system', content: system_prompt })
}
requestMessages.push(...messages)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const payload: any = {
model: modelId,
messages: requestMessages,
temperature
}
if (max_tokens) {
payload.max_tokens = max_tokens
}
if (use_structured_output) {
payload.response_format = { type: 'json_object' }
if (json_schema) {
const schemaText = JSON.stringify(json_schema)
const schemaPrompt = `You must return a valid JSON object that matches this schema:\n${schemaText}`
payload.messages = [
{ role: 'system', content: schemaPrompt },
...requestMessages
]
}
}
try {
const response = await this.network.request({
url: '/chat/completions',
method: 'POST',
headers: {
Authorization: `Bearer ${this.api_key}`,
'Content-Type': 'application/json'
},
data: payload
})
return {
success: true,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: response.data as any,
model_used: modelId
}
} catch (error: unknown) {
return {
success: false,
error: `Cerebras API error: ${(error as Error).message}`,
status_code:
error instanceof NetworkError ? error.response.statusCode : undefined
}
}
}
/**
* General text completion for any use case
*/
async completion(options: CompletionOptions): Promise<ApiResponse> {
const {
prompt,
model,
temperature = 0.7,
max_tokens,
system_prompt,
use_structured_output = false,
json_schema
} = options
const messages = [{ role: 'user', content: prompt }]
const response = await this.chatCompletion({
messages,
model: model || this.model,
temperature,
max_tokens,
system_prompt,
use_structured_output,
json_schema
})
if (!response.success) {
return response
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const content = (response.data as any).choices[0].message.content
return {
success: true,
data: { content },
model_used: response.model_used
}
} catch (error: unknown) {
return {
success: false,
error: `Failed to extract completion: ${(error as Error).message}`
}
}
}
/**
* Generate structured JSON output using Cerebras structured outputs
*/
async structuredCompletion(
options: StructuredCompletionOptions
): Promise<ApiResponse> {
const {
prompt,
json_schema,
model,
temperature = 0.7,
max_tokens,
system_prompt
} = options
const messages = [{ role: 'user', content: prompt }]
const response = await this.chatCompletion({
messages,
model: model || this.model,
temperature,
max_tokens,
system_prompt,
use_structured_output: true,
json_schema
})
if (!response.success) {
return response
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const content = (response.data as any).choices[0].message.content
const parsedData = JSON.parse(content)
return {
success: true,
data: parsedData,
model_used: response.model_used
}
} catch (error: unknown) {
if (error instanceof SyntaxError) {
return {
success: false,
error: `Failed to parse JSON response: ${error.message}`
}
}
return {
success: false,
error: `Failed to extract completion: ${(error as Error).message}`
}
}
}
/**
* Get list of available models from Cerebras API
*/
async listModels(): Promise<ApiResponse> {
if (!this.api_key) {
return {
success: false,
error: 'Cerebras API key not configured'
}
}
try {
const response = await this.network.request({
url: '/models',
method: 'GET',
headers: {
Authorization: `Bearer ${this.api_key}`
}
})
return {
success: true,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: { models: (response.data as any).data }
}
} catch (error: unknown) {
return {
success: false,
error: `Failed to fetch models: ${(error as Error).message}`
}
}
}
}
@@ -0,0 +1 @@
export { default } from './cerebras-tool'
@@ -0,0 +1,290 @@
import json
from typing import Dict, Any, Optional, List
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.network import Network, NetworkError
# Hardcoded default settings for Cerebras tool
CEREBRAS_API_KEY = None
CEREBRAS_MODEL = "zai-glm-4.7"
DEFAULT_SETTINGS = {
"CEREBRAS_API_KEY": CEREBRAS_API_KEY,
"CEREBRAS_MODEL": CEREBRAS_MODEL,
}
REQUIRED_SETTINGS = ["CEREBRAS_API_KEY"]
class CerebrasTool(BaseTool):
"""Cerebras tool for LLM API access (e.g., GLM 4.7)"""
TOOLKIT = "communication"
def __init__(self, api_key: Optional[str] = None):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
tool_settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.settings = tool_settings
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
# Priority: skill-provided api_key > toolkit settings > hardcoded default
self.api_key = api_key or self.settings.get(
"CEREBRAS_API_KEY", CEREBRAS_API_KEY
)
# Load model settings
self.model = self.settings.get("CEREBRAS_MODEL", CEREBRAS_MODEL)
self.network = Network({"base_url": "https://api.cerebras.ai/v1"})
# Popular Cerebras-hosted models (override with full model IDs if needed)
self.popular_models = {
"zai-glm-4.7": "zai-glm-4.7",
"qwen-3-235b-a22b-instruct-2507": "qwen-3-235b-a22b-instruct-2507",
"qwen-3-32b": "qwen-3-32b",
}
@property
def tool_name(self) -> str:
return "cerebras"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def set_api_key(self, api_key: str) -> None:
"""Set the Cerebras API key"""
self.api_key = api_key
def get_available_models(self) -> List[str]:
"""Get list of popular available models"""
return list(self.popular_models.keys())
def get_model_id(self, model_name: str) -> str:
"""Convert friendly model name to Cerebras model ID"""
return self.popular_models.get(model_name, model_name)
def chat_completion(
self,
messages: List[Dict[str, str]],
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
use_structured_output: bool = False,
json_schema: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Send a chat completion request to Cerebras
Args:
messages: List of message dictionaries with 'role' and 'content'
model: Model name (friendly name or full model ID)
temperature: Sampling temperature (0-2)
max_tokens: Maximum tokens to generate
system_prompt: System prompt to prepend
use_structured_output: Whether to use structured outputs
json_schema: JSON schema for structured output (required if use_structured_output=True)
Returns:
Dict with response data or error information
"""
if not self.api_key:
return {"success": False, "error": "Cerebras API key not configured"}
# Use default model if none provided
model = model or self.model
model_id = self.get_model_id(model)
request_messages: List[Dict[str, str]] = []
if system_prompt:
request_messages.append({"role": "system", "content": system_prompt})
request_messages.extend(messages)
payload: Dict[str, Any] = {
"model": model_id,
"messages": request_messages,
"temperature": temperature,
}
if max_tokens:
payload["max_tokens"] = max_tokens
if use_structured_output:
payload["response_format"] = {"type": "json_object"}
if json_schema:
schema_text = json.dumps(json_schema)
schema_prompt = (
"You must return a valid JSON object that matches this schema:\n"
f"{schema_text}"
)
payload["messages"] = [
{"role": "system", "content": schema_prompt}
] + request_messages
try:
response = self.network.request(
{
"url": "/chat/completions",
"method": "POST",
"headers": {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
},
"data": payload,
}
)
return {"success": True, "data": response["data"], "model_used": model_id}
except NetworkError as e:
return {
"success": False,
"error": f"Cerebras API error: {str(e)}",
"status_code": getattr(e.response, "status_code", None),
}
def completion(
self,
prompt: str,
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
use_structured_output: bool = False,
json_schema: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
General text completion for any use case
Args:
prompt: Text prompt to complete
model: LLM model to use
temperature: Sampling temperature
max_tokens: Maximum tokens to generate
system_prompt: Optional system prompt
use_structured_output: Whether to use structured outputs
json_schema: JSON schema for structured output
Returns:
Dict with completion result
"""
messages = [{"role": "user", "content": prompt}]
response = self.chat_completion(
messages=messages,
model=model,
temperature=temperature,
max_tokens=max_tokens,
system_prompt=system_prompt,
use_structured_output=use_structured_output,
json_schema=json_schema,
)
if not response["success"]:
return response
try:
content = response["data"]["choices"][0]["message"]["content"]
return {
"success": True,
"content": content,
"model_used": response["model_used"],
}
except (KeyError, IndexError) as e:
return {
"success": False,
"error": f"Failed to extract completion: {str(e)}",
}
def structured_completion(
self,
prompt: str,
json_schema: Dict[str, Any],
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
) -> Dict[str, Any]:
"""
Generate structured JSON output using Cerebras structured outputs
Args:
prompt: Text prompt to complete
json_schema: JSON schema defining the required output structure
model: LLM model to use
temperature: Sampling temperature
max_tokens: Maximum tokens to generate
system_prompt: Optional system prompt
Returns:
Dict with parsed JSON result or error
"""
messages = [{"role": "user", "content": prompt}]
response = self.chat_completion(
messages=messages,
model=model,
temperature=temperature,
max_tokens=max_tokens,
system_prompt=system_prompt,
use_structured_output=True,
json_schema=json_schema,
)
if not response["success"]:
return response
try:
content = response["data"]["choices"][0]["message"]["content"]
parsed_data = json.loads(content)
return {
"success": True,
"data": parsed_data,
"model_used": response["model_used"],
}
except (KeyError, IndexError) as e:
return {
"success": False,
"error": f"Failed to extract completion: {str(e)}",
}
except json.JSONDecodeError as e:
return {
"success": False,
"error": f"Failed to parse JSON response: {str(e)}",
}
def list_models(self) -> Dict[str, Any]:
"""
Get list of available models from Cerebras API
Returns:
Dict with models list or error
"""
if not self.api_key:
return {"success": False, "error": "Cerebras API key not configured"}
try:
response = self.network.request(
{
"url": "/models",
"method": "GET",
"headers": {"Authorization": f"Bearer {self.api_key}"},
}
)
return {
"success": True,
"models": response["data"].get("data", response["data"]),
}
except NetworkError as e:
return {"success": False, "error": f"Failed to fetch models: {str(e)}"}
+161
View File
@@ -0,0 +1,161 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "cerebras",
"toolkit_id": "communication",
"name": "Cerebras",
"description": "A tool for interacting with Cerebras LLM APIs (e.g., GLM 4.7).",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"chatCompletion": {
"description": "Generate a chat completion using the Cerebras API.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"messages": {
"type": "array",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string"
},
"content": {
"type": "string"
}
},
"required": [
"role",
"content"
],
"additionalProperties": false
}
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
},
"use_structured_output": {
"type": "boolean"
},
"json_schema": {
"type": "object",
"additionalProperties": true
}
},
"required": [
"messages"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"completion": {
"description": "Generate a completion using the Cerebras API.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
},
"use_structured_output": {
"type": "boolean"
},
"json_schema": {
"type": "object",
"additionalProperties": true
}
},
"required": [
"prompt"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"structuredCompletion": {
"description": "Generate a structured completion using a JSON schema.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"json_schema": {
"type": "object",
"additionalProperties": true
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
}
},
"required": [
"prompt",
"json_schema"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"listModels": {
"description": "List available Cerebras models.",
"parameters": {
"type": "object",
"properties": {}
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.inference_tool import InferenceTool
__all__ = ["InferenceTool"]
@@ -0,0 +1 @@
{}
@@ -0,0 +1 @@
export { default } from './inference-tool'
@@ -0,0 +1,97 @@
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network } from '@sdk/network'
interface CompletionOptions {
prompt: string
system_prompt?: string
temperature?: number
max_tokens?: number
thought_tokens_budget?: number
disable_thinking?: boolean
reasoning_mode?: 'off' | 'guarded' | 'on'
track_provider_errors?: boolean
}
interface StructuredCompletionOptions extends CompletionOptions {
json_schema: Record<string, unknown>
}
interface InferenceResponse {
success: boolean
output?: unknown
reasoning?: string
usedInputTokens?: number
usedOutputTokens?: number
generationDurationMs?: number
providerDecodeDurationMs?: number
providerTokensPerSecond?: number
error?: string
}
export default class InferenceTool extends Tool {
private static readonly TOOLKIT = 'communication'
private readonly config: ReturnType<typeof ToolkitConfig.load>
private readonly network: Network
constructor() {
super()
this.config = ToolkitConfig.load(InferenceTool.TOOLKIT, this.toolName)
this.network = new Network({
baseURL: `${process.env['LEON_HOST']}:${process.env['LEON_PORT']}/api/v1`
})
}
get toolName(): string {
return 'inference'
}
get toolkit(): string {
return InferenceTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
async completion(options: CompletionOptions): Promise<InferenceResponse> {
const response = await this.network.request<InferenceResponse>({
url: '/inference',
method: 'POST',
data: {
prompt: options.prompt,
systemPrompt: options.system_prompt,
temperature: options.temperature,
maxTokens: options.max_tokens,
thoughtTokensBudget: options.thought_tokens_budget,
disableThinking: options.disable_thinking,
reasoningMode: options.reasoning_mode,
trackProviderErrors: options.track_provider_errors
}
})
return response.data
}
async structuredCompletion(
options: StructuredCompletionOptions
): Promise<InferenceResponse> {
const response = await this.network.request<InferenceResponse>({
url: '/inference',
method: 'POST',
data: {
prompt: options.prompt,
systemPrompt: options.system_prompt,
temperature: options.temperature,
maxTokens: options.max_tokens,
thoughtTokensBudget: options.thought_tokens_budget,
jsonSchema: options.json_schema,
disableThinking: options.disable_thinking,
reasoningMode: options.reasoning_mode,
trackProviderErrors: options.track_provider_errors
}
})
return response.data
}
}
@@ -0,0 +1,93 @@
import os
from typing import Any, Dict, Optional, Literal
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.network import Network
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
class InferenceTool(BaseTool):
TOOLKIT = "communication"
def __init__(self):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
self.network = Network(
{
"base_url": f"{os.environ.get('LEON_HOST')}:{os.environ.get('LEON_PORT')}/api/v1"
}
)
@property
def tool_name(self) -> str:
return "inference"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config.get("description", "")
def completion(
self,
prompt: str,
system_prompt: Optional[str] = None,
temperature: Optional[float] = None,
max_tokens: Optional[int] = None,
thought_tokens_budget: Optional[int] = None,
disable_thinking: Optional[bool] = None,
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
track_provider_errors: Optional[bool] = None,
) -> Dict[str, Any]:
response = self.network.request(
{
"url": "/inference",
"method": "POST",
"data": {
"prompt": prompt,
"systemPrompt": system_prompt,
"temperature": temperature,
"maxTokens": max_tokens,
"thoughtTokensBudget": thought_tokens_budget,
"disableThinking": disable_thinking,
"reasoningMode": reasoning_mode,
"trackProviderErrors": track_provider_errors,
},
}
)
return response["data"]
def structured_completion(
self,
prompt: str,
json_schema: Dict[str, Any],
system_prompt: Optional[str] = None,
temperature: Optional[float] = None,
max_tokens: Optional[int] = None,
thought_tokens_budget: Optional[int] = None,
disable_thinking: Optional[bool] = None,
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
track_provider_errors: Optional[bool] = None,
) -> Dict[str, Any]:
response = self.network.request(
{
"url": "/inference",
"method": "POST",
"data": {
"prompt": prompt,
"systemPrompt": system_prompt,
"temperature": temperature,
"maxTokens": max_tokens,
"thoughtTokensBudget": thought_tokens_budget,
"jsonSchema": json_schema,
"disableThinking": disable_thinking,
"reasoningMode": reasoning_mode,
"trackProviderErrors": track_provider_errors,
},
}
)
return response["data"]
+101
View File
@@ -0,0 +1,101 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "inference",
"toolkit_id": "communication",
"name": "Inference",
"description": "A generic Leon workflow inference tool backed by the active workflow LLM provider.",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"completion": {
"description": "Generate a workflow inference completion.",
"parameters": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"options": {
"type": "object",
"properties": {
"system_prompt": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"thought_tokens_budget": {
"type": "number"
},
"disable_thinking": {
"type": "boolean"
},
"reasoning_mode": {
"type": "string"
},
"track_provider_errors": {
"type": "boolean"
}
},
"additionalProperties": false
}
},
"required": [
"prompt"
]
}
},
"structuredCompletion": {
"description": "Generate a structured workflow inference completion using a JSON schema.",
"parameters": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"json_schema": {
"type": "object",
"additionalProperties": true
},
"options": {
"type": "object",
"properties": {
"system_prompt": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"thought_tokens_budget": {
"type": "number"
},
"disable_thinking": {
"type": "boolean"
},
"reasoning_mode": {
"type": "string"
},
"track_provider_errors": {
"type": "boolean"
}
},
"additionalProperties": false
}
},
"required": [
"prompt",
"json_schema"
]
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.openrouter_tool import OpenRouterTool
__all__ = ["OpenRouterTool"]
@@ -0,0 +1,4 @@
{
"OPENROUTER_API_KEY": null,
"OPENROUTER_MODEL": "google/gemini-3.1-flash-lite"
}
@@ -0,0 +1 @@
export { default } from './openrouter-tool'
@@ -0,0 +1,340 @@
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network, NetworkError } from '@sdk/network'
// Hardcoded default settings for OpenRouter tool
const OPENROUTER_API_KEY: string | null = null
const OPENROUTER_MODEL = 'google/gemini-3.1-flash-lite'
const DEFAULT_SETTINGS: Record<string, unknown> = {
OPENROUTER_API_KEY,
OPENROUTER_MODEL
}
const REQUIRED_SETTINGS = ['OPENROUTER_API_KEY']
interface ChatMessage {
role: string
content: string
}
interface ChatCompletionOptions {
messages: ChatMessage[]
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
use_structured_output?: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema?: Record<string, any>
}
interface CompletionOptions {
prompt: string
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
use_structured_output?: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema?: Record<string, any>
}
interface StructuredCompletionOptions {
prompt: string
// eslint-disable-next-line @typescript-eslint/no-explicit-any
json_schema: Record<string, any>
model?: string
temperature?: number
max_tokens?: number
system_prompt?: string
}
interface ApiResponse {
success: boolean
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data?: any
model_used?: string
error?: string
status_code?: number
}
export default class OpenRouterTool extends Tool {
private static readonly TOOLKIT = 'communication'
private readonly config: ReturnType<typeof ToolkitConfig.load>
private api_key: string | null
private model: string
private readonly network: Network
constructor(apiKey?: string) {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(OpenRouterTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
OpenRouterTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = apiKey ? [] : REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
// Priority: skill-provided apiKey > toolkit settings > hardcoded default
this.api_key =
apiKey ||
(this.settings['OPENROUTER_API_KEY'] as string) ||
OPENROUTER_API_KEY
// Load model from toolkit settings or hardcoded default
this.model =
(this.settings['OPENROUTER_MODEL'] as string) || OPENROUTER_MODEL
this.network = new Network({ baseURL: 'https://openrouter.ai/api' })
}
get toolName(): string {
return 'openrouter'
}
get toolkit(): string {
return OpenRouterTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Set the OpenRouter API key
*/
setApiKey(apiKey: string): void {
this.api_key = apiKey
}
/**
* Send a chat completion request to OpenRouter
*/
async chatCompletion(options: ChatCompletionOptions): Promise<ApiResponse> {
const {
messages,
model,
temperature = 0.7,
max_tokens,
system_prompt,
use_structured_output = false,
json_schema
} = options
if (!this.api_key) {
return {
success: false,
error: 'OpenRouter API key not configured'
}
}
// Use default model if none provided
const finalModel = model || this.model
// Prepare messages with system prompt if provided
const requestMessages = []
if (system_prompt) {
requestMessages.push({ role: 'system', content: system_prompt })
}
requestMessages.push(...messages)
// Prepare request payload
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const payload: any = {
model: finalModel,
messages: requestMessages,
temperature
}
if (max_tokens) {
payload.max_tokens = max_tokens
}
// Add structured output configuration if requested
if (use_structured_output && json_schema) {
payload.response_format = {
type: 'json_schema',
json_schema: {
name: json_schema['name'] || 'response',
strict: true,
schema: json_schema['schema']
}
}
}
try {
const response = await this.network.request({
url: '/v1/chat/completions',
method: 'POST',
headers: {
Authorization: `Bearer ${this.api_key}`,
'Content-Type': 'application/json'
},
data: payload
})
return {
success: true,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: response.data as any,
model_used: finalModel
}
} catch (error: unknown) {
return {
success: false,
error: `OpenRouter API error: ${(error as Error).message}`,
status_code:
error instanceof NetworkError ? error.response.statusCode : undefined
}
}
}
/**
* General text completion for any use case
*/
async completion(options: CompletionOptions): Promise<ApiResponse> {
const {
prompt,
model,
temperature = 0.7,
max_tokens,
system_prompt,
use_structured_output = false,
json_schema
} = options
const messages = [{ role: 'user', content: prompt }]
const response = await this.chatCompletion({
messages,
model: model || this.model,
temperature,
max_tokens,
system_prompt,
use_structured_output,
json_schema
})
if (!response.success) {
return response
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const content = (response.data as any).choices[0].message.content
return {
success: true,
data: { content },
model_used: response.model_used
}
} catch (error: unknown) {
return {
success: false,
error: `Failed to extract completion: ${(error as Error).message}`
}
}
}
/**
* Generate structured JSON output using OpenRouter's structured outputs feature
*/
async structuredCompletion(
options: StructuredCompletionOptions
): Promise<ApiResponse> {
const {
prompt,
json_schema,
model,
temperature = 0.7,
max_tokens,
system_prompt
} = options
const messages = [{ role: 'user', content: prompt }]
const response = await this.chatCompletion({
messages,
model: model || this.model,
temperature,
max_tokens,
system_prompt,
use_structured_output: true,
json_schema
})
if (!response.success) {
return response
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const content = (response.data as any).choices[0].message.content
const parsedData =
typeof content === 'string' ? JSON.parse(content) : content
return {
success: true,
data: parsedData,
model_used: response.model_used
}
} catch (error: unknown) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const content = (response.data as any).choices[0]?.message?.content
if (error instanceof SyntaxError) {
// Show raw response preview to help debug JSON parsing errors
const preview =
typeof content === 'string'
? content.substring(0, 500)
: JSON.stringify(content ?? 'null').substring(0, 500)
return {
success: false,
error: `Failed to parse JSON response: ${error.message}. Response preview: ${preview}`
}
} else {
return {
success: false,
error: `Failed to extract completion: ${(error as Error).message}`
}
}
}
}
/**
* Get list of available models from OpenRouter API
*/
async listModels(): Promise<ApiResponse> {
if (!this.api_key) {
return {
success: false,
error: 'OpenRouter API key not configured'
}
}
try {
const response = await this.network.request({
url: '/v1/models',
method: 'GET',
headers: {
Authorization: `Bearer ${this.api_key}`
}
})
return {
success: true,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: { models: (response.data as any).data }
}
} catch (error: unknown) {
return {
success: false,
error: `Failed to fetch models: ${(error as Error).message}`
}
}
}
}
@@ -0,0 +1,272 @@
import json
from typing import Dict, Any, Optional, List
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.network import Network, NetworkError
# Hardcoded default settings for OpenRouter tool
OPENROUTER_API_KEY = None
OPENROUTER_MODEL = "google/gemini-3.1-flash-lite"
DEFAULT_SETTINGS = {
"OPENROUTER_API_KEY": OPENROUTER_API_KEY,
"OPENROUTER_MODEL": OPENROUTER_MODEL,
}
REQUIRED_SETTINGS = ["OPENROUTER_API_KEY"]
class OpenRouterTool(BaseTool):
"""OpenRouter tool for unified LLM API access across all skills"""
TOOLKIT = "communication"
def __init__(self, api_key: Optional[str] = None):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
tool_settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.settings = tool_settings
self.required_settings = [] if api_key else REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
# Priority: skill-provided api_key > toolkit settings > hardcoded default
self.api_key = api_key or self.settings.get(
"OPENROUTER_API_KEY", OPENROUTER_API_KEY
)
# Load model settings
self.model = self.settings.get("OPENROUTER_MODEL", OPENROUTER_MODEL)
self.network = Network({"base_url": "https://openrouter.ai/api"})
@property
def tool_name(self) -> str:
return "openrouter"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def set_api_key(self, api_key: str) -> None:
"""Set the OpenRouter API key"""
self.api_key = api_key
def chat_completion(
self,
messages: List[Dict[str, str]],
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
use_structured_output: bool = False,
json_schema: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Send a chat completion request to OpenRouter
Args:
messages: List of message dictionaries with 'role' and 'content'
model: Model ID (full OpenRouter model ID, e.g. 'google/gemini-3.1-flash-lite')
temperature: Sampling temperature (0-2)
max_tokens: Maximum tokens to generate
system_prompt: System prompt to prepend
use_structured_output: Whether to use OpenRouter's structured outputs
json_schema: JSON schema for structured output (required if use_structured_output=True)
Returns:
Dict with response data or error information
"""
if not self.api_key:
return {"success": False, "error": "OpenRouter API key not configured"}
# Use default model if none provided
model = model or self.model
# Prepare messages with system prompt if provided
request_messages = []
if system_prompt:
request_messages.append({"role": "system", "content": system_prompt})
request_messages.extend(messages)
# Prepare request payload
payload = {
"model": model,
"messages": request_messages,
"temperature": temperature,
}
if max_tokens:
payload["max_tokens"] = max_tokens
# Add structured output configuration if requested
if use_structured_output and json_schema:
payload["response_format"] = {
"type": "json_schema",
"json_schema": {
"name": json_schema.get("name", "response"),
"strict": True,
"schema": json_schema["schema"],
},
}
try:
response = self.network.request(
{
"url": "/v1/chat/completions",
"method": "POST",
"headers": {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
},
"data": payload,
}
)
return {"success": True, "data": response["data"], "model_used": model}
except NetworkError as e:
return {
"success": False,
"error": f"OpenRouter API error: {str(e)}",
"status_code": getattr(e.response, "status_code", None),
}
def completion(
self,
prompt: str,
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
use_structured_output: bool = False,
json_schema: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
General text completion for any use case
Args:
prompt: Text prompt to complete
model: Model ID (full OpenRouter model ID)
temperature: Sampling temperature
max_tokens: Maximum tokens to generate
system_prompt: Optional system prompt
use_structured_output: Whether to use structured outputs
json_schema: JSON schema for structured output
Returns:
Dict with completion result
"""
messages = [{"role": "user", "content": prompt}]
response = self.chat_completion(
messages=messages,
model=model,
temperature=temperature,
max_tokens=max_tokens,
system_prompt=system_prompt,
use_structured_output=use_structured_output,
json_schema=json_schema,
)
if not response["success"]:
return response
try:
content = response["data"]["choices"][0]["message"]["content"]
return {
"success": True,
"content": content,
"model_used": response["model_used"],
}
except (KeyError, IndexError) as e:
return {
"success": False,
"error": f"Failed to extract completion: {str(e)}",
}
def structured_completion(
self,
prompt: str,
json_schema: Dict[str, Any],
model: Optional[str] = None,
temperature: float = 0.7,
max_tokens: Optional[int] = None,
system_prompt: Optional[str] = None,
) -> Dict[str, Any]:
"""
Generate structured JSON output using OpenRouter's structured outputs feature
Args:
prompt: Text prompt to complete
json_schema: JSON schema defining the required output structure
model: Model ID (full OpenRouter model ID)
temperature: Sampling temperature
max_tokens: Maximum tokens to generate
system_prompt: Optional system prompt
Returns:
Dict with parsed JSON result or error
"""
messages = [{"role": "user", "content": prompt}]
response = self.chat_completion(
messages=messages,
model=model,
temperature=temperature,
max_tokens=max_tokens,
system_prompt=system_prompt,
use_structured_output=True,
json_schema=json_schema,
)
if not response["success"]:
return response
try:
content = response["data"]["choices"][0]["message"]["content"]
# With structured outputs, content is already valid JSON
parsed_data = json.loads(content)
return {
"success": True,
"data": parsed_data,
"model_used": response["model_used"],
}
except (KeyError, IndexError) as e:
return {
"success": False,
"error": f"Failed to extract completion: {str(e)}",
}
except json.JSONDecodeError as e:
return {
"success": False,
"error": f"Failed to parse JSON response: {str(e)}",
}
def list_models(self) -> Dict[str, Any]:
"""
Get list of available models from OpenRouter API
Returns:
Dict with models list or error
"""
if not self.api_key:
return {"success": False, "error": "OpenRouter API key not configured"}
try:
response = self.network.request(
{
"url": "/v1/models",
"method": "GET",
"headers": {"Authorization": f"Bearer {self.api_key}"},
}
)
return {"success": True, "models": response["data"]["data"]}
except NetworkError as e:
return {"success": False, "error": f"Failed to fetch models: {str(e)}"}
+162
View File
@@ -0,0 +1,162 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "openrouter",
"toolkit_id": "communication",
"name": "OpenRouter",
"description": "A tool for interacting with various LLMs through the OpenRouter API gateway.",
"icon_name": "route-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"chatCompletion": {
"description": "Generate a chat completion using OpenRouter.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"messages": {
"type": "array",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string"
},
"content": {
"type": "string"
}
},
"required": [
"role",
"content"
],
"additionalProperties": false
}
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
},
"use_structured_output": {
"type": "boolean"
},
"json_schema": {
"type": "object",
"additionalProperties": true
}
},
"required": [
"messages"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"completion": {
"description": "Generate a completion using OpenRouter.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
},
"use_structured_output": {
"type": "boolean"
},
"json_schema": {
"type": "object",
"additionalProperties": true
}
},
"required": [
"prompt"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"structuredCompletion": {
"description": "Generate a structured completion using a JSON schema.",
"parameters": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"prompt": {
"type": "string"
},
"json_schema": {
"type": "object",
"additionalProperties": true
},
"model": {
"type": "string"
},
"temperature": {
"type": "number"
},
"max_tokens": {
"type": "number"
},
"system_prompt": {
"type": "string"
}
},
"required": [
"prompt",
"json_schema"
],
"additionalProperties": false
}
},
"required": [
"options"
]
}
},
"listModels": {
"description": "List available OpenRouter models.",
"parameters": {
"type": "object",
"properties": {}
}
}
}
}
+12
View File
@@ -0,0 +1,12 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Communication",
"description": "Tools for communication and language model interactions.",
"icon_name": "chat-3-line",
"context_files": [
"LEON.md",
"ARCHITECTURE.md",
"MEDIA_PROFILE.md"
],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Dialog",
"description": "Tools for dialog and conversation handling.",
"icon_name": "discuss-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "File System",
"description": "Tools for file system operations.",
"icon_name": "folders-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Food & Drink",
"description": "Tools for food and drink queries.",
"icon_name": "restaurant-2-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Games",
"description": "Tools for games and entertainment.",
"icon_name": "gamepad-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Health & Fitness",
"description": "Tools for health and fitness information.",
"icon_name": "heart-pulse-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Media Generation",
"description": "Tools for media generation and creative workflows.",
"icon_name": "sparkling-2-line",
"context_files": [],
"tools": []
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
"name": "Movies & TV",
"description": "Tools for movies and TV information.",
"icon_name": "movie-2-line",
"context_files": [],
"tools": []
}
@@ -0,0 +1,3 @@
from .src.python.assemblyai_audio_tool import AssemblyAIAudioTool
__all__ = ["AssemblyAIAudioTool"]
@@ -0,0 +1,3 @@
{
"ASSEMBLYAI_AUDIO_API_KEY": null
}
@@ -0,0 +1,276 @@
import fs from 'node:fs'
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network } from '@sdk/network'
// Hardcoded default setting for AssemblyAI audio tool
const ASSEMBLYAI_AUDIO_API_KEY: string | null = null
const DEFAULT_SETTINGS: Record<string, unknown> = {
ASSEMBLYAI_AUDIO_API_KEY
}
const REQUIRED_SETTINGS = ['ASSEMBLYAI_AUDIO_API_KEY']
interface AssemblyAIUploadResponse {
upload_url: string
}
interface AssemblyAITranscriptionResponse {
id: string
status: 'queued' | 'processing' | 'completed' | 'error'
text: string
words?: {
text: string
start: number
end: number
confidence: number
speaker?: string
}[]
utterances?: {
text: string
start: number
end: number
confidence: number
speaker: string
words: {
text: string
start: number
end: number
confidence: number
}[]
}[]
audio_duration?: number
error?: string
}
export default class AssemblyAIAudioTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
readonly apiKey: string | null
constructor() {
super()
this.config = ToolkitConfig.load(AssemblyAIAudioTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
AssemblyAIAudioTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
// Priority: toolkit settings > hardcoded default
this.apiKey =
(this.settings['ASSEMBLYAI_AUDIO_API_KEY'] as string) ||
ASSEMBLYAI_AUDIO_API_KEY
}
get toolName(): string {
return 'assemblyai_audio'
}
get toolkit(): string {
return AssemblyAIAudioTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Transcribe audio to a file using AssemblyAI's audio transcription API via SDK Network
* @param inputPath Path to the audio file to transcribe
* @param outputPath Path to save the JSON transcription
* @param apiKey AssemblyAI API key (uses env/hardcoded default if not provided)
* @param speakerLabels Enable speaker diarization (default: true)
*/
async transcribeToFile(
inputPath: string,
outputPath: string,
apiKey?: string,
speakerLabels = true
): Promise<string> {
// Use provided apiKey, instance apiKey, or error
const finalApiKey = apiKey || this.apiKey
if (!finalApiKey) {
throw new Error('AssemblyAI API key is missing')
}
const network = new Network({ baseURL: 'https://api.assemblyai.com' })
// Step 1: Upload the audio file
const audioData = await fs.promises.readFile(inputPath)
const uploadResponse = await network.request({
url: '/v2/upload',
method: 'POST',
data: audioData,
headers: {
Authorization: finalApiKey,
'Content-Type': 'application/octet-stream'
}
})
const uploadUrl = (uploadResponse.data as AssemblyAIUploadResponse)
.upload_url
// Step 2: Submit transcription request
const transcriptionResponse = await network.request({
url: '/v2/transcript',
method: 'POST',
data: {
audio_url: uploadUrl,
speaker_labels: speakerLabels,
language_detection: true
},
headers: {
Authorization: finalApiKey,
'Content-Type': 'application/json'
}
})
const transcriptId = (
transcriptionResponse.data as AssemblyAITranscriptionResponse
).id
// Step 3: Poll for completion
let transcriptData: AssemblyAITranscriptionResponse
let attempts = 0
const maxAttempts = 180 // 15 minutes with 5 second intervals
while (attempts < maxAttempts) {
const statusResponse = await network.request({
url: `/v2/transcript/${transcriptId}`,
method: 'GET',
headers: {
Authorization: finalApiKey
}
})
transcriptData = statusResponse.data as AssemblyAITranscriptionResponse
if (transcriptData.status === 'completed') {
break
} else if (transcriptData.status === 'error') {
throw new Error(
`AssemblyAI transcription failed: ${
transcriptData.error || 'Unknown error'
}`
)
}
// Wait 5 seconds before polling again
await new Promise((resolve) => setTimeout(resolve, 5000))
attempts++
}
if (attempts >= maxAttempts) {
throw new Error('AssemblyAI transcription timed out')
}
// Step 4: Parse and save the transcription
const parsedOutput = this.parseTranscription(transcriptData!)
await fs.promises.writeFile(
outputPath,
JSON.stringify(parsedOutput, null, 2),
'utf8'
)
return outputPath
}
private parseTranscription(
rawOutput: AssemblyAITranscriptionResponse
): TranscriptionOutput {
const segments: {
from: number
to: number
text: string
speaker: string | null
}[] = []
const speakers: Set<string> = new Set()
// Use utterances for speaker-labeled segments if available
if (rawOutput.utterances && rawOutput.utterances.length > 0) {
for (const utterance of rawOutput.utterances) {
segments.push({
from: utterance.start / 1_000, // Convert milliseconds to seconds
to: utterance.end / 1_000,
text: utterance.text,
speaker: utterance.speaker
})
speakers.add(utterance.speaker)
}
} else if (rawOutput.words && rawOutput.words.length > 0) {
// Fallback to word-level data if utterances are not available
// Group consecutive words by speaker (if available)
let currentSegment: {
from: number
to: number
text: string
speaker: string | null
} | null = null
for (const word of rawOutput.words) {
const speaker = word.speaker || null
if (
currentSegment &&
currentSegment.speaker === speaker &&
word.start / 1_000 - currentSegment.to < 1.0 // Max 1 second gap
) {
// Extend current segment
currentSegment.to = word.end / 1_000
currentSegment.text += ` ${word.text}`
} else {
// Start a new segment
if (currentSegment) {
segments.push(currentSegment)
}
currentSegment = {
from: word.start / 1_000,
to: word.end / 1_000,
text: word.text,
speaker: speaker
}
}
if (speaker) {
speakers.add(speaker)
}
}
// Push the last segment
if (currentSegment) {
segments.push(currentSegment)
}
} else {
// Fallback: create a single segment with the full text
segments.push({
from: 0,
to: (rawOutput.audio_duration || 0) / 1_000,
text: rawOutput.text,
speaker: null
})
}
// Calculate duration
let duration = rawOutput.audio_duration ? rawOutput.audio_duration : 0
if (!duration && segments.length > 0) {
duration = segments[segments.length - 1]?.to || 0
}
return {
duration,
speakers: Array.from(speakers),
speaker_count: speakers.size,
segments,
metadata: {
tool: this.toolName
}
}
}
}
@@ -0,0 +1 @@
export { default } from './assemblyai_audio-tool'
@@ -0,0 +1,240 @@
import json
import time
from typing import List, Dict, Any, Optional
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.network import Network
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
# Hardcoded default settings for AssemblyAI audio tool
ASSEMBLYAI_AUDIO_API_KEY = None
DEFAULT_SETTINGS = {
"ASSEMBLYAI_AUDIO_API_KEY": ASSEMBLYAI_AUDIO_API_KEY,
}
REQUIRED_SETTINGS = ["ASSEMBLYAI_AUDIO_API_KEY"]
class AssemblyAIAudioTool(BaseTool):
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
tool_settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.settings = tool_settings
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
# Priority: toolkit settings > hardcoded default
self.api_key = self.settings.get(
"ASSEMBLYAI_AUDIO_API_KEY", ASSEMBLYAI_AUDIO_API_KEY
)
self.network = Network({"base_url": "https://api.assemblyai.com"})
@property
def tool_name(self) -> str:
return "assemblyai_audio"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def transcribe_to_file(
self,
input_path: str,
output_path: str,
api_key: Optional[str] = None,
speaker_labels: bool = True,
) -> str:
"""
Transcribe audio to a file using AssemblyAI's audio transcription API via SDK Network
Args:
input_path: Path to the audio file to transcribe
output_path: Path to save the JSON transcription (unified format)
api_key: AssemblyAI API key (uses env/hardcoded default if not provided)
speaker_labels: Enable speaker diarization (default: True)
Returns:
The path to the transcription file
"""
# Use provided api_key, instance api_key, or error
api_key = api_key or self.api_key
if not api_key:
raise Exception("AssemblyAI API key is missing")
try:
# Step 1: Upload the audio file
with open(input_path, "rb") as audio_file:
audio_data = audio_file.read()
upload_response = self.network.request(
{
"url": "/v2/upload",
"method": "POST",
"headers": {
"Authorization": api_key,
"Content-Type": "application/octet-stream",
},
"data": audio_data,
}
)
upload_url = upload_response["data"]["upload_url"]
# Step 2: Submit transcription request
transcription_response = self.network.request(
{
"url": "/v2/transcript",
"method": "POST",
"headers": {
"Authorization": api_key,
"Content-Type": "application/json",
},
"data": {
"audio_url": upload_url,
"speaker_labels": speaker_labels,
"language_detection": True,
},
"use_json": True,
}
)
transcript_id = transcription_response["data"]["id"]
# Step 3: Poll for completion
max_attempts = 180 # 15 minutes with 5 second intervals
attempts = 0
transcript_data = None
while attempts < max_attempts:
status_response = self.network.request(
{
"url": f"/v2/transcript/{transcript_id}",
"method": "GET",
"headers": {"Authorization": api_key},
"use_json": True,
}
)
transcript_data = status_response["data"]
if transcript_data["status"] == "completed":
break
elif transcript_data["status"] == "error":
error_msg = transcript_data.get("error", "Unknown error")
raise Exception(f"AssemblyAI transcription failed: {error_msg}")
# Wait 5 seconds before polling again
time.sleep(5)
attempts += 1
if attempts >= max_attempts:
raise Exception("AssemblyAI transcription timed out")
# Step 4: Parse and save the transcription
parsed_output = self._parse_transcription(transcript_data)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
return output_path
except Exception as e:
raise Exception(f"AssemblyAI transcription failed: {str(e)}")
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
segments: List[TranscriptionSegment] = []
speakers_set = set()
# Use utterances for speaker-labeled segments if available
utterances = raw_output.get("utterances", [])
words = raw_output.get("words", [])
if utterances and len(utterances) > 0:
for utterance in utterances:
speaker = utterance.get("speaker")
segments.append(
{
"from": float(utterance.get("start", 0))
/ 1000.0, # Convert ms to seconds
"to": float(utterance.get("end", 0)) / 1000.0,
"text": utterance.get("text", ""),
"speaker": speaker,
}
)
if speaker:
speakers_set.add(speaker)
elif words and len(words) > 0:
# Fallback to word-level data if utterances are not available
# Group consecutive words by speaker (if available)
current_segment = None
for word in words:
speaker = word.get("speaker", None)
word_start = float(word.get("start", 0)) / 1000.0
word_end = float(word.get("end", 0)) / 1000.0
word_text = word.get("text", "")
if (
current_segment
and current_segment["speaker"] == speaker
and word_start - current_segment["to"] < 1.0 # Max 1 second gap
):
# Extend current segment
current_segment["to"] = word_end
current_segment["text"] += f" {word_text}"
else:
# Start a new segment
if current_segment:
segments.append(current_segment)
current_segment = {
"from": word_start,
"to": word_end,
"text": word_text,
"speaker": speaker,
}
if speaker:
speakers_set.add(speaker)
# Push the last segment
if current_segment:
segments.append(current_segment)
else:
# Fallback: create a single segment with the full text
audio_duration = raw_output.get("audio_duration", 0)
segments.append(
{
"from": 0.0,
"to": audio_duration if audio_duration else 0.0,
"text": raw_output.get("text", ""),
"speaker": None,
}
)
# Calculate duration
audio_duration = raw_output.get("audio_duration")
if audio_duration:
duration = float(audio_duration) / 1000.0
elif len(segments) > 0:
duration = segments[-1]["to"]
else:
duration = 0.0
return {
"duration": duration,
"speakers": list(speakers_set),
"speaker_count": len(speakers_set),
"segments": segments,
"metadata": {"tool": self.tool_name},
}
@@ -0,0 +1,39 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "assemblyai_audio",
"toolkit_id": "music_audio",
"name": "AssemblyAI Audio",
"description": "A tool for audio processing using AssemblyAI's API.",
"icon_name": "mic-2-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"transcribeToFile": {
"description": "Transcribe audio to a file using AssemblyAI's audio transcription API.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"apiKey": {
"type": "string"
},
"speakerLabels": {
"type": "boolean"
}
},
"required": [
"inputPath",
"outputPath"
]
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.chatterbox_onnx_tool import ChatterboxONNXTool
__all__ = ["ChatterboxONNXTool"]
@@ -0,0 +1 @@
{}
@@ -0,0 +1,235 @@
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { NVIDIA_LIBS_PATH } from '@bridge/constants'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { getPlatformName } from '@sdk/utils'
const MODEL_NAME = 'chatterbox-multilingual-onnx'
const DEFAULT_MAX_CHARS = 272 // Character limit to avoid hallucination
const DEFAULT_SETTINGS: Record<string, unknown> = {}
const REQUIRED_SETTINGS: string[] = []
interface SynthesisTask {
text: string
target_language?: string
audio_path: string
// @see https://github.com/leon-ai/leon-binaries/tree/main/bins/chatterbox_onnx/default_voices
voice_name?: string
speaker_reference_path?: string
cfg_strength?: number
exaggeration?: number
temperature?: number
// Control automatic text splitting (default: true)
auto_split?: boolean
}
/**
* Split text at natural punctuation boundaries to avoid hallucination.
*
* This function ensures no text segment exceeds maxChars by breaking at
* punctuation marks when possible, falling back to spaces or forced splits.
*
* @param text The text to split
* @param maxChars Maximum characters per segment (default: 272)
* @returns Array of text chunks split at natural boundaries
*/
function splitTextAtPunctuation(
text: string,
maxChars: number = DEFAULT_MAX_CHARS
): string[] {
const trimmedText = text.trim()
if (trimmedText.length <= maxChars) {
return [trimmedText]
}
const chunks: string[] = []
let remaining = trimmedText
while (remaining.length > maxChars) {
// Get segment up to maxChars
const segment = remaining.substring(0, maxChars + 1)
// Look for punctuation followed by space (natural break)
const punctuationPattern = /[.!?,;:]\s/g
let lastMatch = -1
let match: RegExpExecArray | null
while ((match = punctuationPattern.exec(segment)) !== null) {
lastMatch = match.index + 1 // Include the punctuation but not the space
}
// Check if we found punctuation in a reasonable position (latter half)
if (lastMatch > maxChars * 0.5) {
chunks.push(remaining.substring(0, lastMatch).trim())
remaining = remaining.substring(lastMatch).trim()
continue
}
// No good punctuation found, look for last space
const lastSpace = segment.substring(0, maxChars).lastIndexOf(' ')
if (lastSpace > maxChars * 0.3) {
chunks.push(remaining.substring(0, lastSpace).trim())
remaining = remaining.substring(lastSpace).trim()
} else {
// Force split at maxChars
chunks.push(remaining.substring(0, maxChars).trim())
remaining = remaining.substring(maxChars).trim()
}
}
if (remaining.length > 0) {
chunks.push(remaining.trim())
}
return chunks
}
export default class ChatterboxONNXTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
constructor() {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(ChatterboxONNXTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
ChatterboxONNXTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
}
get toolName(): string {
// Use the actual config name for toolkit lookup
return 'chatterbox_onnx'
}
get toolkit(): string {
return ChatterboxONNXTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Synthesize speech from text using Chatterbox ONNX
*
* By default, automatically splits long text (>272 chars) at punctuation boundaries
* to prevent hallucination. Split segments generate separate audio files with
* _part_N suffixes (e.g., output_part_0.wav, output_part_1.wav).
*
* @param tasks Array of synthesis tasks or a single task
* @param cudaRuntimePath Optional path to CUDA runtime for GPU acceleration (auto-detected if not provided)
* @returns A promise that resolves with the list of processed tasks (may include split tasks)
*/
async synthesizeSpeechToFiles(
tasks: SynthesisTask | SynthesisTask[],
cudaRuntimePath?: string
): Promise<Omit<SynthesisTask, 'auto_split'>[]> {
try {
// Normalize tasks to array
const taskArray = Array.isArray(tasks) ? tasks : [tasks]
// Process tasks: split long text into multiple tasks with _part_N suffixes
const tasksToSynthesize: Omit<SynthesisTask, 'auto_split'>[] = []
for (const task of taskArray) {
const autoSplit = task.auto_split !== undefined ? task.auto_split : true // Default: enabled
const text = task.text.trim()
const maxChars = DEFAULT_MAX_CHARS
// If auto_split disabled or text is short, pass through as-is
if (!autoSplit || text.length <= maxChars) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { auto_split, ...cleanTask } = task
tasksToSynthesize.push(cleanTask)
continue
}
// Split long text at punctuation boundaries
const textChunks = splitTextAtPunctuation(text, maxChars)
// If only one chunk after splitting, no need for special handling
if (textChunks.length === 1) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { auto_split, ...cleanTask } = task
tasksToSynthesize.push(cleanTask)
continue
}
// Multiple chunks: create separate tasks with _part_N suffixes
const audioPath = task.audio_path
const parsedPath = path.parse(audioPath)
const basePath = path.join(parsedPath.dir, parsedPath.name)
const ext = parsedPath.ext
for (let i = 0; i < textChunks.length; i += 1) {
const chunk = textChunks[i]
if (!chunk) continue
const baseTask = {
...task,
text: chunk,
audio_path: `${basePath}_part_${i}${ext}`
}
delete baseTask.auto_split
tasksToSynthesize.push(baseTask)
}
}
// Get model path using the generic resource system
const modelPath = await this.getResourcePath(MODEL_NAME)
// Create a temporary JSON file for the tasks
const tempDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'chatterbox_onnx_tasks_')
)
const jsonFilePath = path.join(tempDir, 'tasks.json')
await fs.promises.writeFile(
jsonFilePath,
JSON.stringify(tasksToSynthesize, null, 2),
'utf8'
)
const args = [
'--function',
'synthesize_speech',
'--json_file',
jsonFilePath,
'--resource_path',
modelPath
]
// Auto-detect CUDA runtime path if not provided
const platformName = getPlatformName()
const shouldUseCuda =
platformName === 'linux-x86_64' || platformName === 'win-amd64'
const finalCudaRuntimePath =
cudaRuntimePath ?? (shouldUseCuda ? NVIDIA_LIBS_PATH : undefined)
if (finalCudaRuntimePath) {
args.push('--cuda_runtime_path', finalCudaRuntimePath)
}
await this.executeCommand({
binaryName: 'chatterbox_onnx',
args,
options: { sync: true }
})
// Return the processed tasks so caller knows which files were created
return tasksToSynthesize
} catch (error: unknown) {
throw new Error(`Speech synthesis failed: ${(error as Error).message}`)
}
}
}
@@ -0,0 +1 @@
export { default } from './chatterbox_onnx-tool'
@@ -0,0 +1,241 @@
import json
import os
import re
import tempfile
from typing import Optional, Union, List, TypedDict
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.utils import get_platform_name
from bridges.python.src.constants import NVIDIA_LIBS_PATH
MODEL_NAME = "chatterbox-multilingual-onnx"
DEFAULT_MAX_CHARS = 272 # Character limit to avoid hallucination
DEFAULT_SETTINGS = {}
REQUIRED_SETTINGS = []
def split_text_at_punctuation(
text: str, max_chars: int = DEFAULT_MAX_CHARS
) -> List[str]:
"""
Split text at natural punctuation boundaries to avoid hallucination.
This function ensures no text segment exceeds max_chars by breaking at
punctuation marks when possible, falling back to spaces or forced splits.
Args:
text: The text to split
max_chars: Maximum characters per segment (default: 272)
Returns:
List of text chunks split at natural boundaries
"""
text = text.strip()
if len(text) <= max_chars:
return [text]
chunks = []
remaining = text
while len(remaining) > max_chars:
# Get segment up to max_chars
segment = remaining[: max_chars + 1]
# Look for punctuation followed by space (natural break)
punctuation_pattern = re.compile(r"[.!?,;:]\s")
matches = list(punctuation_pattern.finditer(segment))
if matches:
# Use the last punctuation match within max_chars
last_match = matches[-1]
break_point = (
last_match.end() - 1
) # Don't include the space after punctuation
# Check if it's in a reasonable position (latter half)
if break_point > max_chars * 0.5:
chunks.append(remaining[:break_point].strip())
remaining = remaining[break_point:].strip()
continue
# No good punctuation found, look for last space
last_space = segment[:max_chars].rfind(" ")
if last_space > max_chars * 0.3:
chunks.append(remaining[:last_space].strip())
remaining = remaining[last_space:].strip()
else:
# Force split at max_chars
chunks.append(remaining[:max_chars].strip())
remaining = remaining[max_chars:].strip()
if remaining:
chunks.append(remaining.strip())
return chunks
class SynthesisTask(TypedDict, total=False):
"""Type definition for a synthesis task"""
text: str
target_language: Optional[str]
audio_path: str
# Voice names: https://github.com/leon-ai/leon-binaries/tree/main/bins/chatterbox_onnx/default_voices
voice_name: Optional[str]
speaker_reference_path: Optional[str]
cfg_strength: Optional[float]
exaggeration: Optional[float]
temperature: Optional[float]
# Control automatic text splitting (default: True)
auto_split: Optional[bool]
class ChatterboxONNXTool(BaseTool):
"""
Tool for text-to-speech synthesis using Chatterbox ONNX model.
Supports multilingual synthesis with voice cloning capabilities.
"""
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
# Load configuration from central toolkits directory
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
self.settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
@property
def tool_name(self) -> str:
# Use the actual config name for toolkit lookup
return "chatterbox_onnx"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def synthesize_speech_to_files(
self,
tasks: Union[SynthesisTask, List[SynthesisTask]],
cuda_runtime_path: Optional[str] = None,
) -> List[dict]:
"""
Synthesize speech from text using Chatterbox ONNX
By default, automatically splits long text (>272 chars) at punctuation boundaries
to prevent hallucination. Split segments generate separate audio files with
_part_N suffixes (e.g., output_part_0.wav, output_part_1.wav).
Args:
tasks: A single synthesis task or a list of synthesis tasks.
Each task should contain:
- text: The text to synthesize
- audio_path: Output path for the generated audio file
- target_language: Optional language code (e.g., 'en', 'zh', 'ja')
- voice_name: Optional name of the voice to use
- speaker_reference_path: Optional path to a reference audio file for voice cloning
- cfg_strength: Optional classifier-free guidance strength (default: 0.5)
- exaggeration: Optional exaggeration factor (default: 0.5)
- temperature: Optional temperature for sampling (controls randomness)
- auto_split: Optional flag to enable/disable automatic text splitting (default: True)
cuda_runtime_path: Optional path to CUDA runtime for GPU acceleration (auto-detected if not provided)
Returns:
List of processed tasks (may include split tasks with _part_N suffixes)
"""
try:
# Normalize tasks to list
task_list = tasks if isinstance(tasks, list) else [tasks]
# Process tasks: split long text into multiple tasks with _part_N suffixes
tasks_to_synthesize = []
for task in task_list:
auto_split = task.get("auto_split", True) # Default: enabled
text = task.get("text")
if not text:
raise ValueError("Missing text in synthesis task")
text = text.strip()
max_chars = DEFAULT_MAX_CHARS
# If auto_split disabled or text is short, pass through as-is
if not auto_split or len(text) <= max_chars:
clean_task = {k: v for k, v in task.items() if k != "auto_split"}
tasks_to_synthesize.append(clean_task)
continue
# Split long text at punctuation boundaries
text_chunks = split_text_at_punctuation(text, max_chars)
# If only one chunk after splitting, no need for special handling
if len(text_chunks) == 1:
clean_task = {k: v for k, v in task.items() if k != "auto_split"}
tasks_to_synthesize.append(clean_task)
continue
# Multiple chunks: create separate tasks with _part_N suffixes
audio_path = task.get("audio_path")
if not audio_path:
raise ValueError("Missing audio_path in synthesis task")
base_path, ext = os.path.splitext(audio_path)
for i, chunk in enumerate(text_chunks):
chunk_task = {
k: v
for k, v in task.items()
if k not in ["text", "audio_path", "auto_split"]
}
chunk_task["text"] = chunk
chunk_task["audio_path"] = f"{base_path}_part_{i}{ext}"
tasks_to_synthesize.append(chunk_task)
# Get model path using the generic resource system
model_path = self.get_resource_path(MODEL_NAME)
# Create a temporary JSON file for the tasks
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json", delete=False, encoding="utf-8"
) as temp_file:
json_file_path = temp_file.name
json.dump(tasks_to_synthesize, temp_file, indent=2, ensure_ascii=False)
args = [
"--function",
"synthesize_speech",
"--json_file",
json_file_path,
"--resource_path",
model_path,
]
# Auto-detect CUDA runtime path if not provided
platform_name = get_platform_name()
should_use_cuda = platform_name in ["linux-x86_64", "win-amd64"]
final_cuda_runtime_path = (
cuda_runtime_path
if cuda_runtime_path is not None
else (NVIDIA_LIBS_PATH if should_use_cuda else None)
)
if final_cuda_runtime_path:
args.extend(["--cuda_runtime_path", final_cuda_runtime_path])
self.execute_command(
ExecuteCommandOptions(
binary_name="chatterbox_onnx", args=args, options={"sync": True}
)
)
# Return the processed tasks so caller knows which files were created
return tasks_to_synthesize
except Exception as e:
raise Exception(f"Speech synthesis failed: {str(e)}")
+132
View File
@@ -0,0 +1,132 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "chatterbox_onnx",
"toolkit_id": "music_audio",
"name": "Chatterbox ONNX",
"description": "A tool for text-to-speech synthesis and voice cloning using the Chatterbox ONNX model.",
"icon_name": "chat-voice-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"binaries": {
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-linux-x86_64",
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-linux-aarch64",
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-macosx-x86_64",
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-macosx-arm64",
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-win-amd64.exe"
},
"resources": {
"chatterbox-multilingual-onnx": [
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/Cangjie5_TC.json?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/default_voice.wav?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/tokenizer.json?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/conditional_decoder.onnx?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/conditional_decoder.onnx_data?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/embed_tokens.onnx?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/embed_tokens.onnx_data?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/language_model_q4.onnx?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/language_model_q4.onnx_data?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/speech_encoder.onnx?download=true",
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/speech_encoder.onnx_data?download=true"
]
},
"functions": {
"synthesizeSpeechToFiles": {
"description": "Synthesize speech from text using Chatterbox ONNX.",
"parameters": {
"type": "object",
"properties": {
"tasks": {
"oneOf": [
{
"type": "object",
"properties": {
"text": {
"type": "string"
},
"target_language": {
"type": "string"
},
"audio_path": {
"type": "string"
},
"voice_name": {
"type": "string"
},
"speaker_reference_path": {
"type": "string"
},
"cfg_strength": {
"type": "number"
},
"exaggeration": {
"type": "number"
},
"temperature": {
"type": "number"
},
"auto_split": {
"type": "boolean"
}
},
"required": [
"text",
"audio_path"
],
"additionalProperties": false
},
{
"type": "array",
"items": {
"type": "object",
"properties": {
"text": {
"type": "string"
},
"target_language": {
"type": "string"
},
"audio_path": {
"type": "string"
},
"voice_name": {
"type": "string"
},
"speaker_reference_path": {
"type": "string"
},
"cfg_strength": {
"type": "number"
},
"exaggeration": {
"type": "number"
},
"temperature": {
"type": "number"
},
"auto_split": {
"type": "boolean"
}
},
"required": [
"text",
"audio_path"
],
"additionalProperties": false
}
}
]
},
"cudaRuntimePath": {
"type": "string"
}
},
"required": [
"tasks"
]
}
}
}
}
+3
View File
@@ -0,0 +1,3 @@
from .src.python.ecapa_tool import ECAPATool
__all__ = ["ECAPATool"]
@@ -0,0 +1 @@
{}
@@ -0,0 +1,112 @@
import fs from 'node:fs'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
const MODEL_NAME = 'ecapa-voice_gender_classifier'
const DEFAULT_SETTINGS: Record<string, unknown> = {}
const REQUIRED_SETTINGS: string[] = []
export default class ECAPATool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
constructor() {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(ECAPATool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
ECAPATool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
}
get toolName(): string {
// Use the actual config name for toolkit lookup
return 'ecapa'
}
get toolkit(): string {
return ECAPATool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Detect gender from audio file using ECAPA-TDNN voice gender classifier
* @param inputPath The file path of the audio to be analyzed
* @param device Device to use for processing (cpu, cuda)
* @returns A promise that resolves with the detected gender: "male", "female", or "unknown"
*/
async detectGender(inputPath: string, device = 'cpu'): Promise<string> {
try {
// Validate input file exists
if (!fs.existsSync(inputPath)) {
throw new Error(`Input file does not exist: ${inputPath}`)
}
// Get model path using the generic resource system
const modelPath = await this.getResourcePath(MODEL_NAME)
const args = [
'--function',
'detect_gender',
'--input',
inputPath,
'--model_path',
modelPath,
'--device',
device
]
const result = await this.executeCommand({
binaryName: 'ecapa-voice_gender_classifier',
args,
options: { sync: true }
})
// Parse the output to extract gender
const gender = this.parseGenderOutput(result)
return gender
} catch (error: unknown) {
throw new Error(
`Voice gender detection failed: ${(error as Error).message}`
)
}
}
/**
* Parse the gender detection output
*/
private parseGenderOutput(rawOutput: string): string {
const lines = rawOutput.split('\n')
// Look for gender result in the output
for (const line of lines) {
const lowerLine = line.toLowerCase().trim()
if (lowerLine.includes('gender:')) {
// Extract gender from line like "Gender: male"
const match = lowerLine.match(/gender:\s*(male|female|unknown)/i)
if (match && match[1]) {
return match[1].toLowerCase()
}
}
// Also check for direct gender output
if (lowerLine === 'male' || lowerLine === 'female') {
return lowerLine
}
}
// If no clear gender found, return unknown
return 'unknown'
}
}
@@ -0,0 +1 @@
export { default } from './ecapa-tool'
@@ -0,0 +1,120 @@
import os
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
MODEL_NAME = "ecapa-voice_gender_classifier"
DEFAULT_SETTINGS = {}
REQUIRED_SETTINGS = []
class ECAPATool(BaseTool):
"""
Tool for voice gender classification using ECAPA-TDNN model.
Example output format:
Gender: male
"""
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
# Load configuration from central toolkits directory
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
self.settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
@property
def tool_name(self) -> str:
# Use the actual config name for toolkit lookup
return "ecapa"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def detect_gender(self, input_path: str, device: str = "cpu") -> str:
"""
Detect gender from audio file using ECAPA-TDNN voice gender classifier
Args:
input_path: The file path of the audio to be analyzed
device: Device to use for processing (cpu, cuda)
Returns:
The detected gender: "male", "female", or "unknown"
"""
try:
# Validate input file exists
if not os.path.exists(input_path):
raise Exception(f"Input file does not exist: {input_path}")
# Get model path using the generic resource system
model_path = self.get_resource_path(MODEL_NAME)
args = [
"--function",
"detect_gender",
"--input",
input_path,
"--model_path",
model_path,
"--device",
device,
]
result = self.execute_command(
ExecuteCommandOptions(
binary_name="ecapa-voice_gender_classifier",
args=args,
options={"sync": True},
)
)
# Parse the output to extract gender
gender = self._parse_gender_output(result)
return gender
except Exception as e:
raise Exception(f"Voice gender detection failed: {str(e)}")
def _parse_gender_output(self, raw_output: str) -> str:
"""
Parse the gender detection output
Args:
raw_output: Raw output from the gender detection binary
Returns:
Detected gender: "male", "female", or "unknown"
"""
lines = raw_output.split("\n")
# Look for gender result in the output
for line in lines:
lower_line = line.lower().strip()
if "gender:" in lower_line:
# Extract gender from line like "Gender: male"
import re
match = re.search(
r"gender:\s*(male|female|unknown)", lower_line, re.IGNORECASE
)
if match:
return match.group(1).lower()
# Also check for direct gender output
if lower_line in ["male", "female"]:
return lower_line
# If no clear gender found, return unknown
return "unknown"
+45
View File
@@ -0,0 +1,45 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "ecapa",
"toolkit_id": "music_audio",
"name": "ECAPA",
"description": "A tool for voice gender classification using ECAPA-TDNN model.",
"icon_name": "voice-recognition-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"binaries": {
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-linux-x86_64",
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-linux-aarch64",
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-macosx-x86_64",
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-macosx-arm64",
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-win-amd64.exe"
},
"resources": {
"ecapa-voice_gender_classifier": [
"https://huggingface.co/JaesungHuh/voice-gender-classifier/resolve/main/config.json?download=true",
"https://huggingface.co/JaesungHuh/voice-gender-classifier/resolve/main/model.safetensors?download=true"
]
},
"functions": {
"detectGender": {
"description": "Detect gender from an audio file using ECAPA-TDNN.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"device": {
"type": "string"
}
},
"required": [
"inputPath"
]
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.elevenlabs_audio_tool import ElevenLabsAudioTool
__all__ = ["ElevenLabsAudioTool"]
@@ -0,0 +1,4 @@
{
"ELEVENLABS_AUDIO_API_KEY": null,
"ELEVENLABS_AUDIO_MODEL": "scribe_v1"
}
@@ -0,0 +1,275 @@
import fs from 'node:fs'
import path from 'node:path'
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network } from '@sdk/network'
// Hardcoded default settings for ElevenLabs audio tool
const ELEVENLABS_AUDIO_API_KEY: string | null = null
const ELEVENLABS_AUDIO_MODEL = 'scribe_v1'
const DEFAULT_SETTINGS: Record<string, unknown> = {
ELEVENLABS_AUDIO_API_KEY,
ELEVENLABS_AUDIO_MODEL
}
const REQUIRED_SETTINGS = ['ELEVENLABS_AUDIO_API_KEY']
interface ElevenLabsWord {
text: string
start: number
end: number
type: 'word' | 'spacing' | 'audio_event'
speaker_id?: string
}
interface ElevenLabsTranscriptionResponse {
language_code: string
language_probability: number
text: string
words: ElevenLabsWord[]
}
interface ElevenLabsDubbingCreateResponse {
dubbing_id: string
expected_duration_sec: number
}
interface ElevenLabsDubbingStatusResponse {
dubbing_id: string
name: string
status: 'dubbing' | 'dubbed' | 'failed'
target_languages: string[]
error?: string | null
created_at?: string
editable?: boolean | null
}
export default class ElevenLabsAudioTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
readonly apiKey: string | null
readonly model: string
constructor() {
super()
this.config = ToolkitConfig.load(ElevenLabsAudioTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
ElevenLabsAudioTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
// Priority: toolkit settings > hardcoded default
this.apiKey =
(this.settings['ELEVENLABS_AUDIO_API_KEY'] as string) ||
ELEVENLABS_AUDIO_API_KEY
this.model =
(this.settings['ELEVENLABS_AUDIO_MODEL'] as string) ||
ELEVENLABS_AUDIO_MODEL
}
get toolName(): string {
return 'elevenlabs_audio'
}
get toolkit(): string {
return ElevenLabsAudioTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Transcribe audio to a file using ElevenLabs' Scribe v1 API
* @param inputPath Path to the audio file to transcribe
* @param outputPath Path to save the JSON transcription (unified format)
* @param apiKey ElevenLabs API key (uses env/hardcoded default if not provided)
* @param model Transcription model (defaults to tool default)
* @param diarize Whether to enable speaker diarization (defaults to true)
*/
async transcribeToFile(
inputPath: string,
outputPath: string,
apiKey?: string,
model?: string,
diarize = true
): Promise<string> {
// Use provided values, instance values, or error
const finalApiKey = apiKey || this.apiKey
const finalModel = model || this.model
if (!finalApiKey) {
throw new Error('ElevenLabs API key is missing')
}
const form = new FormData()
const audioFile = await fs.openAsBlob(inputPath)
form.append('file', audioFile, path.basename(inputPath))
form.append('model_id', finalModel)
form.append('diarize', diarize.toString())
form.append('tag_audio_events', 'true')
form.append('timestamps_granularity', 'word')
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
const response = await network.request<ElevenLabsTranscriptionResponse>({
url: '/v1/speech-to-text',
method: 'POST',
data: form,
headers: {
'xi-api-key': finalApiKey
}
})
const normalizedOutput: TranscriptionOutput = this.parseTranscription(
response.data
)
await fs.promises.writeFile(
outputPath,
JSON.stringify(normalizedOutput, null, 2),
'utf8'
)
return outputPath
}
/**
* Create a dubbing project using ElevenLabs' Dubbing API
* @param inputPath Path to the audio/video file to dub
* @param targetLang Target language code (e.g., 'es', 'fr', 'zh')
* @param apiKey ElevenLabs API key
* @param sourceLang Source language code (defaults to 'auto')
* @param numSpeakers Number of speakers (0 for auto-detect)
* @param watermark Whether to add watermark to output video
* @returns Dubbing project ID and expected duration
*/
async createDubbing(
inputPath: string,
targetLang: string,
apiKey: string,
sourceLang = 'auto',
numSpeakers = 0,
watermark = false
): Promise<ElevenLabsDubbingCreateResponse> {
if (!apiKey) {
throw new Error('ElevenLabs API key is missing')
}
const form = new FormData()
const mediaFile = await fs.openAsBlob(inputPath)
form.append('file', mediaFile, path.basename(inputPath))
form.append('target_lang', targetLang)
form.append('source_lang', sourceLang)
form.append('num_speakers', numSpeakers.toString())
form.append('watermark', watermark.toString())
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
const response = await network.request<ElevenLabsDubbingCreateResponse>({
url: '/v1/dubbing',
method: 'POST',
data: form,
headers: {
'xi-api-key': apiKey
}
})
return response.data
}
/**
* Get the status of a dubbing project
* @param dubbingId The dubbing project ID
* @param apiKey ElevenLabs API key
* @returns Dubbing project status information
*/
async getDubbingStatus(
dubbingId: string,
apiKey: string
): Promise<ElevenLabsDubbingStatusResponse> {
if (!apiKey) {
throw new Error('ElevenLabs API key is missing')
}
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
const response = await network.request<ElevenLabsDubbingStatusResponse>({
url: `/v1/dubbing/${dubbingId}`,
method: 'GET',
headers: {
'xi-api-key': apiKey
}
})
return response.data
}
/**
* Download the dubbed file
* @param dubbingId The dubbing project ID
* @param targetLang Target language code
* @param outputPath Path to save the dubbed file
* @param apiKey ElevenLabs API key
* @returns Path to the downloaded file
*/
async downloadDubbedFile(
dubbingId: string,
targetLang: string,
outputPath: string,
apiKey: string
): Promise<string> {
if (!apiKey) {
throw new Error('ElevenLabs API key is missing')
}
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
const response = await network.request({
url: `/v1/dubbing/${dubbingId}/audio/${targetLang}`,
method: 'GET',
headers: {
'xi-api-key': apiKey
},
responseType: 'arraybuffer'
})
// Write the audio/video file
await fs.promises.writeFile(
outputPath,
Buffer.from(response.data as ArrayBuffer)
)
return outputPath
}
private parseTranscription(
rawOutput: ElevenLabsTranscriptionResponse
): TranscriptionOutput {
const wordItems = rawOutput.words.filter((item) => item.type === 'word')
const uniqueSpeakers = Array.from(
new Set(wordItems.map((word) => word.speaker_id).filter(Boolean))
) as string[]
// Calculate duration from the last word's end time
const duration =
wordItems.length > 0 ? wordItems[wordItems.length - 1]?.end : 0
const segments = wordItems.map((word) => ({
from: word.start,
to: word.end,
text: word.text,
speaker: word.speaker_id || null
}))
return {
duration: duration ?? 0,
speakers: uniqueSpeakers,
speaker_count: uniqueSpeakers.length,
segments,
metadata: {
tool: this.toolName
}
}
}
}
@@ -0,0 +1 @@
export { default } from './elevenlabs_audio-tool'
@@ -0,0 +1,146 @@
import json
from typing import List, Dict, Any, Optional
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.network import Network
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
# Hardcoded default settings for ElevenLabs audio tool
ELEVENLABS_AUDIO_API_KEY = None
ELEVENLABS_AUDIO_MODEL = "scribe_v1"
DEFAULT_SETTINGS = {
"ELEVENLABS_AUDIO_API_KEY": ELEVENLABS_AUDIO_API_KEY,
"ELEVENLABS_AUDIO_MODEL": ELEVENLABS_AUDIO_MODEL,
}
REQUIRED_SETTINGS = ["ELEVENLABS_AUDIO_API_KEY"]
class ElevenLabsAudioTool(BaseTool):
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
tool_settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.settings = tool_settings
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
# Priority: toolkit settings > hardcoded default
self.api_key = self.settings.get(
"ELEVENLABS_AUDIO_API_KEY", ELEVENLABS_AUDIO_API_KEY
)
self.model = self.settings.get("ELEVENLABS_AUDIO_MODEL", ELEVENLABS_AUDIO_MODEL)
self.network = Network({"base_url": "https://api.elevenlabs.io"})
@property
def tool_name(self) -> str:
return "elevenlabs_audio"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def transcribe_to_file(
self,
input_path: str,
output_path: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
diarize: bool = True,
) -> str:
"""
Transcribe audio to a file using ElevenLabs' Scribe v1 API
Args:
input_path: Path to the audio file to transcribe
output_path: Path to save the JSON transcription (unified format)
api_key: ElevenLabs API key (uses env/hardcoded default if not provided)
model: Transcription model (defaults to tool default)
diarize: Whether to enable speaker diarization (defaults to True)
Returns:
The path to the transcription file
"""
# Use provided values, instance values, or error
api_key = api_key or self.api_key
model = model or self.model
if not api_key:
raise Exception("ElevenLabs API key is missing")
try:
files: dict = {"file": open(input_path, "rb")}
data: dict = {
"model_id": model,
"diarize": str(diarize).lower(),
"tag_audio_events": "true",
"timestamps_granularity": "word",
}
response = self.network.request(
{
"url": "/v1/speech-to-text",
"method": "POST",
"headers": {"xi-api-key": api_key},
"data": data,
"files": files,
"use_json": True,
}
)
parsed_output = self._parse_transcription(response["data"])
with open(output_path, "w", encoding="utf-8") as f:
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
return output_path
except Exception as e:
raise Exception(f"ElevenLabs transcription failed: {str(e)}")
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
"""
Parse ElevenLabs transcription response into unified schema format
Args:
raw_output: Raw response from ElevenLabs API
Returns:
Parsed transcription in unified format
"""
words_data = raw_output.get("words", [])
word_items = [word for word in words_data if word.get("type") == "word"]
unique_speakers = list(
set(word.get("speaker_id") for word in word_items if word.get("speaker_id"))
)
# Calculate duration from the last word's end time
duration = float(word_items[-1].get("end", 0)) if word_items else 0.0
segments: List[TranscriptionSegment] = []
for word in word_items:
segments.append(
{
"from": float(word.get("start", 0)),
"to": float(word.get("end", 0)),
"text": word.get("text", ""),
"speaker": word.get("speaker_id") or None,
}
)
return {
"duration": duration,
"speakers": unique_speakers,
"speaker_count": len(unique_speakers),
"segments": segments,
"metadata": {"tool": self.tool_name},
}
@@ -0,0 +1,117 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "elevenlabs_audio",
"toolkit_id": "music_audio",
"name": "ElevenLabs Audio",
"description": "A tool for audio processing using ElevenLabs's API.",
"icon_name": "sound-module-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"transcribeToFile": {
"description": "Transcribe audio to a file using ElevenLabs' Scribe API.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"apiKey": {
"type": "string"
},
"model": {
"type": "string"
},
"diarize": {
"type": "boolean"
}
},
"required": [
"inputPath",
"outputPath"
]
}
},
"createDubbing": {
"description": "Create a dubbing project using ElevenLabs' Dubbing API.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"targetLang": {
"type": "string"
},
"apiKey": {
"type": "string"
},
"sourceLang": {
"type": "string"
},
"numSpeakers": {
"type": "number"
},
"watermark": {
"type": "boolean"
}
},
"required": [
"inputPath",
"targetLang",
"apiKey"
]
}
},
"getDubbingStatus": {
"description": "Get the status of a dubbing project.",
"parameters": {
"type": "object",
"properties": {
"dubbingId": {
"type": "string"
},
"apiKey": {
"type": "string"
}
},
"required": [
"dubbingId",
"apiKey"
]
}
},
"downloadDubbedFile": {
"description": "Download the dubbed audio file for a dubbing project.",
"parameters": {
"type": "object",
"properties": {
"dubbingId": {
"type": "string"
},
"targetLang": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"apiKey": {
"type": "string"
}
},
"required": [
"dubbingId",
"targetLang",
"outputPath",
"apiKey"
]
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.faster_whisper_tool import FasterWhisperTool
__all__ = ["FasterWhisperTool"]
@@ -0,0 +1 @@
{}
@@ -0,0 +1,174 @@
import fs from 'node:fs'
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
/**
* Example:
*
* Detected language: en (probability: 1.00)
* Duration: 26.84 seconds
* ==================================================
*
* [0.00 -> 5.70] DuckDB, an open-source, fast, embeddable, SQL OLAP database that simplifies the way
* [5.70 -> 10.84] developers implement analytics. It was developed in the Netherlands, written in C++, and first
* [10.84 -> 16.78] released in 2019. And the TLDR is that it's like SQLite, but for columnar data. Everybody knows
*/
type FasterWhisperTranscriptionOutput = string
const MODEL_NAME = 'faster-whisper-large-v3'
const DEFAULT_SETTINGS: Record<string, unknown> = {}
const REQUIRED_SETTINGS: string[] = []
export default class FasterWhisperTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
constructor() {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(FasterWhisperTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
FasterWhisperTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
}
get toolName(): string {
// Use the actual config name for toolkit lookup
return 'faster_whisper'
}
get toolkit(): string {
return FasterWhisperTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Transcribe audio to a file using faster-whisper
* @param inputPath The file path of the audio to be transcribed
* @param outputPath The desired file path for the transcription output
* @param device Device to use for processing (cpu, cuda, auto)
* @param cpuThreads Number of CPU threads to use
* @param downloadRoot Root directory for model downloads
* @param localFilesOnly Whether to use only local files
* @returns A promise that resolves with the path to the transcription file
*/
async transcribeToFile(
inputPath: string,
outputPath: string,
device = 'auto',
cpuThreads?: number,
downloadRoot?: string,
localFilesOnly = false
): Promise<string> {
try {
// Get model path using the generic resource system
const modelPath = await this.getResourcePath(MODEL_NAME)
const args = [
'--function',
'transcribe_to_file',
'--input',
inputPath,
'--output',
outputPath,
'--model_size_or_path',
modelPath,
'--device',
device
]
if (cpuThreads) {
args.push('--cpu_threads', cpuThreads.toString())
}
if (downloadRoot) {
args.push('--download_root', downloadRoot)
}
if (localFilesOnly) {
args.push('--local_files_only')
}
await this.executeCommand({
binaryName: 'faster_whisper',
args,
options: { sync: true }
})
const transcriptionContent = await fs.promises.readFile(
outputPath,
'utf-8'
)
const parsedOutput = this.parseTranscription(transcriptionContent)
await fs.promises.writeFile(
outputPath,
JSON.stringify(parsedOutput, null, 2),
'utf8'
)
return outputPath
} catch (error: unknown) {
throw new Error(`Audio transcription failed: ${(error as Error).message}`)
}
}
/**
* Speaker diarization is not supported for Faster Whisper
*/
private parseTranscription(
rawOutput: FasterWhisperTranscriptionOutput
): TranscriptionOutput {
const lines = rawOutput.split('\n')
const durationLine = lines.find((line) => line.startsWith('Duration:'))
let duration = 0
if (durationLine) {
const match = durationLine.match(/Duration:\s+([\d.]+)\s+seconds/)
if (match && match[1]) {
duration = parseFloat(match[1])
}
}
const segments: TranscriptionOutput['segments'] = []
const segmentRegex = /^\[(\d+\.\d+)\s+->\s+(\d+\.\d+)\]\s+(.+)$/
for (const line of lines) {
const match = line.match(segmentRegex)
if (match && match[1] && match[2] && match[3]) {
const start = match[1]
const end = match[2]
const text = match[3]
segments.push({
from: parseFloat(start),
to: parseFloat(end),
text: text.trim(),
speaker: null
})
}
}
return {
duration,
speakers: [],
speaker_count: 0,
segments,
metadata: {
tool: this.toolName
}
}
}
}
@@ -0,0 +1 @@
export { default } from './faster_whisper-tool'
@@ -0,0 +1,155 @@
import json
import re
from typing import Optional
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
MODEL_NAME = "faster-whisper-large-v3"
DEFAULT_SETTINGS = {}
REQUIRED_SETTINGS = []
class FasterWhisperTool(BaseTool):
"""
Example output format:
Detected language: en (probability: 1.00)
Duration: 26.84 seconds
==================================================
[0.00 -> 5.70] DuckDB, an open-source, fast, embeddable, SQL OLAP database that simplifies the way
[5.70 -> 10.84] developers implement analytics. It was developed in the Netherlands, written in C++, and first
[10.84 -> 16.78] released in 2019. And the TLDR is that it's like SQLite, but for columnar data. Everybody knows
"""
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
# Load configuration from central toolkits directory
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
self.settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
@property
def tool_name(self) -> str:
# Use the actual config name for toolkit lookup
return "faster_whisper"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def transcribe_to_file(
self,
input_path: str,
output_path: str,
device: str = "auto",
cpu_threads: Optional[int] = None,
download_root: Optional[str] = None,
local_files_only: bool = False,
) -> str:
"""
Transcribe audio to a file using faster-whisper
Args:
input_path: The file path of the audio to be transcribed
output_path: The desired file path for the transcription output
device: Device to use for processing (cpu, cuda, auto)
cpu_threads: Number of CPU threads to use
download_root: Root directory for model downloads
local_files_only: Whether to use only local files
Returns:
The path to the transcription file
"""
try:
# Get model path using the generic resource system
model_path = self.get_resource_path(MODEL_NAME)
args = [
"--function",
"transcribe_to_file",
"--input",
input_path,
"--output",
output_path,
"--model_size_or_path",
model_path,
"--device",
device,
]
if cpu_threads:
args.extend(["--cpu_threads", str(cpu_threads)])
if download_root:
args.extend(["--download_root", download_root])
if local_files_only:
args.append("--local_files_only")
self.execute_command(
ExecuteCommandOptions(
binary_name="faster_whisper", args=args, options={"sync": True}
)
)
with open(output_path, "r", encoding="utf-8") as f:
transcription_content = f.read()
parsed_output = self._parse_transcription(transcription_content)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
return output_path
except Exception as e:
raise Exception(f"Audio transcription failed: {str(e)}")
def _parse_transcription(self, raw_output: str) -> TranscriptionOutput:
lines = raw_output.split("\n")
duration = 0.0
for line in lines:
if line.startswith("Duration:"):
match = re.search(r"Duration:\s+([\d.]+)\s+seconds", line)
if match:
duration = float(match.group(1))
break
segments: list[TranscriptionSegment] = []
segment_regex = re.compile(r"^\[(\d+\.\d+)\s+->\s+(\d+\.\d+)\]\s+(.+)$")
for line in lines:
match = segment_regex.match(line)
if match:
start = match.group(1)
end = match.group(2)
text = match.group(3)
segments.append(
{
"from": float(start),
"to": float(end),
"text": text.strip(),
"speaker": None,
}
)
return {
"duration": duration,
"speakers": [],
"speaker_count": 0,
"segments": segments,
"metadata": {"tool": self.tool_name},
}
@@ -0,0 +1,61 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "faster_whisper",
"toolkit_id": "music_audio",
"name": "Faster Whisper",
"description": "A tool for speech recognition and audio transcription using the Faster Whisper model.",
"icon_name": "mic-2-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"binaries": {
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-linux-x86_64",
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-linux-aarch64",
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-macosx-x86_64",
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-macosx-arm64",
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-win-amd64.exe"
},
"resources": {
"faster-whisper-large-v3": [
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/config.json?download=true",
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/model.bin?download=true",
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/preprocessor_config.json?download=true",
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/tokenizer.json?download=true",
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/vocabulary.json?download=true"
]
},
"functions": {
"transcribeToFile": {
"description": "Transcribe audio to a file using faster-whisper.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"device": {
"type": "string"
},
"cpuThreads": {
"type": "number"
},
"downloadRoot": {
"type": "string"
},
"localFilesOnly": {
"type": "boolean"
}
},
"required": [
"inputPath",
"outputPath"
]
}
}
}
}
@@ -0,0 +1,3 @@
from .src.python.openai_audio_tool import OpenAIAudioTool
__all__ = ["OpenAIAudioTool"]
@@ -0,0 +1,4 @@
{
"OPENAI_AUDIO_API_KEY": null,
"OPENAI_AUDIO_MODEL": "whisper-1"
}
@@ -0,0 +1 @@
export { default } from './openai_audio-tool'
@@ -0,0 +1,157 @@
import fs from 'node:fs'
import path from 'node:path'
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { Network } from '@sdk/network'
// Hardcoded default settings for OpenAI audio tool
const OPENAI_AUDIO_API_KEY: string | null = null
const OPENAI_AUDIO_MODEL = 'whisper-1'
const DEFAULT_SETTINGS: Record<string, unknown> = {
OPENAI_AUDIO_API_KEY,
OPENAI_AUDIO_MODEL
}
const REQUIRED_SETTINGS = ['OPENAI_AUDIO_API_KEY']
interface OpenAITranscriptionOutput {
task: string
duration: number
text: string
segments: {
type: string
id: string
start: number
end: number
text: string
speaker: string
}[]
usage: {
type: string
seconds: number
}
}
export default class OpenAIAudioTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
readonly apiKey: string | null
readonly model: string
constructor() {
super()
this.config = ToolkitConfig.load(OpenAIAudioTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
OpenAIAudioTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
// Priority: toolkit settings > hardcoded default
this.apiKey =
(this.settings['OPENAI_AUDIO_API_KEY'] as string) || OPENAI_AUDIO_API_KEY
this.model =
(this.settings['OPENAI_AUDIO_MODEL'] as string) || OPENAI_AUDIO_MODEL
}
get toolName(): string {
// Use the actual config name for toolkit lookup
return 'openai_audio'
}
get toolkit(): string {
return OpenAIAudioTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Transcribe audio to a file using OpenAI's audio transcription API via SDK Network
* @param inputPath Path to the audio file to transcribe
* @param outputPath Path to save the plain text transcription
* @param apiKey OpenAI API key (uses env/hardcoded default if not provided)
* @param model Transcription model (defaults to tool default)
*/
async transcribeToFile(
inputPath: string,
outputPath: string,
apiKey?: string,
model?: string
): Promise<string> {
// Use provided values, instance values, or error
const finalApiKey = apiKey || this.apiKey
const finalModel = model || this.model
if (!finalApiKey) {
throw new Error('OpenAI API key is missing')
}
const form = new FormData()
const audioFile = await fs.openAsBlob(inputPath)
form.append('file', audioFile, path.basename(inputPath))
form.append('model', finalModel)
form.append('chunking_strategy', 'auto')
form.append('response_format', 'diarized_json')
const network = new Network({ baseURL: 'https://api.openai.com' })
const response = await network.request({
url: '/v1/audio/transcriptions',
method: 'POST',
data: form,
headers: {
Authorization: `Bearer ${finalApiKey}`
}
})
const parsedOutput = this.parseTranscription(
response.data as OpenAITranscriptionOutput
)
await fs.promises.writeFile(
outputPath,
JSON.stringify(parsedOutput, null, 2),
'utf8'
)
return outputPath
}
private parseTranscription(
rawOutput: OpenAITranscriptionOutput
): TranscriptionOutput {
const speakers = Array.from(
new Set(rawOutput.segments.map((segment) => segment.speaker))
)
const segments = rawOutput.segments.map((segment) => {
return {
from: segment.start,
to: segment.end,
text: segment.text,
speaker: segment.speaker || null
}
})
// If duration is not found, use the "to" property from the last segment
let duration = rawOutput.duration
if (!duration && segments.length > 0) {
duration = segments[segments.length - 1]?.to || 0
}
return {
duration: duration || 0,
speakers: speakers,
speaker_count: speakers.length,
segments,
metadata: {
tool: this.toolName
}
}
}
}
@@ -0,0 +1,133 @@
import json
from typing import List, Dict, Any, Optional
from bridges.python.src.sdk.base_tool import BaseTool
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from bridges.python.src.sdk.network import Network
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
# Hardcoded default settings for OpenAI audio tool
OPENAI_AUDIO_API_KEY = None
OPENAI_AUDIO_MODEL = "whisper-1"
DEFAULT_SETTINGS = {
"OPENAI_AUDIO_API_KEY": OPENAI_AUDIO_API_KEY,
"OPENAI_AUDIO_MODEL": OPENAI_AUDIO_MODEL,
}
REQUIRED_SETTINGS = ["OPENAI_AUDIO_API_KEY"]
class OpenAIAudioTool(BaseTool):
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
tool_settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.settings = tool_settings
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
# Priority: toolkit settings > hardcoded default
self.api_key = self.settings.get("OPENAI_AUDIO_API_KEY", OPENAI_AUDIO_API_KEY)
self.model = self.settings.get("OPENAI_AUDIO_MODEL", OPENAI_AUDIO_MODEL)
self.network = Network({"base_url": "https://api.openai.com"})
@property
def tool_name(self) -> str:
# Use the actual config name for toolkit lookup
return "openai_audio"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def transcribe_to_file(
self,
input_path: str,
output_path: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
) -> str:
"""
Transcribe audio to a file using OpenAI's audio transcription API via SDK Network
Args:
input_path: Path to the audio file to transcribe
output_path: Path to save the JSON transcription (unified format)
api_key: OpenAI API key (uses env/hardcoded default if not provided)
model: Transcription model (defaults to tool default)
Returns:
The path to the transcription file
"""
# Use provided values, instance values, or error
api_key = api_key or self.api_key
model = model or self.model
if not api_key:
raise Exception("OpenAI API key is missing")
try:
files: dict = {"file": open(input_path, "rb")}
data: dict = {
"model": model,
"chunking_strategy": "auto",
"response_format": "diarized_json",
}
response = self.network.request(
{
"url": "/v1/audio/transcriptions",
"method": "POST",
"headers": {"Authorization": f"Bearer {api_key}"},
"data": data,
"files": files,
"use_json": True,
}
)
parsed_output = self._parse_transcription(response["data"])
with open(output_path, "w", encoding="utf-8") as f:
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
return output_path
except Exception as e:
raise Exception(f"OpenAI transcription failed: {str(e)}")
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
segments_data = raw_output.get("segments", [])
unique_speakers = list(
set(seg.get("speaker") for seg in segments_data if seg.get("speaker"))
)
segments: List[TranscriptionSegment] = []
for segment in segments_data:
segments.append(
{
"from": float(segment.get("start", 0)),
"to": float(segment.get("end", 0)),
"text": segment.get("text", ""),
"speaker": segment.get("speaker") or None,
}
)
# If duration is not found, use the "to" property from the last segment
duration = raw_output.get("duration")
if not duration and len(segments) > 0:
duration = segments[-1]["to"] or 0.0
return {
"duration": float(duration) if duration else 0.0,
"speakers": unique_speakers,
"speaker_count": len(unique_speakers),
"segments": segments,
"metadata": {"tool": self.tool_name},
}
+39
View File
@@ -0,0 +1,39 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "openai_audio",
"toolkit_id": "music_audio",
"name": "OpenAI Audio",
"description": "A tool for audio processing using OpenAI's API.",
"icon_name": "openai-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"functions": {
"transcribeToFile": {
"description": "Transcribe audio to a file using OpenAI's audio transcription API.",
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"apiKey": {
"type": "string"
},
"model": {
"type": "string"
}
},
"required": [
"inputPath",
"outputPath"
]
}
}
}
}
+3
View File
@@ -0,0 +1,3 @@
from .src.python.qwen3_asr_tool import Qwen3ASRTool
__all__ = ["Qwen3ASRTool"]
@@ -0,0 +1 @@
{}
@@ -0,0 +1 @@
export { default } from './qwen3_asr-tool'
@@ -0,0 +1,227 @@
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
import { Tool } from '@sdk/base-tool'
import { ToolkitConfig } from '@sdk/toolkit-config'
import { NVIDIA_LIBS_PATH, PYTORCH_TORCH_PATH } from '@bridge/constants'
type Qwen3ASRTranscriptionOutput = string
const MODEL_NAME = 'qwen3-asr-1.7b'
const FORCED_ALIGNER_MODEL_NAME = 'qwen3-forcedaligner-0.6b'
const DEFAULT_SETTINGS: Record<string, unknown> = {}
const REQUIRED_SETTINGS: string[] = []
interface Qwen3ASRTask {
audio_path: string
output_path?: string
}
export default class Qwen3ASRTool extends Tool {
private static readonly TOOLKIT = 'music_audio'
private readonly config: ReturnType<typeof ToolkitConfig.load>
constructor() {
super()
// Load configuration from central toolkits directory
this.config = ToolkitConfig.load(Qwen3ASRTool.TOOLKIT, this.toolName)
const toolSettings = ToolkitConfig.loadToolSettings(
Qwen3ASRTool.TOOLKIT,
this.toolName,
DEFAULT_SETTINGS
)
this.settings = toolSettings
this.requiredSettings = REQUIRED_SETTINGS
this.checkRequiredSettings(this.toolName)
}
get toolName(): string {
// Use the actual config name for toolkit lookup
return 'qwen3_asr'
}
get toolkit(): string {
return Qwen3ASRTool.TOOLKIT
}
get description(): string {
return this.config['description']
}
/**
* Transcribe audio to a file using Qwen3-ASR
* @param inputPath The file path of the audio to be transcribed
* @param outputPath The desired file path for the transcription output
* @param device Device to use for processing (cpu, cuda, auto)
* @param batchSize Batch size for processing
* @param language Language code for transcription (auto, en, fr, etc.)
* @param returnTimestamps Whether to return timestamps in output
* @param useForcedAligner Whether to use the forced aligner model
* @param cudaRuntimePath Path to CUDA runtime directory (Linux/Windows only)
* @param torchPath Path to PyTorch installation directory
* @param chunkDuration Chunk duration in seconds for long audio
* @param cpuBatchSize CPU batch size for long audio
* @returns A promise that resolves with the path to the transcription file
*/
async transcribeToFile(
inputPath: string,
outputPath: string,
device = 'auto',
batchSize = 4,
language = 'auto',
returnTimestamps = true,
useForcedAligner = true,
cudaRuntimePath?: string,
torchPath?: string,
chunkDuration = 30,
cpuBatchSize?: number
): Promise<string> {
let tempDir: string | null = null
let jsonFilePath: string | null = null
try {
const inputStats = await fs.promises.stat(inputPath).catch(() => null)
if (!inputStats?.isFile()) {
throw new Error(`Input audio file does not exist: ${inputPath}`)
}
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true })
const modelPath = await this.getResourcePath(MODEL_NAME)
const forcedAlignerPath =
returnTimestamps && useForcedAligner
? await this.getResourcePath(FORCED_ALIGNER_MODEL_NAME)
: undefined
const nvidiaLibsPath = cudaRuntimePath ?? NVIDIA_LIBS_PATH
const torchLibsPath = torchPath ?? PYTORCH_TORCH_PATH
const tasks: Qwen3ASRTask[] = [
{
audio_path: inputPath,
output_path: outputPath
}
]
tempDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'qwen3_asr_tasks_')
)
jsonFilePath = path.join(tempDir, 'tasks.json')
await fs.promises.writeFile(
jsonFilePath,
JSON.stringify(tasks, null, 2),
'utf8'
)
const args = [
'--function',
'transcribe_audio',
'--json_file',
jsonFilePath,
'--model_path',
modelPath,
'--device',
device,
'--batch_size',
batchSize.toString(),
'--language',
language,
'--return_timestamps',
returnTimestamps ? 'true' : 'false',
'--chunk_duration',
chunkDuration.toString()
]
if (nvidiaLibsPath) {
args.push('--cuda_runtime_path', nvidiaLibsPath)
}
if (torchLibsPath) {
args.push('--torch_path', torchLibsPath)
}
if (forcedAlignerPath) {
args.push('--forced_aligner_model_path', forcedAlignerPath)
}
if (cpuBatchSize) {
args.push('--cpu_batch_size', cpuBatchSize.toString())
}
await this.executeCommand({
binaryName: 'qwen3_asr',
args,
options: { sync: true }
})
const transcriptionContent = await fs.promises.readFile(
outputPath,
'utf-8'
)
const parsedOutput = this.parseTranscription(transcriptionContent)
await fs.promises.writeFile(
outputPath,
JSON.stringify(parsedOutput, null, 2),
'utf8'
)
return outputPath
} catch (error: unknown) {
throw new Error(`Audio transcription failed: ${(error as Error).message}`)
}
}
private parseTranscription(
rawOutput: Qwen3ASRTranscriptionOutput
): TranscriptionOutput {
const lines = rawOutput
.split('\n')
.map((line) => line.trim())
.filter((line) => line.length > 0)
const segments: TranscriptionOutput['segments'] = []
const segmentRegex = /^\[(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)s\]\s+(.+)$/
let duration = 0
for (const line of lines) {
const match = line.match(segmentRegex)
if (match && match[1] && match[2] && match[3]) {
const start = parseFloat(match[1])
const end = parseFloat(match[2])
segments.push({
from: start,
to: end,
text: match[3].trim(),
speaker: null
})
if (end > duration) {
duration = end
}
}
}
if (segments.length === 0 && lines.length > 0) {
segments.push({
from: 0,
to: 0,
text: lines[0] ?? '',
speaker: null
})
}
return {
duration,
speakers: [],
speaker_count: 0,
segments,
metadata: {
tool: this.toolName
}
}
}
}
@@ -0,0 +1,195 @@
import json
import os
import re
import tempfile
from typing import Optional
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
from bridges.python.src.constants import NVIDIA_LIBS_PATH, PYTORCH_TORCH_PATH
MODEL_NAME = "qwen3-asr-1.7b"
FORCED_ALIGNER_MODEL_NAME = "qwen3-forcedaligner-0.6b"
DEFAULT_SETTINGS = {}
REQUIRED_SETTINGS = []
class Qwen3ASRTool(BaseTool):
"""
Example output format:
I noticed the app has a very mobile-first feel.
[0.08-0.16s] I
[0.16-0.64s] noticed
"""
TOOLKIT = "music_audio"
def __init__(self):
super().__init__()
# Load configuration from central toolkits directory
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
self.settings = ToolkitConfig.load_tool_settings(
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
)
self.required_settings = REQUIRED_SETTINGS
self._check_required_settings(self.tool_name)
@property
def tool_name(self) -> str:
# Use the actual config name for toolkit lookup
return "qwen3_asr"
@property
def toolkit(self) -> str:
return self.TOOLKIT
@property
def description(self) -> str:
return self.config["description"]
def transcribe_to_file(
self,
input_path: str,
output_path: str,
device: str = "auto",
batch_size: int = 4,
language: str = "auto",
return_timestamps: bool = True,
use_forced_aligner: bool = True,
cuda_runtime_path: Optional[str] = None,
torch_path: Optional[str] = None,
chunk_duration: int = 30,
cpu_batch_size: Optional[int] = None,
) -> str:
"""
Transcribe audio to a file using Qwen3-ASR
Args:
input_path: The file path of the audio to be transcribed
output_path: The desired file path for the transcription output
device: Device to use for processing (cpu, cuda, auto)
batch_size: Batch size for processing
language: Language code for transcription (auto, en, fr, etc.)
return_timestamps: Whether to return timestamps in output
use_forced_aligner: Whether to use the forced aligner model
cuda_runtime_path: Path to CUDA runtime directory (Linux/Windows only)
torch_path: Path to PyTorch installation directory
chunk_duration: Chunk duration in seconds for long audio
cpu_batch_size: CPU batch size for long audio
Returns:
The path to the transcription file
"""
try:
if not os.path.isfile(input_path):
raise FileNotFoundError(f"Input audio file does not exist: {input_path}")
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
model_path = self.get_resource_path(MODEL_NAME)
forced_aligner_path = None
nvidia_libs_path = (
cuda_runtime_path if cuda_runtime_path is not None else NVIDIA_LIBS_PATH
)
torch_libs_path = (
torch_path if torch_path is not None else PYTORCH_TORCH_PATH
)
if return_timestamps and use_forced_aligner:
forced_aligner_path = self.get_resource_path(FORCED_ALIGNER_MODEL_NAME)
tasks = [
{
"audio_path": input_path,
"output_path": output_path,
}
]
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json", delete=False, encoding="utf-8"
) as temp_file:
json_file_path = temp_file.name
json.dump(tasks, temp_file, indent=2, ensure_ascii=False)
args = [
"--function",
"transcribe_audio",
"--json_file",
json_file_path,
"--model_path",
model_path,
"--device",
device,
"--batch_size",
str(batch_size),
"--language",
language,
"--return_timestamps",
"true" if return_timestamps else "false",
"--chunk_duration",
str(chunk_duration),
]
if nvidia_libs_path:
args.extend(["--cuda_runtime_path", nvidia_libs_path])
if torch_libs_path:
args.extend(["--torch_path", torch_libs_path])
if forced_aligner_path:
args.extend(["--forced_aligner_model_path", forced_aligner_path])
if cpu_batch_size is not None:
args.extend(["--cpu_batch_size", str(cpu_batch_size)])
self.execute_command(
ExecuteCommandOptions(
binary_name="qwen3_asr", args=args, options={"sync": True}
)
)
with open(output_path, "r", encoding="utf-8") as f:
transcription_content = f.read()
parsed_output = self.parse_transcription(transcription_content)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
return output_path
except Exception as e:
raise Exception(f"Audio transcription failed: {str(e)}")
def parse_transcription(self, raw_output: str) -> TranscriptionOutput:
lines = [line.strip() for line in raw_output.split("\n") if line.strip()]
segments: list[TranscriptionSegment] = []
segment_regex = re.compile(r"^\[(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)s\]\s+(.+)$")
duration = 0.0
for line in lines:
match = segment_regex.match(line)
if match:
start = float(match.group(1))
end = float(match.group(2))
text = match.group(3)
segments.append(
{"from": start, "to": end, "text": text.strip(), "speaker": None}
)
if end > duration:
duration = end
if not segments and lines:
segments.append({"from": 0.0, "to": 0.0, "text": lines[0], "speaker": None})
return {
"duration": duration,
"speakers": [],
"speaker_count": 0,
"segments": segments,
"metadata": {"tool": self.tool_name},
}
+72
View File
@@ -0,0 +1,72 @@
{
"$schema": "../../../schemas/tool-schemas/tool.json",
"tool_id": "qwen3_asr",
"toolkit_id": "music_audio",
"name": "Qwen3-ASR",
"description": "A tool for speech recognition and timestamped transcription using the Qwen3 ASR models.",
"icon_name": "qwen-ai-line",
"author": {
"name": "Louis Grenard",
"email": "louis@getleon.ai",
"url": "https://twitter.com/grenlouis"
},
"binaries": {
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-linux-x86_64",
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-linux-aarch64",
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-macosx-x86_64",
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-macosx-arm64",
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-win-amd64.exe"
},
"resources": {
"qwen3-asr-1.7b": [
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/chat_template.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/generation_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/merges.txt?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model-00001-of-00002.safetensors?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model-00002-of-00002.safetensors?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model.safetensors.index.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/preprocessor_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/tokenizer_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/vocab.json?download=true"
],
"qwen3-forcedaligner-0.6b": [
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/chat_template.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/generation_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/merges.txt?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/model.safetensors?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/preprocessor_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/tokenizer_config.json?download=true",
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/vocab.json?download=true"
]
},
"functions": {
"transcribeToFile": {
"description": "Transcribe audio to a file using Qwen3 ASR.",
"hooks": {
"post_execution": {
"response_jq": "[.result.segments[].text] | map(select(type == \"string\" and length > 0)) | join(\" \")"
}
},
"parameters": {
"type": "object",
"properties": {
"inputPath": {
"type": "string"
},
"outputPath": {
"type": "string"
},
"device": {
"type": "string"
}
},
"required": [
"inputPath",
"outputPath"
]
}
}
}
}
+3
View File
@@ -0,0 +1,3 @@
from .src.python.qwen3_tts_tool import Qwen3TTSTool
__all__ = ["Qwen3TTSTool"]

Some files were not shown because too many files have changed in this diff Show More