chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Business & Finance",
|
||||
"description": "Tools for business and finance.",
|
||||
"icon_name": "money-dollar-circle-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Calendar & Scheduling",
|
||||
"description": "Tools for calendars and scheduling.",
|
||||
"icon_name": "calendar-2-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.opencode_tool import OpenCodeTool
|
||||
|
||||
__all__ = ["OpenCodeTool"]
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"OPENCODE_OPENROUTER_API_KEY": null,
|
||||
"OPENCODE_OPENROUTER_MODEL": "openrouter/openai/gpt-5.2-codex"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './opencode-tool'
|
||||
@@ -0,0 +1,68 @@
|
||||
{{SYSTEM_PROMPT_SECTION}}
|
||||
{{REPO_SNAPSHOT}}
|
||||
{{TOOLKIT_INFO}}
|
||||
|
||||
# Leon Skill Creation (Concise)
|
||||
|
||||
You are generating a Leon skill in **{{LANGUAGE}}**.
|
||||
|
||||
## Core Rules
|
||||
|
||||
- Use the **{{BRIDGE}}** bridge for all source files.
|
||||
- Skills live directly under `skills/` (no subfolders).
|
||||
- All source files use `{{FILE_EXTENSION}}`.
|
||||
- Validate JSON files against `schemas/skill-schemas/*`.
|
||||
- Write all required files to disk under the chosen `skills/<name>_skill` folder.
|
||||
|
||||
## Required Structure
|
||||
|
||||
```
|
||||
skills/skill_name/
|
||||
skill.json
|
||||
locales/en.json
|
||||
src/
|
||||
settings.sample.json
|
||||
settings.json
|
||||
actions/
|
||||
widgets/ (optional)
|
||||
```
|
||||
|
||||
## skill.json Rules
|
||||
|
||||
- `actions` required, `flow` optional.
|
||||
- If `flow` exists, only the first action receives user parameters.
|
||||
- Use `"skill_name:action_name"` for cross-skill flow steps.
|
||||
- Set `author.name` to `Leon` unless explicitly specified.
|
||||
|
||||
## Settings Files
|
||||
|
||||
- `src/settings.sample.json` and `src/settings.json` must both exist and start identical.
|
||||
- Use `{}` if no settings.
|
||||
|
||||
## Toolkits (Plan First)
|
||||
|
||||
- Choose relevant toolkits from above **before** writing code.
|
||||
- Use existing tools instead of duplicating functionality.
|
||||
|
||||
## leon.answer Basics
|
||||
|
||||
{{LEON_ANSWER_BASIC_EXAMPLE}}
|
||||
|
||||
## Passing Data Between Actions
|
||||
|
||||
{{CONTEXT_DATA_EXAMPLE}}
|
||||
|
||||
## Settings Usage
|
||||
|
||||
{{SETTINGS_USAGE_EXAMPLE}}
|
||||
|
||||
## Widget Rules
|
||||
|
||||
- Do not use `Card` as the parent component. The `WidgetWrapper` is already applied by default.
|
||||
- For icons, use only the icon name without the `ri-` prefix and `-line` suffix. The system automatically completes them to `ri-{icon-name}-line`. For example, use `snow` instead of `ri-snow-line`.
|
||||
|
||||
## Action Parameters
|
||||
|
||||
{{ACTION_PARAMS_EXAMPLE}}
|
||||
|
||||
{{REFERENCE_FILES_SECTION}}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
{{SYSTEM_PROMPT_SECTION}}
|
||||
{{REPO_SNAPSHOT}}
|
||||
{{TOOLKIT_INFO}}
|
||||
|
||||
# Leon Skill Creation (Concise)
|
||||
|
||||
You are generating a Leon skill in **{{LANGUAGE}}**.
|
||||
|
||||
## Core Rules
|
||||
|
||||
- Use the **{{BRIDGE}}** bridge for all source files.
|
||||
- Skills live directly under `skills/` (no subfolders).
|
||||
- All source files use `{{FILE_EXTENSION}}`.
|
||||
- Validate JSON files against `schemas/skill-schemas/*`.
|
||||
- Write all required files to disk under the chosen `skills/<name>_skill` folder.
|
||||
|
||||
## Required Structure
|
||||
|
||||
```
|
||||
skills/skill_name/
|
||||
skill.json
|
||||
locales/en.json
|
||||
src/
|
||||
settings.sample.json
|
||||
settings.json
|
||||
actions/
|
||||
widgets/ (optional)
|
||||
```
|
||||
|
||||
## skill.json Rules
|
||||
|
||||
- `actions` required, `flow` optional.
|
||||
- If `flow` exists, only the first action receives user parameters.
|
||||
- Use `"skill_name:action_name"` for cross-skill flow steps.
|
||||
- Set `author.name` to `Leon` unless explicitly specified.
|
||||
|
||||
## Settings Files
|
||||
|
||||
- `src/settings.sample.json` and `src/settings.json` must both exist and start identical.
|
||||
- Use `{}` if no settings.
|
||||
|
||||
## Toolkits (Plan First)
|
||||
|
||||
- Choose relevant toolkits from above **before** writing code.
|
||||
- Use existing tools instead of duplicating functionality.
|
||||
|
||||
## leon.answer Basics
|
||||
|
||||
{{LEON_ANSWER_BASIC_EXAMPLE}}
|
||||
|
||||
## Passing Data Between Actions
|
||||
|
||||
{{CONTEXT_DATA_EXAMPLE}}
|
||||
|
||||
## Settings Usage
|
||||
|
||||
{{SETTINGS_USAGE_EXAMPLE}}
|
||||
|
||||
## Widget Rules
|
||||
|
||||
- Do not use `Card` as the parent component. The `WidgetWrapper` is already applied by default.
|
||||
- For icons, use only the icon name without the `ri-` prefix and `-line` suffix. The system automatically completes them to `ri-{icon-name}-line`. For example, use `snow` instead of `ri-snow-line`.
|
||||
|
||||
## Action Parameters
|
||||
|
||||
{{ACTION_PARAMS_EXAMPLE}}
|
||||
|
||||
{{REFERENCE_FILES_SECTION}}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "opencode",
|
||||
"toolkit_id": "coding_development",
|
||||
"name": "OpenCode",
|
||||
"description": "An AI-powered coding agent tool that generates skills using multiple LLM providers (Cerebras, MiniMax, Anthropic, OpenAI, Gemini).",
|
||||
"icon_name": "code-box-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"binaries": {
|
||||
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-linux-x86_64.tar.gz",
|
||||
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-linux-aarch64.tar.gz",
|
||||
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-macosx-x86_64.zip",
|
||||
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-macosx-arm64.zip",
|
||||
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/opencode-v1.14.29/opencode_1.14.29-win-amd64.zip"
|
||||
},
|
||||
"functions": {
|
||||
"configureProvider": {
|
||||
"description": "Configure a provider with an API key and optional model.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"provider",
|
||||
"apiKey"
|
||||
]
|
||||
}
|
||||
},
|
||||
"getConfiguredProviders": {
|
||||
"description": "List the providers currently configured with API keys.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
},
|
||||
"getAvailableProviders": {
|
||||
"description": "List providers supported by OpenCode.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
},
|
||||
"getDefaultModel": {
|
||||
"description": "Get the default model name for a provider.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"provider"
|
||||
]
|
||||
}
|
||||
},
|
||||
"generateSkill": {
|
||||
"description": "Generate a new skill using OpenCode CLI with an agentic loop.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"provider": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"api_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"context_files": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"bridge": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"description",
|
||||
"provider",
|
||||
"target_path"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Coding & Development",
|
||||
"description": "Tools for code generation, development, and automation.",
|
||||
"icon_name": "code-s-slash-line",
|
||||
"context_files": [
|
||||
"ARCHITECTURE.md",
|
||||
"WORKSPACE_INTELLIGENCE.md",
|
||||
"LEON_RUNTIME.md",
|
||||
"HOME.md"
|
||||
],
|
||||
"tools": [
|
||||
"opencode"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.cerebras_tool import CerebrasTool
|
||||
|
||||
__all__ = ["CerebrasTool"]
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"CEREBRAS_API_KEY": null,
|
||||
"CEREBRAS_MODEL": "zai-glm-4.7"
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network, NetworkError } from '@sdk/network'
|
||||
|
||||
// Hardcoded default settings for Cerebras tool
|
||||
const CEREBRAS_API_KEY: string | null = null
|
||||
const CEREBRAS_MODEL = 'zai-glm-4.7'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {
|
||||
CEREBRAS_API_KEY,
|
||||
CEREBRAS_MODEL
|
||||
}
|
||||
const REQUIRED_SETTINGS = ['CEREBRAS_API_KEY']
|
||||
|
||||
interface ChatMessage {
|
||||
role: string
|
||||
content: string
|
||||
}
|
||||
|
||||
interface ChatCompletionOptions {
|
||||
messages: ChatMessage[]
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
use_structured_output?: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema?: Record<string, any>
|
||||
}
|
||||
|
||||
interface CompletionOptions {
|
||||
prompt: string
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
use_structured_output?: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema?: Record<string, any>
|
||||
}
|
||||
|
||||
interface StructuredCompletionOptions {
|
||||
prompt: string
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema: Record<string, any>
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
}
|
||||
|
||||
interface ApiResponse {
|
||||
success: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data?: any
|
||||
model_used?: string
|
||||
error?: string
|
||||
status_code?: number
|
||||
}
|
||||
|
||||
export default class CerebrasTool extends Tool {
|
||||
private static readonly TOOLKIT = 'communication'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
private api_key: string | null
|
||||
private model: string
|
||||
private readonly network: Network
|
||||
|
||||
// Popular Cerebras-hosted models (override with full model IDs if needed)
|
||||
private readonly popular_models = {
|
||||
'zai-glm-4.7': 'zai-glm-4.7',
|
||||
'qwen-3-235b-a22b-instruct-2507': 'qwen-3-235b-a22b-instruct-2507',
|
||||
'qwen-3-32b': 'qwen-3-32b'
|
||||
}
|
||||
|
||||
constructor(apiKey?: string) {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(CerebrasTool.TOOLKIT, this.toolName)
|
||||
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
CerebrasTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
|
||||
// Priority: skill-provided apiKey > toolkit settings > hardcoded default
|
||||
this.api_key =
|
||||
apiKey ||
|
||||
(this.settings['CEREBRAS_API_KEY'] as string) ||
|
||||
CEREBRAS_API_KEY
|
||||
|
||||
// Load model from toolkit settings or hardcoded default
|
||||
this.model = (this.settings['CEREBRAS_MODEL'] as string) || CEREBRAS_MODEL
|
||||
|
||||
this.network = new Network({ baseURL: 'https://api.cerebras.ai/v1' })
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'cerebras'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return CerebrasTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the Cerebras API key
|
||||
*/
|
||||
setApiKey(apiKey: string): void {
|
||||
this.api_key = apiKey
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of popular available models
|
||||
*/
|
||||
getAvailableModels(): string[] {
|
||||
return Object.keys(this.popular_models)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert friendly model name to Cerebras model ID
|
||||
*/
|
||||
getModelId(modelName: string): string {
|
||||
return (
|
||||
this.popular_models[modelName as keyof typeof this.popular_models] ||
|
||||
modelName
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a chat completion request to Cerebras
|
||||
*/
|
||||
async chatCompletion(options: ChatCompletionOptions): Promise<ApiResponse> {
|
||||
const {
|
||||
messages,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output = false,
|
||||
json_schema
|
||||
} = options
|
||||
|
||||
if (!this.api_key) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Cerebras API key not configured'
|
||||
}
|
||||
}
|
||||
|
||||
// Use default model if none provided
|
||||
const finalModel = model || this.model
|
||||
const modelId = this.getModelId(finalModel)
|
||||
|
||||
const requestMessages = []
|
||||
if (system_prompt) {
|
||||
requestMessages.push({ role: 'system', content: system_prompt })
|
||||
}
|
||||
requestMessages.push(...messages)
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const payload: any = {
|
||||
model: modelId,
|
||||
messages: requestMessages,
|
||||
temperature
|
||||
}
|
||||
|
||||
if (max_tokens) {
|
||||
payload.max_tokens = max_tokens
|
||||
}
|
||||
|
||||
if (use_structured_output) {
|
||||
payload.response_format = { type: 'json_object' }
|
||||
if (json_schema) {
|
||||
const schemaText = JSON.stringify(json_schema)
|
||||
const schemaPrompt = `You must return a valid JSON object that matches this schema:\n${schemaText}`
|
||||
payload.messages = [
|
||||
{ role: 'system', content: schemaPrompt },
|
||||
...requestMessages
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.network.request({
|
||||
url: '/chat/completions',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.api_key}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
data: payload
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: response.data as any,
|
||||
model_used: modelId
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Cerebras API error: ${(error as Error).message}`,
|
||||
status_code:
|
||||
error instanceof NetworkError ? error.response.statusCode : undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* General text completion for any use case
|
||||
*/
|
||||
async completion(options: CompletionOptions): Promise<ApiResponse> {
|
||||
const {
|
||||
prompt,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output = false,
|
||||
json_schema
|
||||
} = options
|
||||
|
||||
const messages = [{ role: 'user', content: prompt }]
|
||||
|
||||
const response = await this.chatCompletion({
|
||||
messages,
|
||||
model: model || this.model,
|
||||
temperature,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output,
|
||||
json_schema
|
||||
})
|
||||
|
||||
if (!response.success) {
|
||||
return response
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const content = (response.data as any).choices[0].message.content
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: { content },
|
||||
model_used: response.model_used
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to extract completion: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate structured JSON output using Cerebras structured outputs
|
||||
*/
|
||||
async structuredCompletion(
|
||||
options: StructuredCompletionOptions
|
||||
): Promise<ApiResponse> {
|
||||
const {
|
||||
prompt,
|
||||
json_schema,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt
|
||||
} = options
|
||||
|
||||
const messages = [{ role: 'user', content: prompt }]
|
||||
|
||||
const response = await this.chatCompletion({
|
||||
messages,
|
||||
model: model || this.model,
|
||||
temperature,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output: true,
|
||||
json_schema
|
||||
})
|
||||
|
||||
if (!response.success) {
|
||||
return response
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const content = (response.data as any).choices[0].message.content
|
||||
const parsedData = JSON.parse(content)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: parsedData,
|
||||
model_used: response.model_used
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof SyntaxError) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to parse JSON response: ${error.message}`
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to extract completion: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of available models from Cerebras API
|
||||
*/
|
||||
async listModels(): Promise<ApiResponse> {
|
||||
if (!this.api_key) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Cerebras API key not configured'
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.network.request({
|
||||
url: '/models',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.api_key}`
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: { models: (response.data as any).data }
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to fetch models: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './cerebras-tool'
|
||||
@@ -0,0 +1,290 @@
|
||||
import json
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.network import Network, NetworkError
|
||||
|
||||
# Hardcoded default settings for Cerebras tool
|
||||
CEREBRAS_API_KEY = None
|
||||
CEREBRAS_MODEL = "zai-glm-4.7"
|
||||
DEFAULT_SETTINGS = {
|
||||
"CEREBRAS_API_KEY": CEREBRAS_API_KEY,
|
||||
"CEREBRAS_MODEL": CEREBRAS_MODEL,
|
||||
}
|
||||
REQUIRED_SETTINGS = ["CEREBRAS_API_KEY"]
|
||||
|
||||
|
||||
class CerebrasTool(BaseTool):
|
||||
"""Cerebras tool for LLM API access (e.g., GLM 4.7)"""
|
||||
|
||||
TOOLKIT = "communication"
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
|
||||
tool_settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.settings = tool_settings
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
# Priority: skill-provided api_key > toolkit settings > hardcoded default
|
||||
self.api_key = api_key or self.settings.get(
|
||||
"CEREBRAS_API_KEY", CEREBRAS_API_KEY
|
||||
)
|
||||
|
||||
# Load model settings
|
||||
self.model = self.settings.get("CEREBRAS_MODEL", CEREBRAS_MODEL)
|
||||
|
||||
self.network = Network({"base_url": "https://api.cerebras.ai/v1"})
|
||||
|
||||
# Popular Cerebras-hosted models (override with full model IDs if needed)
|
||||
self.popular_models = {
|
||||
"zai-glm-4.7": "zai-glm-4.7",
|
||||
"qwen-3-235b-a22b-instruct-2507": "qwen-3-235b-a22b-instruct-2507",
|
||||
"qwen-3-32b": "qwen-3-32b",
|
||||
}
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "cerebras"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def set_api_key(self, api_key: str) -> None:
|
||||
"""Set the Cerebras API key"""
|
||||
self.api_key = api_key
|
||||
|
||||
def get_available_models(self) -> List[str]:
|
||||
"""Get list of popular available models"""
|
||||
return list(self.popular_models.keys())
|
||||
|
||||
def get_model_id(self, model_name: str) -> str:
|
||||
"""Convert friendly model name to Cerebras model ID"""
|
||||
return self.popular_models.get(model_name, model_name)
|
||||
|
||||
def chat_completion(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
use_structured_output: bool = False,
|
||||
json_schema: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Send a chat completion request to Cerebras
|
||||
|
||||
Args:
|
||||
messages: List of message dictionaries with 'role' and 'content'
|
||||
model: Model name (friendly name or full model ID)
|
||||
temperature: Sampling temperature (0-2)
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: System prompt to prepend
|
||||
use_structured_output: Whether to use structured outputs
|
||||
json_schema: JSON schema for structured output (required if use_structured_output=True)
|
||||
|
||||
Returns:
|
||||
Dict with response data or error information
|
||||
"""
|
||||
if not self.api_key:
|
||||
return {"success": False, "error": "Cerebras API key not configured"}
|
||||
|
||||
# Use default model if none provided
|
||||
model = model or self.model
|
||||
|
||||
model_id = self.get_model_id(model)
|
||||
|
||||
request_messages: List[Dict[str, str]] = []
|
||||
if system_prompt:
|
||||
request_messages.append({"role": "system", "content": system_prompt})
|
||||
request_messages.extend(messages)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model_id,
|
||||
"messages": request_messages,
|
||||
"temperature": temperature,
|
||||
}
|
||||
|
||||
if max_tokens:
|
||||
payload["max_tokens"] = max_tokens
|
||||
|
||||
if use_structured_output:
|
||||
payload["response_format"] = {"type": "json_object"}
|
||||
if json_schema:
|
||||
schema_text = json.dumps(json_schema)
|
||||
schema_prompt = (
|
||||
"You must return a valid JSON object that matches this schema:\n"
|
||||
f"{schema_text}"
|
||||
)
|
||||
payload["messages"] = [
|
||||
{"role": "system", "content": schema_prompt}
|
||||
] + request_messages
|
||||
|
||||
try:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/chat/completions",
|
||||
"method": "POST",
|
||||
"headers": {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"data": payload,
|
||||
}
|
||||
)
|
||||
|
||||
return {"success": True, "data": response["data"], "model_used": model_id}
|
||||
|
||||
except NetworkError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Cerebras API error: {str(e)}",
|
||||
"status_code": getattr(e.response, "status_code", None),
|
||||
}
|
||||
|
||||
def completion(
|
||||
self,
|
||||
prompt: str,
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
use_structured_output: bool = False,
|
||||
json_schema: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
General text completion for any use case
|
||||
|
||||
Args:
|
||||
prompt: Text prompt to complete
|
||||
model: LLM model to use
|
||||
temperature: Sampling temperature
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: Optional system prompt
|
||||
use_structured_output: Whether to use structured outputs
|
||||
json_schema: JSON schema for structured output
|
||||
|
||||
Returns:
|
||||
Dict with completion result
|
||||
"""
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
response = self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
system_prompt=system_prompt,
|
||||
use_structured_output=use_structured_output,
|
||||
json_schema=json_schema,
|
||||
)
|
||||
|
||||
if not response["success"]:
|
||||
return response
|
||||
|
||||
try:
|
||||
content = response["data"]["choices"][0]["message"]["content"]
|
||||
return {
|
||||
"success": True,
|
||||
"content": content,
|
||||
"model_used": response["model_used"],
|
||||
}
|
||||
except (KeyError, IndexError) as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to extract completion: {str(e)}",
|
||||
}
|
||||
|
||||
def structured_completion(
|
||||
self,
|
||||
prompt: str,
|
||||
json_schema: Dict[str, Any],
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate structured JSON output using Cerebras structured outputs
|
||||
|
||||
Args:
|
||||
prompt: Text prompt to complete
|
||||
json_schema: JSON schema defining the required output structure
|
||||
model: LLM model to use
|
||||
temperature: Sampling temperature
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: Optional system prompt
|
||||
|
||||
Returns:
|
||||
Dict with parsed JSON result or error
|
||||
"""
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
response = self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
system_prompt=system_prompt,
|
||||
use_structured_output=True,
|
||||
json_schema=json_schema,
|
||||
)
|
||||
|
||||
if not response["success"]:
|
||||
return response
|
||||
|
||||
try:
|
||||
content = response["data"]["choices"][0]["message"]["content"]
|
||||
parsed_data = json.loads(content)
|
||||
return {
|
||||
"success": True,
|
||||
"data": parsed_data,
|
||||
"model_used": response["model_used"],
|
||||
}
|
||||
except (KeyError, IndexError) as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to extract completion: {str(e)}",
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to parse JSON response: {str(e)}",
|
||||
}
|
||||
|
||||
def list_models(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get list of available models from Cerebras API
|
||||
|
||||
Returns:
|
||||
Dict with models list or error
|
||||
"""
|
||||
if not self.api_key:
|
||||
return {"success": False, "error": "Cerebras API key not configured"}
|
||||
|
||||
try:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/models",
|
||||
"method": "GET",
|
||||
"headers": {"Authorization": f"Bearer {self.api_key}"},
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"models": response["data"].get("data", response["data"]),
|
||||
}
|
||||
except NetworkError as e:
|
||||
return {"success": False, "error": f"Failed to fetch models: {str(e)}"}
|
||||
@@ -0,0 +1,161 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "cerebras",
|
||||
"toolkit_id": "communication",
|
||||
"name": "Cerebras",
|
||||
"description": "A tool for interacting with Cerebras LLM APIs (e.g., GLM 4.7).",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"chatCompletion": {
|
||||
"description": "Generate a chat completion using the Cerebras API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"messages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"use_structured_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"messages"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"completion": {
|
||||
"description": "Generate a completion using the Cerebras API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"use_structured_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"structuredCompletion": {
|
||||
"description": "Generate a structured completion using a JSON schema.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt",
|
||||
"json_schema"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"listModels": {
|
||||
"description": "List available Cerebras models.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.inference_tool import InferenceTool
|
||||
|
||||
__all__ = ["InferenceTool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './inference-tool'
|
||||
@@ -0,0 +1,97 @@
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network } from '@sdk/network'
|
||||
|
||||
interface CompletionOptions {
|
||||
prompt: string
|
||||
system_prompt?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
thought_tokens_budget?: number
|
||||
disable_thinking?: boolean
|
||||
reasoning_mode?: 'off' | 'guarded' | 'on'
|
||||
track_provider_errors?: boolean
|
||||
}
|
||||
|
||||
interface StructuredCompletionOptions extends CompletionOptions {
|
||||
json_schema: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface InferenceResponse {
|
||||
success: boolean
|
||||
output?: unknown
|
||||
reasoning?: string
|
||||
usedInputTokens?: number
|
||||
usedOutputTokens?: number
|
||||
generationDurationMs?: number
|
||||
providerDecodeDurationMs?: number
|
||||
providerTokensPerSecond?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export default class InferenceTool extends Tool {
|
||||
private static readonly TOOLKIT = 'communication'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
private readonly network: Network
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.config = ToolkitConfig.load(InferenceTool.TOOLKIT, this.toolName)
|
||||
this.network = new Network({
|
||||
baseURL: `${process.env['LEON_HOST']}:${process.env['LEON_PORT']}/api/v1`
|
||||
})
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'inference'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return InferenceTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
async completion(options: CompletionOptions): Promise<InferenceResponse> {
|
||||
const response = await this.network.request<InferenceResponse>({
|
||||
url: '/inference',
|
||||
method: 'POST',
|
||||
data: {
|
||||
prompt: options.prompt,
|
||||
systemPrompt: options.system_prompt,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.max_tokens,
|
||||
thoughtTokensBudget: options.thought_tokens_budget,
|
||||
disableThinking: options.disable_thinking,
|
||||
reasoningMode: options.reasoning_mode,
|
||||
trackProviderErrors: options.track_provider_errors
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
|
||||
async structuredCompletion(
|
||||
options: StructuredCompletionOptions
|
||||
): Promise<InferenceResponse> {
|
||||
const response = await this.network.request<InferenceResponse>({
|
||||
url: '/inference',
|
||||
method: 'POST',
|
||||
data: {
|
||||
prompt: options.prompt,
|
||||
systemPrompt: options.system_prompt,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.max_tokens,
|
||||
thoughtTokensBudget: options.thought_tokens_budget,
|
||||
jsonSchema: options.json_schema,
|
||||
disableThinking: options.disable_thinking,
|
||||
reasoningMode: options.reasoning_mode,
|
||||
trackProviderErrors: options.track_provider_errors
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Literal
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.network import Network
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
|
||||
|
||||
class InferenceTool(BaseTool):
|
||||
TOOLKIT = "communication"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.network = Network(
|
||||
{
|
||||
"base_url": f"{os.environ.get('LEON_HOST')}:{os.environ.get('LEON_PORT')}/api/v1"
|
||||
}
|
||||
)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "inference"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config.get("description", "")
|
||||
|
||||
def completion(
|
||||
self,
|
||||
prompt: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
thought_tokens_budget: Optional[int] = None,
|
||||
disable_thinking: Optional[bool] = None,
|
||||
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
|
||||
track_provider_errors: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/inference",
|
||||
"method": "POST",
|
||||
"data": {
|
||||
"prompt": prompt,
|
||||
"systemPrompt": system_prompt,
|
||||
"temperature": temperature,
|
||||
"maxTokens": max_tokens,
|
||||
"thoughtTokensBudget": thought_tokens_budget,
|
||||
"disableThinking": disable_thinking,
|
||||
"reasoningMode": reasoning_mode,
|
||||
"trackProviderErrors": track_provider_errors,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return response["data"]
|
||||
|
||||
def structured_completion(
|
||||
self,
|
||||
prompt: str,
|
||||
json_schema: Dict[str, Any],
|
||||
system_prompt: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
thought_tokens_budget: Optional[int] = None,
|
||||
disable_thinking: Optional[bool] = None,
|
||||
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
|
||||
track_provider_errors: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/inference",
|
||||
"method": "POST",
|
||||
"data": {
|
||||
"prompt": prompt,
|
||||
"systemPrompt": system_prompt,
|
||||
"temperature": temperature,
|
||||
"maxTokens": max_tokens,
|
||||
"thoughtTokensBudget": thought_tokens_budget,
|
||||
"jsonSchema": json_schema,
|
||||
"disableThinking": disable_thinking,
|
||||
"reasoningMode": reasoning_mode,
|
||||
"trackProviderErrors": track_provider_errors,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return response["data"]
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "inference",
|
||||
"toolkit_id": "communication",
|
||||
"name": "Inference",
|
||||
"description": "A generic Leon workflow inference tool backed by the active workflow LLM provider.",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"completion": {
|
||||
"description": "Generate a workflow inference completion.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"thought_tokens_budget": {
|
||||
"type": "number"
|
||||
},
|
||||
"disable_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"reasoning_mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"track_provider_errors": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt"
|
||||
]
|
||||
}
|
||||
},
|
||||
"structuredCompletion": {
|
||||
"description": "Generate a structured workflow inference completion using a JSON schema.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"thought_tokens_budget": {
|
||||
"type": "number"
|
||||
},
|
||||
"disable_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"reasoning_mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"track_provider_errors": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt",
|
||||
"json_schema"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.openrouter_tool import OpenRouterTool
|
||||
|
||||
__all__ = ["OpenRouterTool"]
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"OPENROUTER_API_KEY": null,
|
||||
"OPENROUTER_MODEL": "google/gemini-3.1-flash-lite"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './openrouter-tool'
|
||||
@@ -0,0 +1,340 @@
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network, NetworkError } from '@sdk/network'
|
||||
|
||||
// Hardcoded default settings for OpenRouter tool
|
||||
const OPENROUTER_API_KEY: string | null = null
|
||||
const OPENROUTER_MODEL = 'google/gemini-3.1-flash-lite'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {
|
||||
OPENROUTER_API_KEY,
|
||||
OPENROUTER_MODEL
|
||||
}
|
||||
const REQUIRED_SETTINGS = ['OPENROUTER_API_KEY']
|
||||
|
||||
interface ChatMessage {
|
||||
role: string
|
||||
content: string
|
||||
}
|
||||
|
||||
interface ChatCompletionOptions {
|
||||
messages: ChatMessage[]
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
use_structured_output?: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema?: Record<string, any>
|
||||
}
|
||||
|
||||
interface CompletionOptions {
|
||||
prompt: string
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
use_structured_output?: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema?: Record<string, any>
|
||||
}
|
||||
|
||||
interface StructuredCompletionOptions {
|
||||
prompt: string
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
json_schema: Record<string, any>
|
||||
model?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
system_prompt?: string
|
||||
}
|
||||
|
||||
interface ApiResponse {
|
||||
success: boolean
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data?: any
|
||||
model_used?: string
|
||||
error?: string
|
||||
status_code?: number
|
||||
}
|
||||
|
||||
export default class OpenRouterTool extends Tool {
|
||||
private static readonly TOOLKIT = 'communication'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
private api_key: string | null
|
||||
private model: string
|
||||
private readonly network: Network
|
||||
|
||||
constructor(apiKey?: string) {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(OpenRouterTool.TOOLKIT, this.toolName)
|
||||
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
OpenRouterTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = apiKey ? [] : REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
|
||||
// Priority: skill-provided apiKey > toolkit settings > hardcoded default
|
||||
this.api_key =
|
||||
apiKey ||
|
||||
(this.settings['OPENROUTER_API_KEY'] as string) ||
|
||||
OPENROUTER_API_KEY
|
||||
|
||||
// Load model from toolkit settings or hardcoded default
|
||||
this.model =
|
||||
(this.settings['OPENROUTER_MODEL'] as string) || OPENROUTER_MODEL
|
||||
|
||||
this.network = new Network({ baseURL: 'https://openrouter.ai/api' })
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'openrouter'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return OpenRouterTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the OpenRouter API key
|
||||
*/
|
||||
setApiKey(apiKey: string): void {
|
||||
this.api_key = apiKey
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a chat completion request to OpenRouter
|
||||
*/
|
||||
async chatCompletion(options: ChatCompletionOptions): Promise<ApiResponse> {
|
||||
const {
|
||||
messages,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output = false,
|
||||
json_schema
|
||||
} = options
|
||||
|
||||
if (!this.api_key) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'OpenRouter API key not configured'
|
||||
}
|
||||
}
|
||||
|
||||
// Use default model if none provided
|
||||
const finalModel = model || this.model
|
||||
|
||||
// Prepare messages with system prompt if provided
|
||||
const requestMessages = []
|
||||
if (system_prompt) {
|
||||
requestMessages.push({ role: 'system', content: system_prompt })
|
||||
}
|
||||
requestMessages.push(...messages)
|
||||
|
||||
// Prepare request payload
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const payload: any = {
|
||||
model: finalModel,
|
||||
messages: requestMessages,
|
||||
temperature
|
||||
}
|
||||
|
||||
if (max_tokens) {
|
||||
payload.max_tokens = max_tokens
|
||||
}
|
||||
|
||||
// Add structured output configuration if requested
|
||||
if (use_structured_output && json_schema) {
|
||||
payload.response_format = {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: json_schema['name'] || 'response',
|
||||
strict: true,
|
||||
schema: json_schema['schema']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.network.request({
|
||||
url: '/v1/chat/completions',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.api_key}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
data: payload
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: response.data as any,
|
||||
model_used: finalModel
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `OpenRouter API error: ${(error as Error).message}`,
|
||||
status_code:
|
||||
error instanceof NetworkError ? error.response.statusCode : undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* General text completion for any use case
|
||||
*/
|
||||
async completion(options: CompletionOptions): Promise<ApiResponse> {
|
||||
const {
|
||||
prompt,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output = false,
|
||||
json_schema
|
||||
} = options
|
||||
|
||||
const messages = [{ role: 'user', content: prompt }]
|
||||
|
||||
const response = await this.chatCompletion({
|
||||
messages,
|
||||
model: model || this.model,
|
||||
temperature,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output,
|
||||
json_schema
|
||||
})
|
||||
|
||||
if (!response.success) {
|
||||
return response
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const content = (response.data as any).choices[0].message.content
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: { content },
|
||||
model_used: response.model_used
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to extract completion: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate structured JSON output using OpenRouter's structured outputs feature
|
||||
*/
|
||||
async structuredCompletion(
|
||||
options: StructuredCompletionOptions
|
||||
): Promise<ApiResponse> {
|
||||
const {
|
||||
prompt,
|
||||
json_schema,
|
||||
model,
|
||||
temperature = 0.7,
|
||||
max_tokens,
|
||||
system_prompt
|
||||
} = options
|
||||
|
||||
const messages = [{ role: 'user', content: prompt }]
|
||||
|
||||
const response = await this.chatCompletion({
|
||||
messages,
|
||||
model: model || this.model,
|
||||
temperature,
|
||||
max_tokens,
|
||||
system_prompt,
|
||||
use_structured_output: true,
|
||||
json_schema
|
||||
})
|
||||
|
||||
if (!response.success) {
|
||||
return response
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const content = (response.data as any).choices[0].message.content
|
||||
const parsedData =
|
||||
typeof content === 'string' ? JSON.parse(content) : content
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: parsedData,
|
||||
model_used: response.model_used
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const content = (response.data as any).choices[0]?.message?.content
|
||||
|
||||
if (error instanceof SyntaxError) {
|
||||
// Show raw response preview to help debug JSON parsing errors
|
||||
const preview =
|
||||
typeof content === 'string'
|
||||
? content.substring(0, 500)
|
||||
: JSON.stringify(content ?? 'null').substring(0, 500)
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to parse JSON response: ${error.message}. Response preview: ${preview}`
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to extract completion: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of available models from OpenRouter API
|
||||
*/
|
||||
async listModels(): Promise<ApiResponse> {
|
||||
if (!this.api_key) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'OpenRouter API key not configured'
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.network.request({
|
||||
url: '/v1/models',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.api_key}`
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: { models: (response.data as any).data }
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to fetch models: ${(error as Error).message}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
import json
|
||||
from typing import Dict, Any, Optional, List
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.network import Network, NetworkError
|
||||
|
||||
# Hardcoded default settings for OpenRouter tool
|
||||
OPENROUTER_API_KEY = None
|
||||
OPENROUTER_MODEL = "google/gemini-3.1-flash-lite"
|
||||
DEFAULT_SETTINGS = {
|
||||
"OPENROUTER_API_KEY": OPENROUTER_API_KEY,
|
||||
"OPENROUTER_MODEL": OPENROUTER_MODEL,
|
||||
}
|
||||
REQUIRED_SETTINGS = ["OPENROUTER_API_KEY"]
|
||||
|
||||
|
||||
class OpenRouterTool(BaseTool):
|
||||
"""OpenRouter tool for unified LLM API access across all skills"""
|
||||
|
||||
TOOLKIT = "communication"
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
|
||||
tool_settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.settings = tool_settings
|
||||
self.required_settings = [] if api_key else REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
# Priority: skill-provided api_key > toolkit settings > hardcoded default
|
||||
self.api_key = api_key or self.settings.get(
|
||||
"OPENROUTER_API_KEY", OPENROUTER_API_KEY
|
||||
)
|
||||
|
||||
# Load model settings
|
||||
self.model = self.settings.get("OPENROUTER_MODEL", OPENROUTER_MODEL)
|
||||
|
||||
self.network = Network({"base_url": "https://openrouter.ai/api"})
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "openrouter"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def set_api_key(self, api_key: str) -> None:
|
||||
"""Set the OpenRouter API key"""
|
||||
self.api_key = api_key
|
||||
|
||||
def chat_completion(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
use_structured_output: bool = False,
|
||||
json_schema: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Send a chat completion request to OpenRouter
|
||||
|
||||
Args:
|
||||
messages: List of message dictionaries with 'role' and 'content'
|
||||
model: Model ID (full OpenRouter model ID, e.g. 'google/gemini-3.1-flash-lite')
|
||||
temperature: Sampling temperature (0-2)
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: System prompt to prepend
|
||||
use_structured_output: Whether to use OpenRouter's structured outputs
|
||||
json_schema: JSON schema for structured output (required if use_structured_output=True)
|
||||
|
||||
Returns:
|
||||
Dict with response data or error information
|
||||
"""
|
||||
if not self.api_key:
|
||||
return {"success": False, "error": "OpenRouter API key not configured"}
|
||||
|
||||
# Use default model if none provided
|
||||
model = model or self.model
|
||||
|
||||
# Prepare messages with system prompt if provided
|
||||
request_messages = []
|
||||
if system_prompt:
|
||||
request_messages.append({"role": "system", "content": system_prompt})
|
||||
request_messages.extend(messages)
|
||||
|
||||
# Prepare request payload
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": request_messages,
|
||||
"temperature": temperature,
|
||||
}
|
||||
|
||||
if max_tokens:
|
||||
payload["max_tokens"] = max_tokens
|
||||
|
||||
# Add structured output configuration if requested
|
||||
if use_structured_output and json_schema:
|
||||
payload["response_format"] = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": json_schema.get("name", "response"),
|
||||
"strict": True,
|
||||
"schema": json_schema["schema"],
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/v1/chat/completions",
|
||||
"method": "POST",
|
||||
"headers": {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"data": payload,
|
||||
}
|
||||
)
|
||||
|
||||
return {"success": True, "data": response["data"], "model_used": model}
|
||||
|
||||
except NetworkError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"OpenRouter API error: {str(e)}",
|
||||
"status_code": getattr(e.response, "status_code", None),
|
||||
}
|
||||
|
||||
def completion(
|
||||
self,
|
||||
prompt: str,
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
use_structured_output: bool = False,
|
||||
json_schema: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
General text completion for any use case
|
||||
|
||||
Args:
|
||||
prompt: Text prompt to complete
|
||||
model: Model ID (full OpenRouter model ID)
|
||||
temperature: Sampling temperature
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: Optional system prompt
|
||||
use_structured_output: Whether to use structured outputs
|
||||
json_schema: JSON schema for structured output
|
||||
|
||||
Returns:
|
||||
Dict with completion result
|
||||
"""
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
response = self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
system_prompt=system_prompt,
|
||||
use_structured_output=use_structured_output,
|
||||
json_schema=json_schema,
|
||||
)
|
||||
|
||||
if not response["success"]:
|
||||
return response
|
||||
|
||||
try:
|
||||
content = response["data"]["choices"][0]["message"]["content"]
|
||||
return {
|
||||
"success": True,
|
||||
"content": content,
|
||||
"model_used": response["model_used"],
|
||||
}
|
||||
except (KeyError, IndexError) as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to extract completion: {str(e)}",
|
||||
}
|
||||
|
||||
def structured_completion(
|
||||
self,
|
||||
prompt: str,
|
||||
json_schema: Dict[str, Any],
|
||||
model: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
max_tokens: Optional[int] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate structured JSON output using OpenRouter's structured outputs feature
|
||||
|
||||
Args:
|
||||
prompt: Text prompt to complete
|
||||
json_schema: JSON schema defining the required output structure
|
||||
model: Model ID (full OpenRouter model ID)
|
||||
temperature: Sampling temperature
|
||||
max_tokens: Maximum tokens to generate
|
||||
system_prompt: Optional system prompt
|
||||
|
||||
Returns:
|
||||
Dict with parsed JSON result or error
|
||||
"""
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
response = self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
system_prompt=system_prompt,
|
||||
use_structured_output=True,
|
||||
json_schema=json_schema,
|
||||
)
|
||||
|
||||
if not response["success"]:
|
||||
return response
|
||||
|
||||
try:
|
||||
content = response["data"]["choices"][0]["message"]["content"]
|
||||
# With structured outputs, content is already valid JSON
|
||||
parsed_data = json.loads(content)
|
||||
return {
|
||||
"success": True,
|
||||
"data": parsed_data,
|
||||
"model_used": response["model_used"],
|
||||
}
|
||||
except (KeyError, IndexError) as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to extract completion: {str(e)}",
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Failed to parse JSON response: {str(e)}",
|
||||
}
|
||||
|
||||
def list_models(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get list of available models from OpenRouter API
|
||||
|
||||
Returns:
|
||||
Dict with models list or error
|
||||
"""
|
||||
if not self.api_key:
|
||||
return {"success": False, "error": "OpenRouter API key not configured"}
|
||||
|
||||
try:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/v1/models",
|
||||
"method": "GET",
|
||||
"headers": {"Authorization": f"Bearer {self.api_key}"},
|
||||
}
|
||||
)
|
||||
|
||||
return {"success": True, "models": response["data"]["data"]}
|
||||
|
||||
except NetworkError as e:
|
||||
return {"success": False, "error": f"Failed to fetch models: {str(e)}"}
|
||||
@@ -0,0 +1,162 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "openrouter",
|
||||
"toolkit_id": "communication",
|
||||
"name": "OpenRouter",
|
||||
"description": "A tool for interacting with various LLMs through the OpenRouter API gateway.",
|
||||
"icon_name": "route-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"chatCompletion": {
|
||||
"description": "Generate a chat completion using OpenRouter.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"messages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"use_structured_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"messages"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"completion": {
|
||||
"description": "Generate a completion using OpenRouter.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"use_structured_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"structuredCompletion": {
|
||||
"description": "Generate a structured completion using a JSON schema.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt",
|
||||
"json_schema"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"options"
|
||||
]
|
||||
}
|
||||
},
|
||||
"listModels": {
|
||||
"description": "List available OpenRouter models.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Communication",
|
||||
"description": "Tools for communication and language model interactions.",
|
||||
"icon_name": "chat-3-line",
|
||||
"context_files": [
|
||||
"LEON.md",
|
||||
"ARCHITECTURE.md",
|
||||
"MEDIA_PROFILE.md"
|
||||
],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Dialog",
|
||||
"description": "Tools for dialog and conversation handling.",
|
||||
"icon_name": "discuss-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "File System",
|
||||
"description": "Tools for file system operations.",
|
||||
"icon_name": "folders-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Food & Drink",
|
||||
"description": "Tools for food and drink queries.",
|
||||
"icon_name": "restaurant-2-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Games",
|
||||
"description": "Tools for games and entertainment.",
|
||||
"icon_name": "gamepad-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Health & Fitness",
|
||||
"description": "Tools for health and fitness information.",
|
||||
"icon_name": "heart-pulse-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Media Generation",
|
||||
"description": "Tools for media generation and creative workflows.",
|
||||
"icon_name": "sparkling-2-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "../../schemas/toolkit-schemas/toolkit.json",
|
||||
"name": "Movies & TV",
|
||||
"description": "Tools for movies and TV information.",
|
||||
"icon_name": "movie-2-line",
|
||||
"context_files": [],
|
||||
"tools": []
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.assemblyai_audio_tool import AssemblyAIAudioTool
|
||||
|
||||
__all__ = ["AssemblyAIAudioTool"]
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"ASSEMBLYAI_AUDIO_API_KEY": null
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
import fs from 'node:fs'
|
||||
|
||||
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network } from '@sdk/network'
|
||||
|
||||
// Hardcoded default setting for AssemblyAI audio tool
|
||||
const ASSEMBLYAI_AUDIO_API_KEY: string | null = null
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {
|
||||
ASSEMBLYAI_AUDIO_API_KEY
|
||||
}
|
||||
const REQUIRED_SETTINGS = ['ASSEMBLYAI_AUDIO_API_KEY']
|
||||
|
||||
interface AssemblyAIUploadResponse {
|
||||
upload_url: string
|
||||
}
|
||||
|
||||
interface AssemblyAITranscriptionResponse {
|
||||
id: string
|
||||
status: 'queued' | 'processing' | 'completed' | 'error'
|
||||
text: string
|
||||
words?: {
|
||||
text: string
|
||||
start: number
|
||||
end: number
|
||||
confidence: number
|
||||
speaker?: string
|
||||
}[]
|
||||
utterances?: {
|
||||
text: string
|
||||
start: number
|
||||
end: number
|
||||
confidence: number
|
||||
speaker: string
|
||||
words: {
|
||||
text: string
|
||||
start: number
|
||||
end: number
|
||||
confidence: number
|
||||
}[]
|
||||
}[]
|
||||
audio_duration?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export default class AssemblyAIAudioTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
readonly apiKey: string | null
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.config = ToolkitConfig.load(AssemblyAIAudioTool.TOOLKIT, this.toolName)
|
||||
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
AssemblyAIAudioTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
|
||||
// Priority: toolkit settings > hardcoded default
|
||||
this.apiKey =
|
||||
(this.settings['ASSEMBLYAI_AUDIO_API_KEY'] as string) ||
|
||||
ASSEMBLYAI_AUDIO_API_KEY
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'assemblyai_audio'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return AssemblyAIAudioTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio to a file using AssemblyAI's audio transcription API via SDK Network
|
||||
* @param inputPath Path to the audio file to transcribe
|
||||
* @param outputPath Path to save the JSON transcription
|
||||
* @param apiKey AssemblyAI API key (uses env/hardcoded default if not provided)
|
||||
* @param speakerLabels Enable speaker diarization (default: true)
|
||||
*/
|
||||
async transcribeToFile(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
apiKey?: string,
|
||||
speakerLabels = true
|
||||
): Promise<string> {
|
||||
// Use provided apiKey, instance apiKey, or error
|
||||
const finalApiKey = apiKey || this.apiKey
|
||||
if (!finalApiKey) {
|
||||
throw new Error('AssemblyAI API key is missing')
|
||||
}
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.assemblyai.com' })
|
||||
|
||||
// Step 1: Upload the audio file
|
||||
const audioData = await fs.promises.readFile(inputPath)
|
||||
const uploadResponse = await network.request({
|
||||
url: '/v2/upload',
|
||||
method: 'POST',
|
||||
data: audioData,
|
||||
headers: {
|
||||
Authorization: finalApiKey,
|
||||
'Content-Type': 'application/octet-stream'
|
||||
}
|
||||
})
|
||||
|
||||
const uploadUrl = (uploadResponse.data as AssemblyAIUploadResponse)
|
||||
.upload_url
|
||||
|
||||
// Step 2: Submit transcription request
|
||||
const transcriptionResponse = await network.request({
|
||||
url: '/v2/transcript',
|
||||
method: 'POST',
|
||||
data: {
|
||||
audio_url: uploadUrl,
|
||||
speaker_labels: speakerLabels,
|
||||
language_detection: true
|
||||
},
|
||||
headers: {
|
||||
Authorization: finalApiKey,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
})
|
||||
|
||||
const transcriptId = (
|
||||
transcriptionResponse.data as AssemblyAITranscriptionResponse
|
||||
).id
|
||||
|
||||
// Step 3: Poll for completion
|
||||
let transcriptData: AssemblyAITranscriptionResponse
|
||||
let attempts = 0
|
||||
const maxAttempts = 180 // 15 minutes with 5 second intervals
|
||||
|
||||
while (attempts < maxAttempts) {
|
||||
const statusResponse = await network.request({
|
||||
url: `/v2/transcript/${transcriptId}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: finalApiKey
|
||||
}
|
||||
})
|
||||
|
||||
transcriptData = statusResponse.data as AssemblyAITranscriptionResponse
|
||||
|
||||
if (transcriptData.status === 'completed') {
|
||||
break
|
||||
} else if (transcriptData.status === 'error') {
|
||||
throw new Error(
|
||||
`AssemblyAI transcription failed: ${
|
||||
transcriptData.error || 'Unknown error'
|
||||
}`
|
||||
)
|
||||
}
|
||||
|
||||
// Wait 5 seconds before polling again
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000))
|
||||
attempts++
|
||||
}
|
||||
|
||||
if (attempts >= maxAttempts) {
|
||||
throw new Error('AssemblyAI transcription timed out')
|
||||
}
|
||||
|
||||
// Step 4: Parse and save the transcription
|
||||
const parsedOutput = this.parseTranscription(transcriptData!)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(parsedOutput, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
return outputPath
|
||||
}
|
||||
|
||||
private parseTranscription(
|
||||
rawOutput: AssemblyAITranscriptionResponse
|
||||
): TranscriptionOutput {
|
||||
const segments: {
|
||||
from: number
|
||||
to: number
|
||||
text: string
|
||||
speaker: string | null
|
||||
}[] = []
|
||||
const speakers: Set<string> = new Set()
|
||||
|
||||
// Use utterances for speaker-labeled segments if available
|
||||
if (rawOutput.utterances && rawOutput.utterances.length > 0) {
|
||||
for (const utterance of rawOutput.utterances) {
|
||||
segments.push({
|
||||
from: utterance.start / 1_000, // Convert milliseconds to seconds
|
||||
to: utterance.end / 1_000,
|
||||
text: utterance.text,
|
||||
speaker: utterance.speaker
|
||||
})
|
||||
speakers.add(utterance.speaker)
|
||||
}
|
||||
} else if (rawOutput.words && rawOutput.words.length > 0) {
|
||||
// Fallback to word-level data if utterances are not available
|
||||
// Group consecutive words by speaker (if available)
|
||||
let currentSegment: {
|
||||
from: number
|
||||
to: number
|
||||
text: string
|
||||
speaker: string | null
|
||||
} | null = null
|
||||
|
||||
for (const word of rawOutput.words) {
|
||||
const speaker = word.speaker || null
|
||||
|
||||
if (
|
||||
currentSegment &&
|
||||
currentSegment.speaker === speaker &&
|
||||
word.start / 1_000 - currentSegment.to < 1.0 // Max 1 second gap
|
||||
) {
|
||||
// Extend current segment
|
||||
currentSegment.to = word.end / 1_000
|
||||
currentSegment.text += ` ${word.text}`
|
||||
} else {
|
||||
// Start a new segment
|
||||
if (currentSegment) {
|
||||
segments.push(currentSegment)
|
||||
}
|
||||
currentSegment = {
|
||||
from: word.start / 1_000,
|
||||
to: word.end / 1_000,
|
||||
text: word.text,
|
||||
speaker: speaker
|
||||
}
|
||||
}
|
||||
|
||||
if (speaker) {
|
||||
speakers.add(speaker)
|
||||
}
|
||||
}
|
||||
|
||||
// Push the last segment
|
||||
if (currentSegment) {
|
||||
segments.push(currentSegment)
|
||||
}
|
||||
} else {
|
||||
// Fallback: create a single segment with the full text
|
||||
segments.push({
|
||||
from: 0,
|
||||
to: (rawOutput.audio_duration || 0) / 1_000,
|
||||
text: rawOutput.text,
|
||||
speaker: null
|
||||
})
|
||||
}
|
||||
|
||||
// Calculate duration
|
||||
let duration = rawOutput.audio_duration ? rawOutput.audio_duration : 0
|
||||
if (!duration && segments.length > 0) {
|
||||
duration = segments[segments.length - 1]?.to || 0
|
||||
}
|
||||
|
||||
return {
|
||||
duration,
|
||||
speakers: Array.from(speakers),
|
||||
speaker_count: speakers.size,
|
||||
segments,
|
||||
metadata: {
|
||||
tool: this.toolName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './assemblyai_audio-tool'
|
||||
@@ -0,0 +1,240 @@
|
||||
import json
|
||||
import time
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.network import Network
|
||||
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
|
||||
|
||||
# Hardcoded default settings for AssemblyAI audio tool
|
||||
ASSEMBLYAI_AUDIO_API_KEY = None
|
||||
DEFAULT_SETTINGS = {
|
||||
"ASSEMBLYAI_AUDIO_API_KEY": ASSEMBLYAI_AUDIO_API_KEY,
|
||||
}
|
||||
REQUIRED_SETTINGS = ["ASSEMBLYAI_AUDIO_API_KEY"]
|
||||
|
||||
|
||||
class AssemblyAIAudioTool(BaseTool):
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
|
||||
tool_settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.settings = tool_settings
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
# Priority: toolkit settings > hardcoded default
|
||||
self.api_key = self.settings.get(
|
||||
"ASSEMBLYAI_AUDIO_API_KEY", ASSEMBLYAI_AUDIO_API_KEY
|
||||
)
|
||||
|
||||
self.network = Network({"base_url": "https://api.assemblyai.com"})
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "assemblyai_audio"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def transcribe_to_file(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
api_key: Optional[str] = None,
|
||||
speaker_labels: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to a file using AssemblyAI's audio transcription API via SDK Network
|
||||
|
||||
Args:
|
||||
input_path: Path to the audio file to transcribe
|
||||
output_path: Path to save the JSON transcription (unified format)
|
||||
api_key: AssemblyAI API key (uses env/hardcoded default if not provided)
|
||||
speaker_labels: Enable speaker diarization (default: True)
|
||||
|
||||
Returns:
|
||||
The path to the transcription file
|
||||
"""
|
||||
# Use provided api_key, instance api_key, or error
|
||||
api_key = api_key or self.api_key
|
||||
if not api_key:
|
||||
raise Exception("AssemblyAI API key is missing")
|
||||
|
||||
try:
|
||||
# Step 1: Upload the audio file
|
||||
with open(input_path, "rb") as audio_file:
|
||||
audio_data = audio_file.read()
|
||||
|
||||
upload_response = self.network.request(
|
||||
{
|
||||
"url": "/v2/upload",
|
||||
"method": "POST",
|
||||
"headers": {
|
||||
"Authorization": api_key,
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
"data": audio_data,
|
||||
}
|
||||
)
|
||||
|
||||
upload_url = upload_response["data"]["upload_url"]
|
||||
|
||||
# Step 2: Submit transcription request
|
||||
transcription_response = self.network.request(
|
||||
{
|
||||
"url": "/v2/transcript",
|
||||
"method": "POST",
|
||||
"headers": {
|
||||
"Authorization": api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
"data": {
|
||||
"audio_url": upload_url,
|
||||
"speaker_labels": speaker_labels,
|
||||
"language_detection": True,
|
||||
},
|
||||
"use_json": True,
|
||||
}
|
||||
)
|
||||
|
||||
transcript_id = transcription_response["data"]["id"]
|
||||
|
||||
# Step 3: Poll for completion
|
||||
max_attempts = 180 # 15 minutes with 5 second intervals
|
||||
attempts = 0
|
||||
transcript_data = None
|
||||
|
||||
while attempts < max_attempts:
|
||||
status_response = self.network.request(
|
||||
{
|
||||
"url": f"/v2/transcript/{transcript_id}",
|
||||
"method": "GET",
|
||||
"headers": {"Authorization": api_key},
|
||||
"use_json": True,
|
||||
}
|
||||
)
|
||||
|
||||
transcript_data = status_response["data"]
|
||||
|
||||
if transcript_data["status"] == "completed":
|
||||
break
|
||||
elif transcript_data["status"] == "error":
|
||||
error_msg = transcript_data.get("error", "Unknown error")
|
||||
raise Exception(f"AssemblyAI transcription failed: {error_msg}")
|
||||
|
||||
# Wait 5 seconds before polling again
|
||||
time.sleep(5)
|
||||
attempts += 1
|
||||
|
||||
if attempts >= max_attempts:
|
||||
raise Exception("AssemblyAI transcription timed out")
|
||||
|
||||
# Step 4: Parse and save the transcription
|
||||
parsed_output = self._parse_transcription(transcript_data)
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"AssemblyAI transcription failed: {str(e)}")
|
||||
|
||||
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
|
||||
segments: List[TranscriptionSegment] = []
|
||||
speakers_set = set()
|
||||
|
||||
# Use utterances for speaker-labeled segments if available
|
||||
utterances = raw_output.get("utterances", [])
|
||||
words = raw_output.get("words", [])
|
||||
|
||||
if utterances and len(utterances) > 0:
|
||||
for utterance in utterances:
|
||||
speaker = utterance.get("speaker")
|
||||
segments.append(
|
||||
{
|
||||
"from": float(utterance.get("start", 0))
|
||||
/ 1000.0, # Convert ms to seconds
|
||||
"to": float(utterance.get("end", 0)) / 1000.0,
|
||||
"text": utterance.get("text", ""),
|
||||
"speaker": speaker,
|
||||
}
|
||||
)
|
||||
if speaker:
|
||||
speakers_set.add(speaker)
|
||||
elif words and len(words) > 0:
|
||||
# Fallback to word-level data if utterances are not available
|
||||
# Group consecutive words by speaker (if available)
|
||||
current_segment = None
|
||||
|
||||
for word in words:
|
||||
speaker = word.get("speaker", None)
|
||||
word_start = float(word.get("start", 0)) / 1000.0
|
||||
word_end = float(word.get("end", 0)) / 1000.0
|
||||
word_text = word.get("text", "")
|
||||
|
||||
if (
|
||||
current_segment
|
||||
and current_segment["speaker"] == speaker
|
||||
and word_start - current_segment["to"] < 1.0 # Max 1 second gap
|
||||
):
|
||||
# Extend current segment
|
||||
current_segment["to"] = word_end
|
||||
current_segment["text"] += f" {word_text}"
|
||||
else:
|
||||
# Start a new segment
|
||||
if current_segment:
|
||||
segments.append(current_segment)
|
||||
current_segment = {
|
||||
"from": word_start,
|
||||
"to": word_end,
|
||||
"text": word_text,
|
||||
"speaker": speaker,
|
||||
}
|
||||
|
||||
if speaker:
|
||||
speakers_set.add(speaker)
|
||||
|
||||
# Push the last segment
|
||||
if current_segment:
|
||||
segments.append(current_segment)
|
||||
else:
|
||||
# Fallback: create a single segment with the full text
|
||||
audio_duration = raw_output.get("audio_duration", 0)
|
||||
segments.append(
|
||||
{
|
||||
"from": 0.0,
|
||||
"to": audio_duration if audio_duration else 0.0,
|
||||
"text": raw_output.get("text", ""),
|
||||
"speaker": None,
|
||||
}
|
||||
)
|
||||
|
||||
# Calculate duration
|
||||
audio_duration = raw_output.get("audio_duration")
|
||||
if audio_duration:
|
||||
duration = float(audio_duration) / 1000.0
|
||||
elif len(segments) > 0:
|
||||
duration = segments[-1]["to"]
|
||||
else:
|
||||
duration = 0.0
|
||||
|
||||
return {
|
||||
"duration": duration,
|
||||
"speakers": list(speakers_set),
|
||||
"speaker_count": len(speakers_set),
|
||||
"segments": segments,
|
||||
"metadata": {"tool": self.tool_name},
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "assemblyai_audio",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "AssemblyAI Audio",
|
||||
"description": "A tool for audio processing using AssemblyAI's API.",
|
||||
"icon_name": "mic-2-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"transcribeToFile": {
|
||||
"description": "Transcribe audio to a file using AssemblyAI's audio transcription API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"speakerLabels": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"outputPath"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.chatterbox_onnx_tool import ChatterboxONNXTool
|
||||
|
||||
__all__ = ["ChatterboxONNXTool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1,235 @@
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { NVIDIA_LIBS_PATH } from '@bridge/constants'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { getPlatformName } from '@sdk/utils'
|
||||
|
||||
const MODEL_NAME = 'chatterbox-multilingual-onnx'
|
||||
const DEFAULT_MAX_CHARS = 272 // Character limit to avoid hallucination
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {}
|
||||
const REQUIRED_SETTINGS: string[] = []
|
||||
|
||||
interface SynthesisTask {
|
||||
text: string
|
||||
target_language?: string
|
||||
audio_path: string
|
||||
// @see https://github.com/leon-ai/leon-binaries/tree/main/bins/chatterbox_onnx/default_voices
|
||||
voice_name?: string
|
||||
speaker_reference_path?: string
|
||||
cfg_strength?: number
|
||||
exaggeration?: number
|
||||
temperature?: number
|
||||
// Control automatic text splitting (default: true)
|
||||
auto_split?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Split text at natural punctuation boundaries to avoid hallucination.
|
||||
*
|
||||
* This function ensures no text segment exceeds maxChars by breaking at
|
||||
* punctuation marks when possible, falling back to spaces or forced splits.
|
||||
*
|
||||
* @param text The text to split
|
||||
* @param maxChars Maximum characters per segment (default: 272)
|
||||
* @returns Array of text chunks split at natural boundaries
|
||||
*/
|
||||
function splitTextAtPunctuation(
|
||||
text: string,
|
||||
maxChars: number = DEFAULT_MAX_CHARS
|
||||
): string[] {
|
||||
const trimmedText = text.trim()
|
||||
if (trimmedText.length <= maxChars) {
|
||||
return [trimmedText]
|
||||
}
|
||||
|
||||
const chunks: string[] = []
|
||||
let remaining = trimmedText
|
||||
|
||||
while (remaining.length > maxChars) {
|
||||
// Get segment up to maxChars
|
||||
const segment = remaining.substring(0, maxChars + 1)
|
||||
|
||||
// Look for punctuation followed by space (natural break)
|
||||
const punctuationPattern = /[.!?,;:]\s/g
|
||||
let lastMatch = -1
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = punctuationPattern.exec(segment)) !== null) {
|
||||
lastMatch = match.index + 1 // Include the punctuation but not the space
|
||||
}
|
||||
|
||||
// Check if we found punctuation in a reasonable position (latter half)
|
||||
if (lastMatch > maxChars * 0.5) {
|
||||
chunks.push(remaining.substring(0, lastMatch).trim())
|
||||
remaining = remaining.substring(lastMatch).trim()
|
||||
continue
|
||||
}
|
||||
|
||||
// No good punctuation found, look for last space
|
||||
const lastSpace = segment.substring(0, maxChars).lastIndexOf(' ')
|
||||
if (lastSpace > maxChars * 0.3) {
|
||||
chunks.push(remaining.substring(0, lastSpace).trim())
|
||||
remaining = remaining.substring(lastSpace).trim()
|
||||
} else {
|
||||
// Force split at maxChars
|
||||
chunks.push(remaining.substring(0, maxChars).trim())
|
||||
remaining = remaining.substring(maxChars).trim()
|
||||
}
|
||||
}
|
||||
|
||||
if (remaining.length > 0) {
|
||||
chunks.push(remaining.trim())
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
export default class ChatterboxONNXTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(ChatterboxONNXTool.TOOLKIT, this.toolName)
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
ChatterboxONNXTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
// Use the actual config name for toolkit lookup
|
||||
return 'chatterbox_onnx'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return ChatterboxONNXTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesize speech from text using Chatterbox ONNX
|
||||
*
|
||||
* By default, automatically splits long text (>272 chars) at punctuation boundaries
|
||||
* to prevent hallucination. Split segments generate separate audio files with
|
||||
* _part_N suffixes (e.g., output_part_0.wav, output_part_1.wav).
|
||||
*
|
||||
* @param tasks Array of synthesis tasks or a single task
|
||||
* @param cudaRuntimePath Optional path to CUDA runtime for GPU acceleration (auto-detected if not provided)
|
||||
* @returns A promise that resolves with the list of processed tasks (may include split tasks)
|
||||
*/
|
||||
async synthesizeSpeechToFiles(
|
||||
tasks: SynthesisTask | SynthesisTask[],
|
||||
cudaRuntimePath?: string
|
||||
): Promise<Omit<SynthesisTask, 'auto_split'>[]> {
|
||||
try {
|
||||
// Normalize tasks to array
|
||||
const taskArray = Array.isArray(tasks) ? tasks : [tasks]
|
||||
|
||||
// Process tasks: split long text into multiple tasks with _part_N suffixes
|
||||
const tasksToSynthesize: Omit<SynthesisTask, 'auto_split'>[] = []
|
||||
|
||||
for (const task of taskArray) {
|
||||
const autoSplit = task.auto_split !== undefined ? task.auto_split : true // Default: enabled
|
||||
const text = task.text.trim()
|
||||
const maxChars = DEFAULT_MAX_CHARS
|
||||
|
||||
// If auto_split disabled or text is short, pass through as-is
|
||||
if (!autoSplit || text.length <= maxChars) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { auto_split, ...cleanTask } = task
|
||||
tasksToSynthesize.push(cleanTask)
|
||||
continue
|
||||
}
|
||||
|
||||
// Split long text at punctuation boundaries
|
||||
const textChunks = splitTextAtPunctuation(text, maxChars)
|
||||
|
||||
// If only one chunk after splitting, no need for special handling
|
||||
if (textChunks.length === 1) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { auto_split, ...cleanTask } = task
|
||||
tasksToSynthesize.push(cleanTask)
|
||||
continue
|
||||
}
|
||||
|
||||
// Multiple chunks: create separate tasks with _part_N suffixes
|
||||
const audioPath = task.audio_path
|
||||
const parsedPath = path.parse(audioPath)
|
||||
const basePath = path.join(parsedPath.dir, parsedPath.name)
|
||||
const ext = parsedPath.ext
|
||||
|
||||
for (let i = 0; i < textChunks.length; i += 1) {
|
||||
const chunk = textChunks[i]
|
||||
if (!chunk) continue
|
||||
|
||||
const baseTask = {
|
||||
...task,
|
||||
text: chunk,
|
||||
audio_path: `${basePath}_part_${i}${ext}`
|
||||
}
|
||||
delete baseTask.auto_split
|
||||
|
||||
tasksToSynthesize.push(baseTask)
|
||||
}
|
||||
}
|
||||
|
||||
// Get model path using the generic resource system
|
||||
const modelPath = await this.getResourcePath(MODEL_NAME)
|
||||
|
||||
// Create a temporary JSON file for the tasks
|
||||
const tempDir = await fs.promises.mkdtemp(
|
||||
path.join(os.tmpdir(), 'chatterbox_onnx_tasks_')
|
||||
)
|
||||
const jsonFilePath = path.join(tempDir, 'tasks.json')
|
||||
|
||||
await fs.promises.writeFile(
|
||||
jsonFilePath,
|
||||
JSON.stringify(tasksToSynthesize, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const args = [
|
||||
'--function',
|
||||
'synthesize_speech',
|
||||
'--json_file',
|
||||
jsonFilePath,
|
||||
'--resource_path',
|
||||
modelPath
|
||||
]
|
||||
|
||||
// Auto-detect CUDA runtime path if not provided
|
||||
const platformName = getPlatformName()
|
||||
const shouldUseCuda =
|
||||
platformName === 'linux-x86_64' || platformName === 'win-amd64'
|
||||
const finalCudaRuntimePath =
|
||||
cudaRuntimePath ?? (shouldUseCuda ? NVIDIA_LIBS_PATH : undefined)
|
||||
|
||||
if (finalCudaRuntimePath) {
|
||||
args.push('--cuda_runtime_path', finalCudaRuntimePath)
|
||||
}
|
||||
|
||||
await this.executeCommand({
|
||||
binaryName: 'chatterbox_onnx',
|
||||
args,
|
||||
options: { sync: true }
|
||||
})
|
||||
|
||||
// Return the processed tasks so caller knows which files were created
|
||||
return tasksToSynthesize
|
||||
} catch (error: unknown) {
|
||||
throw new Error(`Speech synthesis failed: ${(error as Error).message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './chatterbox_onnx-tool'
|
||||
@@ -0,0 +1,241 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Optional, Union, List, TypedDict
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.utils import get_platform_name
|
||||
from bridges.python.src.constants import NVIDIA_LIBS_PATH
|
||||
|
||||
MODEL_NAME = "chatterbox-multilingual-onnx"
|
||||
DEFAULT_MAX_CHARS = 272 # Character limit to avoid hallucination
|
||||
DEFAULT_SETTINGS = {}
|
||||
REQUIRED_SETTINGS = []
|
||||
|
||||
|
||||
def split_text_at_punctuation(
|
||||
text: str, max_chars: int = DEFAULT_MAX_CHARS
|
||||
) -> List[str]:
|
||||
"""
|
||||
Split text at natural punctuation boundaries to avoid hallucination.
|
||||
|
||||
This function ensures no text segment exceeds max_chars by breaking at
|
||||
punctuation marks when possible, falling back to spaces or forced splits.
|
||||
|
||||
Args:
|
||||
text: The text to split
|
||||
max_chars: Maximum characters per segment (default: 272)
|
||||
|
||||
Returns:
|
||||
List of text chunks split at natural boundaries
|
||||
"""
|
||||
text = text.strip()
|
||||
if len(text) <= max_chars:
|
||||
return [text]
|
||||
|
||||
chunks = []
|
||||
remaining = text
|
||||
|
||||
while len(remaining) > max_chars:
|
||||
# Get segment up to max_chars
|
||||
segment = remaining[: max_chars + 1]
|
||||
|
||||
# Look for punctuation followed by space (natural break)
|
||||
punctuation_pattern = re.compile(r"[.!?,;:]\s")
|
||||
matches = list(punctuation_pattern.finditer(segment))
|
||||
|
||||
if matches:
|
||||
# Use the last punctuation match within max_chars
|
||||
last_match = matches[-1]
|
||||
break_point = (
|
||||
last_match.end() - 1
|
||||
) # Don't include the space after punctuation
|
||||
|
||||
# Check if it's in a reasonable position (latter half)
|
||||
if break_point > max_chars * 0.5:
|
||||
chunks.append(remaining[:break_point].strip())
|
||||
remaining = remaining[break_point:].strip()
|
||||
continue
|
||||
|
||||
# No good punctuation found, look for last space
|
||||
last_space = segment[:max_chars].rfind(" ")
|
||||
if last_space > max_chars * 0.3:
|
||||
chunks.append(remaining[:last_space].strip())
|
||||
remaining = remaining[last_space:].strip()
|
||||
else:
|
||||
# Force split at max_chars
|
||||
chunks.append(remaining[:max_chars].strip())
|
||||
remaining = remaining[max_chars:].strip()
|
||||
|
||||
if remaining:
|
||||
chunks.append(remaining.strip())
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
class SynthesisTask(TypedDict, total=False):
|
||||
"""Type definition for a synthesis task"""
|
||||
|
||||
text: str
|
||||
target_language: Optional[str]
|
||||
audio_path: str
|
||||
# Voice names: https://github.com/leon-ai/leon-binaries/tree/main/bins/chatterbox_onnx/default_voices
|
||||
voice_name: Optional[str]
|
||||
speaker_reference_path: Optional[str]
|
||||
cfg_strength: Optional[float]
|
||||
exaggeration: Optional[float]
|
||||
temperature: Optional[float]
|
||||
# Control automatic text splitting (default: True)
|
||||
auto_split: Optional[bool]
|
||||
|
||||
|
||||
class ChatterboxONNXTool(BaseTool):
|
||||
"""
|
||||
Tool for text-to-speech synthesis using Chatterbox ONNX model.
|
||||
Supports multilingual synthesis with voice cloning capabilities.
|
||||
"""
|
||||
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Load configuration from central toolkits directory
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
# Use the actual config name for toolkit lookup
|
||||
return "chatterbox_onnx"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def synthesize_speech_to_files(
|
||||
self,
|
||||
tasks: Union[SynthesisTask, List[SynthesisTask]],
|
||||
cuda_runtime_path: Optional[str] = None,
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Synthesize speech from text using Chatterbox ONNX
|
||||
|
||||
By default, automatically splits long text (>272 chars) at punctuation boundaries
|
||||
to prevent hallucination. Split segments generate separate audio files with
|
||||
_part_N suffixes (e.g., output_part_0.wav, output_part_1.wav).
|
||||
|
||||
Args:
|
||||
tasks: A single synthesis task or a list of synthesis tasks.
|
||||
Each task should contain:
|
||||
- text: The text to synthesize
|
||||
- audio_path: Output path for the generated audio file
|
||||
- target_language: Optional language code (e.g., 'en', 'zh', 'ja')
|
||||
- voice_name: Optional name of the voice to use
|
||||
- speaker_reference_path: Optional path to a reference audio file for voice cloning
|
||||
- cfg_strength: Optional classifier-free guidance strength (default: 0.5)
|
||||
- exaggeration: Optional exaggeration factor (default: 0.5)
|
||||
- temperature: Optional temperature for sampling (controls randomness)
|
||||
- auto_split: Optional flag to enable/disable automatic text splitting (default: True)
|
||||
cuda_runtime_path: Optional path to CUDA runtime for GPU acceleration (auto-detected if not provided)
|
||||
|
||||
Returns:
|
||||
List of processed tasks (may include split tasks with _part_N suffixes)
|
||||
"""
|
||||
try:
|
||||
# Normalize tasks to list
|
||||
task_list = tasks if isinstance(tasks, list) else [tasks]
|
||||
|
||||
# Process tasks: split long text into multiple tasks with _part_N suffixes
|
||||
tasks_to_synthesize = []
|
||||
|
||||
for task in task_list:
|
||||
auto_split = task.get("auto_split", True) # Default: enabled
|
||||
text = task.get("text")
|
||||
if not text:
|
||||
raise ValueError("Missing text in synthesis task")
|
||||
text = text.strip()
|
||||
max_chars = DEFAULT_MAX_CHARS
|
||||
|
||||
# If auto_split disabled or text is short, pass through as-is
|
||||
if not auto_split or len(text) <= max_chars:
|
||||
clean_task = {k: v for k, v in task.items() if k != "auto_split"}
|
||||
tasks_to_synthesize.append(clean_task)
|
||||
continue
|
||||
|
||||
# Split long text at punctuation boundaries
|
||||
text_chunks = split_text_at_punctuation(text, max_chars)
|
||||
|
||||
# If only one chunk after splitting, no need for special handling
|
||||
if len(text_chunks) == 1:
|
||||
clean_task = {k: v for k, v in task.items() if k != "auto_split"}
|
||||
tasks_to_synthesize.append(clean_task)
|
||||
continue
|
||||
|
||||
# Multiple chunks: create separate tasks with _part_N suffixes
|
||||
audio_path = task.get("audio_path")
|
||||
if not audio_path:
|
||||
raise ValueError("Missing audio_path in synthesis task")
|
||||
base_path, ext = os.path.splitext(audio_path)
|
||||
|
||||
for i, chunk in enumerate(text_chunks):
|
||||
chunk_task = {
|
||||
k: v
|
||||
for k, v in task.items()
|
||||
if k not in ["text", "audio_path", "auto_split"]
|
||||
}
|
||||
chunk_task["text"] = chunk
|
||||
chunk_task["audio_path"] = f"{base_path}_part_{i}{ext}"
|
||||
tasks_to_synthesize.append(chunk_task)
|
||||
|
||||
# Get model path using the generic resource system
|
||||
model_path = self.get_resource_path(MODEL_NAME)
|
||||
|
||||
# Create a temporary JSON file for the tasks
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, encoding="utf-8"
|
||||
) as temp_file:
|
||||
json_file_path = temp_file.name
|
||||
json.dump(tasks_to_synthesize, temp_file, indent=2, ensure_ascii=False)
|
||||
|
||||
args = [
|
||||
"--function",
|
||||
"synthesize_speech",
|
||||
"--json_file",
|
||||
json_file_path,
|
||||
"--resource_path",
|
||||
model_path,
|
||||
]
|
||||
|
||||
# Auto-detect CUDA runtime path if not provided
|
||||
platform_name = get_platform_name()
|
||||
should_use_cuda = platform_name in ["linux-x86_64", "win-amd64"]
|
||||
final_cuda_runtime_path = (
|
||||
cuda_runtime_path
|
||||
if cuda_runtime_path is not None
|
||||
else (NVIDIA_LIBS_PATH if should_use_cuda else None)
|
||||
)
|
||||
|
||||
if final_cuda_runtime_path:
|
||||
args.extend(["--cuda_runtime_path", final_cuda_runtime_path])
|
||||
|
||||
self.execute_command(
|
||||
ExecuteCommandOptions(
|
||||
binary_name="chatterbox_onnx", args=args, options={"sync": True}
|
||||
)
|
||||
)
|
||||
|
||||
# Return the processed tasks so caller knows which files were created
|
||||
return tasks_to_synthesize
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"Speech synthesis failed: {str(e)}")
|
||||
@@ -0,0 +1,132 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "chatterbox_onnx",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "Chatterbox ONNX",
|
||||
"description": "A tool for text-to-speech synthesis and voice cloning using the Chatterbox ONNX model.",
|
||||
"icon_name": "chat-voice-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"binaries": {
|
||||
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-linux-x86_64",
|
||||
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-linux-aarch64",
|
||||
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-macosx-x86_64",
|
||||
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-macosx-arm64",
|
||||
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/chatterbox_onnx-v1.1.2/chatterbox_onnx_1.1.2-win-amd64.exe"
|
||||
},
|
||||
"resources": {
|
||||
"chatterbox-multilingual-onnx": [
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/Cangjie5_TC.json?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/default_voice.wav?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/tokenizer.json?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/conditional_decoder.onnx?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/conditional_decoder.onnx_data?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/embed_tokens.onnx?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/embed_tokens.onnx_data?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/language_model_q4.onnx?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/language_model_q4.onnx_data?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/speech_encoder.onnx?download=true",
|
||||
"https://huggingface.co/onnx-community/chatterbox-multilingual-ONNX/resolve/main/onnx/speech_encoder.onnx_data?download=true"
|
||||
]
|
||||
},
|
||||
"functions": {
|
||||
"synthesizeSpeechToFiles": {
|
||||
"description": "Synthesize speech from text using Chatterbox ONNX.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tasks": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_language": {
|
||||
"type": "string"
|
||||
},
|
||||
"audio_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"voice_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"speaker_reference_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"cfg_strength": {
|
||||
"type": "number"
|
||||
},
|
||||
"exaggeration": {
|
||||
"type": "number"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"auto_split": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"text",
|
||||
"audio_path"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_language": {
|
||||
"type": "string"
|
||||
},
|
||||
"audio_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"voice_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"speaker_reference_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"cfg_strength": {
|
||||
"type": "number"
|
||||
},
|
||||
"exaggeration": {
|
||||
"type": "number"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"auto_split": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"text",
|
||||
"audio_path"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"cudaRuntimePath": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"tasks"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.ecapa_tool import ECAPATool
|
||||
|
||||
__all__ = ["ECAPATool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1,112 @@
|
||||
import fs from 'node:fs'
|
||||
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
|
||||
const MODEL_NAME = 'ecapa-voice_gender_classifier'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {}
|
||||
const REQUIRED_SETTINGS: string[] = []
|
||||
|
||||
export default class ECAPATool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(ECAPATool.TOOLKIT, this.toolName)
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
ECAPATool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
// Use the actual config name for toolkit lookup
|
||||
return 'ecapa'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return ECAPATool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect gender from audio file using ECAPA-TDNN voice gender classifier
|
||||
* @param inputPath The file path of the audio to be analyzed
|
||||
* @param device Device to use for processing (cpu, cuda)
|
||||
* @returns A promise that resolves with the detected gender: "male", "female", or "unknown"
|
||||
*/
|
||||
async detectGender(inputPath: string, device = 'cpu'): Promise<string> {
|
||||
try {
|
||||
// Validate input file exists
|
||||
if (!fs.existsSync(inputPath)) {
|
||||
throw new Error(`Input file does not exist: ${inputPath}`)
|
||||
}
|
||||
|
||||
// Get model path using the generic resource system
|
||||
const modelPath = await this.getResourcePath(MODEL_NAME)
|
||||
|
||||
const args = [
|
||||
'--function',
|
||||
'detect_gender',
|
||||
'--input',
|
||||
inputPath,
|
||||
'--model_path',
|
||||
modelPath,
|
||||
'--device',
|
||||
device
|
||||
]
|
||||
|
||||
const result = await this.executeCommand({
|
||||
binaryName: 'ecapa-voice_gender_classifier',
|
||||
args,
|
||||
options: { sync: true }
|
||||
})
|
||||
|
||||
// Parse the output to extract gender
|
||||
const gender = this.parseGenderOutput(result)
|
||||
|
||||
return gender
|
||||
} catch (error: unknown) {
|
||||
throw new Error(
|
||||
`Voice gender detection failed: ${(error as Error).message}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the gender detection output
|
||||
*/
|
||||
private parseGenderOutput(rawOutput: string): string {
|
||||
const lines = rawOutput.split('\n')
|
||||
|
||||
// Look for gender result in the output
|
||||
for (const line of lines) {
|
||||
const lowerLine = line.toLowerCase().trim()
|
||||
|
||||
if (lowerLine.includes('gender:')) {
|
||||
// Extract gender from line like "Gender: male"
|
||||
const match = lowerLine.match(/gender:\s*(male|female|unknown)/i)
|
||||
if (match && match[1]) {
|
||||
return match[1].toLowerCase()
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for direct gender output
|
||||
if (lowerLine === 'male' || lowerLine === 'female') {
|
||||
return lowerLine
|
||||
}
|
||||
}
|
||||
|
||||
// If no clear gender found, return unknown
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './ecapa-tool'
|
||||
@@ -0,0 +1,120 @@
|
||||
import os
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
|
||||
MODEL_NAME = "ecapa-voice_gender_classifier"
|
||||
DEFAULT_SETTINGS = {}
|
||||
REQUIRED_SETTINGS = []
|
||||
|
||||
|
||||
class ECAPATool(BaseTool):
|
||||
"""
|
||||
Tool for voice gender classification using ECAPA-TDNN model.
|
||||
|
||||
Example output format:
|
||||
Gender: male
|
||||
"""
|
||||
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Load configuration from central toolkits directory
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
# Use the actual config name for toolkit lookup
|
||||
return "ecapa"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def detect_gender(self, input_path: str, device: str = "cpu") -> str:
|
||||
"""
|
||||
Detect gender from audio file using ECAPA-TDNN voice gender classifier
|
||||
|
||||
Args:
|
||||
input_path: The file path of the audio to be analyzed
|
||||
device: Device to use for processing (cpu, cuda)
|
||||
|
||||
Returns:
|
||||
The detected gender: "male", "female", or "unknown"
|
||||
"""
|
||||
try:
|
||||
# Validate input file exists
|
||||
if not os.path.exists(input_path):
|
||||
raise Exception(f"Input file does not exist: {input_path}")
|
||||
|
||||
# Get model path using the generic resource system
|
||||
model_path = self.get_resource_path(MODEL_NAME)
|
||||
|
||||
args = [
|
||||
"--function",
|
||||
"detect_gender",
|
||||
"--input",
|
||||
input_path,
|
||||
"--model_path",
|
||||
model_path,
|
||||
"--device",
|
||||
device,
|
||||
]
|
||||
|
||||
result = self.execute_command(
|
||||
ExecuteCommandOptions(
|
||||
binary_name="ecapa-voice_gender_classifier",
|
||||
args=args,
|
||||
options={"sync": True},
|
||||
)
|
||||
)
|
||||
|
||||
# Parse the output to extract gender
|
||||
gender = self._parse_gender_output(result)
|
||||
|
||||
return gender
|
||||
except Exception as e:
|
||||
raise Exception(f"Voice gender detection failed: {str(e)}")
|
||||
|
||||
def _parse_gender_output(self, raw_output: str) -> str:
|
||||
"""
|
||||
Parse the gender detection output
|
||||
|
||||
Args:
|
||||
raw_output: Raw output from the gender detection binary
|
||||
|
||||
Returns:
|
||||
Detected gender: "male", "female", or "unknown"
|
||||
"""
|
||||
lines = raw_output.split("\n")
|
||||
|
||||
# Look for gender result in the output
|
||||
for line in lines:
|
||||
lower_line = line.lower().strip()
|
||||
|
||||
if "gender:" in lower_line:
|
||||
# Extract gender from line like "Gender: male"
|
||||
import re
|
||||
|
||||
match = re.search(
|
||||
r"gender:\s*(male|female|unknown)", lower_line, re.IGNORECASE
|
||||
)
|
||||
if match:
|
||||
return match.group(1).lower()
|
||||
|
||||
# Also check for direct gender output
|
||||
if lower_line in ["male", "female"]:
|
||||
return lower_line
|
||||
|
||||
# If no clear gender found, return unknown
|
||||
return "unknown"
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "ecapa",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "ECAPA",
|
||||
"description": "A tool for voice gender classification using ECAPA-TDNN model.",
|
||||
"icon_name": "voice-recognition-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"binaries": {
|
||||
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-linux-x86_64",
|
||||
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-linux-aarch64",
|
||||
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-macosx-x86_64",
|
||||
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-macosx-arm64",
|
||||
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/ecapa_voice_gender_classifier-v1.0.0/ecapa_voice_gender_classifier_1.0.0-win-amd64.exe"
|
||||
},
|
||||
"resources": {
|
||||
"ecapa-voice_gender_classifier": [
|
||||
"https://huggingface.co/JaesungHuh/voice-gender-classifier/resolve/main/config.json?download=true",
|
||||
"https://huggingface.co/JaesungHuh/voice-gender-classifier/resolve/main/model.safetensors?download=true"
|
||||
]
|
||||
},
|
||||
"functions": {
|
||||
"detectGender": {
|
||||
"description": "Detect gender from an audio file using ECAPA-TDNN.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"device": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.elevenlabs_audio_tool import ElevenLabsAudioTool
|
||||
|
||||
__all__ = ["ElevenLabsAudioTool"]
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"ELEVENLABS_AUDIO_API_KEY": null,
|
||||
"ELEVENLABS_AUDIO_MODEL": "scribe_v1"
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network } from '@sdk/network'
|
||||
|
||||
// Hardcoded default settings for ElevenLabs audio tool
|
||||
const ELEVENLABS_AUDIO_API_KEY: string | null = null
|
||||
const ELEVENLABS_AUDIO_MODEL = 'scribe_v1'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {
|
||||
ELEVENLABS_AUDIO_API_KEY,
|
||||
ELEVENLABS_AUDIO_MODEL
|
||||
}
|
||||
const REQUIRED_SETTINGS = ['ELEVENLABS_AUDIO_API_KEY']
|
||||
|
||||
interface ElevenLabsWord {
|
||||
text: string
|
||||
start: number
|
||||
end: number
|
||||
type: 'word' | 'spacing' | 'audio_event'
|
||||
speaker_id?: string
|
||||
}
|
||||
|
||||
interface ElevenLabsTranscriptionResponse {
|
||||
language_code: string
|
||||
language_probability: number
|
||||
text: string
|
||||
words: ElevenLabsWord[]
|
||||
}
|
||||
|
||||
interface ElevenLabsDubbingCreateResponse {
|
||||
dubbing_id: string
|
||||
expected_duration_sec: number
|
||||
}
|
||||
|
||||
interface ElevenLabsDubbingStatusResponse {
|
||||
dubbing_id: string
|
||||
name: string
|
||||
status: 'dubbing' | 'dubbed' | 'failed'
|
||||
target_languages: string[]
|
||||
error?: string | null
|
||||
created_at?: string
|
||||
editable?: boolean | null
|
||||
}
|
||||
|
||||
export default class ElevenLabsAudioTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
readonly apiKey: string | null
|
||||
readonly model: string
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.config = ToolkitConfig.load(ElevenLabsAudioTool.TOOLKIT, this.toolName)
|
||||
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
ElevenLabsAudioTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
|
||||
// Priority: toolkit settings > hardcoded default
|
||||
this.apiKey =
|
||||
(this.settings['ELEVENLABS_AUDIO_API_KEY'] as string) ||
|
||||
ELEVENLABS_AUDIO_API_KEY
|
||||
this.model =
|
||||
(this.settings['ELEVENLABS_AUDIO_MODEL'] as string) ||
|
||||
ELEVENLABS_AUDIO_MODEL
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'elevenlabs_audio'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return ElevenLabsAudioTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio to a file using ElevenLabs' Scribe v1 API
|
||||
* @param inputPath Path to the audio file to transcribe
|
||||
* @param outputPath Path to save the JSON transcription (unified format)
|
||||
* @param apiKey ElevenLabs API key (uses env/hardcoded default if not provided)
|
||||
* @param model Transcription model (defaults to tool default)
|
||||
* @param diarize Whether to enable speaker diarization (defaults to true)
|
||||
*/
|
||||
async transcribeToFile(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
apiKey?: string,
|
||||
model?: string,
|
||||
diarize = true
|
||||
): Promise<string> {
|
||||
// Use provided values, instance values, or error
|
||||
const finalApiKey = apiKey || this.apiKey
|
||||
const finalModel = model || this.model
|
||||
if (!finalApiKey) {
|
||||
throw new Error('ElevenLabs API key is missing')
|
||||
}
|
||||
|
||||
const form = new FormData()
|
||||
const audioFile = await fs.openAsBlob(inputPath)
|
||||
form.append('file', audioFile, path.basename(inputPath))
|
||||
form.append('model_id', finalModel)
|
||||
form.append('diarize', diarize.toString())
|
||||
form.append('tag_audio_events', 'true')
|
||||
form.append('timestamps_granularity', 'word')
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
|
||||
const response = await network.request<ElevenLabsTranscriptionResponse>({
|
||||
url: '/v1/speech-to-text',
|
||||
method: 'POST',
|
||||
data: form,
|
||||
headers: {
|
||||
'xi-api-key': finalApiKey
|
||||
}
|
||||
})
|
||||
|
||||
const normalizedOutput: TranscriptionOutput = this.parseTranscription(
|
||||
response.data
|
||||
)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(normalizedOutput, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
return outputPath
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a dubbing project using ElevenLabs' Dubbing API
|
||||
* @param inputPath Path to the audio/video file to dub
|
||||
* @param targetLang Target language code (e.g., 'es', 'fr', 'zh')
|
||||
* @param apiKey ElevenLabs API key
|
||||
* @param sourceLang Source language code (defaults to 'auto')
|
||||
* @param numSpeakers Number of speakers (0 for auto-detect)
|
||||
* @param watermark Whether to add watermark to output video
|
||||
* @returns Dubbing project ID and expected duration
|
||||
*/
|
||||
async createDubbing(
|
||||
inputPath: string,
|
||||
targetLang: string,
|
||||
apiKey: string,
|
||||
sourceLang = 'auto',
|
||||
numSpeakers = 0,
|
||||
watermark = false
|
||||
): Promise<ElevenLabsDubbingCreateResponse> {
|
||||
if (!apiKey) {
|
||||
throw new Error('ElevenLabs API key is missing')
|
||||
}
|
||||
|
||||
const form = new FormData()
|
||||
const mediaFile = await fs.openAsBlob(inputPath)
|
||||
form.append('file', mediaFile, path.basename(inputPath))
|
||||
form.append('target_lang', targetLang)
|
||||
form.append('source_lang', sourceLang)
|
||||
form.append('num_speakers', numSpeakers.toString())
|
||||
form.append('watermark', watermark.toString())
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
|
||||
const response = await network.request<ElevenLabsDubbingCreateResponse>({
|
||||
url: '/v1/dubbing',
|
||||
method: 'POST',
|
||||
data: form,
|
||||
headers: {
|
||||
'xi-api-key': apiKey
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the status of a dubbing project
|
||||
* @param dubbingId The dubbing project ID
|
||||
* @param apiKey ElevenLabs API key
|
||||
* @returns Dubbing project status information
|
||||
*/
|
||||
async getDubbingStatus(
|
||||
dubbingId: string,
|
||||
apiKey: string
|
||||
): Promise<ElevenLabsDubbingStatusResponse> {
|
||||
if (!apiKey) {
|
||||
throw new Error('ElevenLabs API key is missing')
|
||||
}
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
|
||||
const response = await network.request<ElevenLabsDubbingStatusResponse>({
|
||||
url: `/v1/dubbing/${dubbingId}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'xi-api-key': apiKey
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the dubbed file
|
||||
* @param dubbingId The dubbing project ID
|
||||
* @param targetLang Target language code
|
||||
* @param outputPath Path to save the dubbed file
|
||||
* @param apiKey ElevenLabs API key
|
||||
* @returns Path to the downloaded file
|
||||
*/
|
||||
async downloadDubbedFile(
|
||||
dubbingId: string,
|
||||
targetLang: string,
|
||||
outputPath: string,
|
||||
apiKey: string
|
||||
): Promise<string> {
|
||||
if (!apiKey) {
|
||||
throw new Error('ElevenLabs API key is missing')
|
||||
}
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
|
||||
const response = await network.request({
|
||||
url: `/v1/dubbing/${dubbingId}/audio/${targetLang}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'xi-api-key': apiKey
|
||||
},
|
||||
responseType: 'arraybuffer'
|
||||
})
|
||||
|
||||
// Write the audio/video file
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
Buffer.from(response.data as ArrayBuffer)
|
||||
)
|
||||
|
||||
return outputPath
|
||||
}
|
||||
|
||||
private parseTranscription(
|
||||
rawOutput: ElevenLabsTranscriptionResponse
|
||||
): TranscriptionOutput {
|
||||
const wordItems = rawOutput.words.filter((item) => item.type === 'word')
|
||||
const uniqueSpeakers = Array.from(
|
||||
new Set(wordItems.map((word) => word.speaker_id).filter(Boolean))
|
||||
) as string[]
|
||||
|
||||
// Calculate duration from the last word's end time
|
||||
const duration =
|
||||
wordItems.length > 0 ? wordItems[wordItems.length - 1]?.end : 0
|
||||
const segments = wordItems.map((word) => ({
|
||||
from: word.start,
|
||||
to: word.end,
|
||||
text: word.text,
|
||||
speaker: word.speaker_id || null
|
||||
}))
|
||||
|
||||
return {
|
||||
duration: duration ?? 0,
|
||||
speakers: uniqueSpeakers,
|
||||
speaker_count: uniqueSpeakers.length,
|
||||
segments,
|
||||
metadata: {
|
||||
tool: this.toolName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './elevenlabs_audio-tool'
|
||||
@@ -0,0 +1,146 @@
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.network import Network
|
||||
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
|
||||
|
||||
# Hardcoded default settings for ElevenLabs audio tool
|
||||
ELEVENLABS_AUDIO_API_KEY = None
|
||||
ELEVENLABS_AUDIO_MODEL = "scribe_v1"
|
||||
DEFAULT_SETTINGS = {
|
||||
"ELEVENLABS_AUDIO_API_KEY": ELEVENLABS_AUDIO_API_KEY,
|
||||
"ELEVENLABS_AUDIO_MODEL": ELEVENLABS_AUDIO_MODEL,
|
||||
}
|
||||
REQUIRED_SETTINGS = ["ELEVENLABS_AUDIO_API_KEY"]
|
||||
|
||||
|
||||
class ElevenLabsAudioTool(BaseTool):
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
|
||||
tool_settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.settings = tool_settings
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
# Priority: toolkit settings > hardcoded default
|
||||
self.api_key = self.settings.get(
|
||||
"ELEVENLABS_AUDIO_API_KEY", ELEVENLABS_AUDIO_API_KEY
|
||||
)
|
||||
self.model = self.settings.get("ELEVENLABS_AUDIO_MODEL", ELEVENLABS_AUDIO_MODEL)
|
||||
|
||||
self.network = Network({"base_url": "https://api.elevenlabs.io"})
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "elevenlabs_audio"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def transcribe_to_file(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
api_key: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
diarize: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to a file using ElevenLabs' Scribe v1 API
|
||||
|
||||
Args:
|
||||
input_path: Path to the audio file to transcribe
|
||||
output_path: Path to save the JSON transcription (unified format)
|
||||
api_key: ElevenLabs API key (uses env/hardcoded default if not provided)
|
||||
model: Transcription model (defaults to tool default)
|
||||
diarize: Whether to enable speaker diarization (defaults to True)
|
||||
|
||||
Returns:
|
||||
The path to the transcription file
|
||||
"""
|
||||
# Use provided values, instance values, or error
|
||||
api_key = api_key or self.api_key
|
||||
model = model or self.model
|
||||
if not api_key:
|
||||
raise Exception("ElevenLabs API key is missing")
|
||||
|
||||
try:
|
||||
files: dict = {"file": open(input_path, "rb")}
|
||||
data: dict = {
|
||||
"model_id": model,
|
||||
"diarize": str(diarize).lower(),
|
||||
"tag_audio_events": "true",
|
||||
"timestamps_granularity": "word",
|
||||
}
|
||||
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/v1/speech-to-text",
|
||||
"method": "POST",
|
||||
"headers": {"xi-api-key": api_key},
|
||||
"data": data,
|
||||
"files": files,
|
||||
"use_json": True,
|
||||
}
|
||||
)
|
||||
|
||||
parsed_output = self._parse_transcription(response["data"])
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return output_path
|
||||
except Exception as e:
|
||||
raise Exception(f"ElevenLabs transcription failed: {str(e)}")
|
||||
|
||||
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
|
||||
"""
|
||||
Parse ElevenLabs transcription response into unified schema format
|
||||
|
||||
Args:
|
||||
raw_output: Raw response from ElevenLabs API
|
||||
|
||||
Returns:
|
||||
Parsed transcription in unified format
|
||||
"""
|
||||
words_data = raw_output.get("words", [])
|
||||
word_items = [word for word in words_data if word.get("type") == "word"]
|
||||
|
||||
unique_speakers = list(
|
||||
set(word.get("speaker_id") for word in word_items if word.get("speaker_id"))
|
||||
)
|
||||
|
||||
# Calculate duration from the last word's end time
|
||||
duration = float(word_items[-1].get("end", 0)) if word_items else 0.0
|
||||
|
||||
segments: List[TranscriptionSegment] = []
|
||||
for word in word_items:
|
||||
segments.append(
|
||||
{
|
||||
"from": float(word.get("start", 0)),
|
||||
"to": float(word.get("end", 0)),
|
||||
"text": word.get("text", ""),
|
||||
"speaker": word.get("speaker_id") or None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"duration": duration,
|
||||
"speakers": unique_speakers,
|
||||
"speaker_count": len(unique_speakers),
|
||||
"segments": segments,
|
||||
"metadata": {"tool": self.tool_name},
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "elevenlabs_audio",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "ElevenLabs Audio",
|
||||
"description": "A tool for audio processing using ElevenLabs's API.",
|
||||
"icon_name": "sound-module-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"transcribeToFile": {
|
||||
"description": "Transcribe audio to a file using ElevenLabs' Scribe API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"diarize": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"outputPath"
|
||||
]
|
||||
}
|
||||
},
|
||||
"createDubbing": {
|
||||
"description": "Create a dubbing project using ElevenLabs' Dubbing API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"targetLang": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"sourceLang": {
|
||||
"type": "string"
|
||||
},
|
||||
"numSpeakers": {
|
||||
"type": "number"
|
||||
},
|
||||
"watermark": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"targetLang",
|
||||
"apiKey"
|
||||
]
|
||||
}
|
||||
},
|
||||
"getDubbingStatus": {
|
||||
"description": "Get the status of a dubbing project.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dubbingId": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"dubbingId",
|
||||
"apiKey"
|
||||
]
|
||||
}
|
||||
},
|
||||
"downloadDubbedFile": {
|
||||
"description": "Download the dubbed audio file for a dubbing project.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dubbingId": {
|
||||
"type": "string"
|
||||
},
|
||||
"targetLang": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"dubbingId",
|
||||
"targetLang",
|
||||
"outputPath",
|
||||
"apiKey"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.faster_whisper_tool import FasterWhisperTool
|
||||
|
||||
__all__ = ["FasterWhisperTool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1,174 @@
|
||||
import fs from 'node:fs'
|
||||
|
||||
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
|
||||
/**
|
||||
* Example:
|
||||
*
|
||||
* Detected language: en (probability: 1.00)
|
||||
* Duration: 26.84 seconds
|
||||
* ==================================================
|
||||
*
|
||||
* [0.00 -> 5.70] DuckDB, an open-source, fast, embeddable, SQL OLAP database that simplifies the way
|
||||
* [5.70 -> 10.84] developers implement analytics. It was developed in the Netherlands, written in C++, and first
|
||||
* [10.84 -> 16.78] released in 2019. And the TLDR is that it's like SQLite, but for columnar data. Everybody knows
|
||||
*/
|
||||
type FasterWhisperTranscriptionOutput = string
|
||||
|
||||
const MODEL_NAME = 'faster-whisper-large-v3'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {}
|
||||
const REQUIRED_SETTINGS: string[] = []
|
||||
|
||||
export default class FasterWhisperTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(FasterWhisperTool.TOOLKIT, this.toolName)
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
FasterWhisperTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
// Use the actual config name for toolkit lookup
|
||||
return 'faster_whisper'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return FasterWhisperTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio to a file using faster-whisper
|
||||
* @param inputPath The file path of the audio to be transcribed
|
||||
* @param outputPath The desired file path for the transcription output
|
||||
* @param device Device to use for processing (cpu, cuda, auto)
|
||||
* @param cpuThreads Number of CPU threads to use
|
||||
* @param downloadRoot Root directory for model downloads
|
||||
* @param localFilesOnly Whether to use only local files
|
||||
* @returns A promise that resolves with the path to the transcription file
|
||||
*/
|
||||
async transcribeToFile(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
device = 'auto',
|
||||
cpuThreads?: number,
|
||||
downloadRoot?: string,
|
||||
localFilesOnly = false
|
||||
): Promise<string> {
|
||||
try {
|
||||
// Get model path using the generic resource system
|
||||
const modelPath = await this.getResourcePath(MODEL_NAME)
|
||||
|
||||
const args = [
|
||||
'--function',
|
||||
'transcribe_to_file',
|
||||
'--input',
|
||||
inputPath,
|
||||
'--output',
|
||||
outputPath,
|
||||
'--model_size_or_path',
|
||||
modelPath,
|
||||
'--device',
|
||||
device
|
||||
]
|
||||
|
||||
if (cpuThreads) {
|
||||
args.push('--cpu_threads', cpuThreads.toString())
|
||||
}
|
||||
|
||||
if (downloadRoot) {
|
||||
args.push('--download_root', downloadRoot)
|
||||
}
|
||||
|
||||
if (localFilesOnly) {
|
||||
args.push('--local_files_only')
|
||||
}
|
||||
|
||||
await this.executeCommand({
|
||||
binaryName: 'faster_whisper',
|
||||
args,
|
||||
options: { sync: true }
|
||||
})
|
||||
|
||||
const transcriptionContent = await fs.promises.readFile(
|
||||
outputPath,
|
||||
'utf-8'
|
||||
)
|
||||
const parsedOutput = this.parseTranscription(transcriptionContent)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(parsedOutput, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
return outputPath
|
||||
} catch (error: unknown) {
|
||||
throw new Error(`Audio transcription failed: ${(error as Error).message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Speaker diarization is not supported for Faster Whisper
|
||||
*/
|
||||
private parseTranscription(
|
||||
rawOutput: FasterWhisperTranscriptionOutput
|
||||
): TranscriptionOutput {
|
||||
const lines = rawOutput.split('\n')
|
||||
|
||||
const durationLine = lines.find((line) => line.startsWith('Duration:'))
|
||||
let duration = 0
|
||||
|
||||
if (durationLine) {
|
||||
const match = durationLine.match(/Duration:\s+([\d.]+)\s+seconds/)
|
||||
|
||||
if (match && match[1]) {
|
||||
duration = parseFloat(match[1])
|
||||
}
|
||||
}
|
||||
|
||||
const segments: TranscriptionOutput['segments'] = []
|
||||
const segmentRegex = /^\[(\d+\.\d+)\s+->\s+(\d+\.\d+)\]\s+(.+)$/
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(segmentRegex)
|
||||
if (match && match[1] && match[2] && match[3]) {
|
||||
const start = match[1]
|
||||
const end = match[2]
|
||||
const text = match[3]
|
||||
|
||||
segments.push({
|
||||
from: parseFloat(start),
|
||||
to: parseFloat(end),
|
||||
text: text.trim(),
|
||||
speaker: null
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
duration,
|
||||
speakers: [],
|
||||
speaker_count: 0,
|
||||
segments,
|
||||
metadata: {
|
||||
tool: this.toolName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './faster_whisper-tool'
|
||||
@@ -0,0 +1,155 @@
|
||||
import json
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
|
||||
|
||||
MODEL_NAME = "faster-whisper-large-v3"
|
||||
DEFAULT_SETTINGS = {}
|
||||
REQUIRED_SETTINGS = []
|
||||
|
||||
|
||||
class FasterWhisperTool(BaseTool):
|
||||
"""
|
||||
Example output format:
|
||||
|
||||
Detected language: en (probability: 1.00)
|
||||
Duration: 26.84 seconds
|
||||
==================================================
|
||||
|
||||
[0.00 -> 5.70] DuckDB, an open-source, fast, embeddable, SQL OLAP database that simplifies the way
|
||||
[5.70 -> 10.84] developers implement analytics. It was developed in the Netherlands, written in C++, and first
|
||||
[10.84 -> 16.78] released in 2019. And the TLDR is that it's like SQLite, but for columnar data. Everybody knows
|
||||
"""
|
||||
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Load configuration from central toolkits directory
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
# Use the actual config name for toolkit lookup
|
||||
return "faster_whisper"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def transcribe_to_file(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
device: str = "auto",
|
||||
cpu_threads: Optional[int] = None,
|
||||
download_root: Optional[str] = None,
|
||||
local_files_only: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to a file using faster-whisper
|
||||
|
||||
Args:
|
||||
input_path: The file path of the audio to be transcribed
|
||||
output_path: The desired file path for the transcription output
|
||||
device: Device to use for processing (cpu, cuda, auto)
|
||||
cpu_threads: Number of CPU threads to use
|
||||
download_root: Root directory for model downloads
|
||||
local_files_only: Whether to use only local files
|
||||
|
||||
Returns:
|
||||
The path to the transcription file
|
||||
"""
|
||||
try:
|
||||
# Get model path using the generic resource system
|
||||
model_path = self.get_resource_path(MODEL_NAME)
|
||||
|
||||
args = [
|
||||
"--function",
|
||||
"transcribe_to_file",
|
||||
"--input",
|
||||
input_path,
|
||||
"--output",
|
||||
output_path,
|
||||
"--model_size_or_path",
|
||||
model_path,
|
||||
"--device",
|
||||
device,
|
||||
]
|
||||
|
||||
if cpu_threads:
|
||||
args.extend(["--cpu_threads", str(cpu_threads)])
|
||||
|
||||
if download_root:
|
||||
args.extend(["--download_root", download_root])
|
||||
|
||||
if local_files_only:
|
||||
args.append("--local_files_only")
|
||||
|
||||
self.execute_command(
|
||||
ExecuteCommandOptions(
|
||||
binary_name="faster_whisper", args=args, options={"sync": True}
|
||||
)
|
||||
)
|
||||
|
||||
with open(output_path, "r", encoding="utf-8") as f:
|
||||
transcription_content = f.read()
|
||||
|
||||
parsed_output = self._parse_transcription(transcription_content)
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return output_path
|
||||
except Exception as e:
|
||||
raise Exception(f"Audio transcription failed: {str(e)}")
|
||||
|
||||
def _parse_transcription(self, raw_output: str) -> TranscriptionOutput:
|
||||
lines = raw_output.split("\n")
|
||||
|
||||
duration = 0.0
|
||||
for line in lines:
|
||||
if line.startswith("Duration:"):
|
||||
match = re.search(r"Duration:\s+([\d.]+)\s+seconds", line)
|
||||
if match:
|
||||
duration = float(match.group(1))
|
||||
break
|
||||
|
||||
segments: list[TranscriptionSegment] = []
|
||||
segment_regex = re.compile(r"^\[(\d+\.\d+)\s+->\s+(\d+\.\d+)\]\s+(.+)$")
|
||||
|
||||
for line in lines:
|
||||
match = segment_regex.match(line)
|
||||
if match:
|
||||
start = match.group(1)
|
||||
end = match.group(2)
|
||||
text = match.group(3)
|
||||
|
||||
segments.append(
|
||||
{
|
||||
"from": float(start),
|
||||
"to": float(end),
|
||||
"text": text.strip(),
|
||||
"speaker": None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"duration": duration,
|
||||
"speakers": [],
|
||||
"speaker_count": 0,
|
||||
"segments": segments,
|
||||
"metadata": {"tool": self.tool_name},
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "faster_whisper",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "Faster Whisper",
|
||||
"description": "A tool for speech recognition and audio transcription using the Faster Whisper model.",
|
||||
"icon_name": "mic-2-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"binaries": {
|
||||
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-linux-x86_64",
|
||||
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-linux-aarch64",
|
||||
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-macosx-x86_64",
|
||||
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-macosx-arm64",
|
||||
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/faster_whisper-v1.0.1/faster_whisper_1.0.1-win-amd64.exe"
|
||||
},
|
||||
"resources": {
|
||||
"faster-whisper-large-v3": [
|
||||
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/config.json?download=true",
|
||||
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/model.bin?download=true",
|
||||
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/preprocessor_config.json?download=true",
|
||||
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/tokenizer.json?download=true",
|
||||
"https://huggingface.co/Systran/faster-whisper-large-v3/resolve/main/vocabulary.json?download=true"
|
||||
]
|
||||
},
|
||||
"functions": {
|
||||
"transcribeToFile": {
|
||||
"description": "Transcribe audio to a file using faster-whisper.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"device": {
|
||||
"type": "string"
|
||||
},
|
||||
"cpuThreads": {
|
||||
"type": "number"
|
||||
},
|
||||
"downloadRoot": {
|
||||
"type": "string"
|
||||
},
|
||||
"localFilesOnly": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"outputPath"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.openai_audio_tool import OpenAIAudioTool
|
||||
|
||||
__all__ = ["OpenAIAudioTool"]
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"OPENAI_AUDIO_API_KEY": null,
|
||||
"OPENAI_AUDIO_MODEL": "whisper-1"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './openai_audio-tool'
|
||||
@@ -0,0 +1,157 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network } from '@sdk/network'
|
||||
|
||||
// Hardcoded default settings for OpenAI audio tool
|
||||
const OPENAI_AUDIO_API_KEY: string | null = null
|
||||
const OPENAI_AUDIO_MODEL = 'whisper-1'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {
|
||||
OPENAI_AUDIO_API_KEY,
|
||||
OPENAI_AUDIO_MODEL
|
||||
}
|
||||
const REQUIRED_SETTINGS = ['OPENAI_AUDIO_API_KEY']
|
||||
|
||||
interface OpenAITranscriptionOutput {
|
||||
task: string
|
||||
duration: number
|
||||
text: string
|
||||
segments: {
|
||||
type: string
|
||||
id: string
|
||||
start: number
|
||||
end: number
|
||||
text: string
|
||||
speaker: string
|
||||
}[]
|
||||
usage: {
|
||||
type: string
|
||||
seconds: number
|
||||
}
|
||||
}
|
||||
|
||||
export default class OpenAIAudioTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
readonly apiKey: string | null
|
||||
readonly model: string
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.config = ToolkitConfig.load(OpenAIAudioTool.TOOLKIT, this.toolName)
|
||||
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
OpenAIAudioTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
|
||||
// Priority: toolkit settings > hardcoded default
|
||||
this.apiKey =
|
||||
(this.settings['OPENAI_AUDIO_API_KEY'] as string) || OPENAI_AUDIO_API_KEY
|
||||
this.model =
|
||||
(this.settings['OPENAI_AUDIO_MODEL'] as string) || OPENAI_AUDIO_MODEL
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
// Use the actual config name for toolkit lookup
|
||||
return 'openai_audio'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return OpenAIAudioTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio to a file using OpenAI's audio transcription API via SDK Network
|
||||
* @param inputPath Path to the audio file to transcribe
|
||||
* @param outputPath Path to save the plain text transcription
|
||||
* @param apiKey OpenAI API key (uses env/hardcoded default if not provided)
|
||||
* @param model Transcription model (defaults to tool default)
|
||||
*/
|
||||
async transcribeToFile(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
apiKey?: string,
|
||||
model?: string
|
||||
): Promise<string> {
|
||||
// Use provided values, instance values, or error
|
||||
const finalApiKey = apiKey || this.apiKey
|
||||
const finalModel = model || this.model
|
||||
if (!finalApiKey) {
|
||||
throw new Error('OpenAI API key is missing')
|
||||
}
|
||||
|
||||
const form = new FormData()
|
||||
const audioFile = await fs.openAsBlob(inputPath)
|
||||
form.append('file', audioFile, path.basename(inputPath))
|
||||
form.append('model', finalModel)
|
||||
form.append('chunking_strategy', 'auto')
|
||||
form.append('response_format', 'diarized_json')
|
||||
|
||||
const network = new Network({ baseURL: 'https://api.openai.com' })
|
||||
const response = await network.request({
|
||||
url: '/v1/audio/transcriptions',
|
||||
method: 'POST',
|
||||
data: form,
|
||||
headers: {
|
||||
Authorization: `Bearer ${finalApiKey}`
|
||||
}
|
||||
})
|
||||
|
||||
const parsedOutput = this.parseTranscription(
|
||||
response.data as OpenAITranscriptionOutput
|
||||
)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(parsedOutput, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
return outputPath
|
||||
}
|
||||
|
||||
private parseTranscription(
|
||||
rawOutput: OpenAITranscriptionOutput
|
||||
): TranscriptionOutput {
|
||||
const speakers = Array.from(
|
||||
new Set(rawOutput.segments.map((segment) => segment.speaker))
|
||||
)
|
||||
|
||||
const segments = rawOutput.segments.map((segment) => {
|
||||
return {
|
||||
from: segment.start,
|
||||
to: segment.end,
|
||||
text: segment.text,
|
||||
speaker: segment.speaker || null
|
||||
}
|
||||
})
|
||||
|
||||
// If duration is not found, use the "to" property from the last segment
|
||||
let duration = rawOutput.duration
|
||||
if (!duration && segments.length > 0) {
|
||||
duration = segments[segments.length - 1]?.to || 0
|
||||
}
|
||||
|
||||
return {
|
||||
duration: duration || 0,
|
||||
speakers: speakers,
|
||||
speaker_count: speakers.length,
|
||||
segments,
|
||||
metadata: {
|
||||
tool: this.toolName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from bridges.python.src.sdk.network import Network
|
||||
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
|
||||
|
||||
# Hardcoded default settings for OpenAI audio tool
|
||||
OPENAI_AUDIO_API_KEY = None
|
||||
OPENAI_AUDIO_MODEL = "whisper-1"
|
||||
DEFAULT_SETTINGS = {
|
||||
"OPENAI_AUDIO_API_KEY": OPENAI_AUDIO_API_KEY,
|
||||
"OPENAI_AUDIO_MODEL": OPENAI_AUDIO_MODEL,
|
||||
}
|
||||
REQUIRED_SETTINGS = ["OPENAI_AUDIO_API_KEY"]
|
||||
|
||||
|
||||
class OpenAIAudioTool(BaseTool):
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
|
||||
tool_settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.settings = tool_settings
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
# Priority: toolkit settings > hardcoded default
|
||||
self.api_key = self.settings.get("OPENAI_AUDIO_API_KEY", OPENAI_AUDIO_API_KEY)
|
||||
self.model = self.settings.get("OPENAI_AUDIO_MODEL", OPENAI_AUDIO_MODEL)
|
||||
|
||||
self.network = Network({"base_url": "https://api.openai.com"})
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
# Use the actual config name for toolkit lookup
|
||||
return "openai_audio"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def transcribe_to_file(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
api_key: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to a file using OpenAI's audio transcription API via SDK Network
|
||||
|
||||
Args:
|
||||
input_path: Path to the audio file to transcribe
|
||||
output_path: Path to save the JSON transcription (unified format)
|
||||
api_key: OpenAI API key (uses env/hardcoded default if not provided)
|
||||
model: Transcription model (defaults to tool default)
|
||||
|
||||
Returns:
|
||||
The path to the transcription file
|
||||
"""
|
||||
# Use provided values, instance values, or error
|
||||
api_key = api_key or self.api_key
|
||||
model = model or self.model
|
||||
if not api_key:
|
||||
raise Exception("OpenAI API key is missing")
|
||||
|
||||
try:
|
||||
files: dict = {"file": open(input_path, "rb")}
|
||||
data: dict = {
|
||||
"model": model,
|
||||
"chunking_strategy": "auto",
|
||||
"response_format": "diarized_json",
|
||||
}
|
||||
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/v1/audio/transcriptions",
|
||||
"method": "POST",
|
||||
"headers": {"Authorization": f"Bearer {api_key}"},
|
||||
"data": data,
|
||||
"files": files,
|
||||
"use_json": True,
|
||||
}
|
||||
)
|
||||
|
||||
parsed_output = self._parse_transcription(response["data"])
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return output_path
|
||||
except Exception as e:
|
||||
raise Exception(f"OpenAI transcription failed: {str(e)}")
|
||||
|
||||
def _parse_transcription(self, raw_output: Dict[str, Any]) -> TranscriptionOutput:
|
||||
segments_data = raw_output.get("segments", [])
|
||||
unique_speakers = list(
|
||||
set(seg.get("speaker") for seg in segments_data if seg.get("speaker"))
|
||||
)
|
||||
|
||||
segments: List[TranscriptionSegment] = []
|
||||
for segment in segments_data:
|
||||
segments.append(
|
||||
{
|
||||
"from": float(segment.get("start", 0)),
|
||||
"to": float(segment.get("end", 0)),
|
||||
"text": segment.get("text", ""),
|
||||
"speaker": segment.get("speaker") or None,
|
||||
}
|
||||
)
|
||||
|
||||
# If duration is not found, use the "to" property from the last segment
|
||||
duration = raw_output.get("duration")
|
||||
if not duration and len(segments) > 0:
|
||||
duration = segments[-1]["to"] or 0.0
|
||||
|
||||
return {
|
||||
"duration": float(duration) if duration else 0.0,
|
||||
"speakers": unique_speakers,
|
||||
"speaker_count": len(unique_speakers),
|
||||
"segments": segments,
|
||||
"metadata": {"tool": self.tool_name},
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "openai_audio",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "OpenAI Audio",
|
||||
"description": "A tool for audio processing using OpenAI's API.",
|
||||
"icon_name": "openai-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"transcribeToFile": {
|
||||
"description": "Transcribe audio to a file using OpenAI's audio transcription API.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"outputPath"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.qwen3_asr_tool import Qwen3ASRTool
|
||||
|
||||
__all__ = ["Qwen3ASRTool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './qwen3_asr-tool'
|
||||
@@ -0,0 +1,227 @@
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { TranscriptionOutput } from '@tools/music_audio/transcription-schema'
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { NVIDIA_LIBS_PATH, PYTORCH_TORCH_PATH } from '@bridge/constants'
|
||||
|
||||
type Qwen3ASRTranscriptionOutput = string
|
||||
|
||||
const MODEL_NAME = 'qwen3-asr-1.7b'
|
||||
const FORCED_ALIGNER_MODEL_NAME = 'qwen3-forcedaligner-0.6b'
|
||||
const DEFAULT_SETTINGS: Record<string, unknown> = {}
|
||||
const REQUIRED_SETTINGS: string[] = []
|
||||
|
||||
interface Qwen3ASRTask {
|
||||
audio_path: string
|
||||
output_path?: string
|
||||
}
|
||||
|
||||
export default class Qwen3ASRTool extends Tool {
|
||||
private static readonly TOOLKIT = 'music_audio'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
// Load configuration from central toolkits directory
|
||||
this.config = ToolkitConfig.load(Qwen3ASRTool.TOOLKIT, this.toolName)
|
||||
const toolSettings = ToolkitConfig.loadToolSettings(
|
||||
Qwen3ASRTool.TOOLKIT,
|
||||
this.toolName,
|
||||
DEFAULT_SETTINGS
|
||||
)
|
||||
this.settings = toolSettings
|
||||
this.requiredSettings = REQUIRED_SETTINGS
|
||||
this.checkRequiredSettings(this.toolName)
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
// Use the actual config name for toolkit lookup
|
||||
return 'qwen3_asr'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return Qwen3ASRTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe audio to a file using Qwen3-ASR
|
||||
* @param inputPath The file path of the audio to be transcribed
|
||||
* @param outputPath The desired file path for the transcription output
|
||||
* @param device Device to use for processing (cpu, cuda, auto)
|
||||
* @param batchSize Batch size for processing
|
||||
* @param language Language code for transcription (auto, en, fr, etc.)
|
||||
* @param returnTimestamps Whether to return timestamps in output
|
||||
* @param useForcedAligner Whether to use the forced aligner model
|
||||
* @param cudaRuntimePath Path to CUDA runtime directory (Linux/Windows only)
|
||||
* @param torchPath Path to PyTorch installation directory
|
||||
* @param chunkDuration Chunk duration in seconds for long audio
|
||||
* @param cpuBatchSize CPU batch size for long audio
|
||||
* @returns A promise that resolves with the path to the transcription file
|
||||
*/
|
||||
async transcribeToFile(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
device = 'auto',
|
||||
batchSize = 4,
|
||||
language = 'auto',
|
||||
returnTimestamps = true,
|
||||
useForcedAligner = true,
|
||||
cudaRuntimePath?: string,
|
||||
torchPath?: string,
|
||||
chunkDuration = 30,
|
||||
cpuBatchSize?: number
|
||||
): Promise<string> {
|
||||
let tempDir: string | null = null
|
||||
let jsonFilePath: string | null = null
|
||||
|
||||
try {
|
||||
const inputStats = await fs.promises.stat(inputPath).catch(() => null)
|
||||
if (!inputStats?.isFile()) {
|
||||
throw new Error(`Input audio file does not exist: ${inputPath}`)
|
||||
}
|
||||
|
||||
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true })
|
||||
|
||||
const modelPath = await this.getResourcePath(MODEL_NAME)
|
||||
const forcedAlignerPath =
|
||||
returnTimestamps && useForcedAligner
|
||||
? await this.getResourcePath(FORCED_ALIGNER_MODEL_NAME)
|
||||
: undefined
|
||||
const nvidiaLibsPath = cudaRuntimePath ?? NVIDIA_LIBS_PATH
|
||||
const torchLibsPath = torchPath ?? PYTORCH_TORCH_PATH
|
||||
|
||||
const tasks: Qwen3ASRTask[] = [
|
||||
{
|
||||
audio_path: inputPath,
|
||||
output_path: outputPath
|
||||
}
|
||||
]
|
||||
|
||||
tempDir = await fs.promises.mkdtemp(
|
||||
path.join(os.tmpdir(), 'qwen3_asr_tasks_')
|
||||
)
|
||||
jsonFilePath = path.join(tempDir, 'tasks.json')
|
||||
|
||||
await fs.promises.writeFile(
|
||||
jsonFilePath,
|
||||
JSON.stringify(tasks, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const args = [
|
||||
'--function',
|
||||
'transcribe_audio',
|
||||
'--json_file',
|
||||
jsonFilePath,
|
||||
'--model_path',
|
||||
modelPath,
|
||||
'--device',
|
||||
device,
|
||||
'--batch_size',
|
||||
batchSize.toString(),
|
||||
'--language',
|
||||
language,
|
||||
'--return_timestamps',
|
||||
returnTimestamps ? 'true' : 'false',
|
||||
'--chunk_duration',
|
||||
chunkDuration.toString()
|
||||
]
|
||||
|
||||
if (nvidiaLibsPath) {
|
||||
args.push('--cuda_runtime_path', nvidiaLibsPath)
|
||||
}
|
||||
|
||||
if (torchLibsPath) {
|
||||
args.push('--torch_path', torchLibsPath)
|
||||
}
|
||||
|
||||
if (forcedAlignerPath) {
|
||||
args.push('--forced_aligner_model_path', forcedAlignerPath)
|
||||
}
|
||||
|
||||
if (cpuBatchSize) {
|
||||
args.push('--cpu_batch_size', cpuBatchSize.toString())
|
||||
}
|
||||
|
||||
await this.executeCommand({
|
||||
binaryName: 'qwen3_asr',
|
||||
args,
|
||||
options: { sync: true }
|
||||
})
|
||||
|
||||
const transcriptionContent = await fs.promises.readFile(
|
||||
outputPath,
|
||||
'utf-8'
|
||||
)
|
||||
const parsedOutput = this.parseTranscription(transcriptionContent)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(parsedOutput, null, 2),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
return outputPath
|
||||
} catch (error: unknown) {
|
||||
throw new Error(`Audio transcription failed: ${(error as Error).message}`)
|
||||
}
|
||||
}
|
||||
|
||||
private parseTranscription(
|
||||
rawOutput: Qwen3ASRTranscriptionOutput
|
||||
): TranscriptionOutput {
|
||||
const lines = rawOutput
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0)
|
||||
|
||||
const segments: TranscriptionOutput['segments'] = []
|
||||
const segmentRegex = /^\[(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)s\]\s+(.+)$/
|
||||
let duration = 0
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(segmentRegex)
|
||||
if (match && match[1] && match[2] && match[3]) {
|
||||
const start = parseFloat(match[1])
|
||||
const end = parseFloat(match[2])
|
||||
|
||||
segments.push({
|
||||
from: start,
|
||||
to: end,
|
||||
text: match[3].trim(),
|
||||
speaker: null
|
||||
})
|
||||
|
||||
if (end > duration) {
|
||||
duration = end
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (segments.length === 0 && lines.length > 0) {
|
||||
segments.push({
|
||||
from: 0,
|
||||
to: 0,
|
||||
text: lines[0] ?? '',
|
||||
speaker: null
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
duration,
|
||||
speakers: [],
|
||||
speaker_count: 0,
|
||||
segments,
|
||||
metadata: {
|
||||
tool: this.toolName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool, ExecuteCommandOptions
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
from tools.music_audio.transcription_schema import TranscriptionOutput, TranscriptionSegment
|
||||
from bridges.python.src.constants import NVIDIA_LIBS_PATH, PYTORCH_TORCH_PATH
|
||||
|
||||
MODEL_NAME = "qwen3-asr-1.7b"
|
||||
FORCED_ALIGNER_MODEL_NAME = "qwen3-forcedaligner-0.6b"
|
||||
DEFAULT_SETTINGS = {}
|
||||
REQUIRED_SETTINGS = []
|
||||
|
||||
|
||||
class Qwen3ASRTool(BaseTool):
|
||||
"""
|
||||
Example output format:
|
||||
|
||||
I noticed the app has a very mobile-first feel.
|
||||
[0.08-0.16s] I
|
||||
[0.16-0.64s] noticed
|
||||
"""
|
||||
|
||||
TOOLKIT = "music_audio"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Load configuration from central toolkits directory
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.settings = ToolkitConfig.load_tool_settings(
|
||||
self.TOOLKIT, self.tool_name, DEFAULT_SETTINGS
|
||||
)
|
||||
self.required_settings = REQUIRED_SETTINGS
|
||||
self._check_required_settings(self.tool_name)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
# Use the actual config name for toolkit lookup
|
||||
return "qwen3_asr"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config["description"]
|
||||
|
||||
def transcribe_to_file(
|
||||
self,
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
device: str = "auto",
|
||||
batch_size: int = 4,
|
||||
language: str = "auto",
|
||||
return_timestamps: bool = True,
|
||||
use_forced_aligner: bool = True,
|
||||
cuda_runtime_path: Optional[str] = None,
|
||||
torch_path: Optional[str] = None,
|
||||
chunk_duration: int = 30,
|
||||
cpu_batch_size: Optional[int] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to a file using Qwen3-ASR
|
||||
|
||||
Args:
|
||||
input_path: The file path of the audio to be transcribed
|
||||
output_path: The desired file path for the transcription output
|
||||
device: Device to use for processing (cpu, cuda, auto)
|
||||
batch_size: Batch size for processing
|
||||
language: Language code for transcription (auto, en, fr, etc.)
|
||||
return_timestamps: Whether to return timestamps in output
|
||||
use_forced_aligner: Whether to use the forced aligner model
|
||||
cuda_runtime_path: Path to CUDA runtime directory (Linux/Windows only)
|
||||
torch_path: Path to PyTorch installation directory
|
||||
chunk_duration: Chunk duration in seconds for long audio
|
||||
cpu_batch_size: CPU batch size for long audio
|
||||
|
||||
Returns:
|
||||
The path to the transcription file
|
||||
"""
|
||||
try:
|
||||
if not os.path.isfile(input_path):
|
||||
raise FileNotFoundError(f"Input audio file does not exist: {input_path}")
|
||||
|
||||
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
|
||||
|
||||
model_path = self.get_resource_path(MODEL_NAME)
|
||||
forced_aligner_path = None
|
||||
nvidia_libs_path = (
|
||||
cuda_runtime_path if cuda_runtime_path is not None else NVIDIA_LIBS_PATH
|
||||
)
|
||||
torch_libs_path = (
|
||||
torch_path if torch_path is not None else PYTORCH_TORCH_PATH
|
||||
)
|
||||
|
||||
if return_timestamps and use_forced_aligner:
|
||||
forced_aligner_path = self.get_resource_path(FORCED_ALIGNER_MODEL_NAME)
|
||||
|
||||
tasks = [
|
||||
{
|
||||
"audio_path": input_path,
|
||||
"output_path": output_path,
|
||||
}
|
||||
]
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, encoding="utf-8"
|
||||
) as temp_file:
|
||||
json_file_path = temp_file.name
|
||||
json.dump(tasks, temp_file, indent=2, ensure_ascii=False)
|
||||
|
||||
args = [
|
||||
"--function",
|
||||
"transcribe_audio",
|
||||
"--json_file",
|
||||
json_file_path,
|
||||
"--model_path",
|
||||
model_path,
|
||||
"--device",
|
||||
device,
|
||||
"--batch_size",
|
||||
str(batch_size),
|
||||
"--language",
|
||||
language,
|
||||
"--return_timestamps",
|
||||
"true" if return_timestamps else "false",
|
||||
"--chunk_duration",
|
||||
str(chunk_duration),
|
||||
]
|
||||
|
||||
if nvidia_libs_path:
|
||||
args.extend(["--cuda_runtime_path", nvidia_libs_path])
|
||||
|
||||
if torch_libs_path:
|
||||
args.extend(["--torch_path", torch_libs_path])
|
||||
|
||||
if forced_aligner_path:
|
||||
args.extend(["--forced_aligner_model_path", forced_aligner_path])
|
||||
|
||||
if cpu_batch_size is not None:
|
||||
args.extend(["--cpu_batch_size", str(cpu_batch_size)])
|
||||
|
||||
self.execute_command(
|
||||
ExecuteCommandOptions(
|
||||
binary_name="qwen3_asr", args=args, options={"sync": True}
|
||||
)
|
||||
)
|
||||
|
||||
with open(output_path, "r", encoding="utf-8") as f:
|
||||
transcription_content = f.read()
|
||||
|
||||
parsed_output = self.parse_transcription(transcription_content)
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed_output, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return output_path
|
||||
except Exception as e:
|
||||
raise Exception(f"Audio transcription failed: {str(e)}")
|
||||
|
||||
def parse_transcription(self, raw_output: str) -> TranscriptionOutput:
|
||||
lines = [line.strip() for line in raw_output.split("\n") if line.strip()]
|
||||
|
||||
segments: list[TranscriptionSegment] = []
|
||||
segment_regex = re.compile(r"^\[(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)s\]\s+(.+)$")
|
||||
duration = 0.0
|
||||
|
||||
for line in lines:
|
||||
match = segment_regex.match(line)
|
||||
if match:
|
||||
start = float(match.group(1))
|
||||
end = float(match.group(2))
|
||||
text = match.group(3)
|
||||
|
||||
segments.append(
|
||||
{"from": start, "to": end, "text": text.strip(), "speaker": None}
|
||||
)
|
||||
|
||||
if end > duration:
|
||||
duration = end
|
||||
|
||||
if not segments and lines:
|
||||
segments.append({"from": 0.0, "to": 0.0, "text": lines[0], "speaker": None})
|
||||
|
||||
return {
|
||||
"duration": duration,
|
||||
"speakers": [],
|
||||
"speaker_count": 0,
|
||||
"segments": segments,
|
||||
"metadata": {"tool": self.tool_name},
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "qwen3_asr",
|
||||
"toolkit_id": "music_audio",
|
||||
"name": "Qwen3-ASR",
|
||||
"description": "A tool for speech recognition and timestamped transcription using the Qwen3 ASR models.",
|
||||
"icon_name": "qwen-ai-line",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"binaries": {
|
||||
"linux-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-linux-x86_64",
|
||||
"linux-aarch64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-linux-aarch64",
|
||||
"macosx-x86_64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-macosx-x86_64",
|
||||
"macosx-arm64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-macosx-arm64",
|
||||
"win-amd64": "https://github.com/leon-ai/leon-binaries/releases/download/qwen3_asr-v1.0.0/qwen3_asr_1.0.0-win-amd64.exe"
|
||||
},
|
||||
"resources": {
|
||||
"qwen3-asr-1.7b": [
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/chat_template.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/generation_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/merges.txt?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model-00001-of-00002.safetensors?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model-00002-of-00002.safetensors?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/model.safetensors.index.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/preprocessor_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/tokenizer_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/resolve/main/vocab.json?download=true"
|
||||
],
|
||||
"qwen3-forcedaligner-0.6b": [
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/chat_template.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/generation_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/merges.txt?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/model.safetensors?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/preprocessor_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/tokenizer_config.json?download=true",
|
||||
"https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B/resolve/main/vocab.json?download=true"
|
||||
]
|
||||
},
|
||||
"functions": {
|
||||
"transcribeToFile": {
|
||||
"description": "Transcribe audio to a file using Qwen3 ASR.",
|
||||
"hooks": {
|
||||
"post_execution": {
|
||||
"response_jq": "[.result.segments[].text] | map(select(type == \"string\" and length > 0)) | join(\" \")"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"device": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"inputPath",
|
||||
"outputPath"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
from .src.python.qwen3_tts_tool import Qwen3TTSTool
|
||||
|
||||
__all__ = ["Qwen3TTSTool"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user