chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,3 @@
|
||||
from .src.python.inference_tool import InferenceTool
|
||||
|
||||
__all__ = ["InferenceTool"]
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from './inference-tool'
|
||||
@@ -0,0 +1,97 @@
|
||||
import { Tool } from '@sdk/base-tool'
|
||||
import { ToolkitConfig } from '@sdk/toolkit-config'
|
||||
import { Network } from '@sdk/network'
|
||||
|
||||
interface CompletionOptions {
|
||||
prompt: string
|
||||
system_prompt?: string
|
||||
temperature?: number
|
||||
max_tokens?: number
|
||||
thought_tokens_budget?: number
|
||||
disable_thinking?: boolean
|
||||
reasoning_mode?: 'off' | 'guarded' | 'on'
|
||||
track_provider_errors?: boolean
|
||||
}
|
||||
|
||||
interface StructuredCompletionOptions extends CompletionOptions {
|
||||
json_schema: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface InferenceResponse {
|
||||
success: boolean
|
||||
output?: unknown
|
||||
reasoning?: string
|
||||
usedInputTokens?: number
|
||||
usedOutputTokens?: number
|
||||
generationDurationMs?: number
|
||||
providerDecodeDurationMs?: number
|
||||
providerTokensPerSecond?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export default class InferenceTool extends Tool {
|
||||
private static readonly TOOLKIT = 'communication'
|
||||
private readonly config: ReturnType<typeof ToolkitConfig.load>
|
||||
private readonly network: Network
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.config = ToolkitConfig.load(InferenceTool.TOOLKIT, this.toolName)
|
||||
this.network = new Network({
|
||||
baseURL: `${process.env['LEON_HOST']}:${process.env['LEON_PORT']}/api/v1`
|
||||
})
|
||||
}
|
||||
|
||||
get toolName(): string {
|
||||
return 'inference'
|
||||
}
|
||||
|
||||
get toolkit(): string {
|
||||
return InferenceTool.TOOLKIT
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return this.config['description']
|
||||
}
|
||||
|
||||
async completion(options: CompletionOptions): Promise<InferenceResponse> {
|
||||
const response = await this.network.request<InferenceResponse>({
|
||||
url: '/inference',
|
||||
method: 'POST',
|
||||
data: {
|
||||
prompt: options.prompt,
|
||||
systemPrompt: options.system_prompt,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.max_tokens,
|
||||
thoughtTokensBudget: options.thought_tokens_budget,
|
||||
disableThinking: options.disable_thinking,
|
||||
reasoningMode: options.reasoning_mode,
|
||||
trackProviderErrors: options.track_provider_errors
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
|
||||
async structuredCompletion(
|
||||
options: StructuredCompletionOptions
|
||||
): Promise<InferenceResponse> {
|
||||
const response = await this.network.request<InferenceResponse>({
|
||||
url: '/inference',
|
||||
method: 'POST',
|
||||
data: {
|
||||
prompt: options.prompt,
|
||||
systemPrompt: options.system_prompt,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.max_tokens,
|
||||
thoughtTokensBudget: options.thought_tokens_budget,
|
||||
jsonSchema: options.json_schema,
|
||||
disableThinking: options.disable_thinking,
|
||||
reasoningMode: options.reasoning_mode,
|
||||
trackProviderErrors: options.track_provider_errors
|
||||
}
|
||||
})
|
||||
|
||||
return response.data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Literal
|
||||
|
||||
from bridges.python.src.sdk.base_tool import BaseTool
|
||||
from bridges.python.src.sdk.network import Network
|
||||
from bridges.python.src.sdk.toolkit_config import ToolkitConfig
|
||||
|
||||
|
||||
class InferenceTool(BaseTool):
|
||||
TOOLKIT = "communication"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.config = ToolkitConfig.load(self.TOOLKIT, self.tool_name)
|
||||
self.network = Network(
|
||||
{
|
||||
"base_url": f"{os.environ.get('LEON_HOST')}:{os.environ.get('LEON_PORT')}/api/v1"
|
||||
}
|
||||
)
|
||||
|
||||
@property
|
||||
def tool_name(self) -> str:
|
||||
return "inference"
|
||||
|
||||
@property
|
||||
def toolkit(self) -> str:
|
||||
return self.TOOLKIT
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.config.get("description", "")
|
||||
|
||||
def completion(
|
||||
self,
|
||||
prompt: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
thought_tokens_budget: Optional[int] = None,
|
||||
disable_thinking: Optional[bool] = None,
|
||||
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
|
||||
track_provider_errors: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/inference",
|
||||
"method": "POST",
|
||||
"data": {
|
||||
"prompt": prompt,
|
||||
"systemPrompt": system_prompt,
|
||||
"temperature": temperature,
|
||||
"maxTokens": max_tokens,
|
||||
"thoughtTokensBudget": thought_tokens_budget,
|
||||
"disableThinking": disable_thinking,
|
||||
"reasoningMode": reasoning_mode,
|
||||
"trackProviderErrors": track_provider_errors,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return response["data"]
|
||||
|
||||
def structured_completion(
|
||||
self,
|
||||
prompt: str,
|
||||
json_schema: Dict[str, Any],
|
||||
system_prompt: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
thought_tokens_budget: Optional[int] = None,
|
||||
disable_thinking: Optional[bool] = None,
|
||||
reasoning_mode: Optional[Literal["off", "guarded", "on"]] = None,
|
||||
track_provider_errors: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
response = self.network.request(
|
||||
{
|
||||
"url": "/inference",
|
||||
"method": "POST",
|
||||
"data": {
|
||||
"prompt": prompt,
|
||||
"systemPrompt": system_prompt,
|
||||
"temperature": temperature,
|
||||
"maxTokens": max_tokens,
|
||||
"thoughtTokensBudget": thought_tokens_budget,
|
||||
"jsonSchema": json_schema,
|
||||
"disableThinking": disable_thinking,
|
||||
"reasoningMode": reasoning_mode,
|
||||
"trackProviderErrors": track_provider_errors,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return response["data"]
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"$schema": "../../../schemas/tool-schemas/tool.json",
|
||||
"tool_id": "inference",
|
||||
"toolkit_id": "communication",
|
||||
"name": "Inference",
|
||||
"description": "A generic Leon workflow inference tool backed by the active workflow LLM provider.",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"functions": {
|
||||
"completion": {
|
||||
"description": "Generate a workflow inference completion.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"thought_tokens_budget": {
|
||||
"type": "number"
|
||||
},
|
||||
"disable_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"reasoning_mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"track_provider_errors": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt"
|
||||
]
|
||||
}
|
||||
},
|
||||
"structuredCompletion": {
|
||||
"description": "Generate a structured workflow inference completion using a JSON schema.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"system_prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"thought_tokens_budget": {
|
||||
"type": "number"
|
||||
},
|
||||
"disable_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"reasoning_mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"track_provider_errors": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt",
|
||||
"json_schema"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user