chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
import { FALAI_HOSTED_KEY_MARKUP_MULTIPLIER } from '@/lib/tools/falai-pricing'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
import { parseBooleanParam, parseBooleanParamWithDefault } from '@/tools/video/utils'
|
||||
|
||||
export const falaiVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
id: 'video_falai',
|
||||
name: 'Fal.ai Video Generation',
|
||||
description:
|
||||
'Generate videos using Fal.ai with access to Veo 3.1, Sora 2, Seedance 2.0, Kling 3.0, MiniMax Hailuo 2.3, WAN 2.2, LTX 2.3, and previously supported models',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Video provider (falai)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Fal.ai API key',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Fal.ai model: veo-3.1, veo-3.1-fast, sora-2, sora-2-pro, seedance-2.0, seedance-2.0-fast, kling-v3-pro, kling-v3-4k, kling-o3-pro, kling-o3-4k, minimax-hailuo-2.3-pro, minimax-hailuo-2.3-standard, wan-2.2-a14b-turbo, ltx-2.3, ltx-2.3-fast, plus previously supported model IDs',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Text prompt describing the video to generate',
|
||||
},
|
||||
duration: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video duration in seconds (varies by model)',
|
||||
},
|
||||
aspectRatio: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Aspect ratio (varies by model): 16:9, 9:16, 1:1',
|
||||
},
|
||||
resolution: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Video resolution (varies by model): 480p, 580p, 720p, 1080p, true_1080p, 1440p, 2160p, 4k',
|
||||
},
|
||||
promptOptimizer: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Enable prompt optimization for MiniMax models (default: true)',
|
||||
},
|
||||
generateAudio: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Generate native audio when supported by the selected Fal.ai model',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FALAI_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'falai',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
const providerCostDollars = output.__falaiCostDollars
|
||||
if (typeof providerCostDollars !== 'number' || Number.isNaN(providerCostDollars)) {
|
||||
throw new Error('Fal.ai video response missing cost data')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: providerCostDollars * FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
metadata: {
|
||||
...(typeof output.__falaiBilling === 'object' && output.__falaiBilling !== null
|
||||
? (output.__falaiBilling as Record<string, unknown>)
|
||||
: {}),
|
||||
providerCostDollars,
|
||||
markupMultiplier: FALAI_HOSTED_KEY_MARKUP_MULTIPLIER,
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 40,
|
||||
burstMultiplier: 1,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
__usingHostedKey?: boolean
|
||||
}
|
||||
) => ({
|
||||
provider: 'falai',
|
||||
apiKey: params.apiKey,
|
||||
model: params.model,
|
||||
prompt: params.prompt,
|
||||
duration: params.duration,
|
||||
aspectRatio: params.aspectRatio,
|
||||
resolution: params.resolution,
|
||||
promptOptimizer: parseBooleanParamWithDefault(params.promptOptimizer, true),
|
||||
generateAudio: parseBooleanParam(params.generateAudio),
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
useHostedCostTracking: params.__usingHostedKey === true,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.error || 'Video generation failed',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Missing videoUrl in response',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
videoUrl: data.videoUrl,
|
||||
videoFile: data.videoFile,
|
||||
duration: data.duration,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
provider: 'falai',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
__falaiCostDollars: data.__falaiCostDollars,
|
||||
__falaiBilling: data.__falaiBilling,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
videoUrl: { type: 'string', description: 'Generated video URL' },
|
||||
videoFile: { type: 'file', description: 'Video file object with metadata' },
|
||||
duration: { type: 'number', description: 'Video duration in seconds' },
|
||||
width: { type: 'number', description: 'Video width in pixels' },
|
||||
height: { type: 'number', description: 'Video height in pixels' },
|
||||
provider: { type: 'string', description: 'Provider used (falai)' },
|
||||
model: { type: 'string', description: 'Model used' },
|
||||
jobId: { type: 'string', description: 'Job ID' },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { falaiVideoTool } from '@/tools/video/falai'
|
||||
import { lumaVideoTool } from '@/tools/video/luma'
|
||||
import { minimaxVideoTool } from '@/tools/video/minimax'
|
||||
import { runwayVideoTool } from '@/tools/video/runway'
|
||||
import { veoVideoTool } from '@/tools/video/veo'
|
||||
|
||||
export { runwayVideoTool, veoVideoTool, lumaVideoTool, minimaxVideoTool, falaiVideoTool }
|
||||
@@ -0,0 +1,135 @@
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
|
||||
export const lumaVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
id: 'video_luma',
|
||||
name: 'Luma Dream Machine Video',
|
||||
description: 'Generate videos using Luma Dream Machine with advanced camera controls',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Video provider (luma)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Luma AI API key',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Luma model: ray-2 (default)',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Text prompt describing the video to generate',
|
||||
},
|
||||
duration: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video duration in seconds (5 or 9, default: 5)',
|
||||
},
|
||||
aspectRatio: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Aspect ratio: 16:9 (landscape), 9:16 (portrait), or 1:1 (square)',
|
||||
},
|
||||
resolution: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video resolution: 540p, 720p, or 1080p (default: 1080p)',
|
||||
},
|
||||
cameraControl: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Camera controls as array of concept objects. Format: [{ "key": "concept_name" }]. Valid keys: truck_left, truck_right, pan_left, pan_right, tilt_up, tilt_down, zoom_in, zoom_out, push_in, pull_out, orbit_left, orbit_right, crane_up, crane_down, static, handheld, and 20+ more predefined options',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
}
|
||||
) => ({
|
||||
provider: 'luma',
|
||||
apiKey: params.apiKey,
|
||||
model: params.model || 'ray-2',
|
||||
prompt: params.prompt,
|
||||
duration: params.duration || 5,
|
||||
aspectRatio: params.aspectRatio || '16:9',
|
||||
resolution: params.resolution || '1080p',
|
||||
cameraControl: params.cameraControl,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.error || 'Video generation failed',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Missing videoUrl in response',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
videoUrl: data.videoUrl,
|
||||
videoFile: data.videoFile,
|
||||
duration: data.duration,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
provider: 'luma',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
videoUrl: { type: 'string', description: 'Generated video URL' },
|
||||
videoFile: { type: 'file', description: 'Video file object with metadata' },
|
||||
duration: { type: 'number', description: 'Video duration in seconds' },
|
||||
width: { type: 'number', description: 'Video width in pixels' },
|
||||
height: { type: 'number', description: 'Video height in pixels' },
|
||||
provider: { type: 'string', description: 'Provider used (luma)' },
|
||||
model: { type: 'string', description: 'Model used' },
|
||||
jobId: { type: 'string', description: 'Luma job ID' },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
import { parseBooleanParamWithDefault } from '@/tools/video/utils'
|
||||
|
||||
export const minimaxVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
id: 'video_minimax',
|
||||
name: 'MiniMax Hailuo Video',
|
||||
description:
|
||||
'Generate videos using MiniMax Hailuo through MiniMax Platform API with advanced realism and prompt optimization',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Video provider (minimax)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'MiniMax API key from platform.minimax.io',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'MiniMax model: hailuo-2.3 (default) or hailuo-02',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Text prompt describing the video to generate',
|
||||
},
|
||||
duration: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video duration in seconds (6 or 10, default: 6)',
|
||||
},
|
||||
endpoint: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Quality endpoint: standard (768P) or pro (1080P for 6s videos)',
|
||||
},
|
||||
promptOptimizer: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Enable prompt optimization for better results (default: true)',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
}
|
||||
) => ({
|
||||
provider: 'minimax',
|
||||
apiKey: params.apiKey,
|
||||
model: params.model || 'hailuo-2.3',
|
||||
prompt: params.prompt,
|
||||
duration: params.duration || 6,
|
||||
endpoint: params.endpoint || 'standard',
|
||||
promptOptimizer: parseBooleanParamWithDefault(params.promptOptimizer, true),
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.error || 'Video generation failed',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Missing videoUrl in response',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
videoUrl: data.videoUrl,
|
||||
videoFile: data.videoFile,
|
||||
duration: data.duration,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
provider: 'minimax',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
videoUrl: { type: 'string', description: 'Generated video URL' },
|
||||
videoFile: { type: 'file', description: 'Video file object with metadata' },
|
||||
duration: { type: 'number', description: 'Video duration in seconds' },
|
||||
width: { type: 'number', description: 'Video width in pixels' },
|
||||
height: { type: 'number', description: 'Video height in pixels' },
|
||||
provider: { type: 'string', description: 'Provider used (minimax)' },
|
||||
model: { type: 'string', description: 'Model used' },
|
||||
jobId: { type: 'string', description: 'MiniMax job ID' },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
|
||||
export const runwayVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
id: 'video_runway',
|
||||
name: 'Runway Gen-4 Video',
|
||||
description: 'Generate videos using Runway Gen-4 with world consistency and visual references',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Video provider (runway)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Runway API key',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Runway model: gen-4 (default, higher quality) or gen-4-turbo (faster)',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Text prompt describing the video to generate',
|
||||
},
|
||||
duration: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video duration in seconds (5 or 10, default: 5)',
|
||||
},
|
||||
aspectRatio: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Aspect ratio: 16:9 (landscape), 9:16 (portrait), or 1:1 (square)',
|
||||
},
|
||||
resolution: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video resolution (720p output). Note: Gen-4 Turbo outputs at 720p natively',
|
||||
},
|
||||
visualReference: {
|
||||
type: 'file',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Reference image REQUIRED for Gen-4 (UserFile object). Gen-4 only supports image-to-video, not text-only generation',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
}
|
||||
) => ({
|
||||
provider: 'runway',
|
||||
apiKey: params.apiKey,
|
||||
model: 'gen-4-turbo', // Only gen4_turbo model is supported
|
||||
prompt: params.prompt,
|
||||
duration: params.duration || 5,
|
||||
aspectRatio: params.aspectRatio || '16:9',
|
||||
resolution: params.resolution || '720p',
|
||||
visualReference: params.visualReference,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.error || 'Video generation failed',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Missing videoUrl in response',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
videoUrl: data.videoUrl,
|
||||
videoFile: data.videoFile,
|
||||
duration: data.duration,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
provider: 'runway',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
videoUrl: { type: 'string', description: 'Generated video URL' },
|
||||
videoFile: { type: 'file', description: 'Video file object with metadata' },
|
||||
duration: { type: 'number', description: 'Video duration in seconds' },
|
||||
width: { type: 'number', description: 'Video width in pixels' },
|
||||
height: { type: 'number', description: 'Video height in pixels' },
|
||||
provider: { type: 'string', description: 'Provider used (runway)' },
|
||||
model: { type: 'string', description: 'Model used' },
|
||||
jobId: { type: 'string', description: 'Runway job ID' },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { UserFile } from '@/executor/types'
|
||||
import type { ToolResponse } from '@/tools/types'
|
||||
|
||||
export interface VideoParams {
|
||||
provider: 'runway' | 'veo' | 'luma' | 'minimax' | 'falai'
|
||||
apiKey: string
|
||||
model?: string
|
||||
prompt: string
|
||||
duration?: number
|
||||
aspectRatio?: string
|
||||
resolution?: string
|
||||
/** Runway only, required for Runway generation */
|
||||
visualReference?: UserFile
|
||||
cameraControl?: {
|
||||
pan?: number
|
||||
zoom?: number
|
||||
tilt?: number
|
||||
truck?: number
|
||||
tracking?: boolean
|
||||
}
|
||||
endpoint?: string
|
||||
promptOptimizer?: boolean
|
||||
generateAudio?: boolean
|
||||
}
|
||||
|
||||
export interface VideoResponse extends ToolResponse {
|
||||
output: {
|
||||
videoUrl: string
|
||||
videoFile?: UserFile
|
||||
duration?: number
|
||||
width?: number
|
||||
height?: number
|
||||
provider?: string
|
||||
model?: string
|
||||
jobId?: string
|
||||
__falaiCostDollars?: number
|
||||
__falaiBilling?: {
|
||||
endpointId: string
|
||||
requestId: string
|
||||
source: 'billing_events' | 'historical_estimate' | 'fallback_floor'
|
||||
outputUnits?: number | null
|
||||
unitPrice?: number | null
|
||||
percentDiscount?: number | null
|
||||
currency?: string
|
||||
error?: string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface VideoBlockResponse extends ToolResponse {
|
||||
output: {
|
||||
videoUrl: string
|
||||
videoFile?: UserFile
|
||||
duration?: number
|
||||
width?: number
|
||||
height?: number
|
||||
provider?: string
|
||||
model?: string
|
||||
}
|
||||
}
|
||||
|
||||
interface RunwayParams extends Omit<VideoParams, 'provider'> {
|
||||
/** Only gen4_turbo supports image-to-video */
|
||||
model?: 'gen-4-turbo'
|
||||
/** Required for Gen-4 */
|
||||
visualReference: UserFile
|
||||
/** Gen-4 Turbo outputs at 720p */
|
||||
resolution?: '720p'
|
||||
duration?: 5 | 10
|
||||
}
|
||||
|
||||
interface VeoParams extends Omit<VideoParams, 'provider'> {
|
||||
model?: 'veo-3' | 'veo-3-fast' | 'veo-3.1'
|
||||
aspectRatio?: '16:9' | '9:16'
|
||||
resolution?: '720p' | '1080p'
|
||||
duration?: 4 | 6 | 8
|
||||
}
|
||||
|
||||
interface LumaParams extends Omit<VideoParams, 'provider'> {
|
||||
model?: 'ray3'
|
||||
cameraControl?: {
|
||||
pan?: number
|
||||
zoom?: number
|
||||
tilt?: number
|
||||
truck?: number
|
||||
tracking?: boolean
|
||||
}
|
||||
aspectRatio?: '16:9' | '9:16' | '1:1'
|
||||
resolution?: '540p' | '720p' | '1080p'
|
||||
duration?: 5 | 10
|
||||
}
|
||||
|
||||
interface MinimaxParams extends Omit<VideoParams, 'provider'> {
|
||||
model?: 'hailuo-2.3' | 'hailuo-02'
|
||||
endpoint?: 'pro' | 'standard'
|
||||
promptOptimizer?: boolean
|
||||
duration?: 6 | 10
|
||||
}
|
||||
|
||||
interface VideoRequestBody extends VideoParams {
|
||||
workspaceId?: string
|
||||
workflowId?: string
|
||||
executionId?: string
|
||||
userId?: string
|
||||
}
|
||||
|
||||
interface RunwayJobResponse {
|
||||
id: string
|
||||
status: 'pending' | 'processing' | 'completed' | 'failed'
|
||||
videoUrl?: string
|
||||
progress?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface VeoJobResponse {
|
||||
name: string
|
||||
done: boolean
|
||||
response?: {
|
||||
generatedVideo: {
|
||||
uri: string
|
||||
mimeType: string
|
||||
}
|
||||
}
|
||||
error?: {
|
||||
message: string
|
||||
}
|
||||
}
|
||||
|
||||
interface LumaJobResponse {
|
||||
id: string
|
||||
state: 'queued' | 'processing' | 'completed' | 'failed'
|
||||
video?: {
|
||||
url: string
|
||||
width: number
|
||||
height: number
|
||||
duration: number
|
||||
}
|
||||
failure_reason?: string
|
||||
}
|
||||
|
||||
interface MinimaxJobResponse {
|
||||
request_id: string
|
||||
status: 'pending' | 'processing' | 'completed' | 'failed'
|
||||
video_url?: string
|
||||
error?: string
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
export function parseBooleanParam(value: unknown): boolean | undefined {
|
||||
if (typeof value === 'boolean') return value
|
||||
if (typeof value !== 'string') return undefined
|
||||
|
||||
const normalized = value.trim().toLowerCase()
|
||||
if (normalized === 'true' || normalized === '1') return true
|
||||
if (normalized === 'false' || normalized === '0' || normalized === '') return false
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function parseBooleanParamWithDefault(value: unknown, defaultValue: boolean): boolean {
|
||||
return parseBooleanParam(value) ?? defaultValue
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
import type { VideoParams, VideoResponse } from '@/tools/video/types'
|
||||
|
||||
export const veoVideoTool: ToolConfig<VideoParams, VideoResponse> = {
|
||||
id: 'video_veo',
|
||||
name: 'Google Veo 3 Video',
|
||||
description: 'Generate videos using Google Veo 3/3.1 with native audio generation',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
provider: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Video provider (veo)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Google Gemini API key',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Veo model: veo-3 (default, highest quality), veo-3-fast (faster), or veo-3.1 (latest)',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Text prompt describing the video to generate',
|
||||
},
|
||||
duration: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video duration in seconds (4, 6, or 8, default: 8)',
|
||||
},
|
||||
aspectRatio: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Aspect ratio: 16:9 (landscape) or 9:16 (portrait)',
|
||||
},
|
||||
resolution: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Video resolution: 720p or 1080p (default: 1080p)',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/video',
|
||||
method: 'POST',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (
|
||||
params: VideoParams & {
|
||||
_context?: { workspaceId?: string; workflowId?: string; executionId?: string }
|
||||
}
|
||||
) => ({
|
||||
provider: 'veo',
|
||||
apiKey: params.apiKey,
|
||||
model: params.model || 'veo-3',
|
||||
prompt: params.prompt,
|
||||
duration: params.duration || 8, // Default 8 seconds (valid: 4, 6, or 8)
|
||||
aspectRatio: params.aspectRatio || '16:9',
|
||||
resolution: params.resolution || '1080p',
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!response.ok || data.error) {
|
||||
return {
|
||||
success: false,
|
||||
error: data.error || 'Video generation failed',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Missing videoUrl in response',
|
||||
output: {
|
||||
videoUrl: '',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
videoUrl: data.videoUrl,
|
||||
videoFile: data.videoFile,
|
||||
duration: data.duration,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
provider: 'veo',
|
||||
model: data.model,
|
||||
jobId: data.jobId,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
videoUrl: { type: 'string', description: 'Generated video URL' },
|
||||
videoFile: { type: 'file', description: 'Video file object with metadata' },
|
||||
duration: { type: 'number', description: 'Video duration in seconds' },
|
||||
width: { type: 'number', description: 'Video width in pixels' },
|
||||
height: { type: 'number', description: 'Video height in pixels' },
|
||||
provider: { type: 'string', description: 'Provider used (veo)' },
|
||||
model: { type: 'string', description: 'Model used' },
|
||||
jobId: { type: 'string', description: 'Veo job ID' },
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user