chore: import upstream snapshot with attribution

apps/sim/tools/stt/assemblyai.ts

@@ -0,0 +1,231 @@
+import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types'
+import {
+  STT_ENTITY_OUTPUT_PROPERTIES,
+  STT_SEGMENT_OUTPUT_PROPERTIES,
+  STT_SENTIMENT_OUTPUT_PROPERTIES,
+} from '@/tools/stt/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const assemblyaiSttTool: ToolConfig<SttParams, SttResponse> = {
+  id: 'stt_assemblyai',
+  name: 'AssemblyAI STT',
+  description: 'Transcribe audio to text using AssemblyAI with advanced NLP features',
+  version: '1.0.0',
+
+  params: {
+    provider: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'STT provider (assemblyai)',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'AssemblyAI API key',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'AssemblyAI model to use (default: best)',
+    },
+    audioFile: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Audio or video file to transcribe (e.g., MP3, WAV, M4A, WEBM)',
+    },
+    audioFileReference: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Reference to audio/video file from previous blocks',
+    },
+    audioUrl: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'URL to audio or video file',
+    },
+    language: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Language code (e.g., "en", "es", "fr") or "auto" for auto-detection',
+    },
+    timestamps: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Timestamp granularity: none, sentence, or word',
+    },
+    diarization: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable speaker diarization',
+    },
+    sentiment: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable sentiment analysis',
+    },
+    entityDetection: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable entity detection',
+    },
+    piiRedaction: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable PII redaction',
+    },
+    summarization: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable automatic summarization',
+    },
+  },
+
+  request: {
+    url: '/api/tools/stt',
+    method: 'POST',
+    headers: () => ({
+      'Content-Type': 'application/json',
+    }),
+    body: (
+      params: SttParams & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'assemblyai',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      audioUrl: params.audioUrl,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      diarization: params.diarization || false,
+      sentiment: (params as any).sentiment || false,
+      entityDetection: (params as any).entityDetection || false,
+      piiRedaction: (params as any).piiRedaction || false,
+      summarization: (params as any).summarization || false,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    if (!response.ok || data.error) {
+      return {
+        success: false,
+        error: data.error || 'Transcription failed',
+        output: {
+          transcript: '',
+        },
+      }
+    }
+
+    return {
+      success: true,
+      output: {
+        transcript: data.transcript,
+        segments: data.segments,
+        language: data.language,
+        duration: data.duration,
+        confidence: data.confidence,
+        sentiment: data.sentiment,
+        entities: data.entities,
+        summary: data.summary,
+      },
+    }
+  },
+
+  outputs: {
+    transcript: { type: 'string', description: 'Full transcribed text' },
+    segments: {
+      type: 'array',
+      description: 'Timestamped segments with speaker labels',
+      items: {
+        type: 'object',
+        properties: STT_SEGMENT_OUTPUT_PROPERTIES,
+      },
+    },
+    language: { type: 'string', description: 'Detected or specified language' },
+    duration: { type: 'number', description: 'Audio duration in seconds' },
+    confidence: { type: 'number', description: 'Overall confidence score' },
+    sentiment: {
+      type: 'array',
+      description: 'Sentiment analysis results',
+      items: {
+        type: 'object',
+        properties: STT_SENTIMENT_OUTPUT_PROPERTIES,
+      },
+    },
+    entities: {
+      type: 'array',
+      description: 'Detected entities',
+      items: {
+        type: 'object',
+        properties: STT_ENTITY_OUTPUT_PROPERTIES,
+      },
+    },
+    summary: { type: 'string', description: 'Auto-generated summary' },
+  },
+}
+
+const assemblyaiSttV2Params = {
+  provider: assemblyaiSttTool.params.provider,
+  apiKey: assemblyaiSttTool.params.apiKey,
+  model: assemblyaiSttTool.params.model,
+  audioFile: assemblyaiSttTool.params.audioFile,
+  audioFileReference: assemblyaiSttTool.params.audioFileReference,
+  language: assemblyaiSttTool.params.language,
+  timestamps: assemblyaiSttTool.params.timestamps,
+  diarization: assemblyaiSttTool.params.diarization,
+  sentiment: assemblyaiSttTool.params.sentiment,
+  entityDetection: assemblyaiSttTool.params.entityDetection,
+  piiRedaction: assemblyaiSttTool.params.piiRedaction,
+  summarization: assemblyaiSttTool.params.summarization,
+} satisfies ToolConfig['params']
+
+export const assemblyaiSttV2Tool: ToolConfig<SttV2Params, SttResponse> = {
+  ...assemblyaiSttTool,
+  id: 'stt_assemblyai_v2',
+  name: 'AssemblyAI STT',
+  params: assemblyaiSttV2Params,
+  request: {
+    ...assemblyaiSttTool.request,
+    body: (
+      params: SttV2Params & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'assemblyai',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      diarization: params.diarization || false,
+      sentiment: params.sentiment || false,
+      entityDetection: params.entityDetection || false,
+      piiRedaction: params.piiRedaction || false,
+      summarization: params.summarization || false,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+}

apps/sim/tools/stt/deepgram.ts

@@ -0,0 +1,173 @@
+import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types'
+import { STT_SEGMENT_OUTPUT_PROPERTIES } from '@/tools/stt/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const deepgramSttTool: ToolConfig<SttParams, SttResponse> = {
+  id: 'stt_deepgram',
+  name: 'Deepgram STT',
+  description: 'Transcribe audio to text using Deepgram',
+  version: '1.0.0',
+
+  params: {
+    provider: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'STT provider (deepgram)',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'Deepgram API key',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Deepgram model to use (nova-3, nova-2, whisper-large, etc.)',
+    },
+    audioFile: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Audio or video file to transcribe (e.g., MP3, WAV, M4A, WEBM)',
+    },
+    audioFileReference: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Reference to audio/video file from previous blocks',
+    },
+    audioUrl: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'URL to audio or video file',
+    },
+    language: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Language code (e.g., "en", "es", "fr") or "auto" for auto-detection',
+    },
+    timestamps: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Timestamp granularity: none, sentence, or word',
+    },
+    diarization: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Enable speaker diarization',
+    },
+  },
+
+  request: {
+    url: '/api/tools/stt',
+    method: 'POST',
+    headers: () => ({
+      'Content-Type': 'application/json',
+    }),
+    body: (
+      params: SttParams & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'deepgram',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      audioUrl: params.audioUrl,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      diarization: params.diarization || false,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    if (!response.ok || data.error) {
+      return {
+        success: false,
+        error: data.error || 'Transcription failed',
+        output: {
+          transcript: '',
+        },
+      }
+    }
+
+    return {
+      success: true,
+      output: {
+        transcript: data.transcript,
+        segments: data.segments,
+        language: data.language,
+        duration: data.duration,
+        confidence: data.confidence,
+      },
+    }
+  },
+
+  outputs: {
+    transcript: { type: 'string', description: 'Full transcribed text' },
+    segments: {
+      type: 'array',
+      description: 'Timestamped segments with speaker labels',
+      items: {
+        type: 'object',
+        properties: STT_SEGMENT_OUTPUT_PROPERTIES,
+      },
+    },
+    language: { type: 'string', description: 'Detected or specified language' },
+    duration: { type: 'number', description: 'Audio duration in seconds' },
+    confidence: { type: 'number', description: 'Overall confidence score' },
+  },
+}
+
+const deepgramSttV2Params = {
+  provider: deepgramSttTool.params.provider,
+  apiKey: deepgramSttTool.params.apiKey,
+  model: deepgramSttTool.params.model,
+  audioFile: deepgramSttTool.params.audioFile,
+  audioFileReference: deepgramSttTool.params.audioFileReference,
+  language: deepgramSttTool.params.language,
+  timestamps: deepgramSttTool.params.timestamps,
+  diarization: deepgramSttTool.params.diarization,
+  translateToEnglish: deepgramSttTool.params.translateToEnglish,
+} satisfies ToolConfig['params']
+
+export const deepgramSttV2Tool: ToolConfig<SttV2Params, SttResponse> = {
+  ...deepgramSttTool,
+  id: 'stt_deepgram_v2',
+  name: 'Deepgram STT',
+  params: deepgramSttV2Params,
+  request: {
+    ...deepgramSttTool.request,
+    body: (
+      params: SttV2Params & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'deepgram',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      diarization: params.diarization || false,
+      translateToEnglish: params.translateToEnglish || false,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+}

apps/sim/tools/stt/elevenlabs.ts

@@ -0,0 +1,154 @@
+import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const elevenLabsSttTool: ToolConfig<SttParams, SttResponse> = {
+  id: 'stt_elevenlabs',
+  name: 'ElevenLabs STT',
+  description: 'Transcribe audio to text using ElevenLabs',
+  version: '1.0.0',
+
+  params: {
+    provider: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'STT provider (elevenlabs)',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'ElevenLabs API key',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'ElevenLabs model to use (scribe_v2)',
+    },
+    audioFile: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Audio or video file to transcribe (e.g., MP3, WAV, M4A, WEBM)',
+    },
+    audioFileReference: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Reference to audio/video file from previous blocks',
+    },
+    audioUrl: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'URL to audio or video file',
+    },
+    language: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Language code (e.g., "en", "es", "fr") or "auto" for auto-detection',
+    },
+    timestamps: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Timestamp granularity: none, sentence, or word',
+    },
+  },
+
+  request: {
+    url: '/api/tools/stt',
+    method: 'POST',
+    headers: () => ({
+      'Content-Type': 'application/json',
+    }),
+    body: (
+      params: SttParams & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'elevenlabs',
+      apiKey: params.apiKey,
+      model: 'scribe_v2',
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      audioUrl: params.audioUrl,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    if (!response.ok || data.error) {
+      return {
+        success: false,
+        error: data.error || 'Transcription failed',
+        output: {
+          transcript: '',
+        },
+      }
+    }
+
+    return {
+      success: true,
+      output: {
+        transcript: data.transcript,
+        segments: data.segments,
+        language: data.language,
+        duration: data.duration,
+        confidence: data.confidence,
+      },
+    }
+  },
+
+  outputs: {
+    transcript: { type: 'string', description: 'Full transcribed text' },
+    segments: { type: 'array', description: 'Timestamped segments' },
+    language: { type: 'string', description: 'Detected or specified language' },
+    duration: { type: 'number', description: 'Audio duration in seconds' },
+    confidence: { type: 'number', description: 'Overall confidence score' },
+  },
+}
+
+const elevenLabsSttV2Params = {
+  provider: elevenLabsSttTool.params.provider,
+  apiKey: elevenLabsSttTool.params.apiKey,
+  model: elevenLabsSttTool.params.model,
+  audioFile: elevenLabsSttTool.params.audioFile,
+  audioFileReference: elevenLabsSttTool.params.audioFileReference,
+  language: elevenLabsSttTool.params.language,
+  timestamps: elevenLabsSttTool.params.timestamps,
+} satisfies ToolConfig['params']
+
+export const elevenLabsSttV2Tool: ToolConfig<SttV2Params, SttResponse> = {
+  ...elevenLabsSttTool,
+  id: 'stt_elevenlabs_v2',
+  name: 'ElevenLabs STT',
+  params: elevenLabsSttV2Params,
+  request: {
+    ...elevenLabsSttTool.request,
+    body: (
+      params: SttV2Params & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'elevenlabs',
+      apiKey: params.apiKey,
+      model: 'scribe_v2',
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+}

apps/sim/tools/stt/gemini.ts

@@ -0,0 +1,154 @@
+import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const geminiSttTool: ToolConfig<SttParams, SttResponse> = {
+  id: 'stt_gemini',
+  name: 'Google Gemini STT',
+  description: 'Transcribe audio to text using Google Gemini with multimodal capabilities',
+  version: '1.0.0',
+
+  params: {
+    provider: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'STT provider (gemini)',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'Google API key',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Gemini model to use (default: gemini-2.5-flash)',
+    },
+    audioFile: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Audio or video file to transcribe (e.g., MP3, WAV, M4A, WEBM)',
+    },
+    audioFileReference: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Reference to audio/video file from previous blocks',
+    },
+    audioUrl: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'URL to audio or video file',
+    },
+    language: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Language code (e.g., "en", "es", "fr") or "auto" for auto-detection',
+    },
+    timestamps: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Timestamp granularity: none, sentence, or word',
+    },
+  },
+
+  request: {
+    url: '/api/tools/stt',
+    method: 'POST',
+    headers: () => ({
+      'Content-Type': 'application/json',
+    }),
+    body: (
+      params: SttParams & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'gemini',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      audioUrl: params.audioUrl,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    if (!response.ok || data.error) {
+      return {
+        success: false,
+        error: data.error || 'Transcription failed',
+        output: {
+          transcript: '',
+        },
+      }
+    }
+
+    return {
+      success: true,
+      output: {
+        transcript: data.transcript,
+        segments: data.segments,
+        language: data.language,
+        duration: data.duration,
+        confidence: data.confidence,
+      },
+    }
+  },
+
+  outputs: {
+    transcript: { type: 'string', description: 'Full transcribed text' },
+    segments: { type: 'array', description: 'Timestamped segments' },
+    language: { type: 'string', description: 'Detected or specified language' },
+    duration: { type: 'number', description: 'Audio duration in seconds' },
+    confidence: { type: 'number', description: 'Overall confidence score' },
+  },
+}
+
+const geminiSttV2Params = {
+  provider: geminiSttTool.params.provider,
+  apiKey: geminiSttTool.params.apiKey,
+  model: geminiSttTool.params.model,
+  audioFile: geminiSttTool.params.audioFile,
+  audioFileReference: geminiSttTool.params.audioFileReference,
+  language: geminiSttTool.params.language,
+  timestamps: geminiSttTool.params.timestamps,
+} satisfies ToolConfig['params']
+
+export const geminiSttV2Tool: ToolConfig<SttV2Params, SttResponse> = {
+  ...geminiSttTool,
+  id: 'stt_gemini_v2',
+  name: 'Gemini STT',
+  params: geminiSttV2Params,
+  request: {
+    ...geminiSttTool.request,
+    body: (
+      params: SttV2Params & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'gemini',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+}

apps/sim/tools/stt/index.ts

@@ -0,0 +1,18 @@
+import { assemblyaiSttTool, assemblyaiSttV2Tool } from '@/tools/stt/assemblyai'
+import { deepgramSttTool, deepgramSttV2Tool } from '@/tools/stt/deepgram'
+import { elevenLabsSttTool, elevenLabsSttV2Tool } from '@/tools/stt/elevenlabs'
+import { geminiSttTool, geminiSttV2Tool } from '@/tools/stt/gemini'
+import { whisperSttTool, whisperSttV2Tool } from '@/tools/stt/whisper'
+
+export {
+  whisperSttTool,
+  deepgramSttTool,
+  elevenLabsSttTool,
+  assemblyaiSttTool,
+  geminiSttTool,
+  whisperSttV2Tool,
+  deepgramSttV2Tool,
+  elevenLabsSttV2Tool,
+  assemblyaiSttV2Tool,
+  geminiSttV2Tool,
+}

apps/sim/tools/stt/types.ts

@@ -0,0 +1,147 @@
+import type { UserFile } from '@/executor/types'
+import type { OutputProperty, ToolResponse } from '@/tools/types'
+
+/**
+ * Output property definitions for Speech-to-Text API responses.
+ * Covers multiple providers: OpenAI Whisper, Deepgram, ElevenLabs, AssemblyAI, Google Gemini.
+ */
+
+/**
+ * Output definition for transcript segment objects.
+ * @see https://platform.openai.com/docs/api-reference/audio/verbose-json-object
+ * @see https://developers.deepgram.com/docs/transcription-results
+ * @see https://www.assemblyai.com/docs/api-reference/transcripts
+ */
+export const STT_SEGMENT_OUTPUT_PROPERTIES = {
+  text: { type: 'string', description: 'Transcribed text for this segment' },
+  start: { type: 'number', description: 'Start time in seconds' },
+  end: { type: 'number', description: 'End time in seconds' },
+  speaker: {
+    type: 'string',
+    description: 'Speaker identifier (if diarization enabled)',
+    optional: true,
+  },
+  confidence: { type: 'number', description: 'Confidence score (0-1)', optional: true },
+} as const satisfies Record<string, OutputProperty>
+
+/**
+ * Complete segment output definition
+ */
+export const STT_SEGMENT_OUTPUT: OutputProperty = {
+  type: 'object',
+  description: 'Transcript segment with timing information',
+  properties: STT_SEGMENT_OUTPUT_PROPERTIES,
+}
+
+/**
+ * Output definition for sentiment analysis results (AssemblyAI).
+ * @see https://www.assemblyai.com/docs/audio-intelligence/sentiment-analysis
+ */
+export const STT_SENTIMENT_OUTPUT_PROPERTIES = {
+  text: { type: 'string', description: 'Text that was analyzed' },
+  sentiment: { type: 'string', description: 'Sentiment (POSITIVE, NEGATIVE, NEUTRAL)' },
+  confidence: { type: 'number', description: 'Confidence score' },
+  start: { type: 'number', description: 'Start time in milliseconds', optional: true },
+  end: { type: 'number', description: 'End time in milliseconds', optional: true },
+} as const satisfies Record<string, OutputProperty>
+
+/**
+ * Output definition for entity detection results (AssemblyAI).
+ * @see https://www.assemblyai.com/docs/audio-intelligence/entity-detection
+ */
+export const STT_ENTITY_OUTPUT_PROPERTIES = {
+  entity_type: {
+    type: 'string',
+    description: 'Entity type (e.g., person_name, location, organization)',
+  },
+  text: { type: 'string', description: 'Entity text' },
+  start: { type: 'number', description: 'Start time in milliseconds', optional: true },
+  end: { type: 'number', description: 'End time in milliseconds', optional: true },
+} as const satisfies Record<string, OutputProperty>
+
+export interface SttParams {
+  provider: 'whisper' | 'deepgram' | 'elevenlabs' | 'assemblyai' | 'gemini'
+  apiKey: string
+  model?: string
+  audioFile?: UserFile | UserFile[]
+  audioFileReference?: UserFile | UserFile[]
+  audioUrl?: string
+  language?: string
+  timestamps?: 'none' | 'sentence' | 'word'
+  diarization?: boolean
+  translateToEnglish?: boolean
+  // AssemblyAI-specific options
+  sentiment?: boolean
+  entityDetection?: boolean
+  piiRedaction?: boolean
+  summarization?: boolean
+}
+
+export interface SttV2Params extends Omit<SttParams, 'audioUrl'> {}
+
+export interface TranscriptSegment {
+  text: string
+  start: number
+  end: number
+  speaker?: string
+  confidence?: number
+}
+
+export interface SttResponse extends ToolResponse {
+  output: {
+    transcript: string
+    segments?: TranscriptSegment[]
+    language?: string
+    duration?: number
+    confidence?: number
+    // AssemblyAI-specific outputs
+    sentiment?: any[]
+    entities?: any[]
+    summary?: string
+  }
+}
+
+export interface SttBlockResponse extends ToolResponse {
+  output: {
+    transcript: string
+    segments?: TranscriptSegment[]
+    language?: string
+    duration?: number
+    confidence?: number
+    // AssemblyAI-specific outputs
+    sentiment?: any[]
+    entities?: any[]
+    summary?: string
+  }
+}
+
+// Provider-specific types
+
+interface WhisperParams extends Omit<SttParams, 'provider'> {
+  model?: string
+  responseFormat?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt'
+  temperature?: number
+}
+
+interface DeepgramParams extends Omit<SttParams, 'provider'> {
+  model?: string
+  punctuate?: boolean
+  paragraphs?: boolean
+  utterances?: boolean
+}
+
+interface ElevenLabsSttParams extends Omit<SttParams, 'provider'> {
+  model?: string
+}
+
+interface AssemblyAIParams extends Omit<SttParams, 'provider'> {
+  model?: string
+  sentiment?: boolean
+  entityDetection?: boolean
+  piiRedaction?: boolean
+  summarization?: boolean
+}
+
+interface GeminiParams extends Omit<SttParams, 'provider'> {
+  model?: string
+}

apps/sim/tools/stt/whisper.ts

@@ -0,0 +1,199 @@
+import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types'
+import { STT_SEGMENT_OUTPUT_PROPERTIES } from '@/tools/stt/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const whisperSttTool: ToolConfig<SttParams, SttResponse> = {
+  id: 'stt_whisper',
+  name: 'OpenAI Whisper STT',
+  description: 'Transcribe audio to text using OpenAI Whisper',
+  version: '1.0.0',
+
+  params: {
+    provider: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'STT provider (whisper)',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'OpenAI API key',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Whisper model to use (default: whisper-1)',
+    },
+    audioFile: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Audio or video file to transcribe (e.g., MP3, WAV, M4A, WEBM)',
+    },
+    audioFileReference: {
+      type: 'file',
+      required: false,
+      visibility: 'user-only',
+      description: 'Reference to audio/video file from previous blocks',
+    },
+    audioUrl: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'URL to audio or video file',
+    },
+    language: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Language code (e.g., "en", "es", "fr") or "auto" for auto-detection',
+    },
+    timestamps: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Timestamp granularity: none, sentence, or word',
+    },
+    translateToEnglish: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Translate audio to English',
+    },
+    prompt: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        "Optional text to guide the model's style or continue a previous audio segment. Helps with proper nouns and context.",
+    },
+    temperature: {
+      type: 'number',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        'Sampling temperature between 0 and 1. Higher values make output more random, lower values more focused and deterministic.',
+    },
+    responseFormat: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        'Output format for the transcription (e.g., "json", "text", "srt", "verbose_json", "vtt")',
+    },
+  },
+
+  request: {
+    url: '/api/tools/stt',
+    method: 'POST',
+    headers: () => ({
+      'Content-Type': 'application/json',
+    }),
+    body: (
+      params: SttParams & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'whisper',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      audioUrl: params.audioUrl,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      translateToEnglish: params.translateToEnglish || false,
+      prompt: (params as any).prompt,
+      temperature: (params as any).temperature,
+      responseFormat: (params as any).responseFormat,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    if (!response.ok || data.error) {
+      return {
+        success: false,
+        error: data.error || 'Transcription failed',
+        output: {
+          transcript: '',
+        },
+      }
+    }
+
+    return {
+      success: true,
+      output: {
+        transcript: data.transcript,
+        segments: data.segments,
+        language: data.language,
+        duration: data.duration,
+      },
+    }
+  },
+
+  outputs: {
+    transcript: { type: 'string', description: 'Full transcribed text' },
+    segments: {
+      type: 'array',
+      description: 'Timestamped segments',
+      items: {
+        type: 'object',
+        properties: STT_SEGMENT_OUTPUT_PROPERTIES,
+      },
+    },
+    language: { type: 'string', description: 'Detected or specified language' },
+    duration: { type: 'number', description: 'Audio duration in seconds' },
+  },
+}
+
+const whisperSttV2Params = {
+  provider: whisperSttTool.params.provider,
+  apiKey: whisperSttTool.params.apiKey,
+  model: whisperSttTool.params.model,
+  audioFile: whisperSttTool.params.audioFile,
+  audioFileReference: whisperSttTool.params.audioFileReference,
+  language: whisperSttTool.params.language,
+  timestamps: whisperSttTool.params.timestamps,
+  translateToEnglish: whisperSttTool.params.translateToEnglish,
+  prompt: whisperSttTool.params.prompt,
+  temperature: whisperSttTool.params.temperature,
+  responseFormat: whisperSttTool.params.responseFormat,
+} satisfies ToolConfig['params']
+
+export const whisperSttV2Tool: ToolConfig<SttV2Params, SttResponse> = {
+  ...whisperSttTool,
+  id: 'stt_whisper_v2',
+  name: 'OpenAI Whisper STT',
+  params: whisperSttV2Params,
+  request: {
+    ...whisperSttTool.request,
+    body: (
+      params: SttV2Params & {
+        _context?: { workspaceId?: string; workflowId?: string; executionId?: string }
+      }
+    ) => ({
+      provider: 'whisper',
+      apiKey: params.apiKey,
+      model: params.model,
+      audioFile: params.audioFile,
+      audioFileReference: params.audioFileReference,
+      language: params.language || 'auto',
+      timestamps: params.timestamps || 'none',
+      translateToEnglish: params.translateToEnglish || false,
+      prompt: (params as any).prompt,
+      temperature: (params as any).temperature,
+      responseFormat: (params as any).responseFormat,
+      workspaceId: params._context?.workspaceId,
+      workflowId: params._context?.workflowId,
+      executionId: params._context?.executionId,
+    }),
+  },
+}