chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,143 @@
import { createLogger } from '@sim/logger'
import type {
TinybirdAppendDatasourceParams,
TinybirdAppendDatasourceResponse,
} from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-append-datasource')
/**
* Tinybird Append Data Source Tool
*
* Appends data to an existing Data Source from a remote file URL using the
* Data Sources API (`mode=append`). This is asynchronous and returns an import
* job that can be polled for completion.
*/
export const appendDatasourceTool: ToolConfig<
TinybirdAppendDatasourceParams,
TinybirdAppendDatasourceResponse
> = {
id: 'tinybird_append_datasource',
name: 'Tinybird Append Data Source',
description:
'Append data to a Tinybird Data Source from a remote file URL (CSV, NDJSON, Parquet).',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
datasource: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'Name of the existing Data Source to append to. Example: "events_raw"',
},
url: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'Publicly accessible URL of the file to append. Example: "https://example.com/data.csv"',
},
format: {
type: 'string',
required: false,
visibility: 'user-only',
description: 'Format of the source file: "csv" (default), "ndjson", or "parquet"',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with DATASOURCES:APPEND or DATASOURCES:CREATE scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
return `${baseUrl}/v0/datasources`
},
method: 'POST',
headers: (params) => ({
'Content-Type': 'application/x-www-form-urlencoded',
Authorization: `Bearer ${params.token.trim()}`,
}),
body: (params) => {
const searchParams = new URLSearchParams()
searchParams.set('mode', 'append')
searchParams.set('name', params.datasource.trim())
searchParams.set('url', params.url.trim())
if (params.format) {
searchParams.set('format', params.format)
}
return searchParams.toString()
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
logger.info('Started Tinybird append-from-URL import', {
importId: data.import_id ?? data.job?.import_id,
status: data.status ?? data.job?.status,
})
return {
success: true,
output: {
id: data.id ?? null,
import_id: data.import_id ?? data.job?.import_id ?? null,
job_id: data.job_id ?? data.job?.job_id ?? data.job?.id ?? null,
job_url: data.job_url ?? data.job?.job_url ?? null,
status: data.status ?? data.job?.status ?? null,
job: data.job ?? null,
datasource: data.datasource ?? data.job?.datasource ?? null,
},
}
},
outputs: {
id: {
type: 'string',
description: 'Identifier of the append operation',
optional: true,
},
import_id: {
type: 'string',
description: 'Import identifier for the append job',
optional: true,
},
job_id: {
type: 'string',
description: 'Job identifier used to poll import status',
optional: true,
},
job_url: {
type: 'string',
description: 'URL to query the import job status',
optional: true,
},
status: {
type: 'string',
description: 'Initial job status (e.g., "waiting")',
optional: true,
},
job: {
type: 'json',
description: 'Full import job details (kind, id, status, created_at, datasource, ...)',
optional: true,
},
datasource: {
type: 'json',
description: 'Target Data Source metadata (id, name, ...)',
optional: true,
},
},
}
@@ -0,0 +1,136 @@
import { createLogger } from '@sim/logger'
import type {
TinybirdDeleteDatasourceRowsParams,
TinybirdDeleteDatasourceRowsResponse,
} from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-delete-datasource-rows')
/**
* Tinybird Delete Data Source Rows Tool
*
* Deletes rows from a Data Source that match a SQL condition. This is asynchronous
* and returns a delete job that can be polled for completion. Set `dry_run` to test
* the condition without deleting any data.
*/
export const deleteDatasourceRowsTool: ToolConfig<
TinybirdDeleteDatasourceRowsParams,
TinybirdDeleteDatasourceRowsResponse
> = {
id: 'tinybird_delete_datasource_rows',
name: 'Tinybird Delete Data Source Rows',
description: 'Delete rows from a Tinybird Data Source matching a SQL condition.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
datasource: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'Name of the Data Source to delete rows from. Example: "events_raw"',
},
delete_condition: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'SQL WHERE-clause condition selecting the rows to delete. Example: "country = \'ES\'" or "event_date < \'2024-01-01\'"',
},
dry_run: {
type: 'boolean',
required: false,
visibility: 'user-only',
description:
'When true, returns how many rows would be deleted without deleting them. Defaults to false.',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with DATASOURCES:CREATE scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
return `${baseUrl}/v0/datasources/${encodeURIComponent(params.datasource.trim())}/delete`
},
method: 'POST',
headers: (params) => ({
'Content-Type': 'application/x-www-form-urlencoded',
Authorization: `Bearer ${params.token.trim()}`,
}),
body: (params) => {
const searchParams = new URLSearchParams()
searchParams.set('delete_condition', params.delete_condition.trim())
if (params.dry_run) {
searchParams.set('dry_run', 'true')
}
return searchParams.toString()
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
logger.info('Started Tinybird delete-by-condition job', {
deleteId: data.delete_id,
status: data.status ?? data.job?.status,
})
return {
success: true,
output: {
id: data.id ?? null,
job_id: data.job_id ?? data.job?.job_id ?? data.job?.id ?? null,
delete_id: data.delete_id ?? null,
job_url: data.job_url ?? data.job?.job_url ?? null,
status: data.status ?? data.job?.status ?? null,
job: data.job ?? null,
},
}
},
outputs: {
id: {
type: 'string',
description: 'Identifier of the delete operation',
optional: true,
},
job_id: {
type: 'string',
description: 'Job identifier used to poll delete status',
optional: true,
},
delete_id: {
type: 'string',
description: 'Deletion identifier',
optional: true,
},
job_url: {
type: 'string',
description: 'URL to query the delete job status',
optional: true,
},
status: {
type: 'string',
description: 'Current job status (e.g., "waiting", "done")',
optional: true,
},
job: {
type: 'json',
description:
'Full delete job details (kind, id, status, delete_condition, rows_affected, ...)',
optional: true,
},
},
}
+134
View File
@@ -0,0 +1,134 @@
import { gzipSync } from 'zlib'
import { createLogger } from '@sim/logger'
import type { TinybirdEventsParams, TinybirdEventsResponse } from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-events')
export const eventsTool: ToolConfig<TinybirdEventsParams, TinybirdEventsResponse> = {
id: 'tinybird_events',
name: 'Tinybird Events',
description:
'Send events to a Tinybird Data Source using the Events API. Supports JSON and NDJSON formats with optional gzip compression.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description:
'Tinybird API base URL (e.g., https://api.tinybird.co or https://api.us-east.tinybird.co)',
},
datasource: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'Name of the Tinybird Data Source to send events to. Example: "events_raw", "user_analytics"',
},
data: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'Data to send as NDJSON (newline-delimited JSON) or JSON string. Each event should be a valid JSON object. Example NDJSON: {"user_id": 1, "event": "click"}\\n{"user_id": 2, "event": "view"}',
},
wait: {
type: 'boolean',
required: false,
visibility: 'user-only',
description:
'Wait for database acknowledgment before responding. Enables safer retries but introduces latency. Defaults to false.',
},
format: {
type: 'string',
required: false,
visibility: 'user-only',
description: 'Format of the events data: "ndjson" (default) or "json"',
},
compression: {
type: 'string',
required: false,
visibility: 'user-only',
description: 'Compression format: "none" (default) or "gzip"',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with DATASOURCES:APPEND or DATASOURCES:CREATE scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
const url = new URL(`${baseUrl}/v0/events`)
url.searchParams.set('name', params.datasource.trim())
// Tinybird selects JSON parsing via the `format=json` query parameter, not the
// Content-Type header. Default (omitted) is NDJSON.
if (params.format === 'json') {
url.searchParams.set('format', 'json')
}
if (params.wait) {
url.searchParams.set('wait', 'true')
}
return url.toString()
},
method: 'POST',
headers: (params) => {
const headers: Record<string, string> = {
Authorization: `Bearer ${params.token.trim()}`,
}
if (params.compression === 'gzip') {
headers['Content-Encoding'] = 'gzip'
}
if (params.format === 'json') {
headers['Content-Type'] = 'application/json'
} else {
headers['Content-Type'] = 'application/x-ndjson'
}
return headers
},
body: (params) => {
const data = params.data
if (params.compression === 'gzip') {
return gzipSync(Buffer.from(data, 'utf-8'))
}
return data
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
logger.info('Successfully sent events to Tinybird', {
successful: data.successful_rows,
quarantined: data.quarantined_rows,
})
return {
success: true,
output: {
successful_rows: data.successful_rows ?? 0,
quarantined_rows: data.quarantined_rows ?? 0,
},
}
},
outputs: {
successful_rows: {
type: 'number',
description: 'Number of rows successfully ingested',
},
quarantined_rows: {
type: 'number',
description: 'Number of rows quarantined (failed validation)',
},
},
}
+138
View File
@@ -0,0 +1,138 @@
import { createLogger } from '@sim/logger'
import type { TinybirdGetJobParams, TinybirdGetJobResponse } from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-get-job')
/**
* Tinybird Get Job Tool
*
* Polls the status of an asynchronous job (import, delete, populate, copy) by ID.
* Used to check on jobs started by the Append Data Source and Delete Data Source Rows
* operations, which return a job_id but do not wait for completion.
*/
export const getJobTool: ToolConfig<TinybirdGetJobParams, TinybirdGetJobResponse> = {
id: 'tinybird_get_job',
name: 'Tinybird Get Job',
description: 'Check the status of an asynchronous Tinybird job (import, delete, etc.) by ID.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
job_id: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'ID of the job to check, as returned by an append or delete operation',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with ADMIN scope, or the token that started the job',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
return `${baseUrl}/v0/jobs/${encodeURIComponent(params.job_id.trim())}`
},
method: 'GET',
headers: (params) => ({
Authorization: `Bearer ${params.token.trim()}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
logger.info('Fetched Tinybird job status', {
jobId: data.job_id ?? data.id,
kind: data.kind,
status: data.status,
})
return {
success: true,
output: {
id: data.id ?? null,
job_id: data.job_id ?? data.id ?? null,
kind: data.kind ?? null,
status: data.status ?? null,
job_url: data.job_url ?? null,
created_at: data.created_at ?? null,
started_at: data.started_at ?? null,
updated_at: data.updated_at ?? null,
is_cancellable: data.is_cancellable ?? null,
error: data.error ?? null,
job: data,
},
}
},
outputs: {
id: {
type: 'string',
description: 'Job identifier',
optional: true,
},
job_id: {
type: 'string',
description: 'Job identifier (same as id)',
optional: true,
},
kind: {
type: 'string',
description: 'Job kind (e.g., "import", "delete_data", "populateview", "copy")',
optional: true,
},
status: {
type: 'string',
description: 'Current job status: "waiting", "working", "done", "error", or "cancelled"',
optional: true,
},
job_url: {
type: 'string',
description: 'URL to re-query this job status',
optional: true,
},
created_at: {
type: 'string',
description: 'Timestamp the job was created',
optional: true,
},
started_at: {
type: 'string',
description: 'Timestamp the job started running',
optional: true,
},
updated_at: {
type: 'string',
description: 'Timestamp of the last job status update',
optional: true,
},
is_cancellable: {
type: 'boolean',
description: 'Whether the job can still be cancelled',
optional: true,
},
error: {
type: 'string',
description: 'Error message, present only when status is "error"',
optional: true,
},
job: {
type: 'json',
description:
'Full raw job details, including kind-specific fields (statistics, datasource, delete_condition, etc.)',
optional: true,
},
},
}
+15
View File
@@ -0,0 +1,15 @@
import { appendDatasourceTool } from '@/tools/tinybird/append_datasource'
import { deleteDatasourceRowsTool } from '@/tools/tinybird/delete_datasource_rows'
import { eventsTool } from '@/tools/tinybird/events'
import { getJobTool } from '@/tools/tinybird/get_job'
import { queryTool } from '@/tools/tinybird/query'
import { queryPipeTool } from '@/tools/tinybird/query_pipe'
import { truncateDatasourceTool } from '@/tools/tinybird/truncate_datasource'
export const tinybirdEventsTool = eventsTool
export const tinybirdQueryTool = queryTool
export const tinybirdQueryPipeTool = queryPipeTool
export const tinybirdAppendDatasourceTool = appendDatasourceTool
export const tinybirdTruncateDatasourceTool = truncateDatasourceTool
export const tinybirdDeleteDatasourceRowsTool = deleteDatasourceRowsTool
export const tinybirdGetJobTool = getJobTool
+159
View File
@@ -0,0 +1,159 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import type { TinybirdQueryParams, TinybirdQueryResponse } from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-query')
/**
* Tinybird Query Tool
*
* Executes SQL queries against Tinybird and returns results in the format specified in the query.
* - FORMAT JSON: Returns structured data with rows/statistics metadata
* - FORMAT CSV/TSV/etc: Returns raw text string
*
* The tool automatically detects the response format based on Content-Type headers.
*/
export const queryTool: ToolConfig<TinybirdQueryParams, TinybirdQueryResponse> = {
id: 'tinybird_query',
name: 'Tinybird Query',
description: 'Execute SQL queries against Tinybird Pipes and Data Sources using the Query API.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
query: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'SQL query to execute. Specify your desired output format (e.g., FORMAT JSON, FORMAT CSV, FORMAT TSV). JSON format provides structured data, while other formats return raw text. Example: "SELECT * FROM my_datasource LIMIT 100 FORMAT JSON"',
},
pipeline: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description:
'Optional pipe name. When provided, enables SELECT * FROM _ syntax. Example: "my_pipe", "analytics_pipe"',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with PIPES:READ scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
return `${baseUrl}/v0/sql`
},
method: 'POST',
headers: (params) => ({
'Content-Type': 'application/x-www-form-urlencoded',
Authorization: `Bearer ${params.token.trim()}`,
}),
body: (params) => {
const searchParams = new URLSearchParams()
searchParams.set('q', params.query)
if (params.pipeline) {
searchParams.set('pipeline', params.pipeline.trim())
}
return searchParams.toString()
},
},
transformResponse: async (response: Response) => {
const responseText = await response.text()
const contentType = response.headers.get('content-type') || ''
// Check if response is JSON based on content-type or try parsing
const isJson = contentType.includes('application/json') || contentType.includes('text/json')
if (isJson) {
try {
const data = JSON.parse(responseText)
logger.info('Successfully executed Tinybird query (JSON)', {
rows: data.rows,
elapsed: data.statistics?.elapsed,
})
return {
success: true,
output: {
data: data.data ?? [],
meta: data.meta ?? undefined,
rows: data.rows ?? 0,
rows_before_limit_at_least: data.rows_before_limit_at_least ?? undefined,
statistics: data.statistics
? {
elapsed: data.statistics.elapsed,
rows_read: data.statistics.rows_read,
bytes_read: data.statistics.bytes_read,
}
: undefined,
},
}
} catch (parseError) {
logger.error('Failed to parse JSON response', {
contentType,
parseError: toError(parseError).message,
})
throw new Error(`Invalid JSON response: ${getErrorMessage(parseError, 'Parse error')}`)
}
}
// For non-JSON formats (CSV, TSV, etc.), return as raw text
logger.info('Successfully executed Tinybird query (non-JSON)', { contentType })
return {
success: true,
output: {
data: responseText,
rows: undefined,
statistics: undefined,
},
}
},
outputs: {
data: {
type: 'json',
description:
'Query result data. For FORMAT JSON: array of objects. For other formats (CSV, TSV, etc.): raw text string.',
},
meta: {
type: 'array',
description: 'Column metadata for the result set (only available with FORMAT JSON)',
optional: true,
items: {
type: 'object',
properties: {
name: { type: 'string', description: 'Column name' },
type: { type: 'string', description: 'Column data type' },
},
},
},
rows: {
type: 'number',
description: 'Number of rows returned (only available with FORMAT JSON)',
},
rows_before_limit_at_least: {
type: 'number',
description:
'Minimum number of rows there would be without a LIMIT clause (only available with FORMAT JSON)',
optional: true,
},
statistics: {
type: 'json',
description:
'Query execution statistics - elapsed time, rows read, bytes read (only available with FORMAT JSON)',
},
},
}
+174
View File
@@ -0,0 +1,174 @@
import { createLogger } from '@sim/logger'
import type { TinybirdQueryPipeParams, TinybirdQueryPipeResponse } from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-query-pipe')
/**
* Parses the dynamic-parameters input, which may arrive as a JSON object or a
* JSON string from a code/json subBlock. An omitted or empty value means "no
* parameters"; a non-empty value that is not a valid JSON object throws, so a
* mistyped input fails loudly instead of silently dropping the filters.
*/
function parsePipeParameters(
input: TinybirdQueryPipeParams['parameters']
): Record<string, unknown> {
if (input === undefined || input === null) return {}
if (typeof input === 'object') return input as Record<string, unknown>
const trimmed = input.trim()
if (!trimmed) return {}
let parsed: unknown
try {
parsed = JSON.parse(trimmed)
} catch {
throw new Error(
'Invalid Pipe parameters: expected a JSON object of key/value pairs (e.g. {"limit": 10})'
)
}
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
throw new Error('Invalid Pipe parameters: expected a JSON object, not a primitive or array')
}
return parsed as Record<string, unknown>
}
/**
* Tinybird Query Pipe Tool
*
* Calls a published Tinybird Pipe API Endpoint by name using the `.json` format,
* which is an alias for `SELECT * FROM {pipe}`. Templated Pipe parameters are passed
* as query-string arguments, and an optional `q` lets you run SQL on top of the result
* (using `_` to reference the Pipe).
*/
export const queryPipeTool: ToolConfig<TinybirdQueryPipeParams, TinybirdQueryPipeResponse> = {
id: 'tinybird_query_pipe',
name: 'Tinybird Query Pipe',
description:
'Call a published Tinybird Pipe API Endpoint by name, passing dynamic parameters and receiving structured JSON results.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
pipe: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'Name of the published Pipe API Endpoint to call. Example: "top_pages"',
},
parameters: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description:
'Dynamic Pipe parameters as a JSON object, sent as query-string arguments. Example: {"start_date": "2024-01-01", "limit": 10}',
},
q: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description:
'Optional SQL to run on top of the Pipe result. Use "_" to reference the Pipe. Example: "SELECT count() FROM _"',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with PIPES:READ scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
const url = new URL(`${baseUrl}/v0/pipes/${encodeURIComponent(params.pipe.trim())}.json`)
if (params.q) {
url.searchParams.set('q', params.q)
}
const dynamic = parsePipeParameters(params.parameters)
for (const [key, value] of Object.entries(dynamic)) {
// Don't let a dynamic parameter clobber the reserved `q` set above
if (key === 'q') continue
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value))
}
}
return url.toString()
},
method: 'GET',
headers: (params) => ({
Authorization: `Bearer ${params.token.trim()}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
logger.info('Successfully called Tinybird Pipe endpoint', {
rows: data.rows,
elapsed: data.statistics?.elapsed,
})
return {
success: true,
output: {
data: data.data ?? [],
meta: data.meta ?? undefined,
rows: data.rows ?? undefined,
rows_before_limit_at_least: data.rows_before_limit_at_least ?? undefined,
statistics: data.statistics
? {
elapsed: data.statistics.elapsed,
rows_read: data.statistics.rows_read,
bytes_read: data.statistics.bytes_read,
}
: undefined,
},
}
},
outputs: {
data: {
type: 'json',
description: 'Pipe result data as an array of row objects',
},
meta: {
type: 'array',
description: 'Column metadata for the result set',
optional: true,
items: {
type: 'object',
properties: {
name: { type: 'string', description: 'Column name' },
type: { type: 'string', description: 'Column data type' },
},
},
},
rows: {
type: 'number',
description: 'Number of rows returned',
optional: true,
},
rows_before_limit_at_least: {
type: 'number',
description: 'Minimum number of rows there would be without a LIMIT clause',
optional: true,
},
statistics: {
type: 'json',
description: 'Query execution statistics - elapsed time, rows read, bytes read',
optional: true,
properties: {
elapsed: { type: 'number', description: 'Query execution time in seconds' },
rows_read: { type: 'number', description: 'Number of rows processed' },
bytes_read: { type: 'number', description: 'Number of bytes processed' },
},
},
},
}
@@ -0,0 +1,91 @@
import { createLogger } from '@sim/logger'
import type {
TinybirdTruncateDatasourceParams,
TinybirdTruncateDatasourceResponse,
} from '@/tools/tinybird/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('tinybird-truncate-datasource')
/**
* Tinybird Truncate Data Source Tool
*
* Deletes all rows from a Data Source. Dependent Materialized Views are not
* truncated in cascade. The endpoint returns a minimal (often empty) body on success.
*/
export const truncateDatasourceTool: ToolConfig<
TinybirdTruncateDatasourceParams,
TinybirdTruncateDatasourceResponse
> = {
id: 'tinybird_truncate_datasource',
name: 'Tinybird Truncate Data Source',
description: 'Delete all rows from a Tinybird Data Source.',
version: '1.0.0',
errorExtractor: 'nested-error-object',
params: {
base_url: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API base URL (e.g., https://api.tinybird.co)',
},
datasource: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'Name of the Data Source to truncate. Example: "events_raw"',
},
token: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Tinybird API Token with DATASOURCES:CREATE scope',
},
},
request: {
url: (params) => {
const baseUrl = params.base_url.trim().replace(/\/+$/, '')
return `${baseUrl}/v0/datasources/${encodeURIComponent(params.datasource.trim())}/truncate`
},
method: 'POST',
headers: (params) => ({
Authorization: `Bearer ${params.token.trim()}`,
}),
},
transformResponse: async (response: Response) => {
const text = await response.text()
let result: Record<string, unknown> | null = null
if (text) {
try {
result = JSON.parse(text)
} catch {
result = null
}
}
logger.info('Successfully truncated Tinybird Data Source')
return {
success: true,
output: {
truncated: true,
result,
},
}
},
outputs: {
truncated: {
type: 'boolean',
description: 'Whether the Data Source was truncated successfully',
},
result: {
type: 'json',
description: 'Raw response body from the truncate endpoint, if any',
optional: true,
},
},
}
+190
View File
@@ -0,0 +1,190 @@
import type { ToolResponse } from '@/tools/types'
/**
* Base parameters for Tinybird API tools
*/
interface TinybirdBaseParams {
token: string
}
/**
* Parameters for sending events to Tinybird
*/
export interface TinybirdEventsParams extends TinybirdBaseParams {
base_url: string
datasource: string
data: string
wait?: boolean
format?: 'ndjson' | 'json'
compression?: 'none' | 'gzip'
}
/**
* Response from sending events to Tinybird
*/
export interface TinybirdEventsResponse extends ToolResponse {
output: {
successful_rows: number
quarantined_rows: number
}
}
/**
* Parameters for querying Tinybird
*/
export interface TinybirdQueryParams extends TinybirdBaseParams {
base_url: string
query: string
pipeline?: string
}
/**
* Response from querying Tinybird
*/
export interface TinybirdQueryResponse extends ToolResponse {
output: {
data: unknown[] | string
meta?: Array<{ name: string; type: string }>
rows?: number
rows_before_limit_at_least?: number
statistics?: TinybirdQueryStatistics
}
}
/**
* Query execution statistics returned by the Query API and Pipe endpoints
*/
interface TinybirdQueryStatistics {
elapsed: number
rows_read: number
bytes_read: number
}
/**
* Parameters for calling a published Pipe API Endpoint by name
*/
export interface TinybirdQueryPipeParams extends TinybirdBaseParams {
base_url: string
pipe: string
parameters?: Record<string, unknown> | string
q?: string
}
/**
* Response from calling a published Pipe API Endpoint (`.json` format)
*/
export interface TinybirdQueryPipeResponse extends ToolResponse {
output: {
data: unknown[]
meta?: Array<{ name: string; type: string }>
rows?: number
rows_before_limit_at_least?: number
statistics?: TinybirdQueryStatistics
}
}
/**
* Parameters for appending data to a Data Source from a URL
*/
export interface TinybirdAppendDatasourceParams extends TinybirdBaseParams {
base_url: string
datasource: string
url: string
format?: 'csv' | 'ndjson' | 'parquet'
}
/**
* Response from an append-from-URL import job
*/
export interface TinybirdAppendDatasourceResponse extends ToolResponse {
output: {
id: string | null
import_id: string | null
job_id: string | null
job_url: string | null
status: string | null
job: Record<string, unknown> | null
datasource: Record<string, unknown> | null
}
}
/**
* Parameters for truncating (deleting all rows from) a Data Source
*/
export interface TinybirdTruncateDatasourceParams extends TinybirdBaseParams {
base_url: string
datasource: string
}
/**
* Response from truncating a Data Source
*/
export interface TinybirdTruncateDatasourceResponse extends ToolResponse {
output: {
truncated: boolean
result: Record<string, unknown> | null
}
}
/**
* Parameters for deleting rows from a Data Source by condition
*/
export interface TinybirdDeleteDatasourceRowsParams extends TinybirdBaseParams {
base_url: string
datasource: string
delete_condition: string
dry_run?: boolean
}
/**
* Response from a delete-by-condition job
*/
export interface TinybirdDeleteDatasourceRowsResponse extends ToolResponse {
output: {
id: string | null
job_id: string | null
delete_id: string | null
job_url: string | null
status: string | null
job: Record<string, unknown> | null
}
}
/**
* Parameters for checking the status of an asynchronous job
*/
export interface TinybirdGetJobParams extends TinybirdBaseParams {
base_url: string
job_id: string
}
/**
* Response from checking an asynchronous job's status
*/
export interface TinybirdGetJobResponse extends ToolResponse {
output: {
id: string | null
job_id: string | null
kind: string | null
status: string | null
job_url: string | null
created_at: string | null
started_at: string | null
updated_at: string | null
is_cancellable: boolean | null
error: string | null
job: Record<string, unknown> | null
}
}
/**
* Union type for all possible Tinybird responses
*/
export type TinybirdResponse =
| TinybirdEventsResponse
| TinybirdQueryResponse
| TinybirdQueryPipeResponse
| TinybirdAppendDatasourceResponse
| TinybirdTruncateDatasourceResponse
| TinybirdDeleteDatasourceRowsResponse
| TinybirdGetJobResponse