chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
import type { StorageContext } from '@/lib/uploads/shared/types'
|
||||
|
||||
/**
|
||||
* Server-proxied fallback used only when cloud storage isn't configured (local dev).
|
||||
* Production always takes the presigned PUT path.
|
||||
*/
|
||||
export async function uploadViaApiFallback(
|
||||
file: File,
|
||||
context: StorageContext,
|
||||
workspaceId?: string
|
||||
): Promise<{ path: string; key?: string }> {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
formData.append('context', context)
|
||||
if (workspaceId) {
|
||||
formData.append('workspaceId', workspaceId)
|
||||
}
|
||||
|
||||
// boundary-raw-fetch: local-dev fallback when cloud storage is not configured; multipart upload incompatible with requestJson
|
||||
const response = await fetch('/api/files/upload', { method: 'POST', body: formData })
|
||||
if (!response.ok) {
|
||||
const errorData = (await response.json().catch(() => ({}))) as {
|
||||
message?: string
|
||||
error?: string
|
||||
}
|
||||
throw new Error(
|
||||
errorData.message || errorData.error || `Failed to upload file: ${response.status}`
|
||||
)
|
||||
}
|
||||
const data = (await response.json()) as {
|
||||
fileInfo?: { path?: string; key?: string }
|
||||
path?: string
|
||||
key?: string
|
||||
url?: string
|
||||
}
|
||||
const path = data.fileInfo?.path ?? data.path ?? data.url
|
||||
const key = data.fileInfo?.key ?? data.key
|
||||
if (!path) {
|
||||
throw new Error('Invalid upload response: missing path')
|
||||
}
|
||||
return { path, key }
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/**
|
||||
* @vitest-environment jsdom
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
DirectUploadError,
|
||||
type PresignedUploadInfo,
|
||||
runUploadStrategy,
|
||||
} from '@/lib/uploads/client/direct-upload'
|
||||
|
||||
const ONE_MB = 1024 * 1024
|
||||
const LARGE_THRESHOLD = 50 * ONE_MB
|
||||
|
||||
const makeFile = (size: number, name = 'test.bin', type = 'application/octet-stream'): File => {
|
||||
const file = new File([new Uint8Array(0)], name, { type })
|
||||
Object.defineProperty(file, 'size', { value: size })
|
||||
return file
|
||||
}
|
||||
|
||||
const presigned = (overrides?: Partial<PresignedUploadInfo>): PresignedUploadInfo => ({
|
||||
fileName: 'test.bin',
|
||||
presignedUrl: 'https://s3/presigned',
|
||||
fileInfo: {
|
||||
path: '/api/files/serve/test',
|
||||
key: 'workspace/ws-1/test.bin',
|
||||
name: 'test.bin',
|
||||
size: ONE_MB,
|
||||
type: 'application/octet-stream',
|
||||
},
|
||||
uploadHeaders: undefined,
|
||||
directUploadSupported: true,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
class MockXHR {
|
||||
static instances: MockXHR[] = []
|
||||
upload = { addEventListener: vi.fn() }
|
||||
status = 200
|
||||
statusText = 'OK'
|
||||
private listeners: Record<string, Array<() => void>> = {}
|
||||
open = vi.fn()
|
||||
setRequestHeader = vi.fn()
|
||||
abort = vi.fn()
|
||||
send = vi.fn(() => {
|
||||
queueMicrotask(() => this.listeners.load?.forEach((cb) => cb()))
|
||||
})
|
||||
addEventListener = (event: string, cb: () => void) => {
|
||||
;(this.listeners[event] ??= []).push(cb)
|
||||
}
|
||||
removeEventListener = vi.fn()
|
||||
constructor() {
|
||||
MockXHR.instances.push(this)
|
||||
}
|
||||
}
|
||||
|
||||
describe('runUploadStrategy', () => {
|
||||
let originalXHR: typeof XMLHttpRequest
|
||||
|
||||
beforeEach(() => {
|
||||
MockXHR.instances = []
|
||||
originalXHR = globalThis.XMLHttpRequest
|
||||
globalThis.XMLHttpRequest = MockXHR as unknown as typeof XMLHttpRequest
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.XMLHttpRequest = originalXHR
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('uses presigned PUT for files at or below the multipart threshold', async () => {
|
||||
const file = makeFile(LARGE_THRESHOLD)
|
||||
|
||||
const result = await runUploadStrategy({
|
||||
file,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'workspace',
|
||||
presignedOverride: presigned(),
|
||||
})
|
||||
|
||||
expect(result.key).toBe('workspace/ws-1/test.bin')
|
||||
expect(MockXHR.instances).toHaveLength(1)
|
||||
expect(MockXHR.instances[0].open).toHaveBeenCalledWith('PUT', 'https://s3/presigned')
|
||||
})
|
||||
|
||||
it('throws FALLBACK_REQUIRED when server signals no cloud storage', async () => {
|
||||
const file = makeFile(ONE_MB)
|
||||
|
||||
await expect(
|
||||
runUploadStrategy({
|
||||
file,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'workspace',
|
||||
presignedOverride: presigned({ presignedUrl: '', directUploadSupported: false }),
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
name: 'DirectUploadError',
|
||||
code: 'FALLBACK_REQUIRED',
|
||||
})
|
||||
})
|
||||
|
||||
it('takes the multipart path for files larger than the threshold and posts unified parts', async () => {
|
||||
const file = makeFile(LARGE_THRESHOLD + ONE_MB)
|
||||
const calls: Array<{ url: string; body: unknown }> = []
|
||||
|
||||
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = typeof input === 'string' ? input : input.toString()
|
||||
const rawBody = init?.body
|
||||
const body = typeof rawBody === 'string' ? JSON.parse(rawBody) : undefined
|
||||
calls.push({ url, body })
|
||||
|
||||
if (url.includes('action=initiate')) {
|
||||
return new Response(
|
||||
JSON.stringify({ uploadId: 'u1', key: 'workspace/ws-1/big.bin', uploadToken: 't' }),
|
||||
{ status: 200 }
|
||||
)
|
||||
}
|
||||
if (url.includes('action=get-part-urls')) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
presignedUrls: [
|
||||
{ partNumber: 1, url: 'https://s3/part1' },
|
||||
{ partNumber: 2, url: 'https://s3/part2' },
|
||||
],
|
||||
}),
|
||||
{ status: 200 }
|
||||
)
|
||||
}
|
||||
if (url.startsWith('https://s3/part')) {
|
||||
return new Response(null, { status: 200, headers: { ETag: '"etag-x"' } })
|
||||
}
|
||||
if (url.includes('action=complete')) {
|
||||
return new Response(JSON.stringify({ path: '/api/files/serve/big' }), { status: 200 })
|
||||
}
|
||||
throw new Error(`unexpected url ${url}`)
|
||||
})
|
||||
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
const result = await runUploadStrategy({
|
||||
file,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'workspace',
|
||||
})
|
||||
|
||||
expect(result.path).toBe('/api/files/serve/big')
|
||||
|
||||
const completeCall = calls.find((c) => c.url.includes('action=complete'))!
|
||||
expect(completeCall.body).toMatchObject({
|
||||
uploadToken: 't',
|
||||
parts: [
|
||||
{ partNumber: 1, etag: 'etag-x' },
|
||||
{ partNumber: 2, etag: 'etag-x' },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects with ABORTED when signal is already aborted before PUT begins', async () => {
|
||||
const file = makeFile(ONE_MB)
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await expect(
|
||||
runUploadStrategy({
|
||||
file,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'workspace',
|
||||
presignedOverride: presigned(),
|
||||
signal: controller.signal,
|
||||
})
|
||||
).rejects.toMatchObject({ name: 'DirectUploadError', code: 'ABORTED' })
|
||||
})
|
||||
|
||||
it('fires action=abort when the multipart complete call fails', async () => {
|
||||
const file = makeFile(LARGE_THRESHOLD + ONE_MB)
|
||||
const calls: Array<{ url: string }> = []
|
||||
|
||||
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
|
||||
const url = typeof input === 'string' ? input : input.toString()
|
||||
calls.push({ url })
|
||||
|
||||
if (url.includes('action=initiate')) {
|
||||
return new Response(
|
||||
JSON.stringify({ uploadId: 'u1', key: 'workspace/ws-1/big.bin', uploadToken: 't' }),
|
||||
{ status: 200 }
|
||||
)
|
||||
}
|
||||
if (url.includes('action=get-part-urls')) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
presignedUrls: [
|
||||
{ partNumber: 1, url: 'https://s3/part1' },
|
||||
{ partNumber: 2, url: 'https://s3/part2' },
|
||||
],
|
||||
}),
|
||||
{ status: 200 }
|
||||
)
|
||||
}
|
||||
if (url.startsWith('https://s3/part')) {
|
||||
return new Response(null, { status: 200, headers: { ETag: '"etag-x"' } })
|
||||
}
|
||||
if (url.includes('action=complete')) {
|
||||
return new Response(JSON.stringify({ error: 'kaboom' }), { status: 500 })
|
||||
}
|
||||
if (url.includes('action=abort')) {
|
||||
return new Response(null, { status: 200 })
|
||||
}
|
||||
throw new Error(`unexpected url ${url}`)
|
||||
})
|
||||
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await expect(
|
||||
runUploadStrategy({ file, workspaceId: 'ws-1', context: 'workspace' })
|
||||
).rejects.toBeInstanceOf(DirectUploadError)
|
||||
|
||||
expect(calls.some((c) => c.url.includes('action=abort'))).toBe(true)
|
||||
})
|
||||
|
||||
it('throws when neither presignedEndpoint nor presignedOverride is supplied', async () => {
|
||||
const file = makeFile(ONE_MB)
|
||||
await expect(
|
||||
runUploadStrategy({ file, workspaceId: 'ws-1', context: 'workspace' })
|
||||
).rejects.toBeInstanceOf(DirectUploadError)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,632 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { getFileContentType, isAbortError } from '@/lib/uploads/utils/file-utils'
|
||||
|
||||
const logger = createLogger('DirectUpload')
|
||||
|
||||
const CHUNK_SIZE = 8 * 1024 * 1024
|
||||
export const LARGE_FILE_THRESHOLD = 50 * 1024 * 1024
|
||||
const BASE_TIMEOUT_MS = 2 * 60 * 1000
|
||||
const TIMEOUT_PER_MB_MS = 1500
|
||||
const MAX_TIMEOUT_MS = 10 * 60 * 1000
|
||||
export const MULTIPART_PART_CONCURRENCY = 3
|
||||
export const MULTIPART_MAX_RETRIES = 3
|
||||
export const MULTIPART_RETRY_DELAY_MS = 2000
|
||||
export const MULTIPART_RETRY_BACKOFF = 2
|
||||
export const WHOLE_FILE_PARALLEL_UPLOADS = 3
|
||||
|
||||
interface PresignedFileInfo {
|
||||
path: string
|
||||
key: string
|
||||
name: string
|
||||
size: number
|
||||
type: string
|
||||
}
|
||||
|
||||
export interface PresignedUploadInfo {
|
||||
fileName: string
|
||||
presignedUrl: string
|
||||
fileInfo: PresignedFileInfo
|
||||
uploadHeaders?: Record<string, string>
|
||||
directUploadSupported: boolean
|
||||
}
|
||||
|
||||
export interface UploadStrategyResult {
|
||||
key: string
|
||||
path: string
|
||||
name: string
|
||||
size: number
|
||||
contentType: string
|
||||
}
|
||||
|
||||
export interface UploadProgressEvent {
|
||||
loaded: number
|
||||
total: number
|
||||
percent: number
|
||||
}
|
||||
|
||||
export type DirectUploadErrorCode =
|
||||
| 'PRESIGNED_URL_ERROR'
|
||||
| 'DIRECT_UPLOAD_ERROR'
|
||||
| 'MULTIPART_ERROR'
|
||||
| 'ABORTED'
|
||||
| 'FALLBACK_REQUIRED'
|
||||
|
||||
export class DirectUploadError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: DirectUploadErrorCode,
|
||||
public details?: unknown,
|
||||
public status?: number
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'DirectUploadError'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transport-level upload errors worth retrying at the outer level: timeouts,
|
||||
* network failures, and 5xx from the storage backend. Excludes deterministic
|
||||
* client failures (4xx, `PRESIGNED_URL_ERROR`, `FALLBACK_REQUIRED`) and aborts.
|
||||
*/
|
||||
export const isTransientUploadError = (error: unknown): boolean => {
|
||||
if (!(error instanceof DirectUploadError)) return false
|
||||
if (error.code !== 'DIRECT_UPLOAD_ERROR' && error.code !== 'MULTIPART_ERROR') return false
|
||||
if (error.status === undefined) return true
|
||||
return error.status >= 500 && error.status < 600
|
||||
}
|
||||
|
||||
export const calculateUploadTimeoutMs = (fileSize: number): number => {
|
||||
const sizeInMb = fileSize / (1024 * 1024)
|
||||
const dynamicBudget = BASE_TIMEOUT_MS + sizeInMb * TIMEOUT_PER_MB_MS
|
||||
return Math.min(dynamicBudget, MAX_TIMEOUT_MS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `worker` over `items` with at most `limit` concurrent invocations.
|
||||
* Returns a settled result per item (never rejects), so callers can handle
|
||||
* partial failures explicitly.
|
||||
*/
|
||||
export const runWithConcurrency = async <T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
worker: (item: T, index: number) => Promise<R>
|
||||
): Promise<Array<PromiseSettledResult<R>>> => {
|
||||
const results: Array<PromiseSettledResult<R>> = Array(items.length)
|
||||
if (items.length === 0) return results
|
||||
|
||||
const concurrency = Math.max(1, Math.min(limit, items.length))
|
||||
let nextIndex = 0
|
||||
|
||||
const runners = Array.from({ length: concurrency }, async () => {
|
||||
while (true) {
|
||||
const currentIndex = nextIndex++
|
||||
if (currentIndex >= items.length) break
|
||||
try {
|
||||
const value = await worker(items[currentIndex], currentIndex)
|
||||
results[currentIndex] = { status: 'fulfilled', value }
|
||||
} catch (error) {
|
||||
results[currentIndex] = { status: 'rejected', reason: error }
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
await Promise.all(runners)
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a presigned-upload server response into a {@link PresignedUploadInfo}.
|
||||
* Accepts both single (`/api/files/presigned`) and batch entry shapes, tolerates
|
||||
* `presignedUrl` vs `uploadUrl` aliases, and short-circuits when the server
|
||||
* signals no cloud storage (`directUploadSupported: false`) so callers can fall
|
||||
* back to a server-proxied upload path.
|
||||
*
|
||||
* @throws {@link DirectUploadError} with code `PRESIGNED_URL_ERROR` if the
|
||||
* response is missing a presigned URL or `fileInfo.path`.
|
||||
*/
|
||||
export const normalizePresignedData = (data: unknown, context: string): PresignedUploadInfo => {
|
||||
const d = (data ?? {}) as Record<string, unknown>
|
||||
const presignedUrl = (d.presignedUrl as string) || (d.uploadUrl as string) || ''
|
||||
const fileInfo = d.fileInfo as Record<string, unknown> | undefined
|
||||
const directUploadSupported = d.directUploadSupported !== false
|
||||
|
||||
if (!directUploadSupported) {
|
||||
return {
|
||||
fileName: (d.fileName as string) || context,
|
||||
presignedUrl: '',
|
||||
fileInfo: { path: '', key: '', name: context, size: 0, type: '' },
|
||||
directUploadSupported: false,
|
||||
}
|
||||
}
|
||||
|
||||
if (!presignedUrl || !fileInfo?.path) {
|
||||
throw new DirectUploadError(
|
||||
`Invalid presigned response for ${context}`,
|
||||
'PRESIGNED_URL_ERROR',
|
||||
data
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
fileName: (d.fileName as string) || (fileInfo.name as string) || context,
|
||||
presignedUrl,
|
||||
fileInfo: {
|
||||
path: fileInfo.path as string,
|
||||
key: (fileInfo.key as string) || '',
|
||||
name: (fileInfo.name as string) || context,
|
||||
size: (fileInfo.size as number) || (d.fileSize as number) || 0,
|
||||
type: (fileInfo.type as string) || (d.contentType as string) || '',
|
||||
},
|
||||
uploadHeaders: (d.uploadHeaders as Record<string, string>) || undefined,
|
||||
directUploadSupported: true,
|
||||
}
|
||||
}
|
||||
|
||||
interface GetPresignedOptions {
|
||||
endpoint: string
|
||||
file: File
|
||||
body?: Record<string, unknown>
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single presigned upload URL from a server endpoint that follows the
|
||||
* `{ presignedUrl, fileInfo, uploadHeaders?, directUploadSupported }` contract.
|
||||
*/
|
||||
export const getPresignedUploadInfo = async (
|
||||
opts: GetPresignedOptions
|
||||
): Promise<PresignedUploadInfo> => {
|
||||
const { endpoint, file, body, signal } = opts
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
fileName: file.name,
|
||||
contentType: getFileContentType(file),
|
||||
fileSize: file.size,
|
||||
...body,
|
||||
}),
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
let errorDetails: unknown = null
|
||||
try {
|
||||
errorDetails = await response.json()
|
||||
} catch {}
|
||||
const serverMessage =
|
||||
errorDetails != null &&
|
||||
typeof errorDetails === 'object' &&
|
||||
typeof (errorDetails as Record<string, unknown>).error === 'string'
|
||||
? ((errorDetails as Record<string, unknown>).error as string)
|
||||
: null
|
||||
throw new DirectUploadError(
|
||||
serverMessage ||
|
||||
`Failed to get presigned URL for ${file.name}: ${response.status} ${response.statusText}`,
|
||||
'PRESIGNED_URL_ERROR',
|
||||
errorDetails
|
||||
)
|
||||
}
|
||||
|
||||
return normalizePresignedData(await response.json(), file.name)
|
||||
}
|
||||
|
||||
interface UploadViaPutOptions {
|
||||
file: File
|
||||
presignedUrl: string
|
||||
uploadHeaders?: Record<string, string>
|
||||
signal?: AbortSignal
|
||||
onProgress?: (event: UploadProgressEvent) => void
|
||||
}
|
||||
|
||||
const uploadViaPresignedPut = (opts: UploadViaPutOptions): Promise<void> => {
|
||||
const { file, presignedUrl, uploadHeaders, signal, onProgress } = opts
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest()
|
||||
let isCompleted = false
|
||||
const timeoutMs = calculateUploadTimeoutMs(file.size)
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
if (isCompleted) return
|
||||
isCompleted = true
|
||||
signal?.removeEventListener('abort', abortHandler)
|
||||
xhr.abort()
|
||||
reject(new DirectUploadError(`Upload timeout for ${file.name}`, 'DIRECT_UPLOAD_ERROR'))
|
||||
}, timeoutMs)
|
||||
|
||||
const abortHandler = () => {
|
||||
if (isCompleted) return
|
||||
isCompleted = true
|
||||
clearTimeout(timeoutId)
|
||||
xhr.abort()
|
||||
reject(new DirectUploadError(`Upload aborted for ${file.name}`, 'ABORTED'))
|
||||
}
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
abortHandler()
|
||||
return
|
||||
}
|
||||
signal.addEventListener('abort', abortHandler)
|
||||
}
|
||||
|
||||
xhr.upload.addEventListener('progress', (event) => {
|
||||
if (event.lengthComputable && !isCompleted) {
|
||||
onProgress?.({
|
||||
loaded: event.loaded,
|
||||
total: event.total,
|
||||
percent: Math.round((event.loaded / event.total) * 100),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
xhr.addEventListener('load', () => {
|
||||
if (isCompleted) return
|
||||
isCompleted = true
|
||||
clearTimeout(timeoutId)
|
||||
signal?.removeEventListener('abort', abortHandler)
|
||||
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
onProgress?.({ loaded: file.size, total: file.size, percent: 100 })
|
||||
resolve()
|
||||
} else {
|
||||
reject(
|
||||
new DirectUploadError(
|
||||
`Direct upload failed for ${file.name}: ${xhr.status} ${xhr.statusText}`,
|
||||
'DIRECT_UPLOAD_ERROR',
|
||||
undefined,
|
||||
xhr.status
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
xhr.addEventListener('error', () => {
|
||||
if (isCompleted) return
|
||||
isCompleted = true
|
||||
clearTimeout(timeoutId)
|
||||
signal?.removeEventListener('abort', abortHandler)
|
||||
reject(new DirectUploadError(`Network error uploading ${file.name}`, 'DIRECT_UPLOAD_ERROR'))
|
||||
})
|
||||
|
||||
xhr.open('PUT', presignedUrl)
|
||||
xhr.setRequestHeader('Content-Type', getFileContentType(file))
|
||||
if (uploadHeaders) {
|
||||
for (const [key, value] of Object.entries(uploadHeaders)) {
|
||||
xhr.setRequestHeader(key, value)
|
||||
}
|
||||
}
|
||||
xhr.send(file)
|
||||
})
|
||||
}
|
||||
|
||||
interface MultipartUploadOptions {
|
||||
file: File
|
||||
workspaceId: string
|
||||
context:
|
||||
| 'workspace'
|
||||
| 'knowledge-base'
|
||||
| 'mothership'
|
||||
| 'profile-pictures'
|
||||
| 'workspace-logos'
|
||||
| 'execution'
|
||||
workflowId?: string
|
||||
executionId?: string
|
||||
signal?: AbortSignal
|
||||
onProgress?: (event: UploadProgressEvent) => void
|
||||
}
|
||||
|
||||
interface CompletedPart {
|
||||
partNumber: number
|
||||
etag?: string
|
||||
}
|
||||
|
||||
interface PartUrl {
|
||||
partNumber: number
|
||||
url: string
|
||||
}
|
||||
|
||||
const uploadViaMultipart = async (
|
||||
opts: MultipartUploadOptions
|
||||
): Promise<{ key: string; path: string }> => {
|
||||
const { file, workspaceId, context, workflowId, executionId, signal, onProgress } = opts
|
||||
|
||||
// boundary-raw-fetch: multipart upload control plane uses action query strings; client lifecycle (initiate/get-part-urls/complete/abort) is sequenced manually and not modeled by a single contract
|
||||
const initiateResponse = await fetch('/api/files/multipart?action=initiate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
fileName: file.name,
|
||||
contentType: getFileContentType(file),
|
||||
fileSize: file.size,
|
||||
workspaceId,
|
||||
context,
|
||||
...(workflowId ? { workflowId } : {}),
|
||||
...(executionId ? { executionId } : {}),
|
||||
}),
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!initiateResponse.ok) {
|
||||
let errorBody: { error?: string } | null = null
|
||||
try {
|
||||
errorBody = (await initiateResponse.clone().json()) as { error?: string }
|
||||
} catch {}
|
||||
if (
|
||||
initiateResponse.status === 400 &&
|
||||
typeof errorBody?.error === 'string' &&
|
||||
errorBody.error.toLowerCase().includes('cloud storage')
|
||||
) {
|
||||
throw new DirectUploadError(
|
||||
'Server signaled fallback to API upload',
|
||||
'FALLBACK_REQUIRED',
|
||||
errorBody
|
||||
)
|
||||
}
|
||||
throw new DirectUploadError(
|
||||
`Failed to initiate multipart upload: ${initiateResponse.statusText}`,
|
||||
'MULTIPART_ERROR',
|
||||
undefined,
|
||||
initiateResponse.status
|
||||
)
|
||||
}
|
||||
|
||||
const { key, uploadToken } = (await initiateResponse.json()) as {
|
||||
uploadId: string
|
||||
key: string
|
||||
uploadToken: string
|
||||
}
|
||||
|
||||
const numParts = Math.ceil(file.size / CHUNK_SIZE)
|
||||
const partNumbers = Array.from({ length: numParts }, (_, i) => i + 1)
|
||||
|
||||
const abortMultipart = async () => {
|
||||
try {
|
||||
// boundary-raw-fetch: fire-and-forget abort during multipart cleanup; intentionally avoids contract response parsing so cleanup cannot mask the original error
|
||||
await fetch('/api/files/multipart?action=abort', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ uploadToken }),
|
||||
})
|
||||
} catch (err) {
|
||||
logger.warn('Failed to abort multipart upload:', err)
|
||||
}
|
||||
}
|
||||
|
||||
let presignedUrls: PartUrl[]
|
||||
try {
|
||||
// boundary-raw-fetch: multipart upload control plane uses action query strings; sequenced with initiate/complete/abort outside the contract layer
|
||||
const partUrlsResponse = await fetch('/api/files/multipart?action=get-part-urls', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ uploadToken, partNumbers }),
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!partUrlsResponse.ok) {
|
||||
throw new DirectUploadError(
|
||||
`Failed to get part URLs: ${partUrlsResponse.statusText}`,
|
||||
'MULTIPART_ERROR',
|
||||
undefined,
|
||||
partUrlsResponse.status
|
||||
)
|
||||
}
|
||||
|
||||
;({ presignedUrls } = (await partUrlsResponse.json()) as { presignedUrls: PartUrl[] })
|
||||
} catch (err) {
|
||||
await abortMultipart()
|
||||
throw err
|
||||
}
|
||||
|
||||
const completedBytes = new Array<number>(numParts).fill(0)
|
||||
const reportProgress = () => {
|
||||
const loaded = completedBytes.reduce((a, b) => a + b, 0)
|
||||
onProgress?.({
|
||||
loaded,
|
||||
total: file.size,
|
||||
percent: Math.min(100, Math.round((loaded / file.size) * 100)),
|
||||
})
|
||||
}
|
||||
|
||||
const uploadedParts: CompletedPart[] = []
|
||||
|
||||
try {
|
||||
const uploadPart = async ({ partNumber, url }: PartUrl): Promise<CompletedPart> => {
|
||||
const start = (partNumber - 1) * CHUNK_SIZE
|
||||
const end = Math.min(start + CHUNK_SIZE, file.size)
|
||||
const chunk = file.slice(start, end)
|
||||
|
||||
for (let attempt = 0; attempt <= MULTIPART_MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const partResponse = await fetch(url, {
|
||||
method: 'PUT',
|
||||
body: chunk,
|
||||
signal,
|
||||
headers: { 'Content-Type': getFileContentType(file) },
|
||||
})
|
||||
|
||||
if (!partResponse.ok) {
|
||||
throw new DirectUploadError(
|
||||
`Failed to upload part ${partNumber}: ${partResponse.statusText}`,
|
||||
'MULTIPART_ERROR',
|
||||
undefined,
|
||||
partResponse.status
|
||||
)
|
||||
}
|
||||
|
||||
const etag = partResponse.headers.get('ETag') || undefined
|
||||
completedBytes[partNumber - 1] = end - start
|
||||
reportProgress()
|
||||
|
||||
return { partNumber, etag: etag?.replace(/"/g, '') }
|
||||
} catch (partError) {
|
||||
const isClientError =
|
||||
partError instanceof DirectUploadError &&
|
||||
partError.status !== undefined &&
|
||||
partError.status >= 400 &&
|
||||
partError.status < 500
|
||||
if (isAbortError(partError) || isClientError || attempt >= MULTIPART_MAX_RETRIES) {
|
||||
throw partError
|
||||
}
|
||||
const delay = MULTIPART_RETRY_DELAY_MS * MULTIPART_RETRY_BACKOFF ** attempt
|
||||
logger.warn(
|
||||
`Part ${partNumber} failed (attempt ${attempt + 1}), retrying in ${Math.round(delay / 1000)}s`
|
||||
)
|
||||
await sleep(delay)
|
||||
}
|
||||
}
|
||||
|
||||
throw new DirectUploadError(`Retries exhausted for part ${partNumber}`, 'MULTIPART_ERROR')
|
||||
}
|
||||
|
||||
const partResults = await runWithConcurrency(
|
||||
presignedUrls,
|
||||
MULTIPART_PART_CONCURRENCY,
|
||||
uploadPart
|
||||
)
|
||||
|
||||
for (const result of partResults) {
|
||||
if (result?.status === 'fulfilled') {
|
||||
uploadedParts.push(result.value)
|
||||
} else if (result?.status === 'rejected') {
|
||||
throw result.reason
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await abortMultipart()
|
||||
throw error
|
||||
}
|
||||
|
||||
let path: string
|
||||
try {
|
||||
// boundary-raw-fetch: multipart upload control plane uses action query strings; sequenced with initiate/get-part-urls/abort outside the contract layer
|
||||
const completeResponse = await fetch('/api/files/multipart?action=complete', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ uploadToken, parts: uploadedParts }),
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!completeResponse.ok) {
|
||||
throw new DirectUploadError(
|
||||
`Failed to complete multipart upload: ${completeResponse.statusText}`,
|
||||
'MULTIPART_ERROR',
|
||||
undefined,
|
||||
completeResponse.status
|
||||
)
|
||||
}
|
||||
|
||||
;({ path } = (await completeResponse.json()) as { path: string })
|
||||
} catch (err) {
|
||||
await abortMultipart()
|
||||
throw err
|
||||
}
|
||||
return { key, path }
|
||||
}
|
||||
|
||||
export interface RunUploadStrategyOptions {
|
||||
file: File
|
||||
workspaceId: string
|
||||
context:
|
||||
| 'workspace'
|
||||
| 'knowledge-base'
|
||||
| 'mothership'
|
||||
| 'profile-pictures'
|
||||
| 'workspace-logos'
|
||||
| 'execution'
|
||||
/** Endpoint to mint a presigned PUT URL. Required unless `presignedOverride` is provided. */
|
||||
presignedEndpoint?: string
|
||||
/** Pre-fetched presigned data (e.g. from a batch endpoint). Skips per-file fetch. */
|
||||
presignedOverride?: PresignedUploadInfo
|
||||
/** Extra JSON body fields for the presigned endpoint. */
|
||||
presignedBody?: Record<string, unknown>
|
||||
/** Required when context is `execution`; forwarded to the multipart route to scope the storage key. */
|
||||
workflowId?: string
|
||||
/** Required when context is `execution`; forwarded to the multipart route to scope the storage key. */
|
||||
executionId?: string
|
||||
signal?: AbortSignal
|
||||
onProgress?: (event: UploadProgressEvent) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy ladder for client-side uploads:
|
||||
* - Files larger than {@link LARGE_FILE_THRESHOLD} use multipart S3/Blob with chunked PUTs.
|
||||
* - Smaller files use a presigned PUT URL (fetched per-file, or supplied via
|
||||
* `presignedOverride` for batched flows like KB).
|
||||
* - If the server signals no cloud storage is configured, a {@link DirectUploadError}
|
||||
* with code `FALLBACK_REQUIRED` is thrown so callers can fall back to a server-proxied path.
|
||||
*/
|
||||
export const runUploadStrategy = async (
|
||||
opts: RunUploadStrategyOptions
|
||||
): Promise<UploadStrategyResult> => {
|
||||
const {
|
||||
file,
|
||||
presignedEndpoint,
|
||||
presignedOverride,
|
||||
presignedBody,
|
||||
workspaceId,
|
||||
context,
|
||||
workflowId,
|
||||
executionId,
|
||||
signal,
|
||||
onProgress,
|
||||
} = opts
|
||||
const contentType = getFileContentType(file)
|
||||
|
||||
if (presignedOverride && !presignedOverride.directUploadSupported) {
|
||||
throw new DirectUploadError('Server signaled fallback to API upload', 'FALLBACK_REQUIRED')
|
||||
}
|
||||
|
||||
if (file.size > LARGE_FILE_THRESHOLD) {
|
||||
const { key, path } = await uploadViaMultipart({
|
||||
file,
|
||||
workspaceId,
|
||||
context,
|
||||
workflowId,
|
||||
executionId,
|
||||
signal,
|
||||
onProgress,
|
||||
})
|
||||
return { key, path, name: file.name, size: file.size, contentType }
|
||||
}
|
||||
|
||||
let presigned: PresignedUploadInfo
|
||||
if (presignedOverride) {
|
||||
presigned = presignedOverride
|
||||
} else {
|
||||
if (!presignedEndpoint) {
|
||||
throw new DirectUploadError(
|
||||
'runUploadStrategy requires either presignedEndpoint or presignedOverride',
|
||||
'PRESIGNED_URL_ERROR'
|
||||
)
|
||||
}
|
||||
presigned = await getPresignedUploadInfo({
|
||||
endpoint: presignedEndpoint,
|
||||
file,
|
||||
body: presignedBody,
|
||||
signal,
|
||||
})
|
||||
}
|
||||
|
||||
if (!presigned.directUploadSupported) {
|
||||
throw new DirectUploadError('Server signaled fallback to API upload', 'FALLBACK_REQUIRED')
|
||||
}
|
||||
|
||||
await uploadViaPresignedPut({
|
||||
file,
|
||||
presignedUrl: presigned.presignedUrl,
|
||||
uploadHeaders: presigned.uploadHeaders,
|
||||
signal,
|
||||
onProgress,
|
||||
})
|
||||
|
||||
return {
|
||||
key: presigned.fileInfo.key,
|
||||
path: presigned.fileInfo.path,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
contentType,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
|
||||
|
||||
export async function triggerFileDownload(record: WorkspaceFileRecord): Promise<void> {
|
||||
const isMarkdown =
|
||||
record.type === 'text/markdown' ||
|
||||
record.type === 'text/x-markdown' ||
|
||||
/\.(?:md|markdown)$/i.test(record.name)
|
||||
|
||||
const url = isMarkdown
|
||||
? `/api/files/export/${encodeURIComponent(record.id)}`
|
||||
: `/api/files/serve/${encodeURIComponent(record.key)}?context=workspace&t=${Date.now()}`
|
||||
|
||||
const response = await fetch(url, { cache: 'no-store' })
|
||||
if (!response.ok) throw new Error(`Failed to download file: ${response.statusText}`)
|
||||
|
||||
const blob = await response.blob()
|
||||
const objectUrl = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = objectUrl
|
||||
a.download =
|
||||
response.headers.get('Content-Disposition')?.match(/filename="([^"]+)"/)?.[1] ?? record.name
|
||||
document.body.appendChild(a)
|
||||
a.click()
|
||||
document.body.removeChild(a)
|
||||
URL.revokeObjectURL(objectUrl)
|
||||
}
|
||||
Reference in New Issue
Block a user