chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { authMockFns, permissionsMock, permissionsMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockIsUsingCloudStorage,
|
||||
mockGetStorageProvider,
|
||||
mockGetStorageConfig,
|
||||
mockCompleteS3MultipartUpload,
|
||||
mockCompleteBlobMultipartUpload,
|
||||
mockDeriveBlobBlockId,
|
||||
mockVerifyUploadToken,
|
||||
mockSignUploadToken,
|
||||
} = vi.hoisted(() => ({
|
||||
mockIsUsingCloudStorage: vi.fn(),
|
||||
mockGetStorageProvider: vi.fn(),
|
||||
mockGetStorageConfig: vi.fn(),
|
||||
mockCompleteS3MultipartUpload: vi.fn(),
|
||||
mockCompleteBlobMultipartUpload: vi.fn(),
|
||||
mockDeriveBlobBlockId: vi.fn(),
|
||||
mockVerifyUploadToken: vi.fn(),
|
||||
mockSignUploadToken: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads', () => ({
|
||||
isUsingCloudStorage: mockIsUsingCloudStorage,
|
||||
getStorageProvider: mockGetStorageProvider,
|
||||
getStorageConfig: mockGetStorageConfig,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/core/upload-token', () => ({
|
||||
signUploadToken: mockSignUploadToken,
|
||||
verifyUploadToken: mockVerifyUploadToken,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/providers/s3/client', () => ({
|
||||
completeS3MultipartUpload: mockCompleteS3MultipartUpload,
|
||||
initiateS3MultipartUpload: mockInitiateS3MultipartUpload,
|
||||
getS3MultipartPartUrls: vi.fn(),
|
||||
abortS3MultipartUpload: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/providers/blob/client', () => ({
|
||||
completeMultipartUpload: mockCompleteBlobMultipartUpload,
|
||||
deriveBlobBlockId: mockDeriveBlobBlockId,
|
||||
initiateMultipartUpload: vi.fn(),
|
||||
getMultipartPartUrls: vi.fn(),
|
||||
abortMultipartUpload: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
|
||||
|
||||
const { mockCheckStorageQuota, mockInitiateS3MultipartUpload } = vi.hoisted(() => ({
|
||||
mockCheckStorageQuota: vi.fn(),
|
||||
mockInitiateS3MultipartUpload: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/storage', () => ({
|
||||
checkStorageQuota: mockCheckStorageQuota,
|
||||
}))
|
||||
|
||||
import { POST } from '@/app/api/files/multipart/route'
|
||||
|
||||
const tokenPayload = {
|
||||
uploadId: 'upload-1',
|
||||
key: 'workspace/ws-1/123-abc-file.bin',
|
||||
userId: 'user-1',
|
||||
workspaceId: 'ws-1',
|
||||
context: 'workspace' as const,
|
||||
}
|
||||
|
||||
const makeRequest = (action: string, body: unknown) =>
|
||||
new NextRequest(`http://localhost/api/files/multipart?action=${action}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
describe('POST /api/files/multipart action=complete', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } })
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write')
|
||||
mockIsUsingCloudStorage.mockReturnValue(true)
|
||||
mockGetStorageConfig.mockReturnValue({ bucket: 'b', region: 'r' })
|
||||
mockVerifyUploadToken.mockReturnValue({ valid: true, payload: tokenPayload })
|
||||
mockSignUploadToken.mockReturnValue('signed-token')
|
||||
mockCompleteS3MultipartUpload.mockResolvedValue({
|
||||
location: 'loc',
|
||||
path: '/api/files/serve/...',
|
||||
key: tokenPayload.key,
|
||||
})
|
||||
mockCompleteBlobMultipartUpload.mockResolvedValue({
|
||||
location: 'loc',
|
||||
path: '/api/files/serve/...',
|
||||
key: tokenPayload.key,
|
||||
})
|
||||
mockDeriveBlobBlockId.mockImplementation(
|
||||
(n: number) => `block-${n.toString().padStart(6, '0')}`
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects parts without partNumber', async () => {
|
||||
mockGetStorageProvider.mockReturnValue('s3')
|
||||
const res = await POST(
|
||||
makeRequest('complete', {
|
||||
uploadToken: 'tok',
|
||||
parts: [{ etag: 'abc' }],
|
||||
})
|
||||
)
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockCompleteS3MultipartUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('S3 path requires etag and forwards { ETag, PartNumber }', async () => {
|
||||
mockGetStorageProvider.mockReturnValue('s3')
|
||||
|
||||
const missingEtag = await POST(
|
||||
makeRequest('complete', {
|
||||
uploadToken: 'tok',
|
||||
parts: [{ partNumber: 1 }],
|
||||
})
|
||||
)
|
||||
expect(missingEtag.status).toBe(500)
|
||||
|
||||
mockCompleteS3MultipartUpload.mockClear()
|
||||
|
||||
const ok = await POST(
|
||||
makeRequest('complete', {
|
||||
uploadToken: 'tok',
|
||||
parts: [
|
||||
{ partNumber: 1, etag: 'aaa' },
|
||||
{ partNumber: 2, etag: 'bbb' },
|
||||
],
|
||||
})
|
||||
)
|
||||
expect(ok.status).toBe(200)
|
||||
expect(mockCompleteS3MultipartUpload).toHaveBeenCalledWith(
|
||||
tokenPayload.key,
|
||||
tokenPayload.uploadId,
|
||||
[
|
||||
{ ETag: 'aaa', PartNumber: 1 },
|
||||
{ ETag: 'bbb', PartNumber: 2 },
|
||||
],
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('Blob path derives blockId from partNumber and ignores etag', async () => {
|
||||
mockGetStorageProvider.mockReturnValue('blob')
|
||||
mockGetStorageConfig.mockReturnValue({
|
||||
containerName: 'c',
|
||||
accountName: 'a',
|
||||
accountKey: 'k',
|
||||
})
|
||||
|
||||
const res = await POST(
|
||||
makeRequest('complete', {
|
||||
uploadToken: 'tok',
|
||||
parts: [{ partNumber: 1, etag: 'irrelevant' }, { partNumber: 2 }],
|
||||
})
|
||||
)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockDeriveBlobBlockId).toHaveBeenCalledWith(1)
|
||||
expect(mockDeriveBlobBlockId).toHaveBeenCalledWith(2)
|
||||
expect(mockCompleteBlobMultipartUpload).toHaveBeenCalledWith(
|
||||
tokenPayload.key,
|
||||
[
|
||||
{ partNumber: 1, blockId: 'block-000001' },
|
||||
{ partNumber: 2, blockId: 'block-000002' },
|
||||
],
|
||||
expect.objectContaining({ containerName: 'c' })
|
||||
)
|
||||
})
|
||||
|
||||
it('returns 403 when token is invalid', async () => {
|
||||
mockGetStorageProvider.mockReturnValue('s3')
|
||||
mockVerifyUploadToken.mockReturnValueOnce({ valid: false })
|
||||
const res = await POST(
|
||||
makeRequest('complete', {
|
||||
uploadToken: 'bad',
|
||||
parts: [{ partNumber: 1, etag: 'a' }],
|
||||
})
|
||||
)
|
||||
expect(res.status).toBe(403)
|
||||
})
|
||||
|
||||
it('batch complete normalizes per upload', async () => {
|
||||
mockGetStorageProvider.mockReturnValue('s3')
|
||||
const res = await POST(
|
||||
makeRequest('complete', {
|
||||
uploads: [
|
||||
{
|
||||
uploadToken: 'tok-a',
|
||||
parts: [{ partNumber: 1, etag: 'aaa' }],
|
||||
},
|
||||
{
|
||||
uploadToken: 'tok-b',
|
||||
parts: [{ partNumber: 1, etag: 'bbb' }],
|
||||
},
|
||||
],
|
||||
})
|
||||
)
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockCompleteS3MultipartUpload).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /api/files/multipart action=initiate quota enforcement', () => {
|
||||
const makeInitiateRequest = (body: unknown) =>
|
||||
new NextRequest('http://localhost/api/files/multipart?action=initiate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } })
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write')
|
||||
mockIsUsingCloudStorage.mockReturnValue(true)
|
||||
mockGetStorageProvider.mockReturnValue('s3')
|
||||
mockGetStorageConfig.mockReturnValue({ bucket: 'b', region: 'r' })
|
||||
mockSignUploadToken.mockReturnValue('signed-token')
|
||||
mockCheckStorageQuota.mockResolvedValue({ allowed: true })
|
||||
mockInitiateS3MultipartUpload.mockResolvedValue({ uploadId: 'up-1', key: 'k/file.bin' })
|
||||
})
|
||||
|
||||
it('blocks upload when fileSize: 0 exceeds quota', async () => {
|
||||
mockCheckStorageQuota.mockResolvedValue({ allowed: false, error: 'Storage limit exceeded' })
|
||||
|
||||
const res = await makeInitiateRequest({
|
||||
fileName: 'file.bin',
|
||||
contentType: 'application/octet-stream',
|
||||
fileSize: 0,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'knowledge-base',
|
||||
})
|
||||
|
||||
const response = await POST(res)
|
||||
expect(response.status).toBe(413)
|
||||
const body = await response.json()
|
||||
expect(body.error).toContain('Storage limit exceeded')
|
||||
})
|
||||
|
||||
it('allows quota-enforced contexts that pass the quota check', async () => {
|
||||
const res = await makeInitiateRequest({
|
||||
fileName: 'doc.pdf',
|
||||
contentType: 'application/pdf',
|
||||
fileSize: 99999,
|
||||
workspaceId: 'ws-1',
|
||||
context: 'knowledge-base',
|
||||
})
|
||||
|
||||
const response = await POST(res)
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockCheckStorageQuota).toHaveBeenCalledWith('user-1', 99999)
|
||||
expect(mockInitiateS3MultipartUpload).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['og-images', 'profile-pictures', 'workspace-logos', 'logs'])(
|
||||
'rejects quota-exempt context %s — not allowed via the multipart endpoint',
|
||||
async (context) => {
|
||||
const res = await makeInitiateRequest({
|
||||
fileName: 'asset.png',
|
||||
contentType: 'image/png',
|
||||
fileSize: 100 * 1024 * 1024 * 1024,
|
||||
workspaceId: 'ws-1',
|
||||
context,
|
||||
})
|
||||
|
||||
const response = await POST(res)
|
||||
expect(response.status).toBe(400)
|
||||
const body = await response.json()
|
||||
expect(body.error).toMatch(/invalid storage context/i)
|
||||
expect(mockCheckStorageQuota).not.toHaveBeenCalled()
|
||||
expect(mockInitiateS3MultipartUpload).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,481 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
abortMultipartUploadContract,
|
||||
type CompleteMultipartBody,
|
||||
completeMultipartUploadContract,
|
||||
getMultipartPartUrlsContract,
|
||||
initiateMultipartUploadContract,
|
||||
multipartActionSchema,
|
||||
} from '@/lib/api/contracts/storage-transfer'
|
||||
import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
|
||||
import { getSession } from '@/lib/auth'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
getStorageConfig,
|
||||
getStorageProvider,
|
||||
isUsingCloudStorage,
|
||||
type StorageContext,
|
||||
} from '@/lib/uploads'
|
||||
import { deleteFile } from '@/lib/uploads/core/storage-service'
|
||||
import {
|
||||
signUploadToken,
|
||||
type UploadTokenPayload,
|
||||
verifyUploadToken,
|
||||
} from '@/lib/uploads/core/upload-token'
|
||||
import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata'
|
||||
import { QUOTA_EXEMPT_STORAGE_CONTEXTS, type StorageConfig } from '@/lib/uploads/shared/types'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
|
||||
const logger = createLogger('MultipartUploadAPI')
|
||||
|
||||
/**
|
||||
* Contexts the multipart endpoint accepts. The quota-exempt public-asset
|
||||
* contexts (`profile-pictures`, `workspace-logos`, `og-images`) and the
|
||||
* system-internal `logs` context are deliberately excluded: their uploads are
|
||||
* small images capped far below the multipart threshold and routed through the
|
||||
* presigned endpoint, so they have no large-file flow here. Accepting them would
|
||||
* only expose a path that bypasses the per-user storage quota, since every
|
||||
* context in this set is quota-enforced below.
|
||||
*/
|
||||
const ALLOWED_UPLOAD_CONTEXTS = new Set<StorageContext>([
|
||||
'knowledge-base',
|
||||
'chat',
|
||||
'copilot',
|
||||
'mothership',
|
||||
'execution',
|
||||
'workspace',
|
||||
])
|
||||
|
||||
/**
|
||||
* Unified part identity sent by the client when completing a multipart upload.
|
||||
* `etag` is required for S3 (CompleteMultipartUpload). For Azure the server
|
||||
* derives the block id from `partNumber` via {@link deriveBlobBlockId}.
|
||||
*/
|
||||
interface ClientCompletedPart {
|
||||
partNumber: number
|
||||
etag?: string
|
||||
}
|
||||
|
||||
const isClientCompletedParts = (value: unknown): value is ClientCompletedPart[] =>
|
||||
Array.isArray(value) &&
|
||||
value.every(
|
||||
(p) =>
|
||||
p !== null &&
|
||||
typeof p === 'object' &&
|
||||
typeof (p as ClientCompletedPart).partNumber === 'number' &&
|
||||
((p as ClientCompletedPart).etag === undefined ||
|
||||
typeof (p as ClientCompletedPart).etag === 'string')
|
||||
)
|
||||
|
||||
const buildS3CustomConfig = (config: StorageConfig) =>
|
||||
config.bucket && config.region ? { bucket: config.bucket, region: config.region } : undefined
|
||||
|
||||
const buildBlobCustomConfig = (config: StorageConfig) => ({
|
||||
containerName: config.containerName!,
|
||||
accountName: config.accountName!,
|
||||
accountKey: config.accountKey,
|
||||
connectionString: config.connectionString,
|
||||
})
|
||||
|
||||
const verifyTokenForUser = (token: string | undefined, userId: string) => {
|
||||
if (!token || typeof token !== 'string') {
|
||||
return null
|
||||
}
|
||||
const result = verifyUploadToken(token)
|
||||
if (!result.valid || result.payload.userId !== userId) {
|
||||
return null
|
||||
}
|
||||
return result.payload
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a trusted storage-key -> workspace ownership binding for completed
|
||||
* knowledge-base uploads. KB file authorization resolves the owning workspace
|
||||
* from this binding, so every KB object must have one. No-op for other contexts.
|
||||
*/
|
||||
const recordKnowledgeBaseOwnership = async (
|
||||
payload: UploadTokenPayload,
|
||||
key: string
|
||||
): Promise<void> => {
|
||||
if (payload.context !== 'knowledge-base' || !payload.workspaceId) {
|
||||
return
|
||||
}
|
||||
await recordKnowledgeBaseFileOwnership({
|
||||
key,
|
||||
userId: payload.userId,
|
||||
workspaceId: payload.workspaceId,
|
||||
originalName: payload.fileName ?? key.split('/').pop() ?? key,
|
||||
contentType: payload.contentType ?? 'application/octet-stream',
|
||||
size: typeof payload.fileSize === 'number' ? payload.fileSize : 0,
|
||||
})
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const session = await getSession()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
const userId = session.user.id
|
||||
|
||||
const actionParam = request.nextUrl.searchParams.get('action')
|
||||
const actionResult = multipartActionSchema.safeParse(actionParam)
|
||||
const action = actionResult.success ? actionResult.data : null
|
||||
|
||||
if (!isUsingCloudStorage()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Multipart upload is only available with cloud storage (S3 or Azure Blob)' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const storageProvider = getStorageProvider()
|
||||
|
||||
switch (action) {
|
||||
case 'initiate': {
|
||||
const parsed = await parseRequest(
|
||||
initiateMultipartUploadContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) =>
|
||||
NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const data = parsed.data.body
|
||||
const { fileName, contentType, fileSize, workspaceId, context = 'knowledge-base' } = data
|
||||
|
||||
if (!workspaceId || typeof workspaceId !== 'string') {
|
||||
return NextResponse.json({ error: 'workspaceId is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (!ALLOWED_UPLOAD_CONTEXTS.has(context as StorageContext)) {
|
||||
return NextResponse.json({ error: 'Invalid storage context' }, { status: 400 })
|
||||
}
|
||||
const storageContext = context as StorageContext
|
||||
|
||||
const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
|
||||
if (permission !== 'write' && permission !== 'admin') {
|
||||
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
|
||||
}
|
||||
|
||||
const config = getStorageConfig(storageContext)
|
||||
|
||||
if (!QUOTA_EXEMPT_STORAGE_CONTEXTS.has(storageContext)) {
|
||||
const { checkStorageQuota } = await import('@/lib/billing/storage')
|
||||
const quotaCheck = await checkStorageQuota(userId, fileSize ?? 0)
|
||||
if (!quotaCheck.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: quotaCheck.error || 'Storage limit exceeded' },
|
||||
{ status: 413 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let customKey: string | undefined
|
||||
if (context === 'workspace' || context === 'mothership') {
|
||||
const { MAX_WORKSPACE_FILE_SIZE } = await import('@/lib/uploads/shared/types')
|
||||
if (typeof fileSize === 'number' && fileSize > MAX_WORKSPACE_FILE_SIZE) {
|
||||
return NextResponse.json(
|
||||
{ error: `File size exceeds maximum of ${MAX_WORKSPACE_FILE_SIZE} bytes` },
|
||||
{ status: 413 }
|
||||
)
|
||||
}
|
||||
|
||||
const { generateWorkspaceFileKey } = await import(
|
||||
'@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
)
|
||||
customKey = generateWorkspaceFileKey(workspaceId, fileName)
|
||||
} else if (context === 'execution') {
|
||||
const workflowId = (data as { workflowId?: unknown }).workflowId
|
||||
const executionId = (data as { executionId?: unknown }).executionId
|
||||
if (typeof workflowId !== 'string' || !workflowId.trim()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'workflowId is required for execution uploads' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
if (typeof executionId !== 'string' || !executionId.trim()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'executionId is required for execution uploads' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const { generateExecutionFileKey } = await import(
|
||||
'@/lib/uploads/contexts/execution/utils'
|
||||
)
|
||||
customKey = generateExecutionFileKey({ workspaceId, workflowId, executionId }, fileName)
|
||||
}
|
||||
|
||||
let uploadId: string
|
||||
let key: string
|
||||
|
||||
if (storageProvider === 's3') {
|
||||
const { initiateS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client')
|
||||
const result = await initiateS3MultipartUpload({
|
||||
fileName,
|
||||
contentType,
|
||||
fileSize,
|
||||
customConfig: buildS3CustomConfig(config),
|
||||
customKey,
|
||||
purpose: context,
|
||||
})
|
||||
uploadId = result.uploadId
|
||||
key = result.key
|
||||
} else if (storageProvider === 'blob') {
|
||||
const { initiateMultipartUpload } = await import('@/lib/uploads/providers/blob/client')
|
||||
const result = await initiateMultipartUpload({
|
||||
fileName,
|
||||
contentType,
|
||||
fileSize,
|
||||
customConfig: buildBlobCustomConfig(config),
|
||||
customKey,
|
||||
})
|
||||
uploadId = result.uploadId
|
||||
key = result.key
|
||||
} else {
|
||||
return NextResponse.json(
|
||||
{ error: `Unsupported storage provider: ${storageProvider}` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const uploadToken = signUploadToken({
|
||||
uploadId,
|
||||
key,
|
||||
userId,
|
||||
workspaceId,
|
||||
context: storageContext,
|
||||
fileName,
|
||||
contentType,
|
||||
...(typeof fileSize === 'number' ? { fileSize } : {}),
|
||||
})
|
||||
|
||||
logger.info(
|
||||
`Initiated ${storageProvider} multipart upload for ${fileName} (context: ${storageContext}, workspace: ${workspaceId}): ${uploadId}`
|
||||
)
|
||||
|
||||
return NextResponse.json({ uploadId, key, uploadToken })
|
||||
}
|
||||
|
||||
case 'get-part-urls': {
|
||||
const parsed = await parseRequest(
|
||||
getMultipartPartUrlsContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) =>
|
||||
NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const data = parsed.data.body
|
||||
const { partNumbers } = data
|
||||
|
||||
const tokenPayload = verifyTokenForUser(data.uploadToken, userId)
|
||||
if (!tokenPayload) {
|
||||
return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 })
|
||||
}
|
||||
|
||||
const { uploadId, key, context } = tokenPayload
|
||||
const config = getStorageConfig(context)
|
||||
|
||||
if (storageProvider === 's3') {
|
||||
const { getS3MultipartPartUrls } = await import('@/lib/uploads/providers/s3/client')
|
||||
const presignedUrls = await getS3MultipartPartUrls(
|
||||
key,
|
||||
uploadId,
|
||||
partNumbers,
|
||||
buildS3CustomConfig(config)
|
||||
)
|
||||
return NextResponse.json({ presignedUrls })
|
||||
}
|
||||
if (storageProvider === 'blob') {
|
||||
const { getMultipartPartUrls } = await import('@/lib/uploads/providers/blob/client')
|
||||
const presignedUrls = await getMultipartPartUrls(
|
||||
key,
|
||||
partNumbers,
|
||||
buildBlobCustomConfig(config)
|
||||
)
|
||||
return NextResponse.json({ presignedUrls })
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: `Unsupported storage provider: ${storageProvider}` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
case 'complete': {
|
||||
const parsed = await parseRequest(
|
||||
completeMultipartUploadContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) =>
|
||||
NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const data: CompleteMultipartBody = parsed.data.body
|
||||
|
||||
const s3Module =
|
||||
storageProvider === 's3' ? await import('@/lib/uploads/providers/s3/client') : null
|
||||
const blobModule =
|
||||
storageProvider === 'blob' ? await import('@/lib/uploads/providers/blob/client') : null
|
||||
|
||||
const completeOne = async (payload: UploadTokenPayload, parts: ClientCompletedPart[]) => {
|
||||
const { uploadId, key, context } = payload
|
||||
const config = getStorageConfig(context)
|
||||
|
||||
let completed: { location: string; path: string; key: string }
|
||||
if (storageProvider === 's3' && s3Module) {
|
||||
const { completeS3MultipartUpload } = s3Module
|
||||
const s3Parts = parts.map((p) => {
|
||||
if (!p.etag) {
|
||||
throw new Error(`Missing etag for S3 part ${p.partNumber}`)
|
||||
}
|
||||
return { ETag: p.etag, PartNumber: p.partNumber }
|
||||
})
|
||||
completed = await completeS3MultipartUpload(
|
||||
key,
|
||||
uploadId,
|
||||
s3Parts,
|
||||
buildS3CustomConfig(config)
|
||||
)
|
||||
} else if (storageProvider === 'blob' && blobModule) {
|
||||
const { completeMultipartUpload, deriveBlobBlockId } = blobModule
|
||||
const blobParts = parts.map((p) => ({
|
||||
partNumber: p.partNumber,
|
||||
blockId: deriveBlobBlockId(p.partNumber),
|
||||
}))
|
||||
completed = await completeMultipartUpload(key, blobParts, buildBlobCustomConfig(config))
|
||||
} else {
|
||||
throw new Error(`Unsupported storage provider: ${storageProvider}`)
|
||||
}
|
||||
|
||||
try {
|
||||
await recordKnowledgeBaseOwnership(payload, completed.key)
|
||||
} catch (error) {
|
||||
// The object is committed, but without an ownership binding a KB file
|
||||
// is unreadable and undeletable via the KB paths. Remove the orphan
|
||||
// best-effort and surface a retryable error so the client re-uploads.
|
||||
if (payload.context === 'knowledge-base') {
|
||||
await deleteFile({ key: completed.key, context: 'knowledge-base' }).catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
return {
|
||||
success: true as const,
|
||||
location: completed.location,
|
||||
path: completed.path,
|
||||
key: completed.key,
|
||||
}
|
||||
}
|
||||
|
||||
if ('uploads' in data && Array.isArray(data.uploads)) {
|
||||
const verified: Array<{ payload: UploadTokenPayload; parts: ClientCompletedPart[] }> = []
|
||||
for (const upload of data.uploads) {
|
||||
const payload = verifyTokenForUser(upload.uploadToken, userId)
|
||||
if (!payload) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid or expired upload token' },
|
||||
{ status: 403 }
|
||||
)
|
||||
}
|
||||
if (!isClientCompletedParts(upload.parts)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid parts payload: expected [{ partNumber, etag? }]' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
verified.push({ payload, parts: upload.parts })
|
||||
}
|
||||
|
||||
const results = await Promise.all(
|
||||
verified.map(({ payload, parts }) => completeOne(payload, parts))
|
||||
)
|
||||
|
||||
logger.info(`Completed ${verified.length} multipart uploads`)
|
||||
return NextResponse.json({ results })
|
||||
}
|
||||
|
||||
const single = data
|
||||
const tokenPayload = verifyTokenForUser(single.uploadToken, userId)
|
||||
if (!tokenPayload) {
|
||||
return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 })
|
||||
}
|
||||
if (!isClientCompletedParts(single.parts)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid parts payload: expected [{ partNumber, etag? }]' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const result = await completeOne(tokenPayload, single.parts)
|
||||
logger.info(
|
||||
`Completed ${storageProvider} multipart upload for key ${tokenPayload.key} (context: ${tokenPayload.context})`
|
||||
)
|
||||
return NextResponse.json(result)
|
||||
}
|
||||
|
||||
case 'abort': {
|
||||
const parsed = await parseRequest(
|
||||
abortMultipartUploadContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) =>
|
||||
NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const data = parsed.data.body
|
||||
const tokenPayload = verifyTokenForUser(data.uploadToken, userId)
|
||||
if (!tokenPayload) {
|
||||
return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 })
|
||||
}
|
||||
|
||||
const { uploadId, key, context } = tokenPayload
|
||||
const config = getStorageConfig(context)
|
||||
|
||||
if (storageProvider === 's3') {
|
||||
const { abortS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client')
|
||||
await abortS3MultipartUpload(key, uploadId, buildS3CustomConfig(config))
|
||||
logger.info(`Aborted S3 multipart upload for key ${key} (context: ${context})`)
|
||||
} else if (storageProvider === 'blob') {
|
||||
const { abortMultipartUpload } = await import('@/lib/uploads/providers/blob/client')
|
||||
await abortMultipartUpload(key, buildBlobCustomConfig(config))
|
||||
logger.info(`Aborted Azure multipart upload for key ${key} (context: ${context})`)
|
||||
} else {
|
||||
return NextResponse.json(
|
||||
{ error: `Unsupported storage provider: ${storageProvider}` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
return NextResponse.json({ success: true })
|
||||
}
|
||||
|
||||
default:
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid action. Use: initiate, get-part-urls, complete, or abort' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Multipart upload error:', error)
|
||||
return NextResponse.json(
|
||||
{ error: getErrorMessage(error, 'Multipart upload failed') },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user