d25d482dc2
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
783 lines
26 KiB
TypeScript
783 lines
26 KiB
TypeScript
import type { Readable } from 'node:stream'
|
||
import type { BlobServiceClient as BlobServiceClientType } from '@azure/storage-blob'
|
||
import { createLogger } from '@sim/logger'
|
||
import { generateId } from '@sim/utils/id'
|
||
import {
|
||
assertKnownSizeWithinLimit,
|
||
readNodeStreamToBufferWithLimit,
|
||
} from '@/lib/core/utils/stream-limits'
|
||
import { BLOB_CONFIG } from '@/lib/uploads/config'
|
||
import type {
|
||
AzureMultipartPart,
|
||
AzureMultipartUploadInit,
|
||
AzurePartUploadUrl,
|
||
BlobConfig,
|
||
} from '@/lib/uploads/providers/blob/types'
|
||
import type { FileInfo } from '@/lib/uploads/shared/types'
|
||
import { sanitizeStorageMetadata } from '@/lib/uploads/utils/file-utils'
|
||
import { sanitizeFileName } from '@/executor/constants'
|
||
|
||
const logger = createLogger('BlobClient')
|
||
|
||
let _blobServiceClient: BlobServiceClientType | null = null
|
||
|
||
interface ParsedCredentials {
|
||
accountName: string
|
||
accountKey: string
|
||
}
|
||
|
||
/**
|
||
* Extract account name and key from an Azure connection string.
|
||
* Connection strings have the format: DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=...
|
||
*/
|
||
export function parseConnectionString(connectionString: string): ParsedCredentials {
|
||
const accountNameMatch = connectionString.match(/AccountName=([^;]+)/)
|
||
if (!accountNameMatch) {
|
||
throw new Error('Cannot extract account name from connection string')
|
||
}
|
||
|
||
const accountKeyMatch = connectionString.match(/AccountKey=([^;]+)/)
|
||
if (!accountKeyMatch) {
|
||
throw new Error('Cannot extract account key from connection string')
|
||
}
|
||
|
||
return {
|
||
accountName: accountNameMatch[1],
|
||
accountKey: accountKeyMatch[1],
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get account credentials from BLOB_CONFIG, extracting from connection string if necessary.
|
||
*/
|
||
function getAccountCredentials(): ParsedCredentials {
|
||
if (BLOB_CONFIG.connectionString) {
|
||
return parseConnectionString(BLOB_CONFIG.connectionString)
|
||
}
|
||
|
||
if (BLOB_CONFIG.accountName && BLOB_CONFIG.accountKey) {
|
||
return {
|
||
accountName: BLOB_CONFIG.accountName,
|
||
accountKey: BLOB_CONFIG.accountKey,
|
||
}
|
||
}
|
||
|
||
throw new Error(
|
||
'Azure Blob Storage credentials are missing – set AZURE_CONNECTION_STRING or both AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY'
|
||
)
|
||
}
|
||
|
||
export async function getBlobServiceClient(): Promise<BlobServiceClientType> {
|
||
if (_blobServiceClient) return _blobServiceClient
|
||
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
const { accountName, accountKey, connectionString } = BLOB_CONFIG
|
||
|
||
if (connectionString) {
|
||
_blobServiceClient = BlobServiceClient.fromConnectionString(connectionString)
|
||
} else if (accountName && accountKey) {
|
||
const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey)
|
||
_blobServiceClient = new BlobServiceClient(
|
||
`https://${accountName}.blob.core.windows.net`,
|
||
sharedKeyCredential
|
||
)
|
||
} else {
|
||
throw new Error(
|
||
'Azure Blob Storage credentials are missing – set AZURE_STORAGE_CONNECTION_STRING or both AZURE_STORAGE_ACCOUNT_NAME and AZURE_STORAGE_ACCOUNT_KEY in your environment.'
|
||
)
|
||
}
|
||
|
||
return _blobServiceClient
|
||
}
|
||
|
||
/**
|
||
* Upload a file to Azure Blob Storage
|
||
* @param file Buffer containing file data
|
||
* @param fileName Original file name
|
||
* @param contentType MIME type of the file
|
||
* @param configOrSize Custom Blob configuration OR file size in bytes (optional)
|
||
* @param size File size in bytes (required if configOrSize is BlobConfig, optional otherwise)
|
||
* @param preserveKey Preserve the fileName as the storage key without adding timestamp prefix (default: false)
|
||
* @param metadata Optional metadata to store with the file
|
||
* @returns Object with file information
|
||
*/
|
||
export async function uploadToBlob(
|
||
file: Buffer,
|
||
fileName: string,
|
||
contentType: string,
|
||
configOrSize?: BlobConfig | number,
|
||
size?: number,
|
||
preserveKey?: boolean,
|
||
metadata?: Record<string, string>
|
||
): Promise<FileInfo> {
|
||
let config: BlobConfig
|
||
let fileSize: number
|
||
let shouldPreserveKey: boolean
|
||
|
||
if (typeof configOrSize === 'object') {
|
||
config = configOrSize
|
||
fileSize = size ?? file.length
|
||
shouldPreserveKey = preserveKey ?? false
|
||
} else {
|
||
config = {
|
||
containerName: BLOB_CONFIG.containerName,
|
||
accountName: BLOB_CONFIG.accountName,
|
||
accountKey: BLOB_CONFIG.accountKey,
|
||
connectionString: BLOB_CONFIG.connectionString,
|
||
}
|
||
fileSize = configOrSize ?? file.length
|
||
shouldPreserveKey = preserveKey ?? false
|
||
}
|
||
|
||
const safeFileName = sanitizeFileName(fileName)
|
||
const uniqueKey = shouldPreserveKey ? fileName : `${Date.now()}-${safeFileName}`
|
||
|
||
const blobServiceClient = await getBlobServiceClient()
|
||
const containerClient = blobServiceClient.getContainerClient(config.containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey)
|
||
|
||
const blobMetadata: Record<string, string> = {
|
||
originalName: encodeURIComponent(fileName), // Encode filename to prevent invalid characters in HTTP headers
|
||
uploadedAt: new Date().toISOString(),
|
||
}
|
||
|
||
if (metadata) {
|
||
Object.assign(blobMetadata, sanitizeStorageMetadata(metadata, 8000))
|
||
}
|
||
|
||
await blockBlobClient.upload(file, fileSize, {
|
||
blobHTTPHeaders: {
|
||
blobContentType: contentType,
|
||
},
|
||
metadata: blobMetadata,
|
||
})
|
||
|
||
const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}`
|
||
|
||
return {
|
||
path: servePath,
|
||
key: uniqueKey,
|
||
name: fileName,
|
||
size: fileSize,
|
||
type: contentType,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Generate a presigned URL for direct file access
|
||
* @param key Blob name
|
||
* @param expiresIn Time in seconds until URL expires
|
||
* @returns Presigned URL
|
||
*/
|
||
export async function getPresignedUrl(key: string, expiresIn = 3600) {
|
||
const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } =
|
||
await import('@azure/storage-blob')
|
||
const blobServiceClient = await getBlobServiceClient()
|
||
const containerClient = blobServiceClient.getContainerClient(BLOB_CONFIG.containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
const { accountName, accountKey } = getAccountCredentials()
|
||
|
||
const sasOptions = {
|
||
containerName: BLOB_CONFIG.containerName,
|
||
blobName: key,
|
||
permissions: BlobSASPermissions.parse('r'), // Read permission
|
||
startsOn: new Date(),
|
||
expiresOn: new Date(Date.now() + expiresIn * 1000),
|
||
}
|
||
|
||
const sasToken = generateBlobSASQueryParameters(
|
||
sasOptions,
|
||
new StorageSharedKeyCredential(accountName, accountKey)
|
||
).toString()
|
||
|
||
return `${blockBlobClient.url}?${sasToken}`
|
||
}
|
||
|
||
/**
|
||
* Generate a presigned URL for direct file access with custom container
|
||
* @param key Blob name
|
||
* @param customConfig Custom Blob configuration
|
||
* @param expiresIn Time in seconds until URL expires
|
||
* @returns Presigned URL
|
||
*/
|
||
export async function getPresignedUrlWithConfig(
|
||
key: string,
|
||
customConfig: BlobConfig,
|
||
expiresIn = 3600
|
||
) {
|
||
const {
|
||
BlobServiceClient,
|
||
BlobSASPermissions,
|
||
generateBlobSASQueryParameters,
|
||
StorageSharedKeyCredential,
|
||
} = await import('@azure/storage-blob')
|
||
let tempBlobServiceClient: BlobServiceClientType
|
||
let accountName: string
|
||
let accountKey: string
|
||
|
||
if (customConfig.connectionString) {
|
||
tempBlobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
const credentials = parseConnectionString(customConfig.connectionString)
|
||
accountName = credentials.accountName
|
||
accountKey = credentials.accountKey
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const sharedKeyCredential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
tempBlobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
sharedKeyCredential
|
||
)
|
||
accountName = customConfig.accountName
|
||
accountKey = customConfig.accountKey
|
||
} else {
|
||
throw new Error(
|
||
'Custom blob config must include either connectionString or accountName + accountKey'
|
||
)
|
||
}
|
||
|
||
const containerClient = tempBlobServiceClient.getContainerClient(customConfig.containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
const sasOptions = {
|
||
containerName: customConfig.containerName,
|
||
blobName: key,
|
||
permissions: BlobSASPermissions.parse('r'), // Read permission
|
||
startsOn: new Date(),
|
||
expiresOn: new Date(Date.now() + expiresIn * 1000),
|
||
}
|
||
|
||
const sasToken = generateBlobSASQueryParameters(
|
||
sasOptions,
|
||
new StorageSharedKeyCredential(accountName, accountKey)
|
||
).toString()
|
||
|
||
return `${blockBlobClient.url}?${sasToken}`
|
||
}
|
||
|
||
/**
|
||
* Download a file from Azure Blob Storage
|
||
* @param key Blob name
|
||
* @returns File buffer
|
||
*/
|
||
export async function downloadFromBlob(key: string): Promise<Buffer>
|
||
|
||
/**
|
||
* Download a file from Azure Blob Storage with custom configuration
|
||
* @param key Blob name
|
||
* @param customConfig Custom Blob configuration
|
||
* @returns File buffer
|
||
*/
|
||
export async function downloadFromBlob(key: string, customConfig: BlobConfig): Promise<Buffer>
|
||
|
||
export async function downloadFromBlob(
|
||
key: string,
|
||
customConfig: BlobConfig,
|
||
maxBytes: number
|
||
): Promise<Buffer>
|
||
|
||
export async function downloadFromBlob(
|
||
key: string,
|
||
customConfig?: BlobConfig,
|
||
maxBytes?: number
|
||
): Promise<Buffer> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
const downloadBlockBlobResponse = await blockBlobClient.download()
|
||
if (maxBytes !== undefined && downloadBlockBlobResponse.contentLength !== undefined) {
|
||
try {
|
||
assertKnownSizeWithinLimit(
|
||
downloadBlockBlobResponse.contentLength,
|
||
maxBytes,
|
||
'storage download'
|
||
)
|
||
} catch (error) {
|
||
const stream = downloadBlockBlobResponse.readableStreamBody as
|
||
| { destroy?: (error?: Error) => void }
|
||
| undefined
|
||
stream?.destroy?.(error instanceof Error ? error : undefined)
|
||
throw error
|
||
}
|
||
}
|
||
|
||
if (!downloadBlockBlobResponse.readableStreamBody) {
|
||
throw new Error('Failed to get readable stream from blob download')
|
||
}
|
||
const downloaded = await readNodeStreamToBufferWithLimit(
|
||
downloadBlockBlobResponse.readableStreamBody,
|
||
{
|
||
maxBytes: maxBytes ?? Number.MAX_SAFE_INTEGER,
|
||
label: 'storage download',
|
||
}
|
||
)
|
||
|
||
return downloaded
|
||
}
|
||
|
||
/**
|
||
* Stream a blob out of storage without buffering it. The caller MUST fully consume or
|
||
* `destroy()` the returned stream. Used by the large-CSV import worker.
|
||
*/
|
||
export async function downloadFromBlobStream(
|
||
key: string,
|
||
customConfig?: BlobConfig
|
||
): Promise<Readable> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
const downloadBlockBlobResponse = await blockBlobClient.download()
|
||
if (!downloadBlockBlobResponse.readableStreamBody) {
|
||
throw new Error('Failed to get readable stream from blob download')
|
||
}
|
||
return downloadBlockBlobResponse.readableStreamBody as Readable
|
||
}
|
||
|
||
/**
|
||
* Check whether a blob exists (and return its size when it does).
|
||
* Returns null when the blob is missing.
|
||
*/
|
||
export async function headBlobObject(
|
||
key: string,
|
||
customConfig?: BlobConfig
|
||
): Promise<{ size: number; contentType?: string } | null> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
try {
|
||
const properties = await blockBlobClient.getProperties()
|
||
return {
|
||
size: properties.contentLength ?? 0,
|
||
contentType: properties.contentType,
|
||
}
|
||
} catch (err) {
|
||
const status = (err as { statusCode?: number }).statusCode
|
||
const code = (err as { code?: string }).code
|
||
if (status === 404 || code === 'BlobNotFound') {
|
||
return null
|
||
}
|
||
throw err
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Delete a file from Azure Blob Storage
|
||
* @param key Blob name
|
||
*/
|
||
export async function deleteFromBlob(key: string): Promise<void>
|
||
|
||
/**
|
||
* Delete a file from Azure Blob Storage with custom configuration
|
||
* @param key Blob name
|
||
* @param customConfig Custom Blob configuration
|
||
*/
|
||
export async function deleteFromBlob(key: string, customConfig: BlobConfig): Promise<void>
|
||
|
||
export async function deleteFromBlob(key: string, customConfig?: BlobConfig): Promise<void> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
await blockBlobClient.deleteIfExists()
|
||
}
|
||
|
||
/**
|
||
* Derive the deterministic Azure block id for a given part number.
|
||
* Block ids must be base64-encoded and equal length within an upload; using a
|
||
* fixed-width zero-padded counter gives both properties for free, and lets the
|
||
* server reconstruct the id from `partNumber` alone when completing an upload.
|
||
*/
|
||
export function deriveBlobBlockId(partNumber: number): string {
|
||
return Buffer.from(`block-${partNumber.toString().padStart(6, '0')}`).toString('base64')
|
||
}
|
||
|
||
/**
|
||
* Initiate a multipart upload for Azure Blob Storage
|
||
*/
|
||
export async function initiateMultipartUpload(
|
||
options: AzureMultipartUploadInit
|
||
): Promise<{ uploadId: string; key: string }> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
const { fileName, contentType, customConfig, customKey } = options
|
||
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const safeFileName = sanitizeFileName(fileName)
|
||
const uniqueKey = customKey || `kb/${generateId()}-${safeFileName}`
|
||
|
||
const uploadId = generateId()
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey)
|
||
|
||
await blockBlobClient.setMetadata({
|
||
uploadId,
|
||
fileName: encodeURIComponent(fileName),
|
||
contentType,
|
||
uploadStarted: new Date().toISOString(),
|
||
multipartUpload: 'true',
|
||
})
|
||
|
||
return {
|
||
uploadId,
|
||
key: uniqueKey,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Generate presigned URLs for uploading parts
|
||
*/
|
||
export async function getMultipartPartUrls(
|
||
key: string,
|
||
partNumbers: number[],
|
||
customConfig?: BlobConfig
|
||
): Promise<AzurePartUploadUrl[]> {
|
||
const {
|
||
BlobServiceClient,
|
||
BlobSASPermissions,
|
||
generateBlobSASQueryParameters,
|
||
StorageSharedKeyCredential,
|
||
} = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
let accountName: string
|
||
let accountKey: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
const credentials = parseConnectionString(customConfig.connectionString)
|
||
accountName = credentials.accountName
|
||
accountKey = credentials.accountKey
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
accountName = customConfig.accountName
|
||
accountKey = customConfig.accountKey
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
const credentials = getAccountCredentials()
|
||
accountName = credentials.accountName
|
||
accountKey = credentials.accountKey
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
return partNumbers.map((partNumber) => {
|
||
const blockId = deriveBlobBlockId(partNumber)
|
||
|
||
const sasOptions = {
|
||
containerName,
|
||
blobName: key,
|
||
permissions: BlobSASPermissions.parse('w'), // Write permission
|
||
startsOn: new Date(),
|
||
expiresOn: new Date(Date.now() + 3600 * 1000), // 1 hour
|
||
}
|
||
|
||
const sasToken = generateBlobSASQueryParameters(
|
||
sasOptions,
|
||
new StorageSharedKeyCredential(accountName, accountKey)
|
||
).toString()
|
||
|
||
return {
|
||
partNumber,
|
||
blockId,
|
||
url: `${blockBlobClient.url}?comp=block&blockid=${encodeURIComponent(blockId)}&${sasToken}`,
|
||
}
|
||
})
|
||
}
|
||
|
||
async function getBlockBlobClientFor(key: string, customConfig?: BlobConfig) {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
return blobServiceClient.getContainerClient(containerName).getBlockBlobClient(key)
|
||
}
|
||
|
||
/**
|
||
* Stage a single block from the server (body in hand), returning its
|
||
* `{ partNumber, blockId }`. The server-side streaming counterpart to the
|
||
* presigned {@link getMultipartPartUrls}.
|
||
*/
|
||
export async function stageBlobPart(
|
||
key: string,
|
||
partNumber: number,
|
||
body: Buffer,
|
||
customConfig?: BlobConfig
|
||
): Promise<AzureMultipartPart> {
|
||
const blockBlobClient = await getBlockBlobClientFor(key, customConfig)
|
||
const blockId = deriveBlobBlockId(partNumber)
|
||
await blockBlobClient.stageBlock(blockId, body, body.length)
|
||
return { partNumber, blockId }
|
||
}
|
||
|
||
/**
|
||
* Commit staged blocks into the final blob, setting the content type. Used by the
|
||
* server-side streaming uploader (the KB flow uses {@link completeMultipartUpload}).
|
||
*/
|
||
export async function commitBlobBlockList(
|
||
key: string,
|
||
parts: AzureMultipartPart[],
|
||
contentType: string,
|
||
customConfig?: BlobConfig
|
||
): Promise<void> {
|
||
const blockBlobClient = await getBlockBlobClientFor(key, customConfig)
|
||
const sortedBlockIds = parts.sort((a, b) => a.partNumber - b.partNumber).map((p) => p.blockId)
|
||
await blockBlobClient.commitBlockList(sortedBlockIds, {
|
||
blobHTTPHeaders: { blobContentType: contentType },
|
||
})
|
||
}
|
||
|
||
/**
|
||
* Complete multipart upload by committing all blocks
|
||
*/
|
||
export async function completeMultipartUpload(
|
||
key: string,
|
||
parts: AzureMultipartPart[],
|
||
customConfig?: BlobConfig
|
||
): Promise<{ location: string; path: string; key: string }> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
const sortedBlockIds = parts
|
||
.sort((a, b) => a.partNumber - b.partNumber)
|
||
.map((part) => part.blockId)
|
||
|
||
await blockBlobClient.commitBlockList(sortedBlockIds, {
|
||
metadata: {
|
||
multipartUpload: 'completed',
|
||
uploadCompletedAt: new Date().toISOString(),
|
||
},
|
||
})
|
||
|
||
const location = blockBlobClient.url
|
||
const path = `/api/files/serve/${encodeURIComponent(key)}`
|
||
|
||
return {
|
||
location,
|
||
path,
|
||
key,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Abort multipart upload by deleting the blob if it exists
|
||
*/
|
||
export async function abortMultipartUpload(key: string, customConfig?: BlobConfig): Promise<void> {
|
||
const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
|
||
let blobServiceClient: BlobServiceClientType
|
||
let containerName: string
|
||
|
||
if (customConfig) {
|
||
if (customConfig.connectionString) {
|
||
blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
|
||
} else if (customConfig.accountName && customConfig.accountKey) {
|
||
const credential = new StorageSharedKeyCredential(
|
||
customConfig.accountName,
|
||
customConfig.accountKey
|
||
)
|
||
blobServiceClient = new BlobServiceClient(
|
||
`https://${customConfig.accountName}.blob.core.windows.net`,
|
||
credential
|
||
)
|
||
} else {
|
||
throw new Error('Invalid custom blob configuration')
|
||
}
|
||
containerName = customConfig.containerName
|
||
} else {
|
||
blobServiceClient = await getBlobServiceClient()
|
||
containerName = BLOB_CONFIG.containerName
|
||
}
|
||
|
||
const containerClient = blobServiceClient.getContainerClient(containerName)
|
||
const blockBlobClient = containerClient.getBlockBlobClient(key)
|
||
|
||
try {
|
||
await blockBlobClient.deleteIfExists()
|
||
} catch (error) {
|
||
logger.warn('Error cleaning up multipart upload:', error)
|
||
}
|
||
}
|