import type { Readable } from 'node:stream' import type { BlobServiceClient as BlobServiceClientType } from '@azure/storage-blob' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { assertKnownSizeWithinLimit, readNodeStreamToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { BLOB_CONFIG } from '@/lib/uploads/config' import type { AzureMultipartPart, AzureMultipartUploadInit, AzurePartUploadUrl, BlobConfig, } from '@/lib/uploads/providers/blob/types' import type { FileInfo } from '@/lib/uploads/shared/types' import { sanitizeStorageMetadata } from '@/lib/uploads/utils/file-utils' import { sanitizeFileName } from '@/executor/constants' const logger = createLogger('BlobClient') let _blobServiceClient: BlobServiceClientType | null = null interface ParsedCredentials { accountName: string accountKey: string } /** * Extract account name and key from an Azure connection string. * Connection strings have the format: DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=... */ export function parseConnectionString(connectionString: string): ParsedCredentials { const accountNameMatch = connectionString.match(/AccountName=([^;]+)/) if (!accountNameMatch) { throw new Error('Cannot extract account name from connection string') } const accountKeyMatch = connectionString.match(/AccountKey=([^;]+)/) if (!accountKeyMatch) { throw new Error('Cannot extract account key from connection string') } return { accountName: accountNameMatch[1], accountKey: accountKeyMatch[1], } } /** * Get account credentials from BLOB_CONFIG, extracting from connection string if necessary. */ function getAccountCredentials(): ParsedCredentials { if (BLOB_CONFIG.connectionString) { return parseConnectionString(BLOB_CONFIG.connectionString) } if (BLOB_CONFIG.accountName && BLOB_CONFIG.accountKey) { return { accountName: BLOB_CONFIG.accountName, accountKey: BLOB_CONFIG.accountKey, } } throw new Error( 'Azure Blob Storage credentials are missing – set AZURE_CONNECTION_STRING or both AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY' ) } export async function getBlobServiceClient(): Promise { if (_blobServiceClient) return _blobServiceClient const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') const { accountName, accountKey, connectionString } = BLOB_CONFIG if (connectionString) { _blobServiceClient = BlobServiceClient.fromConnectionString(connectionString) } else if (accountName && accountKey) { const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey) _blobServiceClient = new BlobServiceClient( `https://${accountName}.blob.core.windows.net`, sharedKeyCredential ) } else { throw new Error( 'Azure Blob Storage credentials are missing – set AZURE_STORAGE_CONNECTION_STRING or both AZURE_STORAGE_ACCOUNT_NAME and AZURE_STORAGE_ACCOUNT_KEY in your environment.' ) } return _blobServiceClient } /** * Upload a file to Azure Blob Storage * @param file Buffer containing file data * @param fileName Original file name * @param contentType MIME type of the file * @param configOrSize Custom Blob configuration OR file size in bytes (optional) * @param size File size in bytes (required if configOrSize is BlobConfig, optional otherwise) * @param preserveKey Preserve the fileName as the storage key without adding timestamp prefix (default: false) * @param metadata Optional metadata to store with the file * @returns Object with file information */ export async function uploadToBlob( file: Buffer, fileName: string, contentType: string, configOrSize?: BlobConfig | number, size?: number, preserveKey?: boolean, metadata?: Record ): Promise { let config: BlobConfig let fileSize: number let shouldPreserveKey: boolean if (typeof configOrSize === 'object') { config = configOrSize fileSize = size ?? file.length shouldPreserveKey = preserveKey ?? false } else { config = { containerName: BLOB_CONFIG.containerName, accountName: BLOB_CONFIG.accountName, accountKey: BLOB_CONFIG.accountKey, connectionString: BLOB_CONFIG.connectionString, } fileSize = configOrSize ?? file.length shouldPreserveKey = preserveKey ?? false } const safeFileName = sanitizeFileName(fileName) const uniqueKey = shouldPreserveKey ? fileName : `${Date.now()}-${safeFileName}` const blobServiceClient = await getBlobServiceClient() const containerClient = blobServiceClient.getContainerClient(config.containerName) const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey) const blobMetadata: Record = { originalName: encodeURIComponent(fileName), // Encode filename to prevent invalid characters in HTTP headers uploadedAt: new Date().toISOString(), } if (metadata) { Object.assign(blobMetadata, sanitizeStorageMetadata(metadata, 8000)) } await blockBlobClient.upload(file, fileSize, { blobHTTPHeaders: { blobContentType: contentType, }, metadata: blobMetadata, }) const servePath = `/api/files/serve/${encodeURIComponent(uniqueKey)}` return { path: servePath, key: uniqueKey, name: fileName, size: fileSize, type: contentType, } } /** * Generate a presigned URL for direct file access * @param key Blob name * @param expiresIn Time in seconds until URL expires * @returns Presigned URL */ export async function getPresignedUrl(key: string, expiresIn = 3600) { const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = await import('@azure/storage-blob') const blobServiceClient = await getBlobServiceClient() const containerClient = blobServiceClient.getContainerClient(BLOB_CONFIG.containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) const { accountName, accountKey } = getAccountCredentials() const sasOptions = { containerName: BLOB_CONFIG.containerName, blobName: key, permissions: BlobSASPermissions.parse('r'), // Read permission startsOn: new Date(), expiresOn: new Date(Date.now() + expiresIn * 1000), } const sasToken = generateBlobSASQueryParameters( sasOptions, new StorageSharedKeyCredential(accountName, accountKey) ).toString() return `${blockBlobClient.url}?${sasToken}` } /** * Generate a presigned URL for direct file access with custom container * @param key Blob name * @param customConfig Custom Blob configuration * @param expiresIn Time in seconds until URL expires * @returns Presigned URL */ export async function getPresignedUrlWithConfig( key: string, customConfig: BlobConfig, expiresIn = 3600 ) { const { BlobServiceClient, BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential, } = await import('@azure/storage-blob') let tempBlobServiceClient: BlobServiceClientType let accountName: string let accountKey: string if (customConfig.connectionString) { tempBlobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) const credentials = parseConnectionString(customConfig.connectionString) accountName = credentials.accountName accountKey = credentials.accountKey } else if (customConfig.accountName && customConfig.accountKey) { const sharedKeyCredential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) tempBlobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, sharedKeyCredential ) accountName = customConfig.accountName accountKey = customConfig.accountKey } else { throw new Error( 'Custom blob config must include either connectionString or accountName + accountKey' ) } const containerClient = tempBlobServiceClient.getContainerClient(customConfig.containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) const sasOptions = { containerName: customConfig.containerName, blobName: key, permissions: BlobSASPermissions.parse('r'), // Read permission startsOn: new Date(), expiresOn: new Date(Date.now() + expiresIn * 1000), } const sasToken = generateBlobSASQueryParameters( sasOptions, new StorageSharedKeyCredential(accountName, accountKey) ).toString() return `${blockBlobClient.url}?${sasToken}` } /** * Download a file from Azure Blob Storage * @param key Blob name * @returns File buffer */ export async function downloadFromBlob(key: string): Promise /** * Download a file from Azure Blob Storage with custom configuration * @param key Blob name * @param customConfig Custom Blob configuration * @returns File buffer */ export async function downloadFromBlob(key: string, customConfig: BlobConfig): Promise export async function downloadFromBlob( key: string, customConfig: BlobConfig, maxBytes: number ): Promise export async function downloadFromBlob( key: string, customConfig?: BlobConfig, maxBytes?: number ): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) const downloadBlockBlobResponse = await blockBlobClient.download() if (maxBytes !== undefined && downloadBlockBlobResponse.contentLength !== undefined) { try { assertKnownSizeWithinLimit( downloadBlockBlobResponse.contentLength, maxBytes, 'storage download' ) } catch (error) { const stream = downloadBlockBlobResponse.readableStreamBody as | { destroy?: (error?: Error) => void } | undefined stream?.destroy?.(error instanceof Error ? error : undefined) throw error } } if (!downloadBlockBlobResponse.readableStreamBody) { throw new Error('Failed to get readable stream from blob download') } const downloaded = await readNodeStreamToBufferWithLimit( downloadBlockBlobResponse.readableStreamBody, { maxBytes: maxBytes ?? Number.MAX_SAFE_INTEGER, label: 'storage download', } ) return downloaded } /** * Stream a blob out of storage without buffering it. The caller MUST fully consume or * `destroy()` the returned stream. Used by the large-CSV import worker. */ export async function downloadFromBlobStream( key: string, customConfig?: BlobConfig ): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) const downloadBlockBlobResponse = await blockBlobClient.download() if (!downloadBlockBlobResponse.readableStreamBody) { throw new Error('Failed to get readable stream from blob download') } return downloadBlockBlobResponse.readableStreamBody as Readable } /** * Check whether a blob exists (and return its size when it does). * Returns null when the blob is missing. */ export async function headBlobObject( key: string, customConfig?: BlobConfig ): Promise<{ size: number; contentType?: string } | null> { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) try { const properties = await blockBlobClient.getProperties() return { size: properties.contentLength ?? 0, contentType: properties.contentType, } } catch (err) { const status = (err as { statusCode?: number }).statusCode const code = (err as { code?: string }).code if (status === 404 || code === 'BlobNotFound') { return null } throw err } } /** * Delete a file from Azure Blob Storage * @param key Blob name */ export async function deleteFromBlob(key: string): Promise /** * Delete a file from Azure Blob Storage with custom configuration * @param key Blob name * @param customConfig Custom Blob configuration */ export async function deleteFromBlob(key: string, customConfig: BlobConfig): Promise export async function deleteFromBlob(key: string, customConfig?: BlobConfig): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) await blockBlobClient.deleteIfExists() } /** * Derive the deterministic Azure block id for a given part number. * Block ids must be base64-encoded and equal length within an upload; using a * fixed-width zero-padded counter gives both properties for free, and lets the * server reconstruct the id from `partNumber` alone when completing an upload. */ export function deriveBlobBlockId(partNumber: number): string { return Buffer.from(`block-${partNumber.toString().padStart(6, '0')}`).toString('base64') } /** * Initiate a multipart upload for Azure Blob Storage */ export async function initiateMultipartUpload( options: AzureMultipartUploadInit ): Promise<{ uploadId: string; key: string }> { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') const { fileName, contentType, customConfig, customKey } = options let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const safeFileName = sanitizeFileName(fileName) const uniqueKey = customKey || `kb/${generateId()}-${safeFileName}` const uploadId = generateId() const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey) await blockBlobClient.setMetadata({ uploadId, fileName: encodeURIComponent(fileName), contentType, uploadStarted: new Date().toISOString(), multipartUpload: 'true', }) return { uploadId, key: uniqueKey, } } /** * Generate presigned URLs for uploading parts */ export async function getMultipartPartUrls( key: string, partNumbers: number[], customConfig?: BlobConfig ): Promise { const { BlobServiceClient, BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential, } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string let accountName: string let accountKey: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) const credentials = parseConnectionString(customConfig.connectionString) accountName = credentials.accountName accountKey = credentials.accountKey } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) accountName = customConfig.accountName accountKey = customConfig.accountKey } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName const credentials = getAccountCredentials() accountName = credentials.accountName accountKey = credentials.accountKey } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) return partNumbers.map((partNumber) => { const blockId = deriveBlobBlockId(partNumber) const sasOptions = { containerName, blobName: key, permissions: BlobSASPermissions.parse('w'), // Write permission startsOn: new Date(), expiresOn: new Date(Date.now() + 3600 * 1000), // 1 hour } const sasToken = generateBlobSASQueryParameters( sasOptions, new StorageSharedKeyCredential(accountName, accountKey) ).toString() return { partNumber, blockId, url: `${blockBlobClient.url}?comp=block&blockid=${encodeURIComponent(blockId)}&${sasToken}`, } }) } async function getBlockBlobClientFor(key: string, customConfig?: BlobConfig) { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } return blobServiceClient.getContainerClient(containerName).getBlockBlobClient(key) } /** * Stage a single block from the server (body in hand), returning its * `{ partNumber, blockId }`. The server-side streaming counterpart to the * presigned {@link getMultipartPartUrls}. */ export async function stageBlobPart( key: string, partNumber: number, body: Buffer, customConfig?: BlobConfig ): Promise { const blockBlobClient = await getBlockBlobClientFor(key, customConfig) const blockId = deriveBlobBlockId(partNumber) await blockBlobClient.stageBlock(blockId, body, body.length) return { partNumber, blockId } } /** * Commit staged blocks into the final blob, setting the content type. Used by the * server-side streaming uploader (the KB flow uses {@link completeMultipartUpload}). */ export async function commitBlobBlockList( key: string, parts: AzureMultipartPart[], contentType: string, customConfig?: BlobConfig ): Promise { const blockBlobClient = await getBlockBlobClientFor(key, customConfig) const sortedBlockIds = parts.sort((a, b) => a.partNumber - b.partNumber).map((p) => p.blockId) await blockBlobClient.commitBlockList(sortedBlockIds, { blobHTTPHeaders: { blobContentType: contentType }, }) } /** * Complete multipart upload by committing all blocks */ export async function completeMultipartUpload( key: string, parts: AzureMultipartPart[], customConfig?: BlobConfig ): Promise<{ location: string; path: string; key: string }> { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) const sortedBlockIds = parts .sort((a, b) => a.partNumber - b.partNumber) .map((part) => part.blockId) await blockBlobClient.commitBlockList(sortedBlockIds, { metadata: { multipartUpload: 'completed', uploadCompletedAt: new Date().toISOString(), }, }) const location = blockBlobClient.url const path = `/api/files/serve/${encodeURIComponent(key)}` return { location, path, key, } } /** * Abort multipart upload by deleting the blob if it exists */ export async function abortMultipartUpload(key: string, customConfig?: BlobConfig): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string if (customConfig) { if (customConfig.connectionString) { blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) } else if (customConfig.accountName && customConfig.accountKey) { const credential = new StorageSharedKeyCredential( customConfig.accountName, customConfig.accountKey ) blobServiceClient = new BlobServiceClient( `https://${customConfig.accountName}.blob.core.windows.net`, credential ) } else { throw new Error('Invalid custom blob configuration') } containerName = customConfig.containerName } else { blobServiceClient = await getBlobServiceClient() containerName = BLOB_CONFIG.containerName } const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) try { await blockBlobClient.deleteIfExists() } catch (error) { logger.warn('Error cleaning up multipart upload:', error) } }