Files
wehub-resource-sync d25d482dc2
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:20:55 +08:00

676 lines
24 KiB
TypeScript

import crypto from 'crypto'
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { isHosted } from '@/lib/core/config/env-flags'
import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { s3ConnectorMeta } from '@/connectors/s3/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
CONNECTOR_MAX_FILE_BYTES,
isSkippedDocument,
markSkipped,
parseTagDate,
readBodyWithLimit,
sizeLimitSkipReason,
stubOrSkipBySize,
takeIndexableWithinCap,
} from '@/connectors/utils'
import { encodeS3PathComponent, getSignatureKey } from '@/tools/s3/utils'
const logger = createLogger('S3Connector')
/** Maximum object size to sync. Larger objects are skipped during listing. */
const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
/** Number of objects requested per ListObjectsV2 page (S3 caps at 1000). */
const LIST_MAX_KEYS = 1000
/**
* Default set of file extensions considered safely text-extractable. Objects
* with any other extension (or no extension) are skipped, since their content
* cannot be reliably decoded to plain text. Users can override this list via
* the `extensions` config field.
*/
const DEFAULT_EXTENSIONS = new Set([
'txt',
'md',
'markdown',
'csv',
'tsv',
'json',
'jsonl',
'ndjson',
'html',
'htm',
'xml',
'yaml',
'yml',
'log',
'rtf',
])
/**
* A single object entry parsed out of a ListObjectsV2 XML response.
*/
interface S3ObjectEntry {
key: string
etag: string
lastModified: string
size: number
}
/**
* A parsed custom S3-compatible endpoint (Cloudflare R2, MinIO, etc.).
*
* `host` is the bare hostname, `hostHeader` is the value used both as the wire
* `Host` header and in the SigV4 canonical headers — it includes the port when
* a non-default port is configured (e.g. `localhost:9000`). When the endpoint
* uses the scheme's default port (443 for https, 80 for http) the port is
* omitted from `hostHeader`, matching what the HTTP client sends on the wire.
*/
interface S3Endpoint {
scheme: 'http' | 'https'
host: string
hostHeader: string
}
/**
* AWS credentials and target resource resolved from sourceConfig + access token.
*
* When `endpoint` is present the connector targets an S3-compatible store using
* path-style addressing (`{endpoint}/{bucket}/{key}`). When absent it targets
* AWS S3 using virtual-hosted-style addressing
* (`{bucket}.s3.{region}.amazonaws.com`), preserving the original behavior.
*/
interface S3Context {
accessKeyId: string
secretAccessKey: string
region: string
bucket: string
endpoint?: S3Endpoint
}
/**
* Parses the comma-separated `extensions` config override into a normalized set
* (lowercased, no leading dot). Returns the built-in default set when the
* override is empty or contains no usable entries.
*/
function resolveExtensions(raw: unknown): Set<string> {
if (typeof raw !== 'string') return DEFAULT_EXTENSIONS
const exts = raw
.split(',')
.map((e) => e.trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean)
return exts.length > 0 ? new Set(exts) : DEFAULT_EXTENSIONS
}
/**
* Extracts the lowercased file extension from an object key, or '' if none.
*/
function getExtension(key: string): string {
const lastSegment = key.split('/').pop() ?? ''
const dotIndex = lastSegment.lastIndexOf('.')
if (dotIndex <= 0 || dotIndex === lastSegment.length - 1) return ''
return lastSegment.slice(dotIndex + 1).toLowerCase()
}
/**
* Returns true when the object key ends in one of the allowed text extensions.
*/
function isSupportedKey(key: string, allowedExtensions: Set<string>): boolean {
return allowedExtensions.has(getExtension(key))
}
/**
* Returns true when the host is a loopback address for which plain `http://` is
* tolerated (local MinIO development on a self-hosted deployment). Any other
* host must use `https://`. This is only an early check — the SSRF boundary is
* enforced at request time by {@link secureFetchWithRetry}, which blocks
* loopback/private targets on hosted Sim regardless of what this parser accepts.
*/
function isLoopbackHost(host: string): boolean {
const bare = host.replace(/^\[|\]$/g, '')
return bare === 'localhost' || bare === '127.0.0.1' || bare === '::1'
}
/**
* Parses and validates a custom S3-compatible endpoint string.
*
* Accepts a full origin such as `https://accountid.r2.cloudflarestorage.com` or
* `http://localhost:9000`. Trailing slashes are stripped. Throws when the value
* is not a valid URL, carries a path/query/fragment beyond `/` (which would
* corrupt the path-style canonical URI), or uses plain `http://` against a
* non-loopback host.
*
* The returned `hostHeader` includes the port only when it differs from the
* scheme default, matching the `Host` header the HTTP client emits — this keeps
* the SigV4 canonical Host byte-identical to the wire Host.
*/
function parseEndpoint(raw: string): S3Endpoint {
let url: URL
try {
url = new URL(raw)
} catch {
throw new Error('Endpoint must be a valid URL, e.g. https://accountid.r2.cloudflarestorage.com')
}
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
throw new Error('Endpoint must use http:// or https://')
}
const scheme = url.protocol === 'https:' ? 'https' : 'http'
if (url.username || url.password) {
throw new Error('Endpoint must not contain credentials')
}
if (url.search || url.hash) {
throw new Error('Endpoint must not contain a query string or fragment')
}
const path = url.pathname.replace(/\/+$/, '')
if (path !== '') {
throw new Error('Endpoint must not contain a path — provide only the host, e.g. https://host')
}
const host = url.hostname
if (!host) throw new Error('Endpoint is missing a host')
if (scheme === 'http' && !(isLoopbackHost(host) && !isHosted)) {
throw new Error(
'Plain http:// endpoints are only allowed for localhost on self-hosted deployments — use https:// otherwise'
)
}
const defaultPort = scheme === 'https' ? '443' : '80'
const port = url.port && url.port !== defaultPort ? url.port : ''
const hostHeader = port ? `${host}:${port}` : host
return { scheme, host, hostHeader }
}
/**
* Resolves AWS credentials and the target bucket from the connector's
* sourceConfig and the encrypted secret (delivered as accessToken). When an
* `endpoint` is configured it is parsed/validated into an {@link S3Endpoint} so
* the connector targets an S3-compatible store via path-style addressing.
*/
function resolveContext(accessToken: string, sourceConfig: Record<string, unknown>): S3Context {
const accessKeyId = ((sourceConfig.accessKeyId as string) ?? '').trim()
const region = ((sourceConfig.region as string) ?? '').trim()
const bucket = ((sourceConfig.bucket as string) ?? '').trim()
const secretAccessKey = (accessToken ?? '').trim()
const rawEndpoint = ((sourceConfig.endpoint as string) ?? '').trim()
if (!accessKeyId) throw new Error('Missing AWS Access Key ID')
if (!secretAccessKey) throw new Error('Missing AWS Secret Access Key')
if (!region) throw new Error('Missing AWS region')
if (!bucket) throw new Error('Missing S3 bucket name')
const endpoint = rawEndpoint ? parseEndpoint(rawEndpoint) : undefined
return { accessKeyId, secretAccessKey, region, bucket, endpoint }
}
/**
* Returns the SigV4 canonical Host header for the request. For AWS this is the
* virtual-hosted-style host; for a custom endpoint it is the endpoint host
* (with port when non-default).
*/
function resolveHost(ctx: S3Context): string {
return ctx.endpoint ? ctx.endpoint.hostHeader : `${ctx.bucket}.s3.${ctx.region}.amazonaws.com`
}
/**
* Returns the request scheme: always `https` for AWS, or the endpoint scheme
* (which may be `http` for local MinIO) for a custom endpoint.
*/
function resolveScheme(ctx: S3Context): string {
return ctx.endpoint ? ctx.endpoint.scheme : 'https'
}
/**
* Builds the canonical URI for an object key.
*
* AWS (virtual-hosted-style): `/{key}` — the bucket lives in the host.
* Custom endpoint (path-style): `/{bucket}/{key}` — the bucket is the first
* path segment. Both the bucket and key are percent-encoded per AWS UriEncode
* rules while preserving `/` separators via {@link encodeS3PathComponent}.
*/
function buildObjectPath(ctx: S3Context, key: string): string {
const encodedKey = encodeS3PathComponent(key)
return ctx.endpoint ? `/${encodeS3PathComponent(ctx.bucket)}/${encodedKey}` : `/${encodedKey}`
}
/**
* Builds the canonical URI for a bucket-level (ListObjectsV2) request.
*
* AWS (virtual-hosted-style): `/`.
* Custom endpoint (path-style): `/{bucket}/`.
*/
function buildBucketPath(ctx: S3Context): string {
return ctx.endpoint ? `/${encodeS3PathComponent(ctx.bucket)}/` : '/'
}
/**
* Builds the full request URL from the canonical path and an optional canonical
* query string. The path passed here is the same canonical, percent-encoded
* string used to compute the SigV4 signature, so the signed URI and the wire
* URI are byte-identical.
*/
function buildUrl(ctx: S3Context, encodedPath: string, canonicalQueryString: string): string {
const base = `${resolveScheme(ctx)}://${resolveHost(ctx)}${encodedPath}`
return canonicalQueryString ? `${base}?${canonicalQueryString}` : base
}
/**
* Builds SigV4 request headers for an S3 REST call.
*
* `canonicalQueryString` must be the already-sorted, percent-encoded query
* string (empty for GetObject) — the caller builds the request URL from this
* exact same string so the signed query and the wire query are byte-identical
* (the classic continuation-token signing mismatch cannot occur here).
* `encodedPath` is the canonical URI path starting with '/' (virtual-hosted
* `/{key}` for AWS, path-style `/{bucket}/{key}` for custom endpoints). The
* canonical Host header is resolved via {@link resolveHost} and includes the
* port for non-default custom-endpoint ports, exactly matching the wire Host.
* Reuses {@link getSignatureKey} from the s3 tool utilities.
*
* The signed headers embed `x-amz-date` and are reused verbatim across
* `secureFetchWithRetry` attempts. S3 allows a 15-minute clock-skew window; the
* retry helper's worst-case total backoff (~31s default, ~10s in validate) is
* far inside that window, so a stale timestamp never triggers
* RequestTimeTooSkewed.
*/
function buildSignedHeaders(
ctx: S3Context,
method: 'GET',
encodedPath: string,
canonicalQueryString: string
): Record<string, string> {
const date = new Date()
const amzDate = date.toISOString().replace(/[:-]|\.\d{3}/g, '')
const dateStamp = amzDate.slice(0, 8)
const host = resolveHost(ctx)
const payloadHash = crypto.createHash('sha256').update('').digest('hex')
const canonicalHeaders =
`host:${host}\n` + `x-amz-content-sha256:${payloadHash}\n` + `x-amz-date:${amzDate}\n`
const signedHeaders = 'host;x-amz-content-sha256;x-amz-date'
const canonicalRequest = `${method}\n${encodedPath}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}`
const algorithm = 'AWS4-HMAC-SHA256'
const credentialScope = `${dateStamp}/${ctx.region}/s3/aws4_request`
const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${crypto
.createHash('sha256')
.update(canonicalRequest)
.digest('hex')}`
const signingKey = getSignatureKey(ctx.secretAccessKey, dateStamp, ctx.region, 's3')
const signature = crypto.createHmac('sha256', signingKey).update(stringToSign).digest('hex')
const authorizationHeader = `${algorithm} Credential=${ctx.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`
return {
Host: host,
'X-Amz-Content-Sha256': payloadHash,
'X-Amz-Date': amzDate,
Authorization: authorizationHeader,
}
}
/**
* Percent-encodes a query parameter name or value per AWS SigV4 canonical rules
* (every byte except the unreserved set `A-Za-z0-9-_.~` is encoded).
* `encodeURIComponent` leaves `!'()*` unencoded, so those are encoded here.
*/
function encodeQueryValue(value: string): string {
return encodeURIComponent(value).replace(
/[!'()*]/g,
(c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`
)
}
/**
* Builds the canonical (sorted, percent-encoded) query string for a
* ListObjectsV2 request. Keys are sorted lexicographically after encoding and
* each name/value pair is encoded individually.
*/
function buildListQueryString(params: Record<string, string>): string {
return Object.keys(params)
.sort()
.map((key) => `${encodeQueryValue(key)}=${encodeQueryValue(params[key])}`)
.join('&')
}
/**
* Decodes XML entities found in S3 response text values. `&amp;` is decoded
* last so sequences like `&amp;lt;` resolve to `&lt;` rather than `<`.
*/
function decodeXmlEntities(value: string): string {
return value
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&')
}
/**
* Normalizes an ETag from either a ListObjectsV2 XML `<ETag>` element or a
* GetObject response header into a stable bare token used in the content hash.
*
* Strips surrounding double quotes and a leading weak-validator prefix (`W/`).
* AWS S3 always returns strong, quoted ETags (including the multipart `-N`
* suffix) identically from List and Get, but S3-compatible stores (MinIO, R2)
* are not contractually bound to that and could emit a weak ETag on one path
* and a strong one on the other. Normalizing both ends keeps the
* `s3:{key}:{etag}` hash invariant between the listing stub and the hydrated
* document so unchanged objects are not re-uploaded every sync.
*/
function normalizeEtag(raw: string): string {
return raw.replace(/^W\//, '').replace(/"/g, '')
}
/**
* Decodes a URL-encoded object key returned when `encoding-type=url` is set.
* Falls back to the raw value if decoding fails (malformed percent sequence).
*/
function decodeObjectKey(value: string): string {
try {
return decodeURIComponent(value)
} catch {
return value
}
}
/**
* Extracts the text content of the first matching XML tag within a fragment.
*/
function extractTag(fragment: string, tag: string): string | undefined {
const match = fragment.match(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`))
return match ? decodeXmlEntities(match[1]) : undefined
}
/**
* Parses a ListObjectsV2 XML response into object entries plus pagination state.
*
* The request is always made with `encoding-type=url`, so the per-`Key` values
* are percent-encoded in the XML (safe for the regex parser even when keys
* contain XML-hostile bytes such as `&`, `<`, or ASCII control characters).
* Each `Key` is XML-entity-decoded then URL-decoded back to its true value.
* `NextContinuationToken` is opaque and is not affected by `encoding-type`, so
* it is used verbatim.
*/
function parseListResponse(xml: string): {
objects: S3ObjectEntry[]
isTruncated: boolean
nextContinuationToken?: string
} {
const objects: S3ObjectEntry[] = []
for (const match of xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)) {
const block = match[1]
const rawKey = extractTag(block, 'Key')
if (!rawKey) continue
const key = decodeObjectKey(rawKey)
const etag = normalizeEtag(extractTag(block, 'ETag') ?? '')
const lastModified = extractTag(block, 'LastModified') ?? ''
const size = Number(extractTag(block, 'Size') ?? '0')
objects.push({ key, etag, lastModified, size: Number.isNaN(size) ? 0 : size })
}
const isTruncated = extractTag(xml, 'IsTruncated') === 'true'
const nextContinuationToken = extractTag(xml, 'NextContinuationToken')
return { objects, isTruncated, nextContinuationToken }
}
/**
* Builds a metadata stub for an S3 object. The content hash combines the key
* and ETag — S3's ETag changes whenever object content changes, making it an
* ideal change indicator. Used by both listDocuments and getDocument to
* guarantee identical hashes.
*/
function objectToStub(ctx: S3Context, entry: S3ObjectEntry): ExternalDocument {
const title = entry.key.split('/').pop() || entry.key
const prefix = entry.key.includes('/') ? entry.key.slice(0, entry.key.lastIndexOf('/')) : ''
return {
externalId: entry.key,
title,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: buildUrl(ctx, buildObjectPath(ctx, entry.key), ''),
contentHash: `s3:${entry.key}:${entry.etag}`,
metadata: {
key: entry.key,
prefix,
etag: entry.etag,
lastModified: entry.lastModified,
fileSize: entry.size,
},
}
}
/**
* Performs a single ListObjectsV2 page request and returns the parsed result.
*/
async function listObjectsPage(
ctx: S3Context,
prefix: string,
continuationToken: string | undefined,
retryOptions?: Parameters<typeof secureFetchWithRetry>[2],
maxKeys: number = LIST_MAX_KEYS
): Promise<{ objects: S3ObjectEntry[]; isTruncated: boolean; nextContinuationToken?: string }> {
const queryParams: Record<string, string> = {
'list-type': '2',
'encoding-type': 'url',
'max-keys': String(maxKeys),
}
if (prefix) queryParams.prefix = prefix
if (continuationToken) queryParams['continuation-token'] = continuationToken
const canonicalQueryString = buildListQueryString(queryParams)
const bucketPath = buildBucketPath(ctx)
const headers = buildSignedHeaders(ctx, 'GET', bucketPath, canonicalQueryString)
const url = buildUrl(ctx, bucketPath, canonicalQueryString)
const response = await secureFetchWithRetry(url, { method: 'GET', headers }, retryOptions)
if (!response.ok) {
const errorText = await response.text()
throw new Error(`S3 ListObjectsV2 failed: ${response.status} ${errorText}`)
}
const xml = await response.text()
return parseListResponse(xml)
}
export const s3Connector: ConnectorConfig = {
...s3ConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const ctx = resolveContext(accessToken, sourceConfig)
const prefix = ((sourceConfig.prefix as string) ?? '').trim()
const allowedExtensions = resolveExtensions(sourceConfig.extensions)
const maxObjects = sourceConfig.maxObjects ? Number(sourceConfig.maxObjects) : 0
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxObjects > 0 && previouslyFetched >= maxObjects) {
return { documents: [], hasMore: false }
}
logger.info('Listing S3 objects', { bucket: ctx.bucket, prefix, cursor: cursor ?? 'initial' })
const { objects, isTruncated, nextContinuationToken } = await listObjectsPage(
ctx,
prefix,
cursor
)
const stubs = objects
.filter((entry) => isSupportedKey(entry.key, allowedExtensions) && entry.size > 0)
.map((entry) => stubOrSkipBySize(objectToStub(ctx, entry), entry.size, MAX_FILE_SIZE))
const { documents, indexableCount, capReached } = takeIndexableWithinCap(
stubs,
isSkippedDocument,
maxObjects,
previouslyFetched
)
const slicedSome = documents.length < stubs.length
const totalFetched = previouslyFetched + indexableCount
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = capReached
const moreAvailable = slicedSome || (isTruncated && Boolean(nextContinuationToken))
if (hitLimit && moreAvailable && syncContext) syncContext.listingCapped = true
return {
documents,
nextCursor: hitLimit ? undefined : isTruncated ? nextContinuationToken : undefined,
hasMore: hitLimit ? false : isTruncated && Boolean(nextContinuationToken),
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const ctx = resolveContext(accessToken, sourceConfig)
const key = externalId
try {
const encodedPath = buildObjectPath(ctx, key)
const headers = buildSignedHeaders(ctx, 'GET', encodedPath, '')
const url = buildUrl(ctx, encodedPath, '')
const response = await secureFetchWithRetry(url, { method: 'GET', headers })
if (response.status === 404) return null
if (!response.ok) {
const errorText = await response.text()
throw new Error(`S3 GetObject failed: ${response.status} ${errorText}`)
}
const etag = normalizeEtag(response.headers.get('etag') ?? '')
const lastModified = response.headers.get('last-modified') ?? ''
const declaredLength = Number(response.headers.get('content-length') ?? '')
if (declaredLength > MAX_FILE_SIZE) {
logger.warn('Skipping oversized S3 object', { key, size: declaredLength })
return markSkipped(
objectToStub(ctx, { key, etag, lastModified, size: declaredLength }),
sizeLimitSkipReason(MAX_FILE_SIZE)
)
}
const body = await readBodyWithLimit(response, MAX_FILE_SIZE)
if (body === null) {
logger.warn('Skipping oversized S3 object (size cap exceeded while streaming)', { key })
return markSkipped(
objectToStub(ctx, {
key,
etag,
lastModified,
size: Number.isFinite(declaredLength) ? declaredLength : 0,
}),
sizeLimitSkipReason(MAX_FILE_SIZE)
)
}
const content = body.toString('utf-8')
if (!content.trim()) return null
const entry: S3ObjectEntry = {
key,
etag,
lastModified,
size:
Number.isNaN(declaredLength) || declaredLength <= 0 ? body.byteLength : declaredLength,
}
const stub = objectToStub(ctx, entry)
return { ...stub, content, contentDeferred: false }
} catch (error) {
logger.warn('Failed to get S3 object', { key, error: toError(error).message })
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
let ctx: S3Context
try {
ctx = resolveContext(accessToken, sourceConfig)
} catch (error) {
return { valid: false, error: getErrorMessage(error, 'Invalid configuration') }
}
const maxObjects = sourceConfig.maxObjects as string | undefined
if (maxObjects && (Number.isNaN(Number(maxObjects)) || Number(maxObjects) <= 0)) {
return { valid: false, error: 'Max objects must be a positive number' }
}
const prefix = ((sourceConfig.prefix as string) ?? '').trim()
try {
await listObjectsPage(ctx, prefix, undefined, VALIDATE_RETRY_OPTIONS, 1)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
const lower = message.toLowerCase()
if (
lower.includes('permanentredirect') ||
lower.includes('authorizationheadermalformed') ||
lower.includes(' 301 ')
) {
return {
valid: false,
error:
'Wrong region for this bucket. Update the region to match where the bucket lives (or use "auto" for Cloudflare R2).',
}
}
if (lower.includes('403') || lower.includes('accessdenied') || lower.includes('signature')) {
return {
valid: false,
error: 'Access denied. Check the access key, secret key, and bucket permissions.',
}
}
if (lower.includes('404') || lower.includes('nosuchbucket')) {
return { valid: false, error: 'Bucket not found. Check the bucket name and region.' }
}
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.prefix === 'string' && metadata.prefix.length > 0) {
result.prefix = metadata.prefix
}
if (metadata.fileSize != null) {
const num = Number(metadata.fileSize)
if (!Number.isNaN(num)) result.fileSize = num
}
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
return result
},
}