d25d482dc2
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
119 lines
3.1 KiB
TypeScript
119 lines
3.1 KiB
TypeScript
import { getErrorMessage } from '@sim/utils/errors'
|
|
import type { FileParseResult } from '@/lib/file-parsers/types'
|
|
|
|
/**
|
|
* Parse JSON files
|
|
*/
|
|
export async function parseJSON(filePath: string): Promise<FileParseResult> {
|
|
const fs = await import('fs/promises')
|
|
const content = await fs.readFile(filePath, 'utf-8')
|
|
|
|
try {
|
|
// Parse to validate JSON
|
|
const jsonData = JSON.parse(content)
|
|
|
|
// Return pretty-printed JSON for better readability
|
|
const formattedContent = JSON.stringify(jsonData, null, 2)
|
|
|
|
// Extract metadata about the JSON structure
|
|
const metadata = {
|
|
type: 'json',
|
|
isArray: Array.isArray(jsonData),
|
|
keys: Array.isArray(jsonData) ? [] : Object.keys(jsonData),
|
|
itemCount: Array.isArray(jsonData) ? jsonData.length : undefined,
|
|
depth: getJsonDepth(jsonData),
|
|
}
|
|
|
|
return {
|
|
content: formattedContent,
|
|
metadata,
|
|
}
|
|
} catch (error) {
|
|
throw new Error(`Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse JSON from buffer
|
|
*/
|
|
export async function parseJSONBuffer(buffer: Buffer): Promise<FileParseResult> {
|
|
const content = buffer.toString('utf-8')
|
|
|
|
try {
|
|
const jsonData = JSON.parse(content)
|
|
const formattedContent = JSON.stringify(jsonData, null, 2)
|
|
|
|
const metadata = {
|
|
type: 'json',
|
|
isArray: Array.isArray(jsonData),
|
|
keys: Array.isArray(jsonData) ? [] : Object.keys(jsonData),
|
|
itemCount: Array.isArray(jsonData) ? jsonData.length : undefined,
|
|
depth: getJsonDepth(jsonData),
|
|
}
|
|
|
|
return {
|
|
content: formattedContent,
|
|
metadata,
|
|
}
|
|
} catch (error) {
|
|
throw new Error(`Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse JSONL (JSON Lines) files — one JSON object per line
|
|
*/
|
|
export async function parseJSONL(filePath: string): Promise<FileParseResult> {
|
|
const fs = await import('fs/promises')
|
|
const content = await fs.readFile(filePath, 'utf-8')
|
|
return parseJSONLContent(content)
|
|
}
|
|
|
|
/**
|
|
* Parse JSONL from buffer
|
|
*/
|
|
export async function parseJSONLBuffer(buffer: Buffer): Promise<FileParseResult> {
|
|
const content = buffer.toString('utf-8')
|
|
return parseJSONLContent(content)
|
|
}
|
|
|
|
function parseJSONLContent(content: string): FileParseResult {
|
|
const lines = content.split('\n').filter((line) => line.trim())
|
|
const items: unknown[] = []
|
|
|
|
for (const line of lines) {
|
|
try {
|
|
items.push(JSON.parse(line))
|
|
} catch {
|
|
throw new Error(`Invalid JSONL: failed to parse line: ${line.slice(0, 100)}`)
|
|
}
|
|
}
|
|
|
|
const formattedContent = JSON.stringify(items, null, 2)
|
|
|
|
return {
|
|
content: formattedContent,
|
|
metadata: {
|
|
type: 'json',
|
|
isArray: true,
|
|
keys: [],
|
|
itemCount: items.length,
|
|
depth: items.length > 0 ? 1 + getJsonDepth(items[0]) : 1,
|
|
},
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate the depth of a JSON object
|
|
*/
|
|
function getJsonDepth(obj: any): number {
|
|
if (obj === null || typeof obj !== 'object') return 0
|
|
|
|
if (Array.isArray(obj)) {
|
|
return obj.length > 0 ? 1 + Math.max(...obj.map(getJsonDepth)) : 1
|
|
}
|
|
|
|
const depths = Object.values(obj).map(getJsonDepth)
|
|
return depths.length > 0 ? 1 + Math.max(...depths) : 1
|
|
}
|