chore: import upstream snapshot with attribution
Docker Publish / docker (web, apps/web/Dockerfile, web) (push) Failing after 0s
Docker Publish / docker (api, apps/api/Dockerfile, api) (push) Failing after 1s
Publish Extension / detect-version (push) Has been skipped
Publish Extension / submit (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:24:41 +08:00
commit d18ada4ee7
567 changed files with 118275 additions and 0 deletions
+41
View File
@@ -0,0 +1,41 @@
import type { Task, TaskInput, TaskPriority, TaskStatus } from '../src/types'
import { EMPTY_PROGRESS } from '../src/types'
let counter = 0
export function makeTaskInput(overrides: Partial<TaskInput> = {}): TaskInput {
return {
url: 'https://example.com/v/abc',
kind: 'video',
...overrides
}
}
export function makeTask(overrides: Partial<Task> = {}): Task {
const id = overrides.id ?? `t-${++counter}`
const now = overrides.createdAt ?? 1_700_000_000_000
const status: TaskStatus = overrides.status ?? 'queued'
return {
id,
kind: 'video',
parentId: null,
input: makeTaskInput(),
priority: 0 as TaskPriority,
groupKey: 'example.com',
status,
prevStatus: null,
statusReason: null,
enteredStatusAt: now,
attempt: 0,
maxAttempts: 5,
nextRetryAt: null,
progress: { ...EMPTY_PROGRESS },
output: null,
lastError: null,
pid: null,
pidStartedAt: null,
createdAt: now,
updatedAt: now,
...overrides
}
}
+141
View File
@@ -0,0 +1,141 @@
import { describe, expect, it } from 'vitest'
import {
CLASSIFIER_RULES,
classify,
defaultMaxAttempts,
parseRetryAfter,
sanitizeOutput,
takeStderrTail,
virtualError
} from '../src/classifier'
describe('ErrorClassifier rules (design §7.1)', () => {
it('rule order matches the design doc', () => {
const order = CLASSIFIER_RULES.map((r) => r.category)
expect(order).toEqual([
'http-429',
'auth-required',
'geo-blocked',
'not-found',
'disk-full',
'permission-denied',
'binary-missing',
'ffmpeg',
'network-transient',
'stalled',
'cancelled-by-user',
'output-missing',
'unknown'
])
})
it.each([
['HTTP Error 429: Too Many Requests', 'http-429'],
['ERROR: Sign in to confirm your age', 'auth-required'],
['This video is not available in your country', 'geo-blocked'],
['HTTP Error 404: not found', 'not-found'],
['ENOSPC: no space left on device', 'disk-full'],
['EACCES: permission denied', 'permission-denied'],
['ffmpeg: not found', 'binary-missing'],
['Postprocessing: ffmpeg failed', 'ffmpeg'],
['ECONNRESET on socket', 'network-transient'],
['HTTP Error 503 Service Unavailable', 'network-transient'],
['nothing in particular', 'unknown']
])('classifies stderr %p as %s', (stderr, category) => {
const r = classify({ stderr })
expect(r.category).toBe(category)
})
it('binary-missing also fires on exit code 127', () => {
const r = classify({ stderr: 'random message', exitCode: 127 })
expect(r.category).toBe('binary-missing')
})
it('http-429 honors Retry-After from the explicit header', () => {
const r = classify({
stderr: 'HTTP Error 429: too many',
retryAfterHeader: '15'
})
expect(r.suggestedRetryAfterMs).toBe(15_000)
})
it('http-429 falls back to Retry-After parsed from stderr', () => {
const stderr = 'HTTP Error 429: too many\nRetry-After: 7\n'
const r = classify({ stderr })
expect(r.suggestedRetryAfterMs).toBe(7_000)
})
it('http-429 default backoff is 30s when no Retry-After present', () => {
const r = classify({ stderr: 'HTTP Error 429' })
expect(r.suggestedRetryAfterMs).toBe(30_000)
})
it('non-retryable categories carry retryable=false', () => {
for (const cat of [
'auth-required',
'geo-blocked',
'not-found',
'disk-full',
'permission-denied',
'binary-missing'
] as const) {
expect(defaultMaxAttempts(cat)).toBe(0)
const v = virtualError(cat, 'x')
expect(v.retryable).toBe(false)
}
})
it('retryable categories carry retryable=true and reasonable maxAttempts', () => {
expect(defaultMaxAttempts('http-429')).toBe(3)
expect(defaultMaxAttempts('network-transient')).toBe(5)
expect(defaultMaxAttempts('stalled')).toBe(3)
expect(defaultMaxAttempts('ffmpeg')).toBe(1)
expect(defaultMaxAttempts('unknown')).toBe(1)
})
})
describe('sanitizeOutput', () => {
it('redacts Authorization headers but keeps the key name', () => {
const out = sanitizeOutput('Authorization: Bearer abc.def')
expect(out).toBe('Authorization: <redacted>')
})
it('redacts cookie/token query-string values', () => {
expect(sanitizeOutput('?token=abc.def&foo=1')).toMatch(/<redacted>/)
expect(sanitizeOutput('password=hunter2')).toMatch(/<redacted>/)
})
})
describe('takeStderrTail', () => {
it('returns the input unchanged if under the budget', () => {
expect(takeStderrTail('hello', 10)).toBe('hello')
})
it('keeps trailing portion when too large', () => {
const big = 'x'.repeat(20_000)
const tail = takeStderrTail(big, 4_096)
expect(tail.length).toBeLessThanOrEqual(big.length)
expect(Buffer.byteLength(tail, 'utf8')).toBeLessThanOrEqual(4_096)
})
})
describe('parseRetryAfter', () => {
it('parses delta-seconds', () => {
expect(parseRetryAfter('120')).toBe(120_000)
expect(parseRetryAfter('0')).toBe(0)
})
it('parses HTTP-date', () => {
const future = new Date(Date.now() + 5_000).toUTCString()
const ms = parseRetryAfter(future)
expect(ms).not.toBeNull()
expect(ms!).toBeGreaterThan(0)
})
it('returns null for nonsense', () => {
expect(parseRetryAfter('')).toBeNull()
expect(parseRetryAfter(null)).toBeNull()
expect(parseRetryAfter('not-a-date')).toBeNull()
})
})
+93
View File
@@ -0,0 +1,93 @@
import { describe, expect, it } from 'vitest'
import { taskQueueContract } from '../src/contract'
import {
AddInputSchema,
TaskInputSchema,
TaskSchema,
TaskStatusSchema
} from '../src/schemas'
describe('taskQueueContract', () => {
it('exposes the routes that adapters must implement', () => {
const expected = [
'add',
'get',
'list',
'cancel',
'pause',
'resume',
'retry',
'setMaxConcurrency',
'setMaxPerGroup',
'removeFromHistory',
'stats'
]
expect(Object.keys(taskQueueContract).sort()).toEqual(expected.sort())
})
})
describe('schemas', () => {
it('TaskInput accepts a minimal payload', () => {
const parsed = TaskInputSchema.parse({
url: 'https://e.com/v',
kind: 'video'
})
expect(parsed.kind).toBe('video')
})
it('TaskStatus enumerates exactly the 8 design statuses', () => {
const expected = [
'queued',
'running',
'processing',
'paused',
'retry-scheduled',
'completed',
'failed',
'cancelled'
]
for (const s of expected) {
expect(() => TaskStatusSchema.parse(s)).not.toThrow()
}
expect(() => TaskStatusSchema.parse('downloading')).toThrow()
})
it('AddInput requires a kind on the inner input', () => {
const r = AddInputSchema.safeParse({ input: { url: 'https://e.com' } })
expect(r.success).toBe(false)
})
it('Task allows null parentId/output/lastError', () => {
const t = {
id: 'x',
kind: 'video' as const,
parentId: null,
input: { url: 'https://e.com', kind: 'video' as const },
priority: 0 as const,
groupKey: 'e.com',
status: 'queued' as const,
prevStatus: null,
statusReason: null,
enteredStatusAt: 1,
attempt: 0,
maxAttempts: 5,
nextRetryAt: null,
progress: {
percent: null,
bytesDownloaded: null,
bytesTotal: null,
speedBps: null,
etaMs: null,
ticks: 0
},
output: null,
lastError: null,
pid: null,
pidStartedAt: null,
createdAt: 1,
updatedAt: 1
}
expect(() => TaskSchema.parse(t)).not.toThrow()
})
})
+152
View File
@@ -0,0 +1,152 @@
import { describe, expect, it } from 'vitest'
import {
IllegalTransitionError,
isLegalTransition,
LEGAL_TRANSITIONS,
transition
} from '../src/fsm'
import type { TaskOutput, TaskStatus } from '../src/types'
import { TERMINAL_STATUSES } from '../src/types'
import { makeTask } from './_fixtures'
const ALL_STATUSES: TaskStatus[] = [
'queued',
'running',
'processing',
'paused',
'retry-scheduled',
'completed',
'failed',
'cancelled'
]
describe('TaskFSM transition table (design §3.1)', () => {
it('legality table matches the design doc one-for-one', () => {
const expected: Record<TaskStatus, ReadonlyArray<TaskStatus>> = {
queued: ['running', 'paused', 'cancelled'],
running: [
'processing',
'completed',
'retry-scheduled',
'failed',
'paused',
'cancelled'
],
processing: [
'completed',
'retry-scheduled',
'failed',
'paused',
'cancelled'
],
paused: ['queued', 'cancelled', 'failed'],
'retry-scheduled': ['queued', 'paused', 'cancelled'],
failed: ['queued'],
cancelled: ['queued'],
completed: []
}
for (const [from, tos] of Object.entries(expected)) {
const set = LEGAL_TRANSITIONS[from as TaskStatus]
expect([...set].sort()).toEqual([...tos].sort())
}
})
it('every illegal From→To combo throws IllegalTransitionError', () => {
for (const from of ALL_STATUSES) {
for (const to of ALL_STATUSES) {
if (from === to) continue
if (isLegalTransition(from, to)) continue
const task = makeTask({ status: from })
expect(() =>
transition(task, to, { trigger: 'fail' })
).toThrowError(IllegalTransitionError)
}
}
})
it('terminal `completed` has no outgoing transitions', () => {
const t = makeTask({ status: 'completed' })
for (const to of ALL_STATUSES) {
if (to === 'completed') continue
expect(() => transition(t, to, { trigger: 'fail' })).toThrow()
}
})
it('completed requires output.size > 0 (design §3.1 guard)', () => {
const t = makeTask({ status: 'running' })
const zero: TaskOutput = {
filePath: '/tmp/x.mp4',
size: 0,
durationMs: null,
sha256: null
}
expect(() =>
transition(t, 'completed', { trigger: 'finalize-success', output: zero })
).toThrow(IllegalTransitionError)
const ok: TaskOutput = { ...zero, size: 1024 }
const next = transition(t, 'completed', {
trigger: 'finalize-success',
output: ok
})
expect(next.status).toBe('completed')
expect(next.output).toEqual(ok)
expect(next.pid).toBeNull()
})
it('queued → running on the first dispatch sets attempt to 1', () => {
const t = makeTask({ status: 'queued', attempt: 0 })
const next = transition(t, 'running', { trigger: 'dispatch' })
expect(next.status).toBe('running')
expect(next.attempt).toBe(1)
})
it('retry-scheduled → queued via tick increments attempt', () => {
const t = makeTask({
status: 'retry-scheduled',
attempt: 2,
nextRetryAt: 1
})
const next = transition(t, 'queued', { trigger: 'retry-tick' })
expect(next.attempt).toBe(3)
expect(next.nextRetryAt).toBeNull()
})
it('failed → queued via manual retry resets attempt to 0', () => {
const t = makeTask({ status: 'failed', attempt: 3 })
const next = transition(t, 'queued', { trigger: 'retry-manual' })
expect(next.attempt).toBe(0)
expect(next.lastError).toBeNull()
})
it('retry-scheduled → retry-scheduled requires nextRetryAt', () => {
const t = makeTask({ status: 'running' })
expect(() =>
transition(t, 'retry-scheduled', {
trigger: 'finalize-error'
})
).toThrow()
const next = transition(t, 'retry-scheduled', {
trigger: 'finalize-error',
nextRetryAt: 1_700_000_010_000
})
expect(next.status).toBe('retry-scheduled')
expect(next.nextRetryAt).toBe(1_700_000_010_000)
})
it('terminal statuses block writes (assert no leak through transition)', () => {
for (const term of TERMINAL_STATUSES) {
const t = makeTask({ status: term })
// Self-transition is illegal except via the explicit failed→queued/
// cancelled→queued paths (already covered by legality table).
for (const to of ALL_STATUSES) {
if (isLegalTransition(term, to)) continue
if (to === term) continue
expect(() =>
transition(t, to, { trigger: 'cancel' })
).toThrow(IllegalTransitionError)
}
}
})
})
@@ -0,0 +1,376 @@
/**
* Integration tests for TaskQueueAPI using a fake executor that scripts
* spawn/progress/finish events deterministically.
*/
import { describe, expect, it } from 'vitest'
import { TaskQueueAPI } from '../src/api'
import type {
Executor,
ExecutorEvents,
ExecutorRun
} from '../src/executor'
import { MemoryPersistAdapter } from '../src/persist'
import type { TaskQueueEvent } from '../src/events'
import type { ClassifiedError } from '../src/types'
interface ScriptStep {
type:
| 'spawn'
| 'progress'
| 'success'
| 'errorRetryable'
| 'errorFatal'
| 'cancelled'
}
function makeFakeExecutor(scriptByUrl: Map<string, ScriptStep[]>): Executor {
let pidCounter = 1000
return {
run(ctx, events: ExecutorEvents): ExecutorRun {
const script = scriptByUrl.get(ctx.input.url) ?? [
{ type: 'spawn' },
{ type: 'success' }
]
const pid = ++pidCounter
let cancelled = false
// Emit synchronously so tests don't have to await microtasks.
queueMicrotask(() => {
for (const step of script) {
if (cancelled) {
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: { type: 'cancelled' },
closedAt: 1,
stdoutTail: '',
stderrTail: ''
})
return
}
switch (step.type) {
case 'spawn':
events.onSpawn({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
pid,
pidStartedAt: 1,
kind: 'yt-dlp',
spawnedAt: 1
})
break
case 'progress':
events.onProgress({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
progress: {
percent: 0.5,
bytesDownloaded: 10,
bytesTotal: 20,
speedBps: 100,
etaMs: 100,
ticks: 1
},
enteredProcessing: false
})
break
case 'success':
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: {
type: 'success',
output: {
filePath: '/fake/file.mp4',
size: 1234,
durationMs: null,
sha256: null
}
},
closedAt: 1,
stdoutTail: '',
stderrTail: ''
})
break
case 'errorRetryable': {
const err: ClassifiedError = {
category: 'network-transient',
exitCode: 1,
rawMessage: 'ECONNRESET',
uiMessageKey: 'task.error.networkTransient',
uiActionHints: ['retry'],
retryable: true,
suggestedRetryAfterMs: null
}
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: { type: 'error', error: err, exitCode: 1 },
closedAt: 1,
stdoutTail: '',
stderrTail: 'ECONNRESET on socket'
})
break
}
case 'errorFatal': {
const err: ClassifiedError = {
category: 'auth-required',
exitCode: 1,
rawMessage: 'Sign in to confirm',
uiMessageKey: 'task.error.authRequired',
uiActionHints: ['import-cookies'],
retryable: false,
suggestedRetryAfterMs: null
}
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: { type: 'error', error: err, exitCode: 1 },
closedAt: 1,
stdoutTail: '',
stderrTail: 'Sign in to confirm'
})
break
}
case 'cancelled':
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: { type: 'cancelled' },
closedAt: 1,
stdoutTail: '',
stderrTail: ''
})
break
}
}
})
return {
cancel: async () => {
cancelled = true
},
pause: async () => {
cancelled = true
}
}
}
}
}
async function flush(ms = 50): Promise<void> {
await new Promise((r) => setTimeout(r, ms))
}
describe('TaskQueueAPI orchestrator', () => {
it('happy path: queued → running → completed', async () => {
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor: makeFakeExecutor(new Map()),
filePresent: () => true,
maxConcurrency: 2
})
await api.start()
const events: TaskQueueEvent[] = []
api.subscribe((e) => events.push(e))
const { id } = await api.add({
input: { url: 'https://example.com/v', kind: 'video' }
})
await flush()
const t = api.get(id)!
expect(t.status).toBe('completed')
expect(t.output?.size).toBe(1234)
const transitions = events
.filter((e) => e.type === 'transition')
.map((e) => (e as { to: string }).to)
expect(transitions).toEqual(['queued', 'running', 'completed'])
})
it('fatal error → failed (no retry)', async () => {
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor: makeFakeExecutor(
new Map([
[
'https://e.com/x',
[{ type: 'spawn' }, { type: 'errorFatal' }]
]
])
),
filePresent: () => true
})
await api.start()
const { id } = await api.add({
input: { url: 'https://e.com/x', kind: 'video' }
})
await flush()
const t = api.get(id)!
expect(t.status).toBe('failed')
expect(t.lastError?.category).toBe('auth-required')
})
it('retryable error → retry-scheduled then queued on tick', async () => {
let now = 1_000_000
const ticks: number[] = []
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor: makeFakeExecutor(
new Map([
[
'https://e.com/x',
// First attempt: fail. Subsequent attempts: succeed.
[{ type: 'spawn' }, { type: 'errorRetryable' }]
]
])
),
filePresent: () => true,
clock: () => now,
setTimer: (fn, ms) => {
ticks.push(ms)
// Force timer to fire on next macrotask using real setTimeout(0).
return setTimeout(fn, 0)
},
clearTimer: (h) => clearTimeout(h as ReturnType<typeof setTimeout>),
rng: () => 0 // deterministic backoff = 0
})
await api.start()
const { id } = await api.add({
input: { url: 'https://e.com/x', kind: 'video' }
})
await flush()
const t = api.get(id)!
// After first attempt fails the orchestrator goes retry-scheduled then
// due to rng=0 the backoff is 0, so the tick fires almost immediately.
// The fake executor still has only one script (errorRetryable), so the
// second attempt also fails → another retry. Eventually the task hits
// the same script. We verify *either* retry-scheduled or further along
// the retry path, but at minimum that the task is not stuck queued and
// that the retry timer fired at least once.
expect(['retry-scheduled', 'queued', 'running', 'failed']).toContain(t.status)
expect(ticks.length).toBeGreaterThan(0)
})
it('cancel on running issues SIGTERM and lands in cancelled', async () => {
let resolveSpawn: (() => void) | null = null
const spawnSeen = new Promise<void>((r) => {
resolveSpawn = r
})
const executor: Executor = {
run(ctx, events) {
let cancelled = false
queueMicrotask(() => {
events.onSpawn({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
pid: 1234,
pidStartedAt: 1,
kind: 'yt-dlp',
spawnedAt: 1
})
resolveSpawn?.()
})
return {
cancel: async () => {
cancelled = true
queueMicrotask(() =>
events.onFinish({
taskId: ctx.taskId,
attemptId: ctx.attemptId,
result: { type: 'cancelled' },
closedAt: 1,
stdoutTail: '',
stderrTail: ''
})
)
},
pause: async () => {
void cancelled
}
}
}
}
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor,
filePresent: () => true
})
await api.start()
const { id } = await api.add({
input: { url: 'https://e.com/c', kind: 'video' }
})
await spawnSeen
await flush(0)
await api.cancel(id, 'user')
await flush()
expect(api.get(id)!.status).toBe('cancelled')
expect(api.stats().running).toBe(0)
})
it('processing → completed guard fails when file missing/empty', async () => {
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor: makeFakeExecutor(new Map()),
filePresent: () => false // pretend the file disappeared
})
await api.start()
const { id } = await api.add({
input: { url: 'https://e.com/v', kind: 'video' }
})
await flush()
const t = api.get(id)!
expect(t.status).toBe('failed')
expect(t.lastError?.category).toBe('output-missing')
})
it('crash recovery: paused/recovered tasks survive restart', async () => {
const persist = new MemoryPersistAdapter()
let api = new TaskQueueAPI({
persist,
executor: makeFakeExecutor(new Map()),
filePresent: () => true
})
await api.start()
const { id } = await api.add({
input: { url: 'https://e.com/r', kind: 'video' }
})
await flush()
expect(api.get(id)!.status).toBe('completed')
// Spin up a brand-new orchestrator on the same persist adapter.
api = new TaskQueueAPI({
persist,
executor: makeFakeExecutor(new Map()),
filePresent: () => true
})
await api.start()
expect(api.get(id)!.status).toBe('completed')
})
// Regression: NEX-124 review screenshot showed two list rows for one
// pasted URL — the renderer's optimistic placeholder (renderer-id) and
// the kernel-generated row (random UUID) coexisted because add() ignored
// the caller-supplied id. Honoring `req.id` makes them merge into one.
it('honors caller-supplied id and is idempotent on re-add', async () => {
const api = new TaskQueueAPI({
persist: new MemoryPersistAdapter(),
executor: makeFakeExecutor(new Map()),
filePresent: () => true
})
await api.start()
const callerId = 'renderer-optimistic-id-123'
const first = await api.add({
id: callerId,
input: { url: 'https://example.com/a', kind: 'video' }
})
expect(first.id).toBe(callerId)
expect(api.get(callerId)).toBeDefined()
// Second add with the same id is a no-op (idempotent), not a duplicate.
const second = await api.add({
id: callerId,
input: { url: 'https://example.com/a', kind: 'video' }
})
expect(second.id).toBe(callerId)
const all = api.list({ limit: 100, cursor: null }).tasks
expect(all.filter((t) => t.id === callerId)).toHaveLength(1)
})
})
@@ -0,0 +1,77 @@
import { describe, expect, it, vi } from 'vitest'
import { MemoryPersistAdapter } from '../src/persist'
import { ProcessRegistry } from '../src/process'
describe('ProcessRegistry', () => {
it('records spawn + close pairs in the journal', async () => {
const persist = new MemoryPersistAdapter()
const reg = new ProcessRegistry({ persist, kill: vi.fn(), sleep: async () => {} })
await reg.recordSpawn({
taskId: 't',
attemptId: 'a',
pid: 100,
pidStartedAt: 50,
kind: 'yt-dlp',
spawnedAt: Date.now()
})
await reg.recordClose('t', 'a', 0, null)
const journal = persist.journalSnapshot()
expect(journal.map((r) => r.op)).toEqual(['spawn', 'close'])
const open = await persist.findOpenSpawns()
expect(open).toHaveLength(0)
})
it('cancel issues SIGTERM then SIGKILL after grace', async () => {
const persist = new MemoryPersistAdapter()
const kill = vi.fn()
const reg = new ProcessRegistry({
persist,
kill,
sleep: async () => {},
// Simulate the process never dying so SIGKILL fires.
killGracePeriodMs: 0
})
await reg.recordSpawn({
taskId: 't',
attemptId: 'a',
pid: 100,
pidStartedAt: 50,
kind: 'yt-dlp',
spawnedAt: Date.now()
})
await reg.cancel('t', 'a')
expect(kill).toHaveBeenCalledWith(100, 'SIGTERM')
// killed row written
const journal = persist.journalSnapshot()
expect(journal.map((r) => r.op)).toContain('killed')
})
it('reconcile finds spawn rows without a matching close and journals killed', async () => {
const persist = new MemoryPersistAdapter()
// Pretend the process has been killed externally by the OS — kill stub is
// a noop but isPidAlive returns false because the pid is fake.
const reg = new ProcessRegistry({
persist,
kill: vi.fn(),
sleep: async () => {},
killGracePeriodMs: 0
})
// Append an orphan spawn directly (simulating a previous app instance).
await persist.appendJournal({
ts: Date.now(),
op: 'spawn',
taskId: 't1',
attemptId: 'a1',
pid: 999_999, // surely dead
pidStartedAt: 0,
exitCode: null,
signal: null
})
const reconciled = await reg.reconcile()
expect(reconciled).toHaveLength(1)
expect(reconciled[0]!.taskId).toBe('t1')
const ops = persist.journalSnapshot().map((r) => r.op)
expect(ops).toContain('killed')
})
})
+236
View File
@@ -0,0 +1,236 @@
import { describe, expect, it } from 'vitest'
import {
legacyDownloadStatusOf,
legacySubStatusOf,
projectTaskToLegacy
} from '../src/projection'
import type { ClassifiedError, Task, TaskStatus } from '../src/types'
const baseTask = (overrides: Partial<Task> = {}): Task => ({
id: 'task-1',
kind: 'video',
parentId: null,
input: {
url: 'https://example.com/v',
kind: 'video',
title: 'Example',
thumbnail: 'https://example.com/t.jpg'
},
priority: 0,
groupKey: 'example.com',
status: 'queued',
prevStatus: null,
statusReason: null,
enteredStatusAt: 1000,
attempt: 0,
maxAttempts: 5,
nextRetryAt: null,
progress: {
percent: null,
bytesDownloaded: null,
bytesTotal: null,
speedBps: null,
etaMs: null,
ticks: 0
},
output: null,
lastError: null,
pid: null,
pidStartedAt: null,
createdAt: 1000,
updatedAt: 1000,
...overrides
})
const sampleError: ClassifiedError = {
category: 'http-429',
exitCode: 1,
rawMessage: 'too many requests',
uiMessageKey: 'errors.rate_limited',
uiActionHints: ['retry-later'],
retryable: true,
suggestedRetryAfterMs: 30_000
}
describe('legacyDownloadStatusOf', () => {
it('maps every internal status to a legacy status', () => {
const cases: Array<[TaskStatus, ReturnType<typeof legacyDownloadStatusOf>]> = [
['queued', 'pending'],
['running', 'downloading'],
['processing', 'processing'],
['paused', 'pending'],
['retry-scheduled', 'pending'],
['completed', 'completed'],
['failed', 'error'],
['cancelled', 'cancelled']
]
for (const [internal, legacy] of cases) {
expect(legacyDownloadStatusOf(internal)).toBe(legacy)
}
})
it('legacySubStatusOf only fires for the collapsed buckets', () => {
expect(legacySubStatusOf('queued')).toBe('queued')
expect(legacySubStatusOf('paused')).toBe('paused')
expect(legacySubStatusOf('retry-scheduled')).toBe('retry-scheduled')
expect(legacySubStatusOf('running')).toBeUndefined()
expect(legacySubStatusOf('processing')).toBeUndefined()
expect(legacySubStatusOf('completed')).toBeUndefined()
expect(legacySubStatusOf('failed')).toBeUndefined()
expect(legacySubStatusOf('cancelled')).toBeUndefined()
})
})
describe('projectTaskToLegacy', () => {
it('projects queued task as pending without progress', () => {
const proj = projectTaskToLegacy(baseTask())
expect(proj.status).toBe('pending')
expect(proj.subStatus).toBe('queued')
expect(proj.internalStatus).toBe('queued')
expect(proj.progress).toBeUndefined()
})
it('projects running task with progress and speed', () => {
const proj = projectTaskToLegacy(
baseTask({
status: 'running',
progress: {
percent: 0.42,
bytesDownloaded: 1024 * 1024,
bytesTotal: 4 * 1024 * 1024,
speedBps: 512 * 1024,
etaMs: 12_345,
ticks: 5
}
})
)
expect(proj.status).toBe('downloading')
expect(proj.subStatus).toBeUndefined()
expect(proj.progress?.percent).toBeCloseTo(42, 5)
expect(proj.progress?.currentSpeed).toBe('512.00KB/s')
expect(proj.speed).toBe('512.00KB/s')
})
it('projects paused task with subStatus and statusReason', () => {
const proj = projectTaskToLegacy(
baseTask({
status: 'paused',
statusReason: 'crash-recovery',
progress: {
percent: 0.5,
bytesDownloaded: 100,
bytesTotal: 200,
speedBps: null,
etaMs: null,
ticks: 1
}
})
)
expect(proj.status).toBe('pending')
expect(proj.subStatus).toBe('paused')
expect(proj.statusReason).toBe('crash-recovery')
expect(proj.progress?.percent).toBeCloseTo(50, 5)
})
it('projects retry-scheduled task with attempt + nextRetryAt + errorCategory', () => {
const proj = projectTaskToLegacy(
baseTask({
status: 'retry-scheduled',
statusReason: 'http-429',
attempt: 2,
maxAttempts: 5,
nextRetryAt: 9_999_999,
lastError: sampleError
})
)
expect(proj.status).toBe('pending')
expect(proj.subStatus).toBe('retry-scheduled')
expect(proj.attempt).toBe(2)
expect(proj.maxAttempts).toBe(5)
expect(proj.nextRetryAt).toBe(9_999_999)
expect(proj.errorCategory).toBe('http-429')
expect(proj.uiMessageKey).toBe('errors.rate_limited')
// Don't surface raw message until status is `error`.
expect(proj.error).toBeUndefined()
})
it('projects completed task with output filename and downloadPath', () => {
const proj = projectTaskToLegacy(
baseTask({
status: 'completed',
output: {
filePath: '/Users/me/Downloads/VidBee/clip.mp4',
size: 1234,
durationMs: 60_000,
sha256: null
}
})
)
expect(proj.status).toBe('completed')
expect(proj.fileSize).toBe(1234)
expect(proj.savedFileName).toBe('clip.mp4')
expect(proj.downloadPath).toBe('/Users/me/Downloads/VidBee')
expect(proj.duration).toBe(60)
})
it('projects failed task and surfaces error/category/uiMessageKey', () => {
const proj = projectTaskToLegacy(
baseTask({
status: 'failed',
statusReason: 'auth-required',
lastError: { ...sampleError, category: 'auth-required', rawMessage: 'login needed' }
})
)
expect(proj.status).toBe('error')
expect(proj.errorCategory).toBe('auth-required')
expect(proj.uiMessageKey).toBe('errors.rate_limited')
expect(proj.error).toBe('login needed')
})
it('projects cancelled task as cancelled', () => {
const proj = projectTaskToLegacy(baseTask({ status: 'cancelled' }))
expect(proj.status).toBe('cancelled')
expect(proj.subStatus).toBeUndefined()
})
it('hoists host metadata from input.options', () => {
const proj = projectTaskToLegacy(
baseTask({
input: {
url: 'https://example.com/v',
kind: 'video',
options: {
description: 'd',
channel: 'c',
uploader: 'u',
viewCount: 100,
tags: ['a', 'b'],
duration: 99,
playlistTitle: 'PL',
playlistSize: 7,
startedAt: 555,
completedAt: 999,
downloadPath: '/tmp'
}
}
})
)
expect(proj.description).toBe('d')
expect(proj.channel).toBe('c')
expect(proj.uploader).toBe('u')
expect(proj.viewCount).toBe(100)
expect(proj.tags).toEqual(['a', 'b'])
expect(proj.duration).toBe(99)
expect(proj.playlistTitle).toBe('PL')
expect(proj.playlistSize).toBe(7)
expect(proj.startedAt).toBe(555)
expect(proj.completedAt).toBe(999)
expect(proj.downloadPath).toBe('/tmp')
})
it('audio TaskKind becomes audio type', () => {
const proj = projectTaskToLegacy(baseTask({ kind: 'audio', input: { url: 'x', kind: 'audio' } }))
expect(proj.type).toBe('audio')
})
})
@@ -0,0 +1,94 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { computeBackoffMs, RetryScheduler } from '../src/scheduler'
describe('computeBackoffMs', () => {
it('honors a non-null suggested value verbatim (e.g. Retry-After)', () => {
expect(computeBackoffMs(7, 12_345)).toBe(12_345)
expect(computeBackoffMs(0, 0)).toBe(0)
})
it('uses exponential + full jitter when no suggestion', () => {
// attempt=2 → exp = 2_000 * 4 = 8_000 → range [0, 8000)
const v = computeBackoffMs(2, null, () => 0.999_999)
expect(v).toBeGreaterThan(0)
expect(v).toBeLessThan(8_000)
})
it('caps at 60s', () => {
const v = computeBackoffMs(20, null, () => 0.999_999)
expect(v).toBeLessThan(60_000)
})
})
describe('RetryScheduler', () => {
afterEach(() => {
vi.useRealTimers()
})
it('drains tasks at their nextRetryAt with fake timers', async () => {
vi.useFakeTimers()
let now = 1_000_000
const due: string[] = []
const sched = new RetryScheduler({
clock: () => now,
setTimer: (fn, ms) => setTimeout(fn, ms),
clearTimer: (h) => clearTimeout(h as ReturnType<typeof setTimeout>),
onDue: (id) => {
due.push(id)
}
})
sched.enqueue('a', 1_000_500)
sched.enqueue('b', 1_000_300)
sched.enqueue('c', 1_000_700)
expect(sched.size()).toBe(3)
now = 1_000_300
await vi.advanceTimersByTimeAsync(0)
await sched.tick()
expect(due).toEqual(['b'])
now = 1_000_500
await sched.tick()
expect(due).toEqual(['b', 'a'])
now = 1_000_700
await sched.tick()
expect(due).toEqual(['b', 'a', 'c'])
expect(sched.size()).toBe(0)
})
it('re-enqueues a task on handler error so the next tick retries', async () => {
let now = 0
let throws = true
const due: string[] = []
const sched = new RetryScheduler({
clock: () => now,
setTimer: (fn, ms) => setTimeout(fn, ms),
clearTimer: (h) => clearTimeout(h as ReturnType<typeof setTimeout>),
onDue: (id) => {
if (throws) {
throws = false
throw new Error('boom')
}
due.push(id)
}
})
sched.enqueue('a', 100)
now = 200
await sched.tick()
expect(sched.size()).toBe(1)
now = 1_500
await sched.tick()
expect(due).toEqual(['a'])
expect(sched.size()).toBe(0)
})
it('remove() drops a pending task', () => {
const sched = new RetryScheduler({ onDue: () => {} })
sched.enqueue('a', Date.now() + 1_000_000)
expect(sched.remove('a')).toBe(true)
expect(sched.remove('a')).toBe(false)
sched.stop()
})
})
+162
View File
@@ -0,0 +1,162 @@
import { describe, expect, it } from 'vitest'
import { Scheduler } from '../src/scheduler'
import type { Task } from '../src/types'
import { makeTask } from './_fixtures'
interface Harness {
scheduler: Scheduler
store: Map<string, Task>
/** taskIds dispatched, in order. */
dispatched: string[]
/** taskIds we issue completions for, simulating finish. */
complete: (id: string) => Promise<void>
}
function makeHarness(opts: {
maxConcurrency?: number
defaultMaxPerGroup?: number | null
failingDispatch?: Set<string>
}): Harness {
const store = new Map<string, Task>()
const dispatched: string[] = []
const scheduler = new Scheduler({
maxConcurrency: opts.maxConcurrency ?? 2,
defaultMaxPerGroup: opts.defaultMaxPerGroup ?? null,
getTask: (id) => store.get(id),
dispatch: (id) => {
if (opts.failingDispatch?.has(id)) return false
dispatched.push(id)
return true
},
demote: (_id) => {}
})
return {
scheduler,
store,
dispatched,
complete: async (id: string) => {
await scheduler.releaseSlot(id)
}
}
}
describe('Scheduler', () => {
it('respects priority then FIFO', async () => {
const h = makeHarness({ maxConcurrency: 1 })
for (let i = 0; i < 4; i++) {
const t = makeTask({
id: `t${i}`,
priority: (i === 2 ? 0 : 10) as Task['priority']
})
h.store.set(t.id, t)
}
// t0 enqueues into an empty heap with the slot free → dispatches immediately.
// t1/t2/t3 wait until t0 finishes; among the queued, t2 (priority 0) wins.
await h.scheduler.enqueue('t0', 10)
await h.scheduler.enqueue('t1', 10)
await h.scheduler.enqueue('t2', 0)
await h.scheduler.enqueue('t3', 10)
// Drain — h.complete awaits releaseSlot which now awaits tryDispatch,
// so dispatched[i] is filled in by the time we read it.
await h.complete(h.dispatched[0]!)
await h.complete(h.dispatched[1]!)
await h.complete(h.dispatched[2]!)
await h.complete(h.dispatched[3]!)
expect(h.dispatched).toEqual(['t0', 't2', 't1', 't3'])
})
it('honors the global concurrency cap', async () => {
const h = makeHarness({ maxConcurrency: 2 })
for (let i = 0; i < 5; i++) {
const t = makeTask({ id: `t${i}` })
h.store.set(t.id, t)
}
for (let i = 0; i < 5; i++) await h.scheduler.enqueue(`t${i}`, 0)
expect(h.dispatched).toHaveLength(2)
expect(h.scheduler.stats().running).toBe(2)
expect(h.scheduler.stats().queued).toBe(3)
await h.complete(h.dispatched[0]!)
expect(h.dispatched).toHaveLength(3)
})
it('honors per-group caps (subscription throttle)', async () => {
const h = makeHarness({ maxConcurrency: 4 })
for (let i = 0; i < 4; i++) {
const t = makeTask({ id: `g${i}`, groupKey: 'sub:foo' })
h.store.set(t.id, t)
}
await h.scheduler.setMaxPerGroup('sub:foo', 1)
for (let i = 0; i < 4; i++) await h.scheduler.enqueue(`g${i}`, 0)
expect(h.dispatched).toHaveLength(1)
await h.complete(h.dispatched[0]!)
expect(h.dispatched).toHaveLength(2)
})
it('1000 concurrent enqueues never over- or under-dispatch', async () => {
const h = makeHarness({ maxConcurrency: 8 })
const N = 1000
for (let i = 0; i < N; i++) {
h.store.set(`x${i}`, makeTask({ id: `x${i}` }))
}
// Enqueue in parallel — the AsyncMutex must serialize slot accounting.
await Promise.all(
Array.from({ length: N }, (_, i) => h.scheduler.enqueue(`x${i}`, 0))
)
// Drain by completing 8 at a time until the heap is empty.
let drained = 0
while (drained < N) {
const inflight = [...h.dispatched.slice(drained, drained + 8)]
if (inflight.length === 0) break
for (const id of inflight) await h.complete(id)
drained += inflight.length
}
expect(h.dispatched).toHaveLength(N)
// No duplicates.
expect(new Set(h.dispatched).size).toBe(N)
})
it('setMaxConcurrency lower demotes lowest-priority running tasks', async () => {
const demoted: string[] = []
const store = new Map<string, Task>()
const dispatched: string[] = []
const scheduler = new Scheduler({
maxConcurrency: 3,
getTask: (id) => store.get(id),
dispatch: (id) => {
dispatched.push(id)
return true
},
demote: (id) => {
demoted.push(id)
}
})
store.set('hi', makeTask({ id: 'hi', priority: 0 }))
store.set('mid', makeTask({ id: 'mid', priority: 10 }))
store.set('lo', makeTask({ id: 'lo', priority: 20 }))
await scheduler.enqueue('hi', 0)
await scheduler.enqueue('mid', 10)
await scheduler.enqueue('lo', 20)
expect(scheduler.stats().running).toBe(3)
await scheduler.setMaxConcurrency(2)
expect(demoted).toEqual(['lo'])
})
it('failing dispatch releases the slot for re-use', async () => {
const h = makeHarness({
maxConcurrency: 1,
failingDispatch: new Set(['boom'])
})
h.store.set('boom', makeTask({ id: 'boom' }))
h.store.set('next', makeTask({ id: 'next' }))
await h.scheduler.enqueue('boom', 0)
await h.scheduler.enqueue('next', 0)
expect(h.dispatched).toContain('next')
})
})
+63
View File
@@ -0,0 +1,63 @@
import { describe, expect, it } from 'vitest'
import { TaskStore } from '../src/store'
import { makeTask } from './_fixtures'
describe('TaskStore', () => {
it('insert/get/list', () => {
const store = new TaskStore()
const a = makeTask({ id: 'a', groupKey: 'g1' })
const b = makeTask({ id: 'b', groupKey: 'g1' })
const c = makeTask({ id: 'c', groupKey: 'g2', status: 'completed' })
store.insert(a)
store.insert(b)
store.insert(c)
expect(store.get('a')!.id).toBe('a')
expect(store.list({ groupKey: 'g1' }).tasks.map((t) => t.id)).toEqual(['a', 'b'])
expect(store.list({ status: 'completed' }).tasks.map((t) => t.id)).toEqual(['c'])
})
it('update maintains status/group indexes', () => {
const store = new TaskStore()
const t = makeTask({ id: 'x', status: 'queued', groupKey: 'g1' })
store.insert(t)
store.update({ ...t, status: 'completed', groupKey: 'g2' })
expect(store.list({ status: 'queued' }).tasks).toHaveLength(0)
expect(store.list({ groupKey: 'g1' }).tasks).toHaveLength(0)
expect(store.list({ status: 'completed' }).tasks).toHaveLength(1)
expect(store.list({ groupKey: 'g2' }).tasks).toHaveLength(1)
})
it('paginates with cursor', () => {
const store = new TaskStore()
for (let i = 0; i < 5; i++) {
store.insert(makeTask({ id: `t${i}`, createdAt: 1000 + i }))
}
const page1 = store.list({ limit: 2 })
expect(page1.tasks.map((t) => t.id)).toEqual(['t0', 't1'])
expect(page1.nextCursor).toBe('t1')
const page2 = store.list({ limit: 2, cursor: page1.nextCursor })
expect(page2.tasks.map((t) => t.id)).toEqual(['t2', 't3'])
const page3 = store.list({ limit: 2, cursor: page2.nextCursor })
expect(page3.tasks.map((t) => t.id)).toEqual(['t4'])
expect(page3.nextCursor).toBeNull()
})
it('rejects duplicate insert', () => {
const store = new TaskStore()
store.insert(makeTask({ id: 'x' }))
expect(() => store.insert(makeTask({ id: 'x' }))).toThrow()
})
it('stats counts by status', () => {
const store = new TaskStore()
store.insert(makeTask({ id: 'a', status: 'queued' }))
store.insert(makeTask({ id: 'b', status: 'completed' }))
store.insert(makeTask({ id: 'c', status: 'completed' }))
const s = store.stats(8)
expect(s.byStatus.queued).toBe(1)
expect(s.byStatus.completed).toBe(2)
expect(s.total).toBe(3)
expect(s.capacity).toBe(8)
})
})