Add atomic task deduplication with partial unique index

- Add partial unique index for active resource-scoped tasks
- Implement createTaskRunRecordAtomic for race-free task creation
- Update findOrEnqueueTask to use atomic insert first
- Add tests for concurrent task creation deduplication
This commit is contained in:
2026-03-15 14:40:38 -04:00
parent a7f7be50b4
commit ed4420b8db
5 changed files with 225 additions and 6 deletions

View File

@@ -68,7 +68,8 @@ describe('task repos', () => {
'0007_company_financial_bundles.sql',
'0008_research_workspace.sql',
'0009_task_notification_context.sql',
'0012_company_overview_cache.sql'
'0012_company_overview_cache.sql',
'0013_task_active_resource_unique.sql'
]) {
applyMigration(sqliteClient, file);
}
@@ -222,4 +223,106 @@ describe('task repos', () => {
expect(failed?.error).toContain('Search indexing could not generate embeddings');
expect(failed?.stage_context?.progress?.current).toBe(2);
});
it('atomically deduplicates concurrent task creation for same resource', async () => {
if (!tasksRepo) {
throw new Error('tasks repo not initialized');
}
const resourceKey = 'sync_filings:AAPL';
const concurrentCount = 10;
const results = await Promise.all(
Array.from({ length: concurrentCount }, (_, i) =>
tasksRepo!.createTaskRunRecordAtomic({
id: `task-race-${i}`,
user_id: TEST_USER_ID,
task_type: 'sync_filings',
payload: { ticker: 'AAPL' },
priority: 50,
max_attempts: 3,
resource_key: resourceKey
})
)
);
const createdResults = results.filter((r) => r.created);
const conflictResults = results.filter((r) => !r.created);
expect(createdResults.length).toBe(1);
expect(conflictResults.length).toBe(concurrentCount - 1);
expect(createdResults[0]?.task.resource_key).toBe(resourceKey);
expect(createdResults[0]?.task.status).toBe('queued');
});
it('allows creating new task for same resource after previous task completes', async () => {
if (!tasksRepo) {
throw new Error('tasks repo not initialized');
}
const resourceKey = 'sync_filings:MSFT';
const first = await tasksRepo.createTaskRunRecordAtomic({
id: 'task-first-msft',
user_id: TEST_USER_ID,
task_type: 'sync_filings',
payload: { ticker: 'MSFT' },
priority: 50,
max_attempts: 3,
resource_key: resourceKey
});
expect(first.created).toBe(true);
if (!first.created) throw new Error('Expected task to be created');
await tasksRepo.completeTask(first.task.id, { ticker: 'MSFT' });
const second = await tasksRepo.createTaskRunRecordAtomic({
id: 'task-second-msft',
user_id: TEST_USER_ID,
task_type: 'sync_filings',
payload: { ticker: 'MSFT' },
priority: 50,
max_attempts: 3,
resource_key: resourceKey
});
expect(second.created).toBe(true);
if (!second.created) throw new Error('Expected second task to be created');
expect(second.task.id).not.toBe(first.task.id);
});
it('allows creating tasks without resource key without deduplication', async () => {
if (!tasksRepo) {
throw new Error('tasks repo not initialized');
}
const results = await Promise.all([
tasksRepo.createTaskRunRecordAtomic({
id: 'task-nokey-1',
user_id: TEST_USER_ID,
task_type: 'sync_filings',
payload: { ticker: 'GOOGL' },
priority: 50,
max_attempts: 3,
resource_key: null
}),
tasksRepo.createTaskRunRecordAtomic({
id: 'task-nokey-2',
user_id: TEST_USER_ID,
task_type: 'sync_filings',
payload: { ticker: 'GOOGL' },
priority: 50,
max_attempts: 3,
resource_key: null
})
]);
expect(results[0]?.created).toBe(true);
expect(results[1]?.created).toBe(true);
if (!results[0]?.created || !results[1]?.created) {
throw new Error('Expected both tasks to be created');
}
expect(results[0].task.id).not.toBe(results[1].task.id);
});
});

View File

@@ -153,6 +153,83 @@ export async function createTaskRunRecord(input: CreateTaskInput) {
});
}
export type AtomicCreateResult =
| { task: Task; created: true }
| { task: null; created: false };
const SQLITE_CONSTRAINT_UNIQUE = 2067;
async function attemptAtomicInsert(
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
input: CreateTaskInput,
now: string
): Promise<{ task: Task; created: true } | null> {
try {
const [row] = await tx
.insert(taskRun)
.values({
id: input.id,
user_id: input.user_id,
task_type: input.task_type,
status: 'queued',
stage: 'queued',
stage_detail: null,
stage_context: null,
resource_key: input.resource_key ?? null,
notification_read_at: null,
notification_silenced_at: null,
priority: input.priority,
payload: input.payload,
result: null,
error: null,
attempts: 0,
max_attempts: input.max_attempts,
workflow_run_id: null,
created_at: now,
updated_at: now,
finished_at: null
})
.returning();
if (!row) {
return null;
}
await insertTaskStageEvent(tx, {
task_id: row.id,
user_id: row.user_id,
stage: row.stage as TaskStage,
stage_detail: row.stage_detail,
stage_context: row.stage_context ?? null,
status: row.status,
created_at: now
});
return { task: toTask(row), created: true };
} catch (error) {
const sqliteError = error as { code?: number; message?: string };
if (
sqliteError.code === SQLITE_CONSTRAINT_UNIQUE ||
sqliteError.message?.includes('UNIQUE constraint failed')
) {
return null;
}
throw error;
}
}
export async function createTaskRunRecordAtomic(input: CreateTaskInput): Promise<AtomicCreateResult> {
const now = new Date().toISOString();
return await db.transaction(async (tx) => {
const result = await attemptAtomicInsert(tx, input, now);
if (result) {
return result;
}
return { task: null, created: false };
});
}
export async function setTaskWorkflowRunId(taskId: string, workflowRunId: string) {
await db
.update(taskRun)