Improve workflow error messaging

This commit is contained in:
2026-03-09 23:51:37 -04:00
parent fa2de3e259
commit f2c25fb9c6
11 changed files with 272 additions and 41 deletions

View File

@@ -1,4 +1,5 @@
import { runTaskProcessor, type TaskExecutionOutcome } from '@/lib/server/task-processors'; import { runTaskProcessor, type TaskExecutionOutcome } from '@/lib/server/task-processors';
import { describeTaskFailure } from '@/lib/server/task-errors';
import { import {
completeTask, completeTask,
getTaskById, getTaskById,
@@ -26,11 +27,9 @@ export async function runTaskWorkflow(taskId: string) {
const outcome = await processTaskStep(refreshedTask); const outcome = await processTaskStep(refreshedTask);
await completeTaskStep(task.id, outcome); await completeTaskStep(task.id, outcome);
} catch (error) { } catch (error) {
const reason = error instanceof Error
? error.message
: 'Task failed unexpectedly';
const latestTask = await loadTaskStep(task.id); const latestTask = await loadTaskStep(task.id);
await markTaskFailureStep(task.id, reason, latestTask); const failure = describeTaskFailure(latestTask ?? task, error);
await markTaskFailureStep(task.id, failure, latestTask ?? task);
throw error; throw error;
} }
} }
@@ -63,10 +62,10 @@ async function completeTaskStep(taskId: string, outcome: TaskExecutionOutcome) {
}); });
} }
async function markTaskFailureStep(taskId: string, reason: string, latestTask: Task | null) { async function markTaskFailureStep(taskId: string, failure: { summary: string; detail: string }, latestTask: Task) {
'use step'; 'use step';
await markTaskFailure(taskId, reason, 'failed', { await markTaskFailure(taskId, failure.detail, latestTask.stage === 'completed' ? 'failed' : latestTask.stage, {
detail: reason, detail: failure.summary,
context: latestTask?.stage_context ?? null context: latestTask.stage_context ?? null
}); });
} }

View File

@@ -125,10 +125,14 @@ export function TaskDetailModal({ isOpen, taskId, onClose }: TaskDetailModalProp
const [expandedStage, setExpandedStage] = useState<string | null>(null); const [expandedStage, setExpandedStage] = useState<string | null>(null);
const defaultExpandedStage = useMemo(() => { const defaultExpandedStage = useMemo(() => {
if (task?.status === 'completed' || task?.status === 'failed') { if (task?.status === 'completed') {
return null; return null;
} }
if (task?.status === 'failed') {
return task.stage;
}
for (const item of timeline) { for (const item of timeline) {
if (item.state === 'active') { if (item.state === 'active') {
return item.stage; return item.stage;
@@ -278,6 +282,8 @@ export function TaskDetailModal({ isOpen, taskId, onClose }: TaskDetailModalProp
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className={item.state === 'active' <span className={item.state === 'active'
? 'text-[11px] uppercase tracking-[0.12em] text-[color:var(--accent)]' ? 'text-[11px] uppercase tracking-[0.12em] text-[color:var(--accent)]'
: item.state === 'failed'
? 'text-[11px] uppercase tracking-[0.12em] text-[#ff8f8f]'
: item.state === 'completed' : item.state === 'completed'
? 'text-[11px] uppercase tracking-[0.12em] text-[color:var(--terminal-muted)]' ? 'text-[11px] uppercase tracking-[0.12em] text-[color:var(--terminal-muted)]'
: 'text-[11px] uppercase tracking-[0.12em] text-[#7f8994]'} : 'text-[11px] uppercase tracking-[0.12em] text-[#7f8994]'}

View File

@@ -720,15 +720,15 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
WHERE id = ?; WHERE id = ?;
`).run( `).run(
'failed', 'failed',
'failed', 'analyze.fetch_document',
'Primary filing document fetch failed.', 'Could not load the primary filing document.',
JSON.stringify({ JSON.stringify({
subject: { subject: {
ticker: 'AAPL', ticker: 'AAPL',
accessionNumber: '0000000000-26-000021' accessionNumber: '0000000000-26-000021'
} }
}), }),
'Primary filing document fetch failed.', 'Could not load the primary filing document for AAPL · 0000000000-26-000021. Retry the job after confirming the SEC source is reachable.',
'2026-03-09T15:01:00.000Z', '2026-03-09T15:01:00.000Z',
'2026-03-09T15:01:00.000Z', '2026-03-09T15:01:00.000Z',
failedTaskId failedTaskId
@@ -739,12 +739,14 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
const failedTask = (failed.json as { const failedTask = (failed.json as {
task: { task: {
notification: { notification: {
statusLine: string;
detailLine: string | null; detailLine: string | null;
actions: Array<{ id: string; href: string | null }>; actions: Array<{ id: string; href: string | null }>;
}; };
}; };
}).task; }).task;
expect(failedTask.notification.detailLine).toBe('Primary filing document fetch failed.'); expect(failedTask.notification.statusLine).toBe('Failed during fetch primary document');
expect(failedTask.notification.detailLine).toBe('Could not load the primary filing document.');
expect(failedTask.notification.actions.some((action) => action.id === 'open_filings')).toBe(true); expect(failedTask.notification.actions.some((action) => action.id === 'open_filings')).toBe(true);
}); });

View File

@@ -202,8 +202,8 @@ describe('task repos', () => {
resource_key: 'index_search:ticker:AAPL' resource_key: 'index_search:ticker:AAPL'
}); });
await tasksRepo.markTaskFailure(failedTask.id, 'Embedding request failed', 'failed', { await tasksRepo.markTaskFailure(failedTask.id, 'Search indexing could not generate embeddings for AAPL · doc-2. The AI provider returned an empty response.', 'search.embed', {
detail: 'Embedding request failed', detail: 'Embedding generation failed.',
context: { context: {
progress: { current: 2, total: 5, unit: 'sources' }, progress: { current: 2, total: 5, unit: 'sources' },
counters: { chunksEmbedded: 20 }, counters: { chunksEmbedded: 20 },
@@ -216,7 +216,9 @@ describe('task repos', () => {
expect(completed?.stage_detail).toContain('Analysis report generated'); expect(completed?.stage_detail).toContain('Analysis report generated');
expect(completed?.stage_context?.subject?.ticker).toBe('AAPL'); expect(completed?.stage_context?.subject?.ticker).toBe('AAPL');
expect(failed?.stage_detail).toBe('Embedding request failed'); expect(failed?.stage).toBe('search.embed');
expect(failed?.stage_detail).toBe('Embedding generation failed.');
expect(failed?.error).toContain('Search indexing could not generate embeddings');
expect(failed?.stage_context?.progress?.current).toBe(2); expect(failed?.stage_context?.progress?.current).toBe(2);
}); });
}); });

View File

@@ -406,11 +406,13 @@ export async function markTaskFailure(
export async function setTaskStatusFromWorkflow( export async function setTaskStatusFromWorkflow(
taskId: string, taskId: string,
status: TaskStatus, status: TaskStatus,
error?: string | null error?: string | null,
detail?: string | null
) { ) {
const isTerminal = status === 'completed' || status === 'failed'; const isTerminal = status === 'completed' || status === 'failed';
const nextStage = statusToStage(status); const nextStage = statusToStage(status);
const nextError = status === 'failed' ? (error ?? 'Workflow run failed') : null; const nextError = status === 'failed' ? (error ?? 'Workflow run failed') : null;
const nextDetail = nextStatusDetail(status, nextError, detail);
return await db.transaction(async (tx) => { return await db.transaction(async (tx) => {
const [current] = await tx const [current] = await tx
@@ -426,7 +428,7 @@ export async function setTaskStatusFromWorkflow(
const hasNoStateChange = current.status === status const hasNoStateChange = current.status === status
&& current.stage === nextStage && current.stage === nextStage
&& (current.error ?? null) === nextError && (current.error ?? null) === nextError
&& (current.stage_detail ?? null) === (nextStatusDetail(status, nextError) ?? null) && (current.stage_detail ?? null) === (nextDetail ?? null)
&& (current.stage_context ?? null) === null && (current.stage_context ?? null) === null
&& (isTerminal ? current.finished_at !== null : current.finished_at === null); && (isTerminal ? current.finished_at !== null : current.finished_at === null);
@@ -440,7 +442,7 @@ export async function setTaskStatusFromWorkflow(
.set({ .set({
status, status,
stage: nextStage, stage: nextStage,
stage_detail: nextStatusDetail(status, nextError), stage_detail: nextDetail,
stage_context: null, stage_context: null,
error: nextError, error: nextError,
updated_at: now, updated_at: now,
@@ -506,9 +508,9 @@ export async function updateTaskNotificationState(
return row ? toTask(row) : null; return row ? toTask(row) : null;
} }
function nextStatusDetail(status: TaskStatus, error?: string | null) { function nextStatusDetail(status: TaskStatus, error?: string | null, detail?: string | null) {
if (status === 'failed') { if (status === 'failed') {
return error ?? 'Workflow run failed'; return detail ?? error ?? 'Workflow run failed';
} }
if (status === 'completed') { if (status === 'completed') {

View File

@@ -0,0 +1,56 @@
import { describe, expect, it } from 'bun:test';
import { describeTaskFailure } from '@/lib/server/task-errors';
describe('task error formatter', () => {
it('formats missing AI credentials for search indexing', () => {
const failure = describeTaskFailure({
task_type: 'index_search',
stage: 'search.embed',
stage_context: {
subject: {
ticker: 'AAPL',
label: 'doc-2'
}
},
payload: {
ticker: 'AAPL'
}
}, 'ZHIPU_API_KEY is required for AI workloads');
expect(failure.summary).toBe('Search indexing could not generate embeddings.');
expect(failure.detail).toContain('ZHIPU_API_KEY is not configured');
expect(failure.detail).toContain('AAPL · doc-2');
});
it('formats empty AI responses with stage context', () => {
const failure = describeTaskFailure({
task_type: 'analyze_filing',
stage: 'analyze.generate_report',
stage_context: {
subject: {
ticker: 'AAPL',
accessionNumber: '0000320193-26-000001'
}
},
payload: {
accessionNumber: '0000320193-26-000001'
}
}, 'AI SDK returned an empty response');
expect(failure.summary).toBe('No usable AI response during generate report.');
expect(failure.detail).toContain('during generate report');
expect(failure.detail).toContain('Retry the job');
});
it('formats workflow cancellations as descriptive operational failures', () => {
const failure = describeTaskFailure({
task_type: 'portfolio_insights',
stage: 'insights.generate',
stage_context: null,
payload: {}
}, 'Workflow run cancelled');
expect(failure.summary).toBe('The background workflow was cancelled.');
expect(failure.detail).toContain('before portfolio insight could finish');
});
});

154
lib/server/task-errors.ts Normal file
View File

@@ -0,0 +1,154 @@
import { stageLabel, taskTypeLabel } from '@/lib/task-workflow';
import type { TaskStage, TaskStageContext, TaskType } from '@/lib/types';
type TaskErrorSource = {
task_type: TaskType;
stage?: TaskStage | null;
stage_context?: TaskStageContext | null;
payload?: Record<string, unknown> | null;
};
export type TaskFailureMessage = {
summary: string;
detail: string;
};
function rawMessage(error: unknown) {
if (error instanceof Error && error.message.trim().length > 0) {
return error.message.trim();
}
if (typeof error === 'string' && error.trim().length > 0) {
return error.trim();
}
return 'Task failed unexpectedly';
}
function normalizeSentence(value: string) {
const collapsed = value
.split('\n')[0]!
.replace(/\s+/g, ' ')
.trim();
if (!collapsed) {
return 'Task failed unexpectedly.';
}
const sentence = collapsed[0]!.toUpperCase() + collapsed.slice(1);
return /[.!?]$/.test(sentence) ? sentence : `${sentence}.`;
}
function asString(value: unknown) {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
}
function subjectLabel(task: TaskErrorSource) {
const subject = task.stage_context?.subject;
const ticker = subject?.ticker ?? asString(task.payload?.ticker);
const accessionNumber = subject?.accessionNumber ?? asString(task.payload?.accessionNumber);
const label = subject?.label ?? asString(task.payload?.label);
const parts = [ticker, accessionNumber, label].filter((part): part is string => Boolean(part));
if (parts.length === 0) {
return null;
}
return parts.join(' · ');
}
function stagePhrase(stage?: TaskStage | null) {
if (!stage || stage === 'queued' || stage === 'running' || stage === 'completed' || stage === 'failed') {
return null;
}
return stageLabel(stage).toLowerCase();
}
function genericFailure(task: TaskErrorSource, message: string): TaskFailureMessage {
const failedStage = stagePhrase(task.stage);
const subject = subjectLabel(task);
const summary = failedStage
? `Failed during ${failedStage}.`
: `${taskTypeLabel(task.task_type)} failed.`;
const detail = [
failedStage
? `${taskTypeLabel(task.task_type)} failed during ${failedStage}`
: `${taskTypeLabel(task.task_type)} failed`,
subject ? `for ${subject}` : null,
'. ',
normalizeSentence(message)
].filter(Boolean).join('');
return {
summary,
detail
};
}
export function describeTaskFailure(task: TaskErrorSource, error: unknown): TaskFailureMessage {
const message = rawMessage(error);
const normalized = message.toLowerCase();
const failedStage = stagePhrase(task.stage);
const subject = subjectLabel(task);
if (normalized.includes('zhipu_api_key is required')) {
if (task.task_type === 'index_search') {
return {
summary: 'Search indexing could not generate embeddings.',
detail: `Search indexing could not generate embeddings${subject ? ` for ${subject}` : ''} because ZHIPU_API_KEY is not configured. Add the API key and retry the job.`
};
}
return {
summary: 'AI configuration is incomplete.',
detail: `${taskTypeLabel(task.task_type)} could not continue${subject ? ` for ${subject}` : ''} because ZHIPU_API_KEY is not configured. Add the API key and retry the job.`
};
}
if (normalized.includes('ai sdk returned an empty response')) {
return {
summary: failedStage
? `No usable AI response during ${failedStage}.`
: 'The AI provider returned an empty response.',
detail: `The AI provider returned an empty response${failedStage ? ` during ${failedStage}` : ''}${subject ? ` for ${subject}` : ''}. Retry the job. If this keeps happening, verify the model configuration and provider health.`
};
}
if (normalized.includes('extraction output invalid json schema')) {
return {
summary: 'The extraction response was not valid JSON.',
detail: `The AI model returned extraction data in an unexpected format${subject ? ` for ${subject}` : ''}, so filing analysis could not continue. Retry the job. If it repeats, inspect the model output or prompt.`
};
}
if (normalized.includes('workflow run cancelled')) {
return {
summary: 'The background workflow was cancelled.',
detail: `The background workflow was cancelled before ${taskTypeLabel(task.task_type).toLowerCase()} could finish${subject ? ` for ${subject}` : ''}. Retry the job if you still need the result.`
};
}
if (normalized.includes('workflow run failed')) {
return {
summary: 'The background workflow stopped unexpectedly.',
detail: `The background workflow stopped unexpectedly before ${taskTypeLabel(task.task_type).toLowerCase()} could finish${subject ? ` for ${subject}` : ''}. Retry the job. If it fails again, inspect the task details for the last completed step.`
};
}
if (normalized.includes('failed to start workflow')) {
return {
summary: 'The background worker could not start this job.',
detail: `The background worker could not start ${taskTypeLabel(task.task_type).toLowerCase()}${subject ? ` for ${subject}` : ''}. The workflow backend may be unavailable. Retry the job once the workflow service is healthy.`
};
}
if (normalized.includes('embedding')) {
return {
summary: 'Embedding generation failed.',
detail: `Search indexing could not generate embeddings${subject ? ` for ${subject}` : ''}. ${normalizeSentence(message)}`
};
}
return genericFailure(task, message);
}

View File

@@ -99,9 +99,9 @@ describe('task notification builder', () => {
const notification = buildTaskNotification(baseTask({ const notification = buildTaskNotification(baseTask({
task_type: 'analyze_filing', task_type: 'analyze_filing',
status: 'failed', status: 'failed',
stage: 'failed', stage: 'analyze.fetch_document',
stage_detail: 'Primary filing document fetch failed.', stage_detail: 'Could not load the primary filing document.',
error: 'Primary filing document fetch failed.', error: 'Could not load the primary filing document for AAPL · 0000320193-26-000001. Retry the job after confirming the SEC source is reachable.',
stage_context: { stage_context: {
subject: { subject: {
ticker: 'AAPL', ticker: 'AAPL',
@@ -116,7 +116,8 @@ describe('task notification builder', () => {
})); }));
expect(notification.tone).toBe('error'); expect(notification.tone).toBe('error');
expect(notification.detailLine).toBe('Primary filing document fetch failed.'); expect(notification.statusLine).toBe('Failed during fetch primary document');
expect(notification.detailLine).toBe('Could not load the primary filing document.');
expect(notification.actions.some((action) => action.id === 'open_filings')).toBe(true); expect(notification.actions.some((action) => action.id === 'open_filings')).toBe(true);
}); });
}); });

View File

@@ -216,13 +216,15 @@ function buildStatusLine(task: TaskCore, progress: TaskNotificationView['progres
case 'completed': case 'completed':
return 'Finished successfully'; return 'Finished successfully';
case 'failed': case 'failed':
return 'Failed'; return task.stage !== 'failed'
? `Failed during ${stageLabel(task.stage).toLowerCase()}`
: 'Failed';
} }
} }
export function buildTaskNotification(task: TaskCore): TaskNotificationView { export function buildTaskNotification(task: TaskCore): TaskNotificationView {
const progress = buildProgress(task); const progress = buildProgress(task);
const detailLine = task.error ?? task.stage_detail; const detailLine = task.stage_detail ?? task.error;
return { return {
title: taskTypeLabel(task.task_type), title: taskTypeLabel(task.task_type),

View File

@@ -3,6 +3,7 @@ import { getRun, start } from 'workflow/api';
import type { WorkflowRunStatus } from '@workflow/world'; import type { WorkflowRunStatus } from '@workflow/world';
import type { Task, TaskStatus, TaskTimeline, TaskType } from '@/lib/types'; import type { Task, TaskStatus, TaskTimeline, TaskType } from '@/lib/types';
import { runTaskWorkflow } from '@/app/workflows/task-runner'; import { runTaskWorkflow } from '@/app/workflows/task-runner';
import { describeTaskFailure } from '@/lib/server/task-errors';
import { import {
countTasksByStatus, countTasksByStatus,
createTaskRunRecord, createTaskRunRecord,
@@ -65,21 +66,24 @@ async function reconcileTaskWithWorkflow(task: Task) {
return task; return task;
} }
const nextError = nextStatus === 'failed' const failure = nextStatus === 'failed'
? workflowStatus === 'cancelled' ? describeTaskFailure(task, workflowStatus === 'cancelled' ? 'Workflow run cancelled' : 'Workflow run failed')
? 'Workflow run cancelled'
: 'Workflow run failed'
: null; : null;
const updated = await setTaskStatusFromWorkflow(task.id, nextStatus, nextError); const updated = await setTaskStatusFromWorkflow(
task.id,
nextStatus,
failure?.detail ?? null,
failure?.summary ?? null
);
const fallbackTask = { const fallbackTask = {
...task, ...task,
status: nextStatus, status: nextStatus,
stage: nextStatus, stage: nextStatus,
stage_detail: nextStatus === 'failed' ? nextError : 'Workflow run completed.', stage_detail: nextStatus === 'failed' ? (failure?.summary ?? 'The background workflow stopped unexpectedly.') : 'Workflow run completed.',
stage_context: null, stage_context: null,
error: nextError, error: failure?.detail ?? null,
finished_at: nextStatus === 'queued' || nextStatus === 'running' finished_at: nextStatus === 'queued' || nextStatus === 'running'
? null ? null
: task.finished_at ?? new Date().toISOString() : task.finished_at ?? new Date().toISOString()
@@ -114,11 +118,14 @@ export async function enqueueTask(input: EnqueueTaskInput) {
workflow_run_id: run.runId workflow_run_id: run.runId
} satisfies Task; } satisfies Task;
} catch (error) { } catch (error) {
const reason = error instanceof Error const failure = describeTaskFailure(task, 'Failed to start workflow');
? error.message await markTaskFailure(task.id, failure.detail, 'failed', {
: 'Failed to start workflow'; detail: failure.summary
await markTaskFailure(task.id, reason, 'failed'); });
throw error;
const wrapped = new Error(failure.detail);
(wrapped as Error & { cause?: unknown }).cause = error;
throw wrapped;
} }
} }

View File

@@ -8,7 +8,7 @@ import type {
export type StageTimelineItem = { export type StageTimelineItem = {
stage: TaskStage; stage: TaskStage;
label: string; label: string;
state: 'completed' | 'active' | 'pending'; state: 'completed' | 'active' | 'pending' | 'failed';
detail: string | null; detail: string | null;
timestamp: string | null; timestamp: string | null;
context: Task['stage_context'] | null; context: Task['stage_context'] | null;
@@ -203,7 +203,7 @@ export function buildStageTimeline(task: Task, events: TaskStageEvent[]): StageT
return { return {
stage, stage,
label: stageLabel(stage), label: stageLabel(stage),
state: 'completed' as const, state: task.status === 'failed' && stage === task.stage ? 'failed' as const : 'completed' as const,
detail: event?.stage_detail ?? task.stage_detail, detail: event?.stage_detail ?? task.stage_detail,
timestamp: event?.created_at ?? task.finished_at, timestamp: event?.created_at ?? task.finished_at,
context: event?.stage_context ?? (stage === task.stage ? task.stage_context : null) ?? null context: event?.stage_context ?? (stage === task.stage ? task.stage_context : null) ?? null