From 953d7c0099b475a9870ca5da0af64b961cde9bba Mon Sep 17 00:00:00 2001 From: francy51 Date: Sat, 28 Feb 2026 22:01:07 -0500 Subject: [PATCH] Fix AI workflow retry loops and improve fallback handling --- app/workflows/task-runner.ts | 5 ++ lib/server/ai.test.ts | 61 +++++++++++++++++++++++ lib/server/ai.ts | 94 +++++++++++++++++++++++++++++++++++- 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/app/workflows/task-runner.ts b/app/workflows/task-runner.ts index 2df7882..a6c0f1e 100644 --- a/app/workflows/task-runner.ts +++ b/app/workflows/task-runner.ts @@ -43,6 +43,11 @@ async function processTaskStep(task: Task) { return await runTaskProcessor(task); } +// Step-level retries duplicate task-level retry handling and can create noisy AI failure loops. +( + processTaskStep as ((task: Task) => Promise>) & { maxRetries?: number } +).maxRetries = 0; + async function completeTaskStep(taskId: string, result: Record) { 'use step'; await completeTask(taskId, result); diff --git a/lib/server/ai.test.ts b/lib/server/ai.test.ts index 1cd8a0c..e611d9d 100644 --- a/lib/server/ai.test.ts +++ b/lib/server/ai.test.ts @@ -119,10 +119,12 @@ describe('ai config and runtime', () => { system?: string; prompt: string; temperature: number; + maxRetries?: number; }) => { expect(input.system).toBe('Use concise style'); expect(input.prompt).toBe('Analyze this filing'); expect(input.temperature).toBe(0.4); + expect(input.maxRetries).toBe(0); return { text: ' Generated insight ' }; }); @@ -216,4 +218,63 @@ describe('ai config and runtime', () => { expect(result.model).toBe('qwen3:8b'); expect(result.text).toContain('AI SDK fallback mode is active'); }); + + it('falls back to local text when report workload fails with insufficient balance', async () => { + const warn = mock((_message: string) => {}); + + const result = await runAiAnalysis('Analyze this filing', 'Use concise style', { + env: { + ZHIPU_API_KEY: 'new-key' + }, + warn, + createModel: () => ({}), + generate: async () => { + throw new Error('AI_RetryError: Failed after 3 attempts. Last error: Insufficient balance or no resource package. Please recharge.'); + } + }); + + expect(result.provider).toBe('local-fallback'); + expect(result.model).toBe('glm-4.7-flashx'); + expect(result.text).toContain('AI SDK fallback mode is active'); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it('falls back to local text when report workload cause contains insufficient balance', async () => { + const warn = mock((_message: string) => {}); + + const result = await runAiAnalysis('Analyze this filing', 'Use concise style', { + env: { + ZHIPU_API_KEY: 'new-key' + }, + warn, + createModel: () => ({}), + generate: async () => { + const retryError = new Error('AI_RetryError: Failed after 3 attempts.'); + (retryError as Error & { cause?: unknown }).cause = new Error( + 'Last error: Insufficient balance or no resource package. Please recharge.' + ); + throw retryError; + } + }); + + expect(result.provider).toBe('local-fallback'); + expect(result.model).toBe('glm-4.7-flashx'); + expect(result.text).toContain('AI SDK fallback mode is active'); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it('keeps throwing unknown report workload errors', async () => { + await expect( + runAiAnalysis('Analyze this filing', 'Use concise style', { + env: { + ZHIPU_API_KEY: 'new-key' + }, + warn: () => {}, + createModel: () => ({}), + generate: async () => { + throw new Error('unexpected schema mismatch'); + } + }) + ).rejects.toThrow('unexpected schema mismatch'); + }); }); diff --git a/lib/server/ai.ts b/lib/server/ai.ts index bfab93a..737de1e 100644 --- a/lib/server/ai.ts +++ b/lib/server/ai.ts @@ -25,6 +25,7 @@ type AiGenerateInput = { system?: string; prompt: string; temperature: number; + maxRetries?: number; }; type AiGenerateOutput = { @@ -108,6 +109,83 @@ function asErrorMessage(error: unknown) { return String(error); } +function errorSearchText(error: unknown) { + const chunks: string[] = []; + const seen = new Set(); + + const visit = (value: unknown) => { + if (value === null || value === undefined) { + return; + } + + if (typeof value === 'string') { + const normalized = value.trim(); + if (normalized.length > 0) { + chunks.push(normalized); + } + + return; + } + + if (typeof value !== 'object') { + chunks.push(String(value)); + return; + } + + if (seen.has(value)) { + return; + } + seen.add(value); + + if (value instanceof Error) { + if (value.message) { + chunks.push(value.message); + } + + const withCause = value as Error & { cause?: unknown }; + if (withCause.cause !== undefined) { + visit(withCause.cause); + } + return; + } + + const record = value as Record; + visit(record.message); + visit(record.error); + visit(record.reason); + visit(record.detail); + visit(record.details); + visit(record.cause); + }; + + visit(error); + return chunks.join('\n'); +} + +const REPORT_FALLBACK_ERROR_PATTERNS: RegExp[] = [ + /insufficient balance/i, + /no resource package/i, + /insufficient quota/i, + /quota exceeded/i, + /insufficient credit/i, + /invalid api key/i, + /authentication/i, + /unauthorized/i, + /forbidden/i, + /payment required/i, + /recharge/i, + /unable to connect/i, + /network/i, + /timeout/i, + /timed out/i, + /econnrefused/i +]; + +function shouldFallbackReportError(error: unknown) { + const searchText = errorSearchText(error) || asErrorMessage(error); + return REPORT_FALLBACK_ERROR_PATTERNS.some((pattern) => pattern.test(searchText)); +} + function defaultCreateModel(config: AiConfig) { if (config.provider === 'zhipu') { const zhipu = createZhipu({ @@ -131,7 +209,8 @@ async function defaultGenerate(input: AiGenerateInput): Promise