Fix AI workflow retry loops and improve fallback handling

This commit is contained in:
2026-02-28 22:01:07 -05:00
parent 3cc085e583
commit 953d7c0099
3 changed files with 158 additions and 2 deletions

View File

@@ -43,6 +43,11 @@ async function processTaskStep(task: Task) {
return await runTaskProcessor(task); return await runTaskProcessor(task);
} }
// Step-level retries duplicate task-level retry handling and can create noisy AI failure loops.
(
processTaskStep as ((task: Task) => Promise<Record<string, unknown>>) & { maxRetries?: number }
).maxRetries = 0;
async function completeTaskStep(taskId: string, result: Record<string, unknown>) { async function completeTaskStep(taskId: string, result: Record<string, unknown>) {
'use step'; 'use step';
await completeTask(taskId, result); await completeTask(taskId, result);

View File

@@ -119,10 +119,12 @@ describe('ai config and runtime', () => {
system?: string; system?: string;
prompt: string; prompt: string;
temperature: number; temperature: number;
maxRetries?: number;
}) => { }) => {
expect(input.system).toBe('Use concise style'); expect(input.system).toBe('Use concise style');
expect(input.prompt).toBe('Analyze this filing'); expect(input.prompt).toBe('Analyze this filing');
expect(input.temperature).toBe(0.4); expect(input.temperature).toBe(0.4);
expect(input.maxRetries).toBe(0);
return { text: ' Generated insight ' }; return { text: ' Generated insight ' };
}); });
@@ -216,4 +218,63 @@ describe('ai config and runtime', () => {
expect(result.model).toBe('qwen3:8b'); expect(result.model).toBe('qwen3:8b');
expect(result.text).toContain('AI SDK fallback mode is active'); expect(result.text).toContain('AI SDK fallback mode is active');
}); });
it('falls back to local text when report workload fails with insufficient balance', async () => {
const warn = mock((_message: string) => {});
const result = await runAiAnalysis('Analyze this filing', 'Use concise style', {
env: {
ZHIPU_API_KEY: 'new-key'
},
warn,
createModel: () => ({}),
generate: async () => {
throw new Error('AI_RetryError: Failed after 3 attempts. Last error: Insufficient balance or no resource package. Please recharge.');
}
});
expect(result.provider).toBe('local-fallback');
expect(result.model).toBe('glm-4.7-flashx');
expect(result.text).toContain('AI SDK fallback mode is active');
expect(warn).toHaveBeenCalledTimes(1);
});
it('falls back to local text when report workload cause contains insufficient balance', async () => {
const warn = mock((_message: string) => {});
const result = await runAiAnalysis('Analyze this filing', 'Use concise style', {
env: {
ZHIPU_API_KEY: 'new-key'
},
warn,
createModel: () => ({}),
generate: async () => {
const retryError = new Error('AI_RetryError: Failed after 3 attempts.');
(retryError as Error & { cause?: unknown }).cause = new Error(
'Last error: Insufficient balance or no resource package. Please recharge.'
);
throw retryError;
}
});
expect(result.provider).toBe('local-fallback');
expect(result.model).toBe('glm-4.7-flashx');
expect(result.text).toContain('AI SDK fallback mode is active');
expect(warn).toHaveBeenCalledTimes(1);
});
it('keeps throwing unknown report workload errors', async () => {
await expect(
runAiAnalysis('Analyze this filing', 'Use concise style', {
env: {
ZHIPU_API_KEY: 'new-key'
},
warn: () => {},
createModel: () => ({}),
generate: async () => {
throw new Error('unexpected schema mismatch');
}
})
).rejects.toThrow('unexpected schema mismatch');
});
}); });

View File

@@ -25,6 +25,7 @@ type AiGenerateInput = {
system?: string; system?: string;
prompt: string; prompt: string;
temperature: number; temperature: number;
maxRetries?: number;
}; };
type AiGenerateOutput = { type AiGenerateOutput = {
@@ -108,6 +109,83 @@ function asErrorMessage(error: unknown) {
return String(error); return String(error);
} }
function errorSearchText(error: unknown) {
const chunks: string[] = [];
const seen = new Set<unknown>();
const visit = (value: unknown) => {
if (value === null || value === undefined) {
return;
}
if (typeof value === 'string') {
const normalized = value.trim();
if (normalized.length > 0) {
chunks.push(normalized);
}
return;
}
if (typeof value !== 'object') {
chunks.push(String(value));
return;
}
if (seen.has(value)) {
return;
}
seen.add(value);
if (value instanceof Error) {
if (value.message) {
chunks.push(value.message);
}
const withCause = value as Error & { cause?: unknown };
if (withCause.cause !== undefined) {
visit(withCause.cause);
}
return;
}
const record = value as Record<string, unknown>;
visit(record.message);
visit(record.error);
visit(record.reason);
visit(record.detail);
visit(record.details);
visit(record.cause);
};
visit(error);
return chunks.join('\n');
}
const REPORT_FALLBACK_ERROR_PATTERNS: RegExp[] = [
/insufficient balance/i,
/no resource package/i,
/insufficient quota/i,
/quota exceeded/i,
/insufficient credit/i,
/invalid api key/i,
/authentication/i,
/unauthorized/i,
/forbidden/i,
/payment required/i,
/recharge/i,
/unable to connect/i,
/network/i,
/timeout/i,
/timed out/i,
/econnrefused/i
];
function shouldFallbackReportError(error: unknown) {
const searchText = errorSearchText(error) || asErrorMessage(error);
return REPORT_FALLBACK_ERROR_PATTERNS.some((pattern) => pattern.test(searchText));
}
function defaultCreateModel(config: AiConfig) { function defaultCreateModel(config: AiConfig) {
if (config.provider === 'zhipu') { if (config.provider === 'zhipu') {
const zhipu = createZhipu({ const zhipu = createZhipu({
@@ -131,7 +209,8 @@ async function defaultGenerate(input: AiGenerateInput): Promise<AiGenerateOutput
model: input.model as never, model: input.model as never,
system: input.system, system: input.system,
prompt: input.prompt, prompt: input.prompt,
temperature: input.temperature temperature: input.temperature,
maxRetries: input.maxRetries ?? 0
}); });
return { text: result.text }; return { text: result.text };
@@ -196,7 +275,8 @@ export async function runAiAnalysis(prompt: string, systemPrompt?: string, optio
model, model,
system: systemPrompt, system: systemPrompt,
prompt, prompt,
temperature: config.temperature temperature: config.temperature,
maxRetries: 0
}); });
const text = result.text.trim(); const text = result.text.trim();
@@ -218,6 +298,16 @@ export async function runAiAnalysis(prompt: string, systemPrompt?: string, optio
text text
}; };
} catch (error) { } catch (error) {
if (workload === 'report' && shouldFallbackReportError(error)) {
warn(`[AI SDK] Report fallback activated: ${asErrorMessage(error)}`);
return {
provider: 'local-fallback',
model: config.model,
text: fallbackResponse(prompt)
};
}
if (workload === 'extraction') { if (workload === 'extraction') {
warn(`[AI SDK] Extraction fallback activated: ${asErrorMessage(error)}`); warn(`[AI SDK] Extraction fallback activated: ${asErrorMessage(error)}`);