Fix AI workflow retry loops and improve fallback handling
This commit is contained in:
@@ -43,6 +43,11 @@ async function processTaskStep(task: Task) {
|
|||||||
return await runTaskProcessor(task);
|
return await runTaskProcessor(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Step-level retries duplicate task-level retry handling and can create noisy AI failure loops.
|
||||||
|
(
|
||||||
|
processTaskStep as ((task: Task) => Promise<Record<string, unknown>>) & { maxRetries?: number }
|
||||||
|
).maxRetries = 0;
|
||||||
|
|
||||||
async function completeTaskStep(taskId: string, result: Record<string, unknown>) {
|
async function completeTaskStep(taskId: string, result: Record<string, unknown>) {
|
||||||
'use step';
|
'use step';
|
||||||
await completeTask(taskId, result);
|
await completeTask(taskId, result);
|
||||||
|
|||||||
@@ -119,10 +119,12 @@ describe('ai config and runtime', () => {
|
|||||||
system?: string;
|
system?: string;
|
||||||
prompt: string;
|
prompt: string;
|
||||||
temperature: number;
|
temperature: number;
|
||||||
|
maxRetries?: number;
|
||||||
}) => {
|
}) => {
|
||||||
expect(input.system).toBe('Use concise style');
|
expect(input.system).toBe('Use concise style');
|
||||||
expect(input.prompt).toBe('Analyze this filing');
|
expect(input.prompt).toBe('Analyze this filing');
|
||||||
expect(input.temperature).toBe(0.4);
|
expect(input.temperature).toBe(0.4);
|
||||||
|
expect(input.maxRetries).toBe(0);
|
||||||
return { text: ' Generated insight ' };
|
return { text: ' Generated insight ' };
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -216,4 +218,63 @@ describe('ai config and runtime', () => {
|
|||||||
expect(result.model).toBe('qwen3:8b');
|
expect(result.model).toBe('qwen3:8b');
|
||||||
expect(result.text).toContain('AI SDK fallback mode is active');
|
expect(result.text).toContain('AI SDK fallback mode is active');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('falls back to local text when report workload fails with insufficient balance', async () => {
|
||||||
|
const warn = mock((_message: string) => {});
|
||||||
|
|
||||||
|
const result = await runAiAnalysis('Analyze this filing', 'Use concise style', {
|
||||||
|
env: {
|
||||||
|
ZHIPU_API_KEY: 'new-key'
|
||||||
|
},
|
||||||
|
warn,
|
||||||
|
createModel: () => ({}),
|
||||||
|
generate: async () => {
|
||||||
|
throw new Error('AI_RetryError: Failed after 3 attempts. Last error: Insufficient balance or no resource package. Please recharge.');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.provider).toBe('local-fallback');
|
||||||
|
expect(result.model).toBe('glm-4.7-flashx');
|
||||||
|
expect(result.text).toContain('AI SDK fallback mode is active');
|
||||||
|
expect(warn).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to local text when report workload cause contains insufficient balance', async () => {
|
||||||
|
const warn = mock((_message: string) => {});
|
||||||
|
|
||||||
|
const result = await runAiAnalysis('Analyze this filing', 'Use concise style', {
|
||||||
|
env: {
|
||||||
|
ZHIPU_API_KEY: 'new-key'
|
||||||
|
},
|
||||||
|
warn,
|
||||||
|
createModel: () => ({}),
|
||||||
|
generate: async () => {
|
||||||
|
const retryError = new Error('AI_RetryError: Failed after 3 attempts.');
|
||||||
|
(retryError as Error & { cause?: unknown }).cause = new Error(
|
||||||
|
'Last error: Insufficient balance or no resource package. Please recharge.'
|
||||||
|
);
|
||||||
|
throw retryError;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.provider).toBe('local-fallback');
|
||||||
|
expect(result.model).toBe('glm-4.7-flashx');
|
||||||
|
expect(result.text).toContain('AI SDK fallback mode is active');
|
||||||
|
expect(warn).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps throwing unknown report workload errors', async () => {
|
||||||
|
await expect(
|
||||||
|
runAiAnalysis('Analyze this filing', 'Use concise style', {
|
||||||
|
env: {
|
||||||
|
ZHIPU_API_KEY: 'new-key'
|
||||||
|
},
|
||||||
|
warn: () => {},
|
||||||
|
createModel: () => ({}),
|
||||||
|
generate: async () => {
|
||||||
|
throw new Error('unexpected schema mismatch');
|
||||||
|
}
|
||||||
|
})
|
||||||
|
).rejects.toThrow('unexpected schema mismatch');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ type AiGenerateInput = {
|
|||||||
system?: string;
|
system?: string;
|
||||||
prompt: string;
|
prompt: string;
|
||||||
temperature: number;
|
temperature: number;
|
||||||
|
maxRetries?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
type AiGenerateOutput = {
|
type AiGenerateOutput = {
|
||||||
@@ -108,6 +109,83 @@ function asErrorMessage(error: unknown) {
|
|||||||
return String(error);
|
return String(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function errorSearchText(error: unknown) {
|
||||||
|
const chunks: string[] = [];
|
||||||
|
const seen = new Set<unknown>();
|
||||||
|
|
||||||
|
const visit = (value: unknown) => {
|
||||||
|
if (value === null || value === undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
const normalized = value.trim();
|
||||||
|
if (normalized.length > 0) {
|
||||||
|
chunks.push(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value !== 'object') {
|
||||||
|
chunks.push(String(value));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (seen.has(value)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
seen.add(value);
|
||||||
|
|
||||||
|
if (value instanceof Error) {
|
||||||
|
if (value.message) {
|
||||||
|
chunks.push(value.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
const withCause = value as Error & { cause?: unknown };
|
||||||
|
if (withCause.cause !== undefined) {
|
||||||
|
visit(withCause.cause);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = value as Record<string, unknown>;
|
||||||
|
visit(record.message);
|
||||||
|
visit(record.error);
|
||||||
|
visit(record.reason);
|
||||||
|
visit(record.detail);
|
||||||
|
visit(record.details);
|
||||||
|
visit(record.cause);
|
||||||
|
};
|
||||||
|
|
||||||
|
visit(error);
|
||||||
|
return chunks.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
const REPORT_FALLBACK_ERROR_PATTERNS: RegExp[] = [
|
||||||
|
/insufficient balance/i,
|
||||||
|
/no resource package/i,
|
||||||
|
/insufficient quota/i,
|
||||||
|
/quota exceeded/i,
|
||||||
|
/insufficient credit/i,
|
||||||
|
/invalid api key/i,
|
||||||
|
/authentication/i,
|
||||||
|
/unauthorized/i,
|
||||||
|
/forbidden/i,
|
||||||
|
/payment required/i,
|
||||||
|
/recharge/i,
|
||||||
|
/unable to connect/i,
|
||||||
|
/network/i,
|
||||||
|
/timeout/i,
|
||||||
|
/timed out/i,
|
||||||
|
/econnrefused/i
|
||||||
|
];
|
||||||
|
|
||||||
|
function shouldFallbackReportError(error: unknown) {
|
||||||
|
const searchText = errorSearchText(error) || asErrorMessage(error);
|
||||||
|
return REPORT_FALLBACK_ERROR_PATTERNS.some((pattern) => pattern.test(searchText));
|
||||||
|
}
|
||||||
|
|
||||||
function defaultCreateModel(config: AiConfig) {
|
function defaultCreateModel(config: AiConfig) {
|
||||||
if (config.provider === 'zhipu') {
|
if (config.provider === 'zhipu') {
|
||||||
const zhipu = createZhipu({
|
const zhipu = createZhipu({
|
||||||
@@ -131,7 +209,8 @@ async function defaultGenerate(input: AiGenerateInput): Promise<AiGenerateOutput
|
|||||||
model: input.model as never,
|
model: input.model as never,
|
||||||
system: input.system,
|
system: input.system,
|
||||||
prompt: input.prompt,
|
prompt: input.prompt,
|
||||||
temperature: input.temperature
|
temperature: input.temperature,
|
||||||
|
maxRetries: input.maxRetries ?? 0
|
||||||
});
|
});
|
||||||
|
|
||||||
return { text: result.text };
|
return { text: result.text };
|
||||||
@@ -196,7 +275,8 @@ export async function runAiAnalysis(prompt: string, systemPrompt?: string, optio
|
|||||||
model,
|
model,
|
||||||
system: systemPrompt,
|
system: systemPrompt,
|
||||||
prompt,
|
prompt,
|
||||||
temperature: config.temperature
|
temperature: config.temperature,
|
||||||
|
maxRetries: 0
|
||||||
});
|
});
|
||||||
|
|
||||||
const text = result.text.trim();
|
const text = result.text.trim();
|
||||||
@@ -218,6 +298,16 @@ export async function runAiAnalysis(prompt: string, systemPrompt?: string, optio
|
|||||||
text
|
text
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (workload === 'report' && shouldFallbackReportError(error)) {
|
||||||
|
warn(`[AI SDK] Report fallback activated: ${asErrorMessage(error)}`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
provider: 'local-fallback',
|
||||||
|
model: config.model,
|
||||||
|
text: fallbackResponse(prompt)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (workload === 'extraction') {
|
if (workload === 'extraction') {
|
||||||
warn(`[AI SDK] Extraction fallback activated: ${asErrorMessage(error)}`);
|
warn(`[AI SDK] Extraction fallback activated: ${asErrorMessage(error)}`);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user