Fix P0 issues in financial ingestion architecture
- Wrap snapshot updates in transactions with error context for each child table - Add sidecar retry with exponential backoff (3 attempts, 2s base, 10s max, 30% jitter) - Add HTTP timeout (30s per request) and SEC rate limiting (10 req/s) in Rust - Add XBRL validation with status reporting (checks root element, tag balance)
This commit is contained in:
59
lib/server/utils/retry.ts
Normal file
59
lib/server/utils/retry.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
export interface RetryOptions {
|
||||
maxRetries: number;
|
||||
baseDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
jitterFactor: number;
|
||||
retryableErrors: RegExp[];
|
||||
}
|
||||
|
||||
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
||||
maxRetries: 3,
|
||||
baseDelayMs: 2000,
|
||||
maxDelayMs: 10000,
|
||||
jitterFactor: 0.3,
|
||||
retryableErrors: [
|
||||
/timeout/i,
|
||||
/ECONNRESET/,
|
||||
/ETIMEDOUT/,
|
||||
/ENOTFOUND/,
|
||||
/exit code 1/,
|
||||
/signal/,
|
||||
/killed/
|
||||
]
|
||||
};
|
||||
|
||||
export async function withRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options?: Partial<RetryOptions>
|
||||
): Promise<T> {
|
||||
const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < opts.maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
|
||||
const isRetryable = opts.retryableErrors.some(
|
||||
(pattern) => pattern.test(lastError!.message)
|
||||
);
|
||||
|
||||
if (!isRetryable || attempt === opts.maxRetries - 1) {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
const baseDelay = opts.baseDelayMs * Math.pow(2, attempt);
|
||||
const jitter = Math.random() * opts.jitterFactor * baseDelay;
|
||||
const delay = Math.min(baseDelay + jitter, opts.maxDelayMs);
|
||||
|
||||
console.warn(
|
||||
`[retry] Attempt ${attempt + 1}/${opts.maxRetries} failed, retrying in ${Math.round(delay)}ms: ${lastError.message}`
|
||||
);
|
||||
|
||||
await Bun.sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
Reference in New Issue
Block a user