Fix P0 issues in financial ingestion architecture
Some checks failed
PR Checks / typecheck-and-build (pull_request) Has been cancelled
Taxonomy Sidecar / taxonomy-sidecar (pull_request) Has been cancelled

- Wrap snapshot updates in transactions with error context for each child table
- Add sidecar retry with exponential backoff (3 attempts, 2s base, 10s max, 30% jitter)
- Add HTTP timeout (30s per request) and SEC rate limiting (10 req/s) in Rust
- Add XBRL validation with status reporting (checks root element, tag balance)
This commit is contained in:
2026-03-15 16:51:32 -04:00
parent edf1cfb421
commit 4313058d65
9 changed files with 468 additions and 142 deletions

59
lib/server/utils/retry.ts Normal file
View File

@@ -0,0 +1,59 @@
export interface RetryOptions {
maxRetries: number;
baseDelayMs: number;
maxDelayMs: number;
jitterFactor: number;
retryableErrors: RegExp[];
}
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
maxRetries: 3,
baseDelayMs: 2000,
maxDelayMs: 10000,
jitterFactor: 0.3,
retryableErrors: [
/timeout/i,
/ECONNRESET/,
/ETIMEDOUT/,
/ENOTFOUND/,
/exit code 1/,
/signal/,
/killed/
]
};
export async function withRetry<T>(
fn: () => Promise<T>,
options?: Partial<RetryOptions>
): Promise<T> {
const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
let lastError: Error | null = null;
for (let attempt = 0; attempt < opts.maxRetries; attempt++) {
try {
return await fn();
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
const isRetryable = opts.retryableErrors.some(
(pattern) => pattern.test(lastError!.message)
);
if (!isRetryable || attempt === opts.maxRetries - 1) {
throw lastError;
}
const baseDelay = opts.baseDelayMs * Math.pow(2, attempt);
const jitter = Math.random() * opts.jitterFactor * baseDelay;
const delay = Math.min(baseDelay + jitter, opts.maxDelayMs);
console.warn(
`[retry] Attempt ${attempt + 1}/${opts.maxRetries} failed, retrying in ${Math.round(delay)}ms: ${lastError.message}`
);
await Bun.sleep(delay);
}
}
throw lastError;
}