Fix P0 issues in financial ingestion architecture
- Wrap snapshot updates in transactions with error context for each child table - Add sidecar retry with exponential backoff (3 attempts, 2s base, 10s max, 30% jitter) - Add HTTP timeout (30s per request) and SEC rate limiting (10 req/s) in Rust - Add XBRL validation with status reporting (checks root element, tag balance)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { TaxonomyHydrationInput, TaxonomyHydrationResult } from '@/lib/server/taxonomy/types';
|
||||
import { withRetry } from '@/lib/server/utils/retry';
|
||||
|
||||
function candidateBinaryPaths() {
|
||||
return [
|
||||
@@ -23,6 +24,10 @@ export function resolveFiscalXbrlBinary() {
|
||||
export async function hydrateFilingTaxonomySnapshotFromSidecar(
|
||||
input: TaxonomyHydrationInput
|
||||
): Promise<TaxonomyHydrationResult> {
|
||||
return withRetry(() => hydrateFromSidecarImpl(input));
|
||||
}
|
||||
|
||||
async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise<TaxonomyHydrationResult> {
|
||||
const binary = resolveFiscalXbrlBinary();
|
||||
const timeoutMs = Math.max(Number(process.env.XBRL_ENGINE_TIMEOUT_MS ?? 45_000), 1_000);
|
||||
const command = [binary, 'hydrate-filing'];
|
||||
|
||||
Reference in New Issue
Block a user