Fix P0 issues in financial ingestion architecture
- Wrap snapshot updates in transactions with error context for each child table - Add sidecar retry with exponential backoff (3 attempts, 2s base, 10s max, 30% jitter) - Add HTTP timeout (30s per request) and SEC rate limiting (10 req/s) in Rust - Add XBRL validation with status reporting (checks root element, tag balance)
This commit is contained in:
@@ -49,6 +49,9 @@ function createHydrationResult(): TaxonomyHydrationResult {
|
||||
unmapped_row_count: 0,
|
||||
material_unmapped_row_count: 0,
|
||||
warnings: ['rust_warning']
|
||||
},
|
||||
xbrl_validation: {
|
||||
status: 'passed'
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { TaxonomyHydrationInput, TaxonomyHydrationResult } from '@/lib/server/taxonomy/types';
|
||||
import { withRetry } from '@/lib/server/utils/retry';
|
||||
|
||||
function candidateBinaryPaths() {
|
||||
return [
|
||||
@@ -23,6 +24,10 @@ export function resolveFiscalXbrlBinary() {
|
||||
export async function hydrateFilingTaxonomySnapshotFromSidecar(
|
||||
input: TaxonomyHydrationInput
|
||||
): Promise<TaxonomyHydrationResult> {
|
||||
return withRetry(() => hydrateFromSidecarImpl(input));
|
||||
}
|
||||
|
||||
async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise<TaxonomyHydrationResult> {
|
||||
const binary = resolveFiscalXbrlBinary();
|
||||
const timeoutMs = Math.max(Number(process.env.XBRL_ENGINE_TIMEOUT_MS ?? 45_000), 1_000);
|
||||
const command = [binary, 'hydrate-filing'];
|
||||
|
||||
@@ -204,6 +204,11 @@ export type TaxonomyHydrationNormalizationSummary = {
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export type XbrlValidationResult = {
|
||||
status: 'passed' | 'warning' | 'error';
|
||||
message?: string;
|
||||
};
|
||||
|
||||
export type TaxonomyHydrationInput = {
|
||||
filingId: number;
|
||||
ticker: string;
|
||||
@@ -279,4 +284,5 @@ export type TaxonomyHydrationResult = {
|
||||
}>;
|
||||
metric_validations: TaxonomyMetricValidationCheck[];
|
||||
normalization_summary: TaxonomyHydrationNormalizationSummary;
|
||||
xbrl_validation: XbrlValidationResult;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user