Fix P0 issues in financial ingestion architecture
Some checks failed
PR Checks / typecheck-and-build (pull_request) Has been cancelled
Taxonomy Sidecar / taxonomy-sidecar (pull_request) Has been cancelled

- Wrap snapshot updates in transactions with error context for each child table
- Add sidecar retry with exponential backoff (3 attempts, 2s base, 10s max, 30% jitter)
- Add HTTP timeout (30s per request) and SEC rate limiting (10 req/s) in Rust
- Add XBRL validation with status reporting (checks root element, tag balance)
This commit is contained in:
2026-03-15 16:51:32 -04:00
parent edf1cfb421
commit 4313058d65
9 changed files with 468 additions and 142 deletions

View File

@@ -49,6 +49,9 @@ function createHydrationResult(): TaxonomyHydrationResult {
unmapped_row_count: 0,
material_unmapped_row_count: 0,
warnings: ['rust_warning']
},
xbrl_validation: {
status: 'passed'
}
};
}

View File

@@ -1,6 +1,7 @@
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import type { TaxonomyHydrationInput, TaxonomyHydrationResult } from '@/lib/server/taxonomy/types';
import { withRetry } from '@/lib/server/utils/retry';
function candidateBinaryPaths() {
return [
@@ -23,6 +24,10 @@ export function resolveFiscalXbrlBinary() {
export async function hydrateFilingTaxonomySnapshotFromSidecar(
input: TaxonomyHydrationInput
): Promise<TaxonomyHydrationResult> {
return withRetry(() => hydrateFromSidecarImpl(input));
}
async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise<TaxonomyHydrationResult> {
const binary = resolveFiscalXbrlBinary();
const timeoutMs = Math.max(Number(process.env.XBRL_ENGINE_TIMEOUT_MS ?? 45_000), 1_000);
const command = [binary, 'hydrate-filing'];

View File

@@ -204,6 +204,11 @@ export type TaxonomyHydrationNormalizationSummary = {
warnings: string[];
};
export type XbrlValidationResult = {
status: 'passed' | 'warning' | 'error';
message?: string;
};
export type TaxonomyHydrationInput = {
filingId: number;
ticker: string;
@@ -279,4 +284,5 @@ export type TaxonomyHydrationResult = {
}>;
metric_validations: TaxonomyMetricValidationCheck[];
normalization_summary: TaxonomyHydrationNormalizationSummary;
xbrl_validation: XbrlValidationResult;
};