Fix financial taxonomy snapshot normalization
This commit is contained in:
@@ -7,7 +7,7 @@ import { db, getSqliteClient } from '@/lib/server/db';
|
||||
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
|
||||
import { companyFinancialBundle } from '@/lib/server/db/schema';
|
||||
|
||||
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
|
||||
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 15;
|
||||
|
||||
export type CompanyFinancialBundleRecord = {
|
||||
id: number;
|
||||
|
||||
291
lib/server/repos/filing-taxonomy.test.ts
Normal file
291
lib/server/repos/filing-taxonomy.test.ts
Normal file
@@ -0,0 +1,291 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import { __filingTaxonomyInternals } from './filing-taxonomy';
|
||||
|
||||
describe('filing taxonomy snapshot normalization', () => {
|
||||
it('normalizes legacy snake_case nested snapshot payloads in toSnapshotRecord', () => {
|
||||
const record = __filingTaxonomyInternals.toSnapshotRecord({
|
||||
id: 1,
|
||||
filing_id: 10,
|
||||
ticker: 'MSFT',
|
||||
filing_date: '2026-01-28',
|
||||
filing_type: '10-Q',
|
||||
parse_status: 'ready',
|
||||
parse_error: null,
|
||||
source: 'xbrl_instance',
|
||||
parser_engine: 'fiscal-xbrl',
|
||||
parser_version: '0.1.0',
|
||||
taxonomy_regime: 'us-gaap',
|
||||
fiscal_pack: 'core',
|
||||
periods: [{
|
||||
id: 'fy-2025',
|
||||
filing_id: 10,
|
||||
accession_number: '0001',
|
||||
filing_date: '2026-01-28',
|
||||
period_start: '2025-01-01',
|
||||
period_end: '2025-12-31',
|
||||
filing_type: '10-Q',
|
||||
period_label: 'FY 2025'
|
||||
}],
|
||||
faithful_rows: {
|
||||
income: [{
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
concept_key: 'us-gaap:Revenue',
|
||||
qname: 'us-gaap:Revenue',
|
||||
namespace_uri: 'http://fasb.org/us-gaap/2025',
|
||||
local_name: 'Revenue',
|
||||
is_extension: false,
|
||||
statement: 'income',
|
||||
role_uri: 'income',
|
||||
order: 10,
|
||||
depth: 0,
|
||||
parent_key: null,
|
||||
values: { 'fy-2025': 10 },
|
||||
units: { 'fy-2025': 'iso4217:USD' },
|
||||
has_dimensions: false,
|
||||
source_fact_ids: [1]
|
||||
}],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
},
|
||||
statement_rows: null,
|
||||
surface_rows: {
|
||||
income: [{
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
category: 'revenue',
|
||||
template_section: 'revenue',
|
||||
order: 10,
|
||||
unit: 'currency',
|
||||
values: { 'fy-2025': 10 },
|
||||
source_concepts: ['us-gaap:Revenue'],
|
||||
source_row_keys: ['revenue'],
|
||||
source_fact_ids: [1],
|
||||
formula_key: null,
|
||||
has_dimensions: false,
|
||||
resolved_source_row_keys: { 'fy-2025': 'revenue' },
|
||||
statement: 'income',
|
||||
detail_count: 1,
|
||||
resolution_method: 'direct',
|
||||
confidence: 'high',
|
||||
warning_codes: ['legacy_surface']
|
||||
}],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
},
|
||||
detail_rows: {
|
||||
income: {
|
||||
revenue: [{
|
||||
key: 'revenue_detail',
|
||||
parent_surface_key: 'revenue',
|
||||
label: 'Revenue Detail',
|
||||
concept_key: 'us-gaap:RevenueDetail',
|
||||
qname: 'us-gaap:RevenueDetail',
|
||||
namespace_uri: 'http://fasb.org/us-gaap/2025',
|
||||
local_name: 'RevenueDetail',
|
||||
unit: 'iso4217:USD',
|
||||
values: { 'fy-2025': 10 },
|
||||
source_fact_ids: [2],
|
||||
is_extension: false,
|
||||
dimensions_summary: ['region:americas'],
|
||||
residual_flag: false
|
||||
}]
|
||||
},
|
||||
balance: {},
|
||||
cash_flow: {},
|
||||
equity: {},
|
||||
comprehensive_income: {}
|
||||
},
|
||||
kpi_rows: [{
|
||||
key: 'cloud_growth',
|
||||
label: 'Cloud Growth',
|
||||
category: 'operating_kpi',
|
||||
unit: 'percent',
|
||||
order: 10,
|
||||
segment: null,
|
||||
axis: null,
|
||||
member: null,
|
||||
values: { 'fy-2025': 0.25 },
|
||||
source_concepts: ['msft:CloudGrowth'],
|
||||
source_fact_ids: [3],
|
||||
provenance_type: 'taxonomy',
|
||||
has_dimensions: false
|
||||
}],
|
||||
derived_metrics: null,
|
||||
validation_result: null,
|
||||
normalization_summary: {
|
||||
surface_row_count: 1,
|
||||
detail_row_count: 1,
|
||||
kpi_row_count: 1,
|
||||
unmapped_row_count: 0,
|
||||
material_unmapped_row_count: 0,
|
||||
warnings: ['legacy_warning']
|
||||
},
|
||||
facts_count: 3,
|
||||
concepts_count: 3,
|
||||
dimensions_count: 1,
|
||||
created_at: '2026-01-28T00:00:00.000Z',
|
||||
updated_at: '2026-01-28T00:00:00.000Z'
|
||||
} as never);
|
||||
|
||||
expect(record.periods[0]).toMatchObject({
|
||||
filingId: 10,
|
||||
accessionNumber: '0001',
|
||||
filingDate: '2026-01-28',
|
||||
periodStart: '2025-01-01',
|
||||
periodEnd: '2025-12-31',
|
||||
periodLabel: 'FY 2025'
|
||||
});
|
||||
expect(record.faithful_rows.income[0]).toMatchObject({
|
||||
conceptKey: 'us-gaap:Revenue',
|
||||
namespaceUri: 'http://fasb.org/us-gaap/2025',
|
||||
localName: 'Revenue',
|
||||
roleUri: 'income',
|
||||
parentKey: null,
|
||||
hasDimensions: false,
|
||||
sourceFactIds: [1]
|
||||
});
|
||||
expect(record.surface_rows.income[0]).toMatchObject({
|
||||
templateSection: 'revenue',
|
||||
sourceConcepts: ['us-gaap:Revenue'],
|
||||
sourceRowKeys: ['revenue'],
|
||||
sourceFactIds: [1],
|
||||
formulaKey: null,
|
||||
hasDimensions: false,
|
||||
resolvedSourceRowKeys: { 'fy-2025': 'revenue' },
|
||||
detailCount: 1,
|
||||
resolutionMethod: 'direct',
|
||||
warningCodes: ['legacy_surface']
|
||||
});
|
||||
expect(record.detail_rows.income.revenue?.[0]).toMatchObject({
|
||||
parentSurfaceKey: 'revenue',
|
||||
conceptKey: 'us-gaap:RevenueDetail',
|
||||
namespaceUri: 'http://fasb.org/us-gaap/2025',
|
||||
sourceFactIds: [2],
|
||||
dimensionsSummary: ['region:americas'],
|
||||
residualFlag: false
|
||||
});
|
||||
expect(record.kpi_rows[0]).toMatchObject({
|
||||
sourceConcepts: ['msft:CloudGrowth'],
|
||||
sourceFactIds: [3],
|
||||
provenanceType: 'taxonomy',
|
||||
hasDimensions: false
|
||||
});
|
||||
expect(record.normalization_summary).toEqual({
|
||||
surfaceRowCount: 1,
|
||||
detailRowCount: 1,
|
||||
kpiRowCount: 1,
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: ['legacy_warning']
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps mixed camelCase and snake_case payloads compatible', () => {
|
||||
const normalized = __filingTaxonomyInternals.normalizeFilingTaxonomySnapshotPayload({
|
||||
periods: [{
|
||||
id: 'fy-2025',
|
||||
filingId: 10,
|
||||
accessionNumber: '0001',
|
||||
filingDate: '2026-01-28',
|
||||
periodStart: '2025-01-01',
|
||||
periodEnd: '2025-12-31',
|
||||
filingType: '10-K',
|
||||
periodLabel: 'FY 2025'
|
||||
}],
|
||||
faithful_rows: {
|
||||
income: [{
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
conceptKey: 'us-gaap:Revenue',
|
||||
qname: 'us-gaap:Revenue',
|
||||
namespaceUri: 'http://fasb.org/us-gaap/2025',
|
||||
localName: 'Revenue',
|
||||
isExtension: false,
|
||||
statement: 'income',
|
||||
roleUri: 'income',
|
||||
order: 10,
|
||||
depth: 0,
|
||||
parentKey: null,
|
||||
values: { 'fy-2025': 10 },
|
||||
units: { 'fy-2025': 'iso4217:USD' },
|
||||
hasDimensions: false,
|
||||
sourceFactIds: [1]
|
||||
}],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
},
|
||||
statement_rows: null,
|
||||
surface_rows: {
|
||||
income: [{
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
category: 'revenue',
|
||||
order: 10,
|
||||
unit: 'currency',
|
||||
values: { 'fy-2025': 10 },
|
||||
source_concepts: ['us-gaap:Revenue'],
|
||||
source_row_keys: ['revenue'],
|
||||
source_fact_ids: [1],
|
||||
formula_key: null,
|
||||
has_dimensions: false,
|
||||
resolved_source_row_keys: { 'fy-2025': 'revenue' }
|
||||
}],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
},
|
||||
detail_rows: {
|
||||
income: {
|
||||
revenue: [{
|
||||
key: 'revenue_detail',
|
||||
parentSurfaceKey: 'revenue',
|
||||
label: 'Revenue Detail',
|
||||
conceptKey: 'us-gaap:RevenueDetail',
|
||||
qname: 'us-gaap:RevenueDetail',
|
||||
namespaceUri: 'http://fasb.org/us-gaap/2025',
|
||||
localName: 'RevenueDetail',
|
||||
unit: 'iso4217:USD',
|
||||
values: { 'fy-2025': 10 },
|
||||
sourceFactIds: [2],
|
||||
isExtension: false,
|
||||
dimensionsSummary: [],
|
||||
residualFlag: false
|
||||
}]
|
||||
},
|
||||
balance: {},
|
||||
cash_flow: {},
|
||||
equity: {},
|
||||
comprehensive_income: {}
|
||||
},
|
||||
kpi_rows: [],
|
||||
normalization_summary: {
|
||||
surfaceRowCount: 1,
|
||||
detail_row_count: 1,
|
||||
kpiRowCount: 0,
|
||||
unmapped_row_count: 0,
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: []
|
||||
}
|
||||
});
|
||||
|
||||
expect(normalized.periods[0]?.filingId).toBe(10);
|
||||
expect(normalized.surface_rows.income[0]?.sourceConcepts).toEqual(['us-gaap:Revenue']);
|
||||
expect(normalized.detail_rows.income.revenue?.[0]?.parentSurfaceKey).toBe('revenue');
|
||||
expect(normalized.normalization_summary).toEqual({
|
||||
surfaceRowCount: 1,
|
||||
detailRowCount: 1,
|
||||
kpiRowCount: 0,
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: []
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { and, desc, eq, gte, inArray, lt, sql } from 'drizzle-orm';
|
||||
import type {
|
||||
DetailFinancialRow,
|
||||
Filing,
|
||||
FinancialStatementKind,
|
||||
MetricValidationResult,
|
||||
@@ -283,6 +284,18 @@ export type UpsertFilingTaxonomySnapshotInput = {
|
||||
}>;
|
||||
};
|
||||
|
||||
const FINANCIAL_STATEMENT_KINDS = [
|
||||
'income',
|
||||
'balance',
|
||||
'cash_flow',
|
||||
'equity',
|
||||
'comprehensive_income'
|
||||
] as const satisfies FinancialStatementKind[];
|
||||
|
||||
type StatementRowMap = Record<FinancialStatementKind, TaxonomyStatementRow[]>;
|
||||
type SurfaceRowMap = Record<FinancialStatementKind, SurfaceFinancialRow[]>;
|
||||
type DetailRowMap = Record<FinancialStatementKind, SurfaceDetailMap>;
|
||||
|
||||
function tenYearsAgoIso() {
|
||||
const date = new Date();
|
||||
date.setUTCFullYear(date.getUTCFullYear() - 10);
|
||||
@@ -310,7 +323,394 @@ function asNumericText(value: number | null) {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function emptyStatementRows(): Record<FinancialStatementKind, TaxonomyStatementRow[]> {
|
||||
function asObject(value: unknown) {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: null;
|
||||
}
|
||||
|
||||
function asString(value: unknown) {
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
|
||||
function asNullableString(value: unknown) {
|
||||
return typeof value === 'string'
|
||||
? value
|
||||
: value === null
|
||||
? null
|
||||
: null;
|
||||
}
|
||||
|
||||
function asBoolean(value: unknown) {
|
||||
return typeof value === 'boolean' ? value : Boolean(value);
|
||||
}
|
||||
|
||||
function asStatementKind(value: unknown): FinancialStatementKind | null {
|
||||
return value === 'income'
|
||||
|| value === 'balance'
|
||||
|| value === 'cash_flow'
|
||||
|| value === 'equity'
|
||||
|| value === 'comprehensive_income'
|
||||
? value
|
||||
: null;
|
||||
}
|
||||
|
||||
function normalizeNumberMap(value: unknown) {
|
||||
const object = asObject(value);
|
||||
if (!object) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(object).map(([key, entry]) => [key, asNumber(entry)])
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeNullableStringMap(value: unknown) {
|
||||
const object = asObject(value);
|
||||
if (!object) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(object).map(([key, entry]) => [key, asNullableString(entry)])
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeStringArray(value: unknown) {
|
||||
return Array.isArray(value)
|
||||
? value.filter((entry): entry is string => typeof entry === 'string')
|
||||
: [];
|
||||
}
|
||||
|
||||
function normalizeNumberArray(value: unknown) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.map((entry) => asNumber(entry))
|
||||
.filter((entry): entry is number => entry !== null);
|
||||
}
|
||||
|
||||
function normalizePeriods(value: unknown): FilingTaxonomyPeriod[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.map((entry) => {
|
||||
const row = asObject(entry);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const id = asString(row.id);
|
||||
const filingId = asNumber(row.filingId ?? row.filing_id);
|
||||
const accessionNumber = asString(row.accessionNumber ?? row.accession_number);
|
||||
const filingDate = asString(row.filingDate ?? row.filing_date);
|
||||
const filingType = row.filingType === '10-K' || row.filing_type === '10-K'
|
||||
? '10-K'
|
||||
: row.filingType === '10-Q' || row.filing_type === '10-Q'
|
||||
? '10-Q'
|
||||
: null;
|
||||
const periodLabel = asString(row.periodLabel ?? row.period_label);
|
||||
|
||||
if (!id || filingId === null || !accessionNumber || !filingDate || !filingType || !periodLabel) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
filingId,
|
||||
accessionNumber,
|
||||
filingDate,
|
||||
periodStart: asNullableString(row.periodStart ?? row.period_start),
|
||||
periodEnd: asNullableString(row.periodEnd ?? row.period_end),
|
||||
filingType,
|
||||
periodLabel
|
||||
} satisfies FilingTaxonomyPeriod;
|
||||
})
|
||||
.filter((entry): entry is FilingTaxonomyPeriod => entry !== null);
|
||||
}
|
||||
|
||||
function normalizeStatementRows(
|
||||
value: unknown,
|
||||
fallbackRows: StatementRowMap = emptyStatementRows()
|
||||
): StatementRowMap {
|
||||
const object = asObject(value);
|
||||
if (!object) {
|
||||
return fallbackRows;
|
||||
}
|
||||
|
||||
const normalized = emptyStatementRows();
|
||||
for (const statement of FINANCIAL_STATEMENT_KINDS) {
|
||||
const rows = Array.isArray(object[statement]) ? object[statement] : [];
|
||||
normalized[statement] = rows
|
||||
.map((entry) => {
|
||||
const row = asObject(entry);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const key = asString(row.key) ?? asString(row.conceptKey ?? row.concept_key);
|
||||
const label = asString(row.label);
|
||||
const conceptKey = asString(row.conceptKey ?? row.concept_key);
|
||||
const qname = asString(row.qname);
|
||||
const namespaceUri = asString(row.namespaceUri ?? row.namespace_uri);
|
||||
const localName = asString(row.localName ?? row.local_name);
|
||||
if (!key || !label || !conceptKey || !qname || !namespaceUri || !localName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
key,
|
||||
label,
|
||||
conceptKey,
|
||||
qname,
|
||||
namespaceUri,
|
||||
localName,
|
||||
isExtension: asBoolean(row.isExtension ?? row.is_extension),
|
||||
statement: asStatementKind(row.statement) ?? statement,
|
||||
roleUri: asNullableString(row.roleUri ?? row.role_uri),
|
||||
order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER,
|
||||
depth: asNumber(row.depth) ?? 0,
|
||||
parentKey: asNullableString(row.parentKey ?? row.parent_key),
|
||||
values: normalizeNumberMap(row.values),
|
||||
units: normalizeNullableStringMap(row.units),
|
||||
hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions),
|
||||
sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids)
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is TaxonomyStatementRow => entry !== null);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeSurfaceRows(
|
||||
value: unknown,
|
||||
fallbackRows: SurfaceRowMap = emptySurfaceRows()
|
||||
): SurfaceRowMap {
|
||||
const object = asObject(value);
|
||||
if (!object) {
|
||||
return fallbackRows;
|
||||
}
|
||||
|
||||
const normalized = emptySurfaceRows();
|
||||
for (const statement of FINANCIAL_STATEMENT_KINDS) {
|
||||
const rows = Array.isArray(object[statement]) ? object[statement] : [];
|
||||
normalized[statement] = rows
|
||||
.map((entry) => {
|
||||
const row = asObject(entry);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const key = asString(row.key);
|
||||
const label = asString(row.label);
|
||||
const category = asString(row.category);
|
||||
const unit = asString(row.unit);
|
||||
if (!key || !label || !category || !unit) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedStatement = asStatementKind(row.statement);
|
||||
const resolutionMethod = row.resolutionMethod ?? row.resolution_method;
|
||||
const confidence = row.confidence;
|
||||
const normalizedRow: SurfaceFinancialRow = {
|
||||
key,
|
||||
label,
|
||||
category: category as SurfaceFinancialRow['category'],
|
||||
order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER,
|
||||
unit: unit as SurfaceFinancialRow['unit'],
|
||||
values: normalizeNumberMap(row.values),
|
||||
sourceConcepts: normalizeStringArray(row.sourceConcepts ?? row.source_concepts),
|
||||
sourceRowKeys: normalizeStringArray(row.sourceRowKeys ?? row.source_row_keys),
|
||||
sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids),
|
||||
formulaKey: asNullableString(row.formulaKey ?? row.formula_key),
|
||||
hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions),
|
||||
resolvedSourceRowKeys: normalizeNullableStringMap(row.resolvedSourceRowKeys ?? row.resolved_source_row_keys)
|
||||
};
|
||||
|
||||
const templateSection = asString(row.templateSection ?? row.template_section);
|
||||
if (templateSection) {
|
||||
normalizedRow.templateSection = templateSection as SurfaceFinancialRow['templateSection'];
|
||||
}
|
||||
if (normalizedStatement === 'income' || normalizedStatement === 'balance' || normalizedStatement === 'cash_flow') {
|
||||
normalizedRow.statement = normalizedStatement;
|
||||
}
|
||||
|
||||
const detailCount = asNumber(row.detailCount ?? row.detail_count);
|
||||
if (detailCount !== null) {
|
||||
normalizedRow.detailCount = detailCount;
|
||||
}
|
||||
|
||||
if (
|
||||
resolutionMethod === 'direct'
|
||||
|| resolutionMethod === 'surface_bridge'
|
||||
|| resolutionMethod === 'formula_derived'
|
||||
|| resolutionMethod === 'not_meaningful'
|
||||
) {
|
||||
normalizedRow.resolutionMethod = resolutionMethod;
|
||||
}
|
||||
|
||||
if (confidence === 'high' || confidence === 'medium' || confidence === 'low') {
|
||||
normalizedRow.confidence = confidence;
|
||||
}
|
||||
|
||||
const warningCodes = normalizeStringArray(row.warningCodes ?? row.warning_codes);
|
||||
if (warningCodes.length > 0) {
|
||||
normalizedRow.warningCodes = warningCodes;
|
||||
}
|
||||
|
||||
return normalizedRow;
|
||||
})
|
||||
.filter((entry): entry is SurfaceFinancialRow => entry !== null);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeDetailRows(
|
||||
value: unknown,
|
||||
fallbackRows: DetailRowMap = emptyDetailRows()
|
||||
): DetailRowMap {
|
||||
const object = asObject(value);
|
||||
if (!object) {
|
||||
return fallbackRows;
|
||||
}
|
||||
|
||||
const normalized = emptyDetailRows();
|
||||
for (const statement of FINANCIAL_STATEMENT_KINDS) {
|
||||
const groups = asObject(object[statement]) ?? {};
|
||||
normalized[statement] = Object.fromEntries(
|
||||
Object.entries(groups).map(([surfaceKey, rows]) => {
|
||||
const normalizedRows = Array.isArray(rows)
|
||||
? rows
|
||||
.map((entry) => {
|
||||
const row = asObject(entry);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const key = asString(row.key) ?? asString(row.conceptKey ?? row.concept_key);
|
||||
const label = asString(row.label);
|
||||
const conceptKey = asString(row.conceptKey ?? row.concept_key);
|
||||
const qname = asString(row.qname);
|
||||
const namespaceUri = asString(row.namespaceUri ?? row.namespace_uri);
|
||||
const localName = asString(row.localName ?? row.local_name);
|
||||
if (!key || !label || !conceptKey || !qname || !namespaceUri || !localName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
key,
|
||||
parentSurfaceKey: asString(row.parentSurfaceKey ?? row.parent_surface_key) ?? surfaceKey,
|
||||
label,
|
||||
conceptKey,
|
||||
qname,
|
||||
namespaceUri,
|
||||
localName,
|
||||
unit: asNullableString(row.unit),
|
||||
values: normalizeNumberMap(row.values),
|
||||
sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids),
|
||||
isExtension: asBoolean(row.isExtension ?? row.is_extension),
|
||||
dimensionsSummary: normalizeStringArray(row.dimensionsSummary ?? row.dimensions_summary),
|
||||
residualFlag: asBoolean(row.residualFlag ?? row.residual_flag)
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is DetailFinancialRow => entry !== null)
|
||||
: [];
|
||||
|
||||
return [surfaceKey, normalizedRows];
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeKpiRows(value: unknown) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.map((entry) => {
|
||||
const row = asObject(entry);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const key = asString(row.key);
|
||||
const label = asString(row.label);
|
||||
const category = asString(row.category);
|
||||
const unit = asString(row.unit);
|
||||
const provenanceType = row.provenanceType ?? row.provenance_type;
|
||||
if (!key || !label || !category || !unit || (provenanceType !== 'taxonomy' && provenanceType !== 'structured_note')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
key,
|
||||
label,
|
||||
category: category as StructuredKpiRow['category'],
|
||||
unit: unit as StructuredKpiRow['unit'],
|
||||
order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER,
|
||||
segment: asNullableString(row.segment),
|
||||
axis: asNullableString(row.axis),
|
||||
member: asNullableString(row.member),
|
||||
values: normalizeNumberMap(row.values),
|
||||
sourceConcepts: normalizeStringArray(row.sourceConcepts ?? row.source_concepts),
|
||||
sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids),
|
||||
provenanceType,
|
||||
hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions)
|
||||
} satisfies StructuredKpiRow;
|
||||
})
|
||||
.filter((entry): entry is StructuredKpiRow => entry !== null);
|
||||
}
|
||||
|
||||
function normalizeNormalizationSummary(value: unknown) {
|
||||
const row = asObject(value);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
surfaceRowCount: asNumber(row.surfaceRowCount ?? row.surface_row_count) ?? 0,
|
||||
detailRowCount: asNumber(row.detailRowCount ?? row.detail_row_count) ?? 0,
|
||||
kpiRowCount: asNumber(row.kpiRowCount ?? row.kpi_row_count) ?? 0,
|
||||
unmappedRowCount: asNumber(row.unmappedRowCount ?? row.unmapped_row_count) ?? 0,
|
||||
materialUnmappedRowCount: asNumber(row.materialUnmappedRowCount ?? row.material_unmapped_row_count) ?? 0,
|
||||
warnings: normalizeStringArray(row.warnings)
|
||||
} satisfies NormalizationSummary;
|
||||
}
|
||||
|
||||
export function normalizeFilingTaxonomySnapshotPayload(input: {
|
||||
periods: unknown;
|
||||
faithful_rows: unknown;
|
||||
statement_rows: unknown;
|
||||
surface_rows: unknown;
|
||||
detail_rows: unknown;
|
||||
kpi_rows: unknown;
|
||||
normalization_summary: unknown;
|
||||
}) {
|
||||
const faithfulRows = normalizeStatementRows(input.faithful_rows);
|
||||
const statementRows = normalizeStatementRows(input.statement_rows, faithfulRows);
|
||||
|
||||
return {
|
||||
periods: normalizePeriods(input.periods),
|
||||
faithful_rows: faithfulRows,
|
||||
statement_rows: statementRows,
|
||||
surface_rows: normalizeSurfaceRows(input.surface_rows),
|
||||
detail_rows: normalizeDetailRows(input.detail_rows),
|
||||
kpi_rows: normalizeKpiRows(input.kpi_rows),
|
||||
normalization_summary: normalizeNormalizationSummary(input.normalization_summary)
|
||||
};
|
||||
}
|
||||
|
||||
function emptyStatementRows(): StatementRowMap {
|
||||
return {
|
||||
income: [],
|
||||
balance: [],
|
||||
@@ -320,7 +720,7 @@ function emptyStatementRows(): Record<FinancialStatementKind, TaxonomyStatementR
|
||||
};
|
||||
}
|
||||
|
||||
function emptySurfaceRows(): Record<FinancialStatementKind, SurfaceFinancialRow[]> {
|
||||
function emptySurfaceRows(): SurfaceRowMap {
|
||||
return {
|
||||
income: [],
|
||||
balance: [],
|
||||
@@ -330,7 +730,7 @@ function emptySurfaceRows(): Record<FinancialStatementKind, SurfaceFinancialRow[
|
||||
};
|
||||
}
|
||||
|
||||
function emptyDetailRows(): Record<FinancialStatementKind, SurfaceDetailMap> {
|
||||
function emptyDetailRows(): DetailRowMap {
|
||||
return {
|
||||
income: {},
|
||||
balance: {},
|
||||
@@ -341,7 +741,15 @@ function emptyDetailRows(): Record<FinancialStatementKind, SurfaceDetailMap> {
|
||||
}
|
||||
|
||||
function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): FilingTaxonomySnapshotRecord {
|
||||
const faithfulRows = row.faithful_rows ?? row.statement_rows ?? emptyStatementRows();
|
||||
const normalized = normalizeFilingTaxonomySnapshotPayload({
|
||||
periods: row.periods,
|
||||
faithful_rows: row.faithful_rows,
|
||||
statement_rows: row.statement_rows,
|
||||
surface_rows: row.surface_rows,
|
||||
detail_rows: row.detail_rows,
|
||||
kpi_rows: row.kpi_rows,
|
||||
normalization_summary: row.normalization_summary
|
||||
});
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
@@ -356,15 +764,15 @@ function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): Fili
|
||||
parser_version: row.parser_version,
|
||||
taxonomy_regime: row.taxonomy_regime,
|
||||
fiscal_pack: row.fiscal_pack,
|
||||
periods: row.periods ?? [],
|
||||
faithful_rows: faithfulRows,
|
||||
statement_rows: faithfulRows,
|
||||
surface_rows: row.surface_rows ?? emptySurfaceRows(),
|
||||
detail_rows: row.detail_rows ?? emptyDetailRows(),
|
||||
kpi_rows: row.kpi_rows ?? [],
|
||||
periods: normalized.periods,
|
||||
faithful_rows: normalized.faithful_rows,
|
||||
statement_rows: normalized.statement_rows,
|
||||
surface_rows: normalized.surface_rows,
|
||||
detail_rows: normalized.detail_rows,
|
||||
kpi_rows: normalized.kpi_rows,
|
||||
derived_metrics: row.derived_metrics ?? null,
|
||||
validation_result: row.validation_result ?? null,
|
||||
normalization_summary: row.normalization_summary ?? null,
|
||||
normalization_summary: normalized.normalization_summary,
|
||||
facts_count: row.facts_count,
|
||||
concepts_count: row.concepts_count,
|
||||
dimensions_count: row.dimensions_count,
|
||||
@@ -552,6 +960,7 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
|
||||
|
||||
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
|
||||
const now = new Date().toISOString();
|
||||
const normalized = normalizeFilingTaxonomySnapshotPayload(input);
|
||||
|
||||
const [saved] = await withFinancialIngestionSchemaRetry({
|
||||
client: getSqliteClient(),
|
||||
@@ -570,15 +979,15 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
|
||||
parser_version: input.parser_version,
|
||||
taxonomy_regime: input.taxonomy_regime,
|
||||
fiscal_pack: input.fiscal_pack,
|
||||
periods: input.periods,
|
||||
faithful_rows: input.faithful_rows,
|
||||
statement_rows: input.statement_rows,
|
||||
surface_rows: input.surface_rows,
|
||||
detail_rows: input.detail_rows,
|
||||
kpi_rows: input.kpi_rows,
|
||||
periods: normalized.periods,
|
||||
faithful_rows: normalized.faithful_rows,
|
||||
statement_rows: normalized.statement_rows,
|
||||
surface_rows: normalized.surface_rows,
|
||||
detail_rows: normalized.detail_rows,
|
||||
kpi_rows: normalized.kpi_rows,
|
||||
derived_metrics: input.derived_metrics,
|
||||
validation_result: input.validation_result,
|
||||
normalization_summary: input.normalization_summary,
|
||||
normalization_summary: normalized.normalization_summary,
|
||||
facts_count: input.facts_count,
|
||||
concepts_count: input.concepts_count,
|
||||
dimensions_count: input.dimensions_count,
|
||||
@@ -598,15 +1007,15 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
|
||||
parser_version: input.parser_version,
|
||||
taxonomy_regime: input.taxonomy_regime,
|
||||
fiscal_pack: input.fiscal_pack,
|
||||
periods: input.periods,
|
||||
faithful_rows: input.faithful_rows,
|
||||
statement_rows: input.statement_rows,
|
||||
surface_rows: input.surface_rows,
|
||||
detail_rows: input.detail_rows,
|
||||
kpi_rows: input.kpi_rows,
|
||||
periods: normalized.periods,
|
||||
faithful_rows: normalized.faithful_rows,
|
||||
statement_rows: normalized.statement_rows,
|
||||
surface_rows: normalized.surface_rows,
|
||||
detail_rows: normalized.detail_rows,
|
||||
kpi_rows: normalized.kpi_rows,
|
||||
derived_metrics: input.derived_metrics,
|
||||
validation_result: input.validation_result,
|
||||
normalization_summary: input.normalization_summary,
|
||||
normalization_summary: normalized.normalization_summary,
|
||||
facts_count: input.facts_count,
|
||||
concepts_count: input.concepts_count,
|
||||
dimensions_count: input.dimensions_count,
|
||||
@@ -906,3 +1315,8 @@ export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) {
|
||||
|
||||
return rows.map(toAssetRecord);
|
||||
}
|
||||
|
||||
export const __filingTaxonomyInternals = {
|
||||
normalizeFilingTaxonomySnapshotPayload,
|
||||
toSnapshotRecord
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user