Fix financial taxonomy snapshot normalization

lib/financials/page-merge.test.ts

@@ -0,0 +1,185 @@
+import { describe, expect, it } from 'bun:test';
+import { __financialPageMergeInternals } from './page-merge';
+import type { CompanyFinancialStatementsResponse } from '@/lib/types';
+
+function createResponse(partial: Partial<CompanyFinancialStatementsResponse>): CompanyFinancialStatementsResponse {
+  return {
+    company: {
+      ticker: 'MSFT',
+      companyName: 'Microsoft Corporation',
+      cik: null
+    },
+    surfaceKind: 'income_statement',
+    cadence: 'annual',
+    displayModes: ['standardized', 'faithful'],
+    defaultDisplayMode: 'standardized',
+    periods: [],
+    statementRows: {
+      faithful: [],
+      standardized: []
+    },
+    statementDetails: null,
+    ratioRows: null,
+    kpiRows: null,
+    trendSeries: [],
+    categories: [],
+    availability: {
+      adjusted: false,
+      customMetrics: false
+    },
+    nextCursor: null,
+    facts: null,
+    coverage: {
+      filings: 0,
+      rows: 0,
+      dimensions: 0,
+      facts: 0
+    },
+    dataSourceStatus: {
+      enabled: true,
+      hydratedFilings: 0,
+      partialFilings: 0,
+      failedFilings: 0,
+      pendingFilings: 0,
+      queuedSync: false
+    },
+    metrics: {
+      taxonomy: null,
+      validation: null
+    },
+    normalization: {
+      parserEngine: 'fiscal-xbrl',
+      regime: 'us-gaap',
+      fiscalPack: 'core',
+      parserVersion: '0.1.0',
+      surfaceRowCount: 0,
+      detailRowCount: 0,
+      kpiRowCount: 0,
+      unmappedRowCount: 0,
+      materialUnmappedRowCount: 0,
+      warnings: []
+    },
+    dimensionBreakdown: null,
+    ...partial
+  };
+}
+
+describe('financial page merge helpers', () => {
+  it('merges detail maps safely when legacy detail rows are missing arrays', () => {
+    const merged = __financialPageMergeInternals.mergeDetailMaps(
+      {
+        revenue: [{
+          key: 'detail',
+          parentSurfaceKey: 'revenue',
+          label: 'Detail',
+          conceptKey: 'detail',
+          qname: 'us-gaap:Detail',
+          namespaceUri: 'http://fasb.org/us-gaap/2024',
+          localName: 'Detail',
+          unit: 'iso4217:USD',
+          values: { p1: 1 },
+          sourceFactIds: undefined,
+          isExtension: false,
+          dimensionsSummary: undefined,
+          residualFlag: false
+        } as never]
+      },
+      {
+        revenue: [{
+          key: 'detail',
+          parentSurfaceKey: 'revenue',
+          label: 'Detail',
+          conceptKey: 'detail',
+          qname: 'us-gaap:Detail',
+          namespaceUri: 'http://fasb.org/us-gaap/2024',
+          localName: 'Detail',
+          unit: 'iso4217:USD',
+          values: { p2: 2 },
+          sourceFactIds: [2],
+          isExtension: false,
+          dimensionsSummary: ['region:americas'],
+          residualFlag: false
+        }]
+      }
+    );
+
+    expect(merged?.revenue?.[0]).toMatchObject({
+      values: { p1: 1, p2: 2 },
+      sourceFactIds: [2],
+      dimensionsSummary: ['region:americas']
+    });
+  });
+
+  it('merges paged financial responses safely when row arrays are partially missing', () => {
+    const base = createResponse({
+      periods: [{
+        id: 'p1',
+        filingId: 1,
+        accessionNumber: '0001',
+        filingDate: '2025-01-01',
+        periodStart: '2024-01-01',
+        periodEnd: '2024-12-31',
+        filingType: '10-K',
+        periodLabel: 'FY 2024'
+      }],
+      statementRows: {
+        faithful: undefined as never,
+        standardized: [{
+          key: 'revenue',
+          label: 'Revenue',
+          category: 'revenue',
+          order: 10,
+          unit: 'currency',
+          values: { p1: 1 },
+          sourceConcepts: [],
+          sourceRowKeys: [],
+          sourceFactIds: [],
+          formulaKey: null,
+          hasDimensions: false,
+          resolvedSourceRowKeys: { p1: 'revenue' }
+        }]
+      }
+    });
+    const next = createResponse({
+      periods: [{
+        id: 'p2',
+        filingId: 2,
+        accessionNumber: '0002',
+        filingDate: '2026-01-01',
+        periodStart: '2025-01-01',
+        periodEnd: '2025-12-31',
+        filingType: '10-K',
+        periodLabel: 'FY 2025'
+      }],
+      statementRows: {
+        faithful: [{
+          key: 'rev',
+          label: 'Revenue',
+          conceptKey: 'rev',
+          qname: 'us-gaap:Revenue',
+          namespaceUri: 'http://fasb.org/us-gaap/2024',
+          localName: 'Revenue',
+          isExtension: false,
+          statement: 'income',
+          roleUri: 'income',
+          order: 10,
+          depth: 0,
+          parentKey: null,
+          values: { p2: 2 },
+          units: { p2: 'iso4217:USD' },
+          hasDimensions: false,
+          sourceFactIds: []
+        }],
+        standardized: undefined as never
+      }
+    });
+
+    const merged = __financialPageMergeInternals.mergeFinancialPages(base, next);
+
+    expect(merged.periods.map((period) => period.id)).toEqual(['p1', 'p2']);
+    expect(merged.statementRows).toMatchObject({
+      faithful: [{ key: 'rev' }],
+      standardized: [{ key: 'revenue', values: { p1: 1 } }]
+    });
+  });
+});

lib/financials/page-merge.ts

@@ -0,0 +1,98 @@
+import type {
+  CompanyFinancialStatementsResponse
+} from '@/lib/types';
+
+export function mergeDetailMaps(
+  base: CompanyFinancialStatementsResponse['statementDetails'],
+  next: CompanyFinancialStatementsResponse['statementDetails']
+) {
+  if (!base) {
+    return next;
+  }
+
+  if (!next) {
+    return base;
+  }
+
+  const merged: NonNullable<CompanyFinancialStatementsResponse['statementDetails']> = structuredClone(base);
+  for (const [surfaceKey, detailRows] of Object.entries(next)) {
+    const existingRows = merged[surfaceKey] ?? [];
+    const rowMap = new Map(existingRows.map((row) => [row.key, row]));
+
+    for (const detailRow of detailRows) {
+      const existing = rowMap.get(detailRow.key);
+      if (!existing) {
+        rowMap.set(detailRow.key, structuredClone(detailRow));
+        continue;
+      }
+
+      existing.values = {
+        ...existing.values,
+        ...detailRow.values
+      };
+      existing.sourceFactIds = [...new Set([...(existing.sourceFactIds ?? []), ...(detailRow.sourceFactIds ?? [])])];
+      existing.dimensionsSummary = [...new Set([...(existing.dimensionsSummary ?? []), ...(detailRow.dimensionsSummary ?? [])])];
+    }
+
+    merged[surfaceKey] = [...rowMap.values()];
+  }
+
+  return merged;
+}
+
+export function mergeFinancialPages(
+  base: CompanyFinancialStatementsResponse | null,
+  next: CompanyFinancialStatementsResponse
+) {
+  if (!base) {
+    return next;
+  }
+
+  const periods = [...base.periods, ...next.periods]
+    .filter((period, index, list) => list.findIndex((item) => item.id === period.id) === index)
+    .sort((left, right) => Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate));
+
+  const mergeRows = <T extends { key: string; values: Record<string, number | null> }>(rows: T[]) => {
+    const map = new Map<string, T>();
+    for (const row of rows) {
+      const existing = map.get(row.key);
+      if (!existing) {
+        map.set(row.key, structuredClone(row));
+        continue;
+      }
+
+      existing.values = {
+        ...existing.values,
+        ...row.values
+      };
+    }
+
+    return [...map.values()];
+  };
+
+  return {
+    ...next,
+    periods,
+    statementRows: next.statementRows && base.statementRows
+      ? {
+        faithful: mergeRows([...(base.statementRows.faithful ?? []), ...(next.statementRows.faithful ?? [])]),
+        standardized: mergeRows([...(base.statementRows.standardized ?? []), ...(next.statementRows.standardized ?? [])])
+      }
+      : next.statementRows,
+    statementDetails: mergeDetailMaps(base.statementDetails, next.statementDetails),
+    ratioRows: next.ratioRows && base.ratioRows
+      ? mergeRows([...(base.ratioRows ?? []), ...(next.ratioRows ?? [])])
+      : next.ratioRows,
+    kpiRows: next.kpiRows && base.kpiRows
+      ? mergeRows([...(base.kpiRows ?? []), ...(next.kpiRows ?? [])])
+      : next.kpiRows,
+    trendSeries: next.trendSeries,
+    categories: next.categories,
+    dimensionBreakdown: next.dimensionBreakdown ?? base.dimensionBreakdown
+  };
+}
+
+export const __financialPageMergeInternals = {
+  mergeDetailMaps,
+  mergeFinancialPages
+};

lib/financials/statement-view-model.test.ts

@@ -160,6 +160,28 @@ describe('statement view model', () => {
     expect(child?.kind === 'surface' && child.expanded).toBe(true);
   });
 
+  it('does not throw when legacy surface rows are missing source arrays', () => {
+    const malformedRow = {
+      ...createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } }),
+      sourceConcepts: undefined,
+      sourceRowKeys: undefined
+    } as unknown as SurfaceFinancialRow;
+
+    const model = buildStatementTree({
+      surfaceKind: 'income_statement',
+      rows: [malformedRow],
+      statementDetails: null,
+      categories: [],
+      searchQuery: 'revenue',
+      expandedRowKeys: new Set()
+    });
+
+    expect(model.sections[0]?.nodes[0]).toMatchObject({
+      kind: 'surface',
+      row: { key: 'revenue' }
+    });
+  });
+
   it('keeps not meaningful rows visible and resolves selections for surface and detail nodes', () => {
     const rows = [
       createSurfaceRow({

lib/financials/statement-view-model.ts

@@ -99,8 +99,8 @@ function searchTextForSurface(row: SurfaceFinancialRow) {
   return [
     row.label,
     row.key,
-    ...row.sourceConcepts,
-    ...row.sourceRowKeys,
+    ...(row.sourceConcepts ?? []),
+    ...(row.sourceRowKeys ?? []),
     ...(row.warningCodes ?? [])
   ]
     .join(' ')
@@ -115,7 +115,7 @@ function searchTextForDetail(row: DetailFinancialRow) {
     row.conceptKey,
     row.qname,
     row.localName,
-    ...row.dimensionsSummary
+    ...(row.dimensionsSummary ?? [])
   ]
     .join(' ')
     .toLowerCase();