Merge branch 't3code/investigate-unmapped-details'

lib/server/financial-taxonomy.test.ts

@@ -719,6 +719,138 @@ describe('financial taxonomy internals', () => {
     expect(standardizedRows.some((row) => row.key.includes('BusinessAcquisitionsProFormaNetIncomeLoss'))).toBe(true);
   });
 
+  it('merges detail rows across taxonomy-version concept drift for Microsoft residuals', () => {
+    const aggregated = __financialTaxonomyInternals.aggregateDetailRows({
+      statement: 'income',
+      selectedPeriodIds: new Set(['2024-fy', '2025-fy']),
+      snapshots: [
+        {
+          ...createSnapshot({
+            filingId: 90,
+            filingType: '10-K',
+            filingDate: '2024-07-30',
+            statement: 'income',
+            periods: [{
+              id: '2024-fy',
+              periodStart: '2023-07-01',
+              periodEnd: '2024-06-30',
+              periodLabel: 'FY24'
+            }]
+          }),
+          detail_rows: {
+            income: {
+              unmapped: [
+                {
+                  key: 'http://fasb.org/us-gaap/2024#AdvertisingExpense',
+                  parentSurfaceKey: 'unmapped',
+                  label: 'Advertising Expense',
+                  conceptKey: 'http://fasb.org/us-gaap/2024#AdvertisingExpense',
+                  qname: 'us-gaap:AdvertisingExpense',
+                  namespaceUri: 'http://fasb.org/us-gaap/2024',
+                  localName: 'AdvertisingExpense',
+                  unit: 'iso4217:USD',
+                  values: { '2024-fy': 1_500_000_000 },
+                  sourceFactIds: [101],
+                  isExtension: false,
+                  dimensionsSummary: [],
+                  residualFlag: true
+                },
+                {
+                  key: 'https://xbrl.microsoft.com/2024#BusinessAcquisitionsProFormaRevenue',
+                  parentSurfaceKey: 'unmapped',
+                  label: 'Business Acquisitions Pro Forma Revenue',
+                  conceptKey: 'https://xbrl.microsoft.com/2024#BusinessAcquisitionsProFormaRevenue',
+                  qname: 'msft:BusinessAcquisitionsProFormaRevenue',
+                  namespaceUri: 'https://xbrl.microsoft.com/2024',
+                  localName: 'BusinessAcquisitionsProFormaRevenue',
+                  unit: 'iso4217:USD',
+                  values: { '2024-fy': 247_442_000_000 },
+                  sourceFactIds: [102],
+                  isExtension: true,
+                  dimensionsSummary: [],
+                  residualFlag: true
+                }
+              ]
+            },
+            balance: {},
+            cash_flow: {},
+            equity: {},
+            comprehensive_income: {}
+          }
+        } satisfies FilingTaxonomySnapshotRecord,
+        {
+          ...createSnapshot({
+            filingId: 91,
+            filingType: '10-K',
+            filingDate: '2025-07-30',
+            statement: 'income',
+            periods: [{
+              id: '2025-fy',
+              periodStart: '2024-07-01',
+              periodEnd: '2025-06-30',
+              periodLabel: 'FY25'
+            }]
+          }),
+          detail_rows: {
+            income: {
+              unmapped: [
+                {
+                  key: 'http://fasb.org/us-gaap/2025#AdvertisingExpense',
+                  parentSurfaceKey: 'unmapped',
+                  label: 'Advertising Expense',
+                  conceptKey: 'http://fasb.org/us-gaap/2025#AdvertisingExpense',
+                  qname: 'us-gaap:AdvertisingExpense',
+                  namespaceUri: 'http://fasb.org/us-gaap/2025',
+                  localName: 'AdvertisingExpense',
+                  unit: 'iso4217:USD',
+                  values: { '2025-fy': 1_700_000_000 },
+                  sourceFactIds: [201],
+                  isExtension: false,
+                  dimensionsSummary: [],
+                  residualFlag: true
+                },
+                {
+                  key: 'https://xbrl.microsoft.com/2025#BusinessAcquisitionsProFormaRevenue',
+                  parentSurfaceKey: 'unmapped',
+                  label: 'Business Acquisitions Pro Forma Revenue',
+                  conceptKey: 'https://xbrl.microsoft.com/2025#BusinessAcquisitionsProFormaRevenue',
+                  qname: 'msft:BusinessAcquisitionsProFormaRevenue',
+                  namespaceUri: 'https://xbrl.microsoft.com/2025',
+                  localName: 'BusinessAcquisitionsProFormaRevenue',
+                  unit: 'iso4217:USD',
+                  values: { '2025-fy': 219_790_000_000 },
+                  sourceFactIds: [202],
+                  isExtension: true,
+                  dimensionsSummary: [],
+                  residualFlag: true
+                }
+              ]
+            },
+            balance: {},
+            cash_flow: {},
+            equity: {},
+            comprehensive_income: {}
+          }
+        } satisfies FilingTaxonomySnapshotRecord
+      ]
+    });
+
+    const unmapped = aggregated.unmapped ?? [];
+    expect(unmapped).toHaveLength(2);
+
+    const advertising = unmapped.find((row) => row.label === 'Advertising Expense');
+    expect(advertising?.values).toEqual({
+      '2024-fy': 1_500_000_000,
+      '2025-fy': 1_700_000_000
+    });
+
+    const proFormaRevenue = unmapped.find((row) => row.label === 'Business Acquisitions Pro Forma Revenue');
+    expect(proFormaRevenue?.values).toEqual({
+      '2024-fy': 247_442_000_000,
+      '2025-fy': 219_790_000_000
+    });
+  });
+
   it('resolves CASY-style income gaps with ordered fallbacks and tighter interest income exclusions', () => {
     const period = createPeriod({
       id: '2025-fy',

lib/server/financial-taxonomy.ts

@@ -315,6 +315,54 @@ function rowHasValues(values: Record<string, number | null>) {
   return Object.values(values).some((value) => value !== null);
 }
 
+function detailConceptIdentity(row: DetailFinancialRow) {
+  const localName = row.localName.trim().toLowerCase();
+  if (localName.length > 0) {
+    if (row.isExtension) {
+      return `extension:${localName}`;
+    }
+
+    if (row.namespaceUri.includes('us-gaap')) {
+      return `us-gaap:${localName}`;
+    }
+
+    if (row.namespaceUri.includes('ifrs')) {
+      return `ifrs:${localName}`;
+    }
+
+    const prefix = row.qname.split(':')[0]?.trim().toLowerCase();
+    if (prefix) {
+      return `${prefix}:${localName}`;
+    }
+  }
+
+  const normalizedQName = row.qname.trim().toLowerCase();
+  if (normalizedQName.length > 0) {
+    return normalizedQName;
+  }
+
+  const normalizedConceptKey = row.conceptKey.trim().toLowerCase();
+  if (normalizedConceptKey.length > 0) {
+    return normalizedConceptKey;
+  }
+
+  return row.key.trim().toLowerCase();
+}
+
+function detailMergeKey(row: DetailFinancialRow) {
+  const dimensionsKey = [...row.dimensionsSummary]
+    .map((value) => value.trim().toLowerCase())
+    .filter((value) => value.length > 0)
+    .sort((left, right) => left.localeCompare(right))
+    .join('|') || 'no-dimensions';
+
+  return [
+    detailConceptIdentity(row),
+    row.unit ?? 'no-unit',
+    dimensionsKey
+  ].join('::');
+}
+
 const PINNED_INCOME_SURFACE_ROWS = new Set([
   'revenue',
   'gross_profit',
@@ -440,9 +488,10 @@ function aggregateDetailRows(input: {
           continue;
         }
 
-        const existing = bucket.get(row.key);
+        const mergeKey = detailMergeKey(row);
+        const existing = bucket.get(mergeKey);
         if (!existing) {
-          bucket.set(row.key, {
+          bucket.set(mergeKey, {
             ...row,
             values: filteredValues,
             sourceFactIds: [...sourceFactIds],
@@ -1216,6 +1265,7 @@ export const __financialTaxonomyInternals = {
   buildDimensionBreakdown,
   buildNormalizationMetadata,
   aggregateSurfaceRows,
+  aggregateDetailRows,
   mergeStructuredKpiRowsByPriority,
   periodSorter,
   selectPrimaryPeriodsByCadence,