From 9f972305e6f688a7a1ac9ff684c5e9af1d47dbda Mon Sep 17 00:00:00 2001 From: francy51 Date: Mon, 9 Mar 2026 18:50:59 -0400 Subject: [PATCH] Fix annual financial selector and QCOM standardization --- lib/server/financial-taxonomy.test.ts | 1144 ++++++++++++- lib/server/financial-taxonomy.ts | 26 +- lib/server/financials/bundles.ts | 7 +- lib/server/financials/cadence.ts | 184 ++- lib/server/financials/standard-template.ts | 1471 +++++++++++++++++ lib/server/financials/standardize.ts | 768 +++++++-- lib/server/repos/company-financial-bundles.ts | 8 +- lib/server/repos/filing-taxonomy.ts | 2 +- lib/types.ts | 1 + 9 files changed, 3385 insertions(+), 226 deletions(-) create mode 100644 lib/server/financials/standard-template.ts diff --git a/lib/server/financial-taxonomy.test.ts b/lib/server/financial-taxonomy.test.ts index a7911d2..771453f 100644 --- a/lib/server/financial-taxonomy.test.ts +++ b/lib/server/financial-taxonomy.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'bun:test'; -import { __financialTaxonomyInternals } from './financial-taxonomy'; +import { + __financialTaxonomyInternals, + getCompanyFinancialTaxonomy +} from './financial-taxonomy'; import type { FilingTaxonomySnapshotRecord } from './repos/filing-taxonomy'; import type { FinancialStatementKind, @@ -15,11 +18,13 @@ function createRow(input: { qname?: string; localName?: string; statement?: FinancialStatementKind; + roleUri?: string | null; order?: number; depth?: number; hasDimensions?: boolean; values: Record; sourceFactIds?: number[]; + unit?: string | null; }): TaxonomyStatementRow { const localName = input.localName ?? 'RevenueFromContractWithCustomerExcludingAssessedTax'; const conceptKey = input.conceptKey ?? `http://fasb.org/us-gaap/2024#${localName}`; @@ -34,12 +39,12 @@ function createRow(input: { localName, isExtension: false, statement: input.statement ?? 'income', - roleUri: input.statement ?? 'income', + roleUri: input.roleUri ?? input.statement ?? 'income', order: input.order ?? 1, depth: input.depth ?? 0, parentKey: null, values: input.values, - units: Object.fromEntries(Object.keys(input.values).map((periodId) => [periodId, 'iso4217:USD'])), + units: Object.fromEntries(Object.keys(input.values).map((periodId) => [periodId, input.unit ?? 'iso4217:USD'])), hasDimensions: input.hasDimensions ?? false, sourceFactIds: input.sourceFactIds ?? [1] }; @@ -157,6 +162,68 @@ function createDimensionFact(input: { }; } +function createFact(input: { + id?: number; + filingId: number; + filingDate: string; + statement?: FinancialStatementKind; + conceptKey?: string; + qname?: string; + localName: string; + periodEnd: string; + periodStart?: string | null; + periodInstant?: string | null; + value: number; + unit?: string | null; +}): TaxonomyFactRow { + const conceptKey = input.conceptKey ?? `http://fasb.org/us-gaap/2024#${input.localName}`; + const qname = input.qname ?? `us-gaap:${input.localName}`; + + return { + id: input.id ?? input.filingId, + snapshotId: input.filingId, + filingId: input.filingId, + filingDate: input.filingDate, + statement: input.statement ?? 'income', + roleUri: input.statement ?? 'income', + conceptKey, + qname, + namespaceUri: 'http://fasb.org/us-gaap/2024', + localName: input.localName, + value: input.value, + contextId: `ctx-${input.filingId}-${input.localName}`, + unit: input.unit ?? 'iso4217:USD', + decimals: null, + periodStart: input.periodStart ?? null, + periodEnd: input.periodEnd, + periodInstant: input.periodInstant ?? null, + dimensions: [], + isDimensionless: true, + sourceFile: null + }; +} + +function findRow(rows: ReturnType, key: string) { + const row = rows.find((entry) => entry.key === key); + expect(row).toBeDefined(); + return row!; +} + +function findPeriodId(periods: FinancialStatementPeriod[], periodEnd: string) { + const period = periods.find((entry) => entry.periodEnd === periodEnd); + expect(period).toBeDefined(); + return period!.id; +} + +function findStandardizedResponseRow( + response: Awaited>, + key: string +) { + const row = response.statementRows?.standardized.find((entry) => entry.key === key); + expect(row).toBeDefined(); + return row!; +} + describe('financial taxonomy internals', () => { it('selects the primary quarter duration for 10-Q income statements', () => { const snapshot = createSnapshot({ @@ -225,6 +292,117 @@ describe('financial taxonomy internals', () => { expect(quarterlyPeriods.map((period) => period.id)).toEqual(['quarter']); }); + it('ignores future-dated fallback note periods for annual income selection', () => { + const snapshot = createSnapshot({ + filingId: 12, + filingType: '10-K', + filingDate: '2025-11-05', + statement: 'income', + periods: [ + { id: 'annual', periodStart: '2024-09-30', periodEnd: '2025-09-28', periodLabel: '2024-09-30 to 2025-09-28' }, + { id: 'future-note', periodStart: '2026-09-28', periodEnd: '2027-09-26', periodLabel: '2026-09-28 to 2027-09-26' } + ], + rows: [ + createRow({ + localName: 'Revenues', + label: 'Revenues', + values: { annual: 44_284_000_000 } + }), + createRow({ + localName: 'CostOfSales', + label: 'Cost of Sales', + values: { annual: 19_738_000_000 }, + order: 2 + }), + createRow({ + localName: 'EffectiveIncomeTaxRateContinuingOperations', + label: 'Effective Income Tax Rate Continuing Operations', + roleUri: null, + order: Number.MAX_SAFE_INTEGER, + values: { 'future-note': 0.16 }, + unit: 'pure' + }) + ] + }); + + const selection = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([snapshot], 'income', 'annual'); + + expect(selection.periods).toHaveLength(1); + expect(selection.periods[0]?.id).toBe('annual'); + }); + + it('prefers broader presented-row coverage over sparser later annual periods', () => { + const snapshot = createSnapshot({ + filingId: 13, + filingType: '10-K', + filingDate: '2025-07-30', + statement: 'income', + periods: [ + { id: 'primary', periodStart: '2024-07-01', periodEnd: '2025-06-30', periodLabel: '2024-07-01 to 2025-06-30' }, + { id: 'sparse-later', periodStart: '2024-07-16', periodEnd: '2025-07-15', periodLabel: '2024-07-16 to 2025-07-15' } + ], + rows: [ + createRow({ + localName: 'Revenues', + label: 'Revenues', + values: { primary: 245_122_000_000, 'sparse-later': 246_000_000_000 } + }), + createRow({ + localName: 'OperatingIncomeLoss', + label: 'Operating Income', + values: { primary: 109_433_000_000 }, + order: 2 + }), + createRow({ + localName: 'NetIncomeLoss', + label: 'Net Income', + values: { primary: 88_136_000_000 }, + order: 3 + }) + ] + }); + + const selection = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([snapshot], 'income', 'annual'); + + expect(selection.periods).toHaveLength(1); + expect(selection.periods[0]?.id).toBe('primary'); + }); + + it('falls back to plausible non-presented periods when no presented rows exist', () => { + const snapshot = createSnapshot({ + filingId: 14, + filingType: '10-K', + filingDate: '2025-11-05', + statement: 'income', + periods: [ + { id: 'annual', periodStart: '2024-09-30', periodEnd: '2025-09-28', periodLabel: '2024-09-30 to 2025-09-28' }, + { id: 'future-note', periodStart: '2026-09-28', periodEnd: '2027-09-26', periodLabel: '2026-09-28 to 2027-09-26' } + ], + rows: [ + createRow({ + localName: 'Revenues', + label: 'Revenues', + roleUri: null, + order: Number.MAX_SAFE_INTEGER, + values: { annual: 44_284_000_000 } + }), + createRow({ + localName: 'EffectiveIncomeTaxRateContinuingOperations', + label: 'Effective Income Tax Rate Continuing Operations', + roleUri: null, + order: Number.MAX_SAFE_INTEGER, + values: { 'future-note': 0.16 }, + unit: 'pure' + }) + ] + }); + + const selection = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([snapshot], 'income', 'annual'); + + expect(selection.periods).toHaveLength(1); + expect(selection.periods[0]?.id).toBe('annual'); + }); + it('maps overlapping GAAP aliases into one standardized COGS row while preserving faithful rows', () => { const period2024 = createPeriod({ id: '2024-q4', @@ -321,14 +499,12 @@ describe('financial taxonomy internals', () => { createRow({ localName: 'CostOfRevenue', label: 'Cost of Revenue', - values: { '2024-q4': 45_000 }, - hasDimensions: true + values: { '2024-q4': 45_000 } }), createRow({ localName: 'CostOfGoodsSold', label: 'Cost of Goods Sold', - values: { '2025-q4': 48_000 }, - hasDimensions: true + values: { '2025-q4': 48_000 } }) ]; const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows( @@ -370,4 +546,958 @@ describe('financial taxonomy internals', () => { 'Cost of Goods Sold' ]); }); + + it('prefers exact Microsoft income aliases over pro forma and reconciliation rows', () => { + const period = createPeriod({ + id: '2024-fy', + filingId: 80, + filingDate: '2024-07-30', + periodStart: '2023-07-01', + periodEnd: '2024-06-30', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'income', + periods: [period], + facts: [], + rows: [ + createRow({ + localName: 'BusinessAcquisitionsProFormaRevenue', + label: 'Business Acquisitions Pro Forma Revenue', + values: { '2024-fy': 247_442_000_000 } + }), + createRow({ + localName: 'RevenueFromContractWithCustomerExcludingAssessedTax', + label: 'Revenue From Contract With Customer Excluding Assessed Tax', + values: { '2024-fy': 245_122_000_000 } + }), + createRow({ + localName: 'NonoperatingIncomeExpense', + label: 'Nonoperating Income Expense', + values: { '2024-fy': -1_646_000_000 } + }), + createRow({ + localName: 'SellingAndMarketingExpense', + label: 'Selling And Marketing Expense', + values: { '2024-fy': 24_456_000_000 } + }), + createRow({ + localName: 'GeneralAndAdministrativeExpense', + label: 'General And Administrative Expense', + values: { '2024-fy': 7_609_000_000 } + }), + createRow({ + localName: 'OperatingIncomeLoss', + label: 'Operating Income Loss', + values: { '2024-fy': 109_433_000_000 } + }), + createRow({ + localName: 'BusinessAcquisitionsProFormaNetIncomeLoss', + label: 'Business Acquisitions Pro Forma Net Income Loss', + values: { '2024-fy': 88_308_000_000 }, + hasDimensions: true + }), + createRow({ + localName: 'NetIncomeLoss', + label: 'Net Income Loss', + values: { '2024-fy': 88_136_000_000 } + }), + createRow({ + localName: 'CurrentIncomeTaxExpenseBenefit', + label: 'Current Income Tax Expense Benefit', + values: { '2024-fy': 24_389_000_000 } + }), + createRow({ + localName: 'IncomeTaxExpenseBenefit', + label: 'Income Tax Expense Benefit', + values: { '2024-fy': 19_651_000_000 } + }), + createRow({ + localName: 'IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest', + label: 'Income Loss From Continuing Operations Before Income Taxes Extraordinary Items Noncontrolling Interest', + values: { '2024-fy': 107_787_000_000 } + }), + createRow({ + localName: 'EffectiveIncomeTaxRateReconciliationInterestIncomeExpense', + label: 'Effective Income Tax Rate Reconciliation Interest Income Expense', + values: { '2024-fy': 0.011 }, + unit: 'pure' + }), + createRow({ + localName: 'InvestmentIncomeNet', + label: 'Investment Income Net', + values: { '2024-fy': 3_157_000_000 } + }), + createRow({ + localName: 'CostOfGoodsAndServicesSold', + label: 'Cost Of Goods And Services Sold', + values: { '2024-fy': 74_114_000_000 } + }) + ] + }); + + const revenue = findRow(standardizedRows, 'revenue'); + expect(revenue.values['2024-fy']).toBe(245_122_000_000); + expect(revenue.resolvedSourceRowKeys['2024-fy']).toContain('RevenueFromContractWithCustomerExcludingAssessedTax'); + + const operatingIncome = findRow(standardizedRows, 'operating_income'); + expect(operatingIncome.values['2024-fy']).toBe(109_433_000_000); + expect(operatingIncome.resolvedSourceRowKeys['2024-fy']).toContain('OperatingIncomeLoss'); + + const sga = findRow(standardizedRows, 'selling_general_and_administrative'); + expect(sga.values['2024-fy']).toBe(32_065_000_000); + expect(sga.formulaKey).toBe('selling_general_and_administrative'); + + const netIncome = findRow(standardizedRows, 'net_income'); + expect(netIncome.values['2024-fy']).toBe(88_136_000_000); + expect(netIncome.resolvedSourceRowKeys['2024-fy']).toContain('NetIncomeLoss'); + + const taxExpense = findRow(standardizedRows, 'income_tax_expense'); + expect(taxExpense.values['2024-fy']).toBe(19_651_000_000); + expect(taxExpense.resolvedSourceRowKeys['2024-fy']).toContain('IncomeTaxExpenseBenefit'); + + const pretaxIncome = findRow(standardizedRows, 'pretax_income'); + expect(pretaxIncome.values['2024-fy']).toBe(107_787_000_000); + + const interestIncome = findRow(standardizedRows, 'interest_income'); + expect(interestIncome.values['2024-fy']).toBe(3_157_000_000); + expect(interestIncome.resolvedSourceRowKeys['2024-fy']).toContain('InvestmentIncomeNet'); + + const cogs = findRow(standardizedRows, 'cost_of_revenue'); + expect(cogs.label).toBe('Cost of Sales'); + expect(cogs.values['2024-fy']).toBe(74_114_000_000); + + expect(standardizedRows.some((row) => row.key.includes('BusinessAcquisitionsProFormaRevenue'))).toBe(true); + expect(standardizedRows.some((row) => row.key.includes('BusinessAcquisitionsProFormaNetIncomeLoss'))).toBe(true); + }); + + it('resolves CASY-style income gaps with ordered fallbacks and tighter interest income exclusions', () => { + const period = createPeriod({ + id: '2025-fy', + filingId: 81, + filingDate: '2025-06-23', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'income', + periods: [period], + facts: [], + rows: [ + createRow({ + localName: 'Revenues', + label: 'Revenues', + values: { '2025-fy': 15_940_899_000 } + }), + createRow({ + localName: 'CostOfGoodsAndServiceExcludingDepreciationDepletionAndAmortization', + label: 'Cost Of Goods And Service Excluding Depreciation Depletion And Amortization', + values: { '2025-fy': 12_188_496_000 } + }), + createRow({ + localName: 'OperatingExpenses', + label: 'Operating Expenses', + values: { '2025-fy': 2_552_356_000 } + }), + createRow({ + localName: 'CostOfGoodsAndServicesSoldDepreciationAndAmortization', + label: 'Cost Of Goods And Services Sold Depreciation And Amortization', + values: { '2025-fy': 403_647_000 } + }), + createRow({ + localName: 'NetIncomeLoss', + label: 'Net Income Loss', + values: { '2025-fy': 546_520_000 } + }), + createRow({ + localName: 'IncomeTaxExpenseBenefit', + label: 'Income Tax Expense Benefit', + values: { '2025-fy': 165_929_000 } + }), + createRow({ + localName: 'InterestExpense', + label: 'Interest Expense', + values: { '2025-fy': 83_951_000 } + }), + createRow({ + localName: 'InterestIncomeExpenseNet', + label: 'Interest Income Expense Net', + values: { '2025-fy': 13_102_000 } + }), + createRow({ + localName: 'EffectiveIncomeTaxRateContinuingOperations', + label: 'Effective Income Tax Rate Continuing Operations', + values: { '2025-fy': 0.233 }, + unit: 'pure' + }) + ] + }); + + expect(findRow(standardizedRows, 'cost_of_revenue').values['2025-fy']).toBe(12_188_496_000); + expect(findRow(standardizedRows, 'gross_profit').values['2025-fy']).toBe(3_752_403_000); + expect(findRow(standardizedRows, 'selling_general_and_administrative').values['2025-fy']).toBe(2_552_356_000); + expect(findRow(standardizedRows, 'pretax_income').values['2025-fy']).toBe(712_449_000); + expect(findRow(standardizedRows, 'operating_income').values['2025-fy']).toBe(796_400_000); + expect(findRow(standardizedRows, 'depreciation_and_amortization').values['2025-fy']).toBe(403_647_000); + expect(findRow(standardizedRows, 'depreciation_and_amortization_expenses').values['2025-fy']).toBe(403_647_000); + expect(findRow(standardizedRows, 'ebitda').values['2025-fy']).toBe(1_200_047_000); + expect(findRow(standardizedRows, 'effective_tax_rate').values['2025-fy']).toBe(0.233); + expect(standardizedRows.some((row) => row.key === 'interest_income')).toBe(false); + }); + + it('keeps EBITDA formula-only instead of mapping it directly from operating income', () => { + const period = createPeriod({ + id: '2024-fy', + filingId: 90, + filingDate: '2024-07-30', + periodStart: '2023-07-01', + periodEnd: '2024-06-30', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'income', + periods: [period], + facts: [], + rows: [ + createRow({ + localName: 'OperatingIncomeLoss', + label: 'Operating Income Loss', + values: { '2024-fy': 109_433_000_000 } + }), + createRow({ + localName: 'DepreciationDepletionAndAmortization', + label: 'Depreciation Depletion And Amortization', + values: { '2024-fy': 22_287_000_000 } + }) + ] + }); + + const ebitda = findRow(standardizedRows, 'ebitda'); + expect(ebitda.values['2024-fy']).toBe(131_720_000_000); + expect(ebitda.formulaKey).toBe('ebitda'); + expect(ebitda.resolvedSourceRowKeys['2024-fy']).toBeNull(); + expect(ebitda.sourceRowKeys).toHaveLength(0); + }); + + it('uses balance template ordering and sections, with equity falling back to assets minus liabilities', () => { + const period = createPeriod({ + id: '2025-balance', + filingId: 95, + filingDate: '2025-07-30', + periodEnd: '2025-06-30', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'balance', + periods: [period], + facts: [], + rows: [ + createRow({ + statement: 'balance', + localName: 'Assets', + label: 'Assets', + values: { '2025-balance': 619_003_000_000 } + }), + createRow({ + statement: 'balance', + localName: 'Liabilities', + label: 'Liabilities', + values: { '2025-balance': 275_524_000_000 } + }), + createRow({ + statement: 'balance', + localName: 'AssetsCurrent', + label: 'Assets Current', + values: { '2025-balance': 191_131_000_000 } + }) + ] + }); + + const currentAssets = findRow(standardizedRows, 'current_assets'); + const totalAssets = findRow(standardizedRows, 'total_assets'); + const totalLiabilities = findRow(standardizedRows, 'total_liabilities'); + const totalEquity = findRow(standardizedRows, 'total_equity'); + + expect(currentAssets.category).toBe('assets'); + expect(totalLiabilities.category).toBe('liabilities'); + expect(totalEquity.category).toBe('equity'); + expect(totalEquity.values['2025-balance']).toBe(343_479_000_000); + expect(totalEquity.formulaKey).toBe('total_equity'); + expect(currentAssets.order).toBeLessThan(totalAssets.order); + expect(totalAssets.order).toBeLessThan(totalLiabilities.order); + expect(totalLiabilities.order).toBeLessThan(totalEquity.order); + }); + + it('maps cash flow capex and computes free cash flow from the template formula', () => { + const period = createPeriod({ + id: '2024-cf', + filingId: 100, + filingDate: '2024-07-30', + periodStart: '2023-07-01', + periodEnd: '2024-06-30', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'cash_flow', + periods: [period], + facts: [], + rows: [ + createRow({ + statement: 'cash_flow', + localName: 'NetCashProvidedByUsedInOperatingActivities', + label: 'Net Cash Provided By Used In Operating Activities', + values: { '2024-cf': 118_548_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'PaymentsToAcquirePropertyPlantAndEquipment', + label: 'Payments To Acquire Property Plant And Equipment', + values: { '2024-cf': 46_937_000_000 } + }) + ] + }); + + const capex = findRow(standardizedRows, 'capital_expenditures'); + expect(capex.values['2024-cf']).toBe(-46_937_000_000); + + const freeCashFlow = findRow(standardizedRows, 'free_cash_flow'); + expect(freeCashFlow.values['2024-cf']).toBe(71_611_000_000); + expect(freeCashFlow.formulaKey).toBe('free_cash_flow'); + }); + + it('prefers dimensionless facts over dimensioned or label-only rows for canonical balance rows', () => { + const period = createPeriod({ + id: '2025-balance', + filingId: 110, + filingDate: '2025-02-05', + periodEnd: '2024-12-31', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'balance', + periods: [period], + rows: [ + createRow({ + statement: 'balance', + localName: 'Assets', + label: 'Assets', + values: { '2025-balance': 8_700_000_000 }, + hasDimensions: true + }), + createRow({ + statement: 'balance', + localName: 'IntangibleAssetsGrossExcludingGoodwill', + label: 'Intangible Assets Gross Excluding Goodwill', + values: { '2025-balance': 2_793_000_000 } + }) + ], + facts: [ + createFact({ + filingId: 110, + filingDate: '2025-02-05', + statement: 'balance', + localName: 'Assets', + periodEnd: '2024-12-31', + periodInstant: '2024-12-31', + value: 450_256_000_000 + }), + createFact({ + id: 111, + filingId: 110, + filingDate: '2025-02-05', + statement: 'balance', + localName: 'Goodwill', + periodEnd: '2024-12-31', + periodInstant: '2024-12-31', + value: 31_885_000_000 + }) + ] + }); + + const totalAssets = findRow(standardizedRows, 'total_assets'); + expect(totalAssets.values['2025-balance']).toBe(450_256_000_000); + expect(totalAssets.resolvedSourceRowKeys['2025-balance']).toContain('Assets'); + expect(standardizedRows.some((row) => row.key === 'other:http://fasb.org/us-gaap/2024#Assets')).toBe(false); + + const goodwill = findRow(standardizedRows, 'goodwill'); + expect(goodwill.values['2025-balance']).toBe(31_885_000_000); + expect(goodwill.resolvedSourceRowKeys['2025-balance']).toContain('Goodwill'); + }); + + it('uses alias priority for Apple-style long-term investments and unearned revenue rows', () => { + const period = createPeriod({ + id: '2025-balance', + filingId: 120, + filingDate: '2025-10-31', + periodEnd: '2025-09-27', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'balance', + periods: [period], + rows: [ + createRow({ + statement: 'balance', + localName: 'AvailableForSaleSecuritiesDebtSecurities', + label: 'Available For Sale Securities Debt Securities', + values: { '2025-balance': 98_027_000_000 } + }), + createRow({ + statement: 'balance', + localName: 'AvailableForSaleSecuritiesDebtMaturitiesSingleMaturityDate', + label: 'Available For Sale Securities Debt Maturities Single Maturity Date', + values: { '2025-balance': 77_723_000_000 } + }), + createRow({ + statement: 'balance', + localName: 'ContractWithCustomerLiability', + label: 'Contract With Customer Liability', + values: { '2025-balance': 13_700_000_000 } + }), + createRow({ + statement: 'balance', + localName: 'ContractWithCustomerLiabilityCurrent', + label: 'Contract With Customer Liability Current', + values: { '2025-balance': 9_055_000_000 } + }) + ], + facts: [] + }); + + const longTermInvestments = findRow(standardizedRows, 'long_term_investments'); + expect(longTermInvestments.values['2025-balance']).toBe(77_723_000_000); + expect(longTermInvestments.resolvedSourceRowKeys['2025-balance']).toContain('AvailableForSaleSecuritiesDebtMaturitiesSingleMaturityDate'); + + const unearnedRevenue = findRow(standardizedRows, 'unearned_revenue'); + expect(unearnedRevenue.values['2025-balance']).toBe(9_055_000_000); + expect(unearnedRevenue.resolvedSourceRowKeys['2025-balance']).toContain('ContractWithCustomerLiabilityCurrent'); + }); + + it('maps WMS extension aliases and direct effective tax rate values', () => { + const period = createPeriod({ + id: '2025-fy', + filingId: 130, + filingDate: '2025-05-15', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'income', + periods: [period], + rows: [ + createRow({ + localName: 'CostOfGoodsAndServicesSold', + label: 'Cost Of Goods And Services Sold', + values: { '2025-fy': 340_800_000 } + }), + createRow({ + localName: 'SellingGeneralAndAdministrativeExpenseEmployeeStockOptionPlanSpecialDividendCompensation', + label: 'Selling General And Administrative Expense Employee Stock Option Plan Special Dividend Compensation', + values: { '2025-fy': 0 } + }) + ], + facts: [ + createFact({ + filingId: 130, + filingDate: '2025-05-15', + localName: 'CostOfGoodsSoldExcludingEmployeeStockOptionPlanSpecialDividendCompensation', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + value: 1_810_004_000 + }), + createFact({ + id: 131, + filingId: 130, + filingDate: '2025-05-15', + localName: 'SellingGeneralAndAdministrativeExpenseExcludingEmployeeStockOptionPlanSpecialDividendCompensation', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + value: 380_378_000 + }), + createFact({ + id: 132, + filingId: 130, + filingDate: '2025-05-15', + localName: 'EffectiveIncomeTaxRateContinuingOperations', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + value: 0.239, + unit: 'pure' + }), + createFact({ + id: 133, + filingId: 130, + filingDate: '2025-05-15', + localName: 'InterestIncomeExpenseNonoperatingNet', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + value: -91_803_000 + }) + ] + }); + + expect(findRow(standardizedRows, 'cost_of_revenue').values['2025-fy']).toBe(1_810_004_000); + expect(findRow(standardizedRows, 'selling_general_and_administrative').values['2025-fy']).toBe(380_378_000); + expect(findRow(standardizedRows, 'effective_tax_rate').values['2025-fy']).toBe(0.239); + expect(findRow(standardizedRows, 'interest_expense').values['2025-fy']).toBe(91_803_000); + }); + + it('inverts Fiscal.ai cash flow outflows and working-capital changes', () => { + const period = createPeriod({ + id: '2025-cf', + filingId: 140, + filingDate: '2025-05-15', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'cash_flow', + periods: [period], + facts: [], + rows: [ + createRow({ + statement: 'cash_flow', + localName: 'NetCashProvidedByUsedInOperatingActivities', + label: 'Net Cash Provided By Used In Operating Activities', + values: { '2025-cf': 1_000_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'PaymentsToAcquirePropertyPlantAndEquipment', + label: 'Payments To Acquire Property Plant And Equipment', + values: { '2025-cf': 250_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'IncreaseDecreaseInReceivables', + label: 'Increase Decrease In Receivables', + values: { '2025-cf': -37_487_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'IncreaseDecreaseInInventories', + label: 'Increase Decrease In Inventories', + values: { '2025-cf': 15_749_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'PaymentsForRepurchaseOfCommonStock', + label: 'Payments For Repurchase Of Common Stock', + values: { '2025-cf': 100_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'PaymentsOfDividends', + label: 'Payments Of Dividends', + values: { '2025-cf': 40_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'OtherInvestingActivitiesNet', + label: 'Other Investing Activities Net', + values: { '2025-cf': 12_000_000 } + }) + ] + }); + + expect(findRow(standardizedRows, 'changes_trade_receivables').values['2025-cf']).toBe(37_487_000); + expect(findRow(standardizedRows, 'changes_inventories').values['2025-cf']).toBe(-15_749_000); + expect(findRow(standardizedRows, 'capital_expenditures').values['2025-cf']).toBe(-250_000_000); + expect(findRow(standardizedRows, 'share_repurchases').values['2025-cf']).toBe(-100_000_000); + expect(findRow(standardizedRows, 'dividends_paid').values['2025-cf']).toBe(-40_000_000); + expect(findRow(standardizedRows, 'other_investing_activities').values['2025-cf']).toBe(-12_000_000); + expect(findRow(standardizedRows, 'free_cash_flow').values['2025-cf']).toBe(750_000_000); + }); + + it('aggregates multiple matching component rows for template rows that require it', () => { + const period = createPeriod({ + id: '2025-cf', + filingId: 150, + filingDate: '2025-05-15', + periodStart: '2024-04-01', + periodEnd: '2025-03-31', + filingType: '10-K' + }); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'cash_flow', + periods: [period], + facts: [], + rows: [ + createRow({ + statement: 'cash_flow', + localName: 'IncreaseDecreaseInOtherOperatingAssets', + label: 'Increase Decrease In Other Operating Assets', + values: { '2025-cf': 25_000_000 } + }), + createRow({ + statement: 'cash_flow', + localName: 'IncreaseDecreaseInOtherOperatingLiabilities', + label: 'Increase Decrease In Other Operating Liabilities', + values: { '2025-cf': -40_000_000 } + }) + ] + }); + + const otherOperating = findRow(standardizedRows, 'changes_other_operating_activities'); + expect(otherOperating.values['2025-cf']).toBe(15_000_000); + expect(otherOperating.resolvedSourceRowKeys['2025-cf']).toBeNull(); + expect(otherOperating.sourceRowKeys).toHaveLength(2); + }); + + it('aggregates CASY balance and cash-flow component rows from dimensionless facts', () => { + const balancePeriod = createPeriod({ + id: '2025-balance', + filingId: 151, + filingDate: '2025-06-23', + periodEnd: '2025-04-30', + filingType: '10-K' + }); + const cashFlowPeriod = createPeriod({ + id: '2025-cf', + filingId: 152, + filingDate: '2025-06-23', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + filingType: '10-K' + }); + + const balanceRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'balance', + periods: [balancePeriod], + facts: [], + rows: [ + createRow({ + statement: 'balance', + localName: 'PropertyPlantAndEquipmentAndFinanceLeaseRightOfUseAssetAfterAccumulatedDepreciationAndAmortization', + label: 'Property Plant And Equipment And Finance Lease Right Of Use Asset After Accumulated Depreciation And Amortization', + values: { '2025-balance': 5_413_244_000 } + }), + createRow({ + statement: 'balance', + localName: 'EmployeeRelatedLiabilitiesCurrent', + label: 'Employee Related Liabilities Current', + values: { '2025-balance': 80_633_000 } + }), + createRow({ + statement: 'balance', + localName: 'OtherLiabilitiesCurrent', + label: 'Other Liabilities Current', + values: { '2025-balance': 189_870_000 } + }), + createRow({ + statement: 'balance', + localName: 'AccruedPropertyTaxes', + qname: 'caseys:AccruedPropertyTaxes', + conceptKey: 'http://www.caseys.com/20250430#AccruedPropertyTaxes', + label: 'Accrued Property Taxes', + values: { '2025-balance': 59_843_000 } + }), + createRow({ + statement: 'balance', + localName: 'DeferredIncomeTaxLiabilitiesNet', + label: 'Deferred Income Tax Liabilities Net', + values: { '2025-balance': 646_905_000 } + }), + createRow({ + statement: 'balance', + localName: 'OtherLiabilitiesNoncurrent', + label: 'Other Liabilities Noncurrent', + values: { '2025-balance': 69_380_000 } + }), + createRow({ + statement: 'balance', + localName: 'LiabilitiesCurrent', + label: 'Liabilities Current', + values: { '2025-balance': 1_101_693_000 } + }), + createRow({ + statement: 'balance', + localName: 'OperatingLeaseLiability', + label: 'Operating Lease Liability', + values: { '2025-balance': 449_354_000 } + }), + createRow({ + statement: 'balance', + localName: 'FinanceLeaseLiability', + label: 'Finance Lease Liability', + values: { '2025-balance': 108_920_000 } + }) + ] + }); + + expect(findRow(balanceRows, 'property_plant_equipment').values['2025-balance']).toBe(5_413_244_000); + expect(findRow(balanceRows, 'accrued_liabilities').values['2025-balance']).toBe(330_346_000); + expect(findRow(balanceRows, 'other_long_term_liabilities').values['2025-balance']).toBe(69_380_000); + expect(findRow(balanceRows, 'total_current_liabilities').values['2025-balance']).toBe(1_101_693_000); + expect(findRow(balanceRows, 'leases').values['2025-balance']).toBe(558_274_000); + + const cashFlowRows = __financialTaxonomyInternals.buildStandardizedRows({ + statement: 'cash_flow', + periods: [cashFlowPeriod], + facts: [ + createFact({ + id: 2001, + filingId: 152, + filingDate: '2025-06-23', + statement: 'income', + localName: 'IncreaseDecreaseInDeferredIncomeTaxes', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: -59_958_000 + }), + createFact({ + id: 2002, + filingId: 152, + filingDate: '2025-06-23', + statement: 'cash_flow', + localName: 'OtherNoncashIncomeExpense', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: 4_054_000 + }), + createFact({ + id: 2003, + filingId: 152, + filingDate: '2025-06-23', + statement: 'income', + localName: 'IncreaseDecreaseInIncomeTaxes', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: -84_000 + }), + createFact({ + id: 2004, + filingId: 152, + filingDate: '2025-06-23', + statement: 'income', + localName: 'IncreaseDecreaseInIncomeTaxesReceivable', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: -15_460_000 + }), + createFact({ + id: 2005, + filingId: 152, + filingDate: '2025-06-23', + statement: 'balance', + localName: 'IncreaseDecreaseInAccruedLiabilities', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: 21_525_000 + }), + createFact({ + id: 2006, + filingId: 152, + filingDate: '2025-06-23', + statement: 'income', + localName: 'IncreaseDecreaseInPrepaidExpense', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: -3_658_000 + }), + createFact({ + id: 2007, + filingId: 152, + filingDate: '2025-06-23', + statement: 'cash_flow', + localName: 'ProceedsFromSaleOfPropertyPlantAndEquipment', + periodStart: '2024-05-01', + periodEnd: '2025-04-30', + value: 18_805_000 + }) + ], + rows: [] + }); + + expect(findRow(cashFlowRows, 'changes_income_taxes_payable').values['2025-cf']).toBe(-84_000); + expect(findRow(cashFlowRows, 'changes_accrued_expenses').values['2025-cf']).toBe(21_525_000); + expect(findRow(cashFlowRows, 'other_adjustments').values['2025-cf']).toBe(55_904_000); + expect(findRow(cashFlowRows, 'changes_other_operating_activities').values['2025-cf']).toBe(63_616_000); + expect(findRow(cashFlowRows, 'proceeds_from_sale_of_property_plant_and_equipment').values['2025-cf']).toBe(18_805_000); + }); + + it('matches local MSFT annual income regression on exact period-end dates', async () => { + const response = await getCompanyFinancialTaxonomy({ + ticker: 'MSFT', + surfaceKind: 'income_statement', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const period2024 = findPeriodId(response.periods, '2024-06-30'); + + expect(response.periods.map((period) => period.periodEnd)).toEqual([ + '2022-06-30', + '2023-06-30', + '2024-06-30', + '2025-06-30' + ]); + expect(findStandardizedResponseRow(response, 'revenue').values[period2024]).toBe(245_122_000_000); + expect(findStandardizedResponseRow(response, 'operating_income').values[period2024]).toBe(109_433_000_000); + expect(findStandardizedResponseRow(response, 'net_income').values[period2024]).toBe(88_136_000_000); + expect(findStandardizedResponseRow(response, 'pretax_income').values[period2024]).toBe(107_787_000_000); + expect(findStandardizedResponseRow(response, 'income_tax_expense').values[period2024]).toBe(19_651_000_000); + expect(findStandardizedResponseRow(response, 'effective_tax_rate').values[period2024]).toBe(0.182); + expect(findStandardizedResponseRow(response, 'revenue').templateSection).toBe('statement'); + }); + + it('matches local QCOM annual income regression on exact period-end dates', async () => { + const response = await getCompanyFinancialTaxonomy({ + ticker: 'QCOM', + surfaceKind: 'income_statement', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const period2024 = findPeriodId(response.periods, '2024-09-29'); + const period2025 = findPeriodId(response.periods, '2025-09-28'); + + expect(response.periods.map((period) => period.periodEnd)).toEqual([ + '2022-09-25', + '2023-09-24', + '2024-09-29', + '2025-09-28' + ]); + expect(response.periods.some((period) => period.periodEnd === '2026-09-28')).toBe(false); + expect(response.periods.some((period) => period.periodEnd === '2027-09-26')).toBe(false); + expect(findStandardizedResponseRow(response, 'revenue').values[period2024]).toBe(38_962_000_000); + expect(findStandardizedResponseRow(response, 'cost_of_revenue').values[period2024]).toBe(17_060_000_000); + expect(findStandardizedResponseRow(response, 'gross_profit').values[period2024]).toBe(21_902_000_000); + expect(findStandardizedResponseRow(response, 'selling_general_and_administrative').values[period2024]).toBe(2_759_000_000); + expect(findStandardizedResponseRow(response, 'research_and_development').values[period2024]).toBe(8_893_000_000); + expect(findStandardizedResponseRow(response, 'pretax_income').values[period2024]).toBe(10_336_000_000); + expect(findStandardizedResponseRow(response, 'ebitda').values[period2024]).toBe(11_777_000_000); + expect(findStandardizedResponseRow(response, 'effective_tax_rate').values[period2024]).toBe(0.02); + expect(findStandardizedResponseRow(response, 'revenue').values[period2025]).toBe(44_284_000_000); + expect(findStandardizedResponseRow(response, 'income_tax_expense').values[period2025]).toBe(7_122_000_000); + expect(findStandardizedResponseRow(response, 'net_income').values[period2025]).toBe(5_541_000_000); + }); + + it('matches local MSFT annual balance regression on the June 30, 2025 balance sheet', async () => { + const response = await getCompanyFinancialTaxonomy({ + ticker: 'MSFT', + surfaceKind: 'balance_sheet', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const period2025 = findPeriodId(response.periods, '2025-06-30'); + + expect(findStandardizedResponseRow(response, 'total_assets').values[period2025]).toBe(619_003_000_000); + expect(findStandardizedResponseRow(response, 'total_liabilities').values[period2025]).toBe(275_524_000_000); + expect(findStandardizedResponseRow(response, 'total_equity').values[period2025]).toBe(343_479_000_000); + expect(findStandardizedResponseRow(response, 'total_assets').templateSection).toBe('assets'); + expect(findStandardizedResponseRow(response, 'total_liabilities').templateSection).toBe('liabilities'); + expect(findStandardizedResponseRow(response, 'total_equity').templateSection).toBe('equity'); + }); + + it('matches local MSFT annual cash flow regression on the June 30, 2025 filing period', async () => { + const response = await getCompanyFinancialTaxonomy({ + ticker: 'MSFT', + surfaceKind: 'cash_flow_statement', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const period2025 = findPeriodId(response.periods, '2025-06-30'); + + expect(findStandardizedResponseRow(response, 'depreciation_and_amortization').values[period2025]).toBe(34_153_000_000); + expect(findStandardizedResponseRow(response, 'stock_based_compensation').values[period2025]).toBe(11_974_000_000); + expect(findStandardizedResponseRow(response, 'free_cash_flow').values[period2025]).toBe(71_611_000_000); + expect(findStandardizedResponseRow(response, 'operating_cash_flow').templateSection).toBe('operating'); + expect(findStandardizedResponseRow(response, 'free_cash_flow').templateSection).toBe('free_cash_flow'); + }); + + it('matches local CASY annual income regression on the April 30, 2025 period', async () => { + const response = await getCompanyFinancialTaxonomy({ + ticker: 'CASY', + surfaceKind: 'income_statement', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const period2025 = findPeriodId(response.periods, '2025-04-30'); + + expect(findStandardizedResponseRow(response, 'revenue').values[period2025]).toBe(15_940_899_000); + expect(findStandardizedResponseRow(response, 'cost_of_revenue').values[period2025]).toBe(12_188_496_000); + expect(findStandardizedResponseRow(response, 'gross_profit').values[period2025]).toBe(3_752_403_000); + expect(findStandardizedResponseRow(response, 'selling_general_and_administrative').values[period2025]).toBe(2_552_356_000); + expect(findStandardizedResponseRow(response, 'operating_income').values[period2025]).toBe(796_400_000); + expect(findStandardizedResponseRow(response, 'pretax_income').values[period2025]).toBe(712_449_000); + expect(findStandardizedResponseRow(response, 'income_tax_expense').values[period2025]).toBe(165_929_000); + expect(findStandardizedResponseRow(response, 'net_income').values[period2025]).toBe(546_520_000); + expect(findStandardizedResponseRow(response, 'ebitda').values[period2025]).toBe(1_200_047_000); + expect(findStandardizedResponseRow(response, 'effective_tax_rate').values[period2025]).toBe(0.233); + }); + + it('matches local CASY annual balance and cash-flow regression on the April 30, 2025 period', async () => { + const balance = await getCompanyFinancialTaxonomy({ + ticker: 'CASY', + surfaceKind: 'balance_sheet', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + const cash = await getCompanyFinancialTaxonomy({ + ticker: 'CASY', + surfaceKind: 'cash_flow_statement', + cadence: 'annual', + includeDimensions: false, + includeFacts: false, + queuedSync: false, + v3Enabled: true + }); + + const balancePeriod2025 = findPeriodId(balance.periods, '2025-04-30'); + const cashPeriod2025 = findPeriodId(cash.periods, '2025-04-30'); + + expect(findStandardizedResponseRow(balance, 'total_assets').values[balancePeriod2025]).toBe(8_208_118_000); + expect(findStandardizedResponseRow(balance, 'total_liabilities').values[balancePeriod2025]).toBe(4_699_448_000); + expect(findStandardizedResponseRow(balance, 'total_equity').values[balancePeriod2025]).toBe(3_508_670_000); + expect(findStandardizedResponseRow(balance, 'cash_and_equivalents').values[balancePeriod2025]).toBe(326_662_000); + expect(findStandardizedResponseRow(balance, 'accrued_liabilities').values[balancePeriod2025]).toBe(330_731_000); + expect(findStandardizedResponseRow(balance, 'other_long_term_liabilities').values[balancePeriod2025]).toBe(121_485_000); + + expect(findStandardizedResponseRow(cash, 'operating_cash_flow').values[cashPeriod2025]).toBe(1_090_854_000); + expect(findStandardizedResponseRow(cash, 'capital_expenditures').values[cashPeriod2025]).toBe(-506_224_000); + expect(findStandardizedResponseRow(cash, 'free_cash_flow').values[cashPeriod2025]).toBe(584_630_000); + expect(findStandardizedResponseRow(cash, 'acquisitions').values[cashPeriod2025]).toBe(-1_239_249_000); + expect(findStandardizedResponseRow(cash, 'long_term_debt_issued').values[cashPeriod2025]).toBe(1_100_000_000); + expect(findStandardizedResponseRow(cash, 'debt_repaid').values[cashPeriod2025]).toBe(-239_492_000); + expect(findStandardizedResponseRow(cash, 'dividends_paid').values[cashPeriod2025]).toBe(-72_309_000); + expect(findStandardizedResponseRow(cash, 'financing_cash_flow').values[cashPeriod2025]).toBe(755_994_000); + expect(findStandardizedResponseRow(cash, 'changes_income_taxes_payable').values[cashPeriod2025]).toBe(-84_000); + expect(findStandardizedResponseRow(cash, 'changes_accrued_expenses').values[cashPeriod2025]).toBe(21_525_000); + expect(findStandardizedResponseRow(cash, 'other_adjustments').values[cashPeriod2025]).toBe(55_904_000); + }); }); diff --git a/lib/server/financial-taxonomy.ts b/lib/server/financial-taxonomy.ts index 5b79294..f234f92 100644 --- a/lib/server/financial-taxonomy.ts +++ b/lib/server/financial-taxonomy.ts @@ -61,6 +61,7 @@ type GetCompanyFinancialsInput = { type StandardizedStatementBundlePayload = { rows: StandardizedFinancialRow[]; trendSeries: CompanyFinancialStatementsResponse['trendSeries']; + categories: CompanyFinancialStatementsResponse['categories']; }; type FilingDocumentRef = { @@ -281,7 +282,8 @@ async function buildStatementSurfaceBundle(input: { if (!statement || (statement !== 'income' && statement !== 'balance' && statement !== 'cash_flow')) { return { rows: [], - trendSeries: [] + trendSeries: [], + categories: [] } satisfies StandardizedStatementBundlePayload; } @@ -289,7 +291,7 @@ async function buildStatementSurfaceBundle(input: { rows: input.faithfulRows, statement, periods: input.periods, - facts: input.facts.filter((fact) => fact.statement === statement) + facts: input.facts }); const payload = { @@ -297,7 +299,8 @@ async function buildStatementSurfaceBundle(input: { trendSeries: buildTrendSeries({ surfaceKind: input.surfaceKind, statementRows: standardizedRows - }) + }), + categories: buildFinancialCategories(standardizedRows, input.surfaceKind) } satisfies StandardizedStatementBundlePayload; await writeFinancialBundle({ @@ -520,7 +523,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr ticker, window: 'all', filingTypes: [...filingTypes], - limit: 2000 + limit: 10000 }); if (isStatementSurface(input.surfaceKind)) { @@ -546,12 +549,13 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr : buildRows(selection.snapshots, statement, selection.selectedPeriodIds); const factsForStatement = allFacts.facts.filter((fact) => fact.statement === statement); + const factsForStandardization = allFacts.facts; const standardizedPayload = await buildStatementSurfaceBundle({ surfaceKind: input.surfaceKind as Extract, cadence: input.cadence, periods, faithfulRows, - facts: factsForStatement, + facts: factsForStandardization, snapshots: selection.snapshots }); @@ -561,7 +565,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr rows: buildRows(selection.snapshots, statement, selection.selectedPeriodIds), statement: statement as Extract, periods: selection.periods, - facts: factsForStatement + facts: factsForStandardization }), selection.periods, periods, @@ -581,7 +585,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr : { facts: [], nextCursor: null }; const dimensionBreakdown = input.includeDimensions - ? buildDimensionBreakdown(factsForStatement, periods, faithfulRows, standardizedRows) + ? buildDimensionBreakdown(factsForStandardization, periods, faithfulRows, standardizedRows) : null; return { @@ -605,7 +609,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr surfaceKind: input.surfaceKind, statementRows: standardizedRows }), - categories: [], + categories: standardizedPayload.categories, availability: { adjusted: false, customMetrics: false @@ -654,19 +658,19 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr rows: buildRows(incomeSelection.snapshots, 'income', incomeSelection.selectedPeriodIds), statement: 'income', periods: incomeSelection.periods, - facts: allFacts.facts.filter((fact) => fact.statement === 'income') + facts: allFacts.facts }); const balanceQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({ rows: buildRows(balanceSelection.snapshots, 'balance', balanceSelection.selectedPeriodIds), statement: 'balance', periods: balanceSelection.periods, - facts: allFacts.facts.filter((fact) => fact.statement === 'balance') + facts: allFacts.facts }), balanceSelection.periods, incomeSelection.periods); const cashFlowQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({ rows: buildRows(cashFlowSelection.snapshots, 'cash_flow', cashFlowSelection.selectedPeriodIds), statement: 'cash_flow', periods: cashFlowSelection.periods, - facts: allFacts.facts.filter((fact) => fact.statement === 'cash_flow') + facts: allFacts.facts }), cashFlowSelection.periods, incomeSelection.periods); const incomeRows = input.cadence === 'ltm' diff --git a/lib/server/financials/bundles.ts b/lib/server/financials/bundles.ts index b3d0d17..c98c2be 100644 --- a/lib/server/financials/bundles.ts +++ b/lib/server/financials/bundles.ts @@ -3,6 +3,7 @@ import type { FinancialSurfaceKind } from '@/lib/types'; import { + CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION, getCompanyFinancialBundle, upsertCompanyFinancialBundle } from '@/lib/server/repos/company-financial-bundles'; @@ -28,7 +29,11 @@ export async function readCachedFinancialBundle(input: { cadence: input.cadence }); - if (!cached || cached.source_signature !== sourceSignature) { + if ( + !cached + || cached.bundle_version !== CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION + || cached.source_signature !== sourceSignature + ) { return null; } diff --git a/lib/server/financials/cadence.ts b/lib/server/financials/cadence.ts index 8045607..1c73c1c 100644 --- a/lib/server/financials/cadence.ts +++ b/lib/server/financials/cadence.ts @@ -13,6 +13,11 @@ type PrimaryPeriodSelection = { snapshots: FilingTaxonomySnapshotRecord[]; }; +type CandidateSelectionInput = { + rows: TaxonomyStatementRow[]; + periods: FinancialStatementPeriod[]; +}; + function parseEpoch(value: string | null) { if (!value) { return Number.NaN; @@ -54,6 +59,147 @@ function preferredDurationDays(filingType: FinancialStatementPeriod['filingType' return filingType === '10-K' ? 365 : 90; } +function isPresentedStatementRow(row: TaxonomyStatementRow) { + return row.roleUri !== null && row.order !== Number.MAX_SAFE_INTEGER; +} + +function isPlausiblePrimaryPeriod(period: FinancialStatementPeriod, filingDate: string) { + const filingEpoch = parseEpoch(filingDate); + const periodEndEpoch = parseEpoch(period.periodEnd ?? period.filingDate); + if (Number.isFinite(filingEpoch) && Number.isFinite(periodEndEpoch) && periodEndEpoch > filingEpoch) { + return false; + } + + const periodStartEpoch = parseEpoch(period.periodStart); + if (Number.isFinite(filingEpoch) && Number.isFinite(periodStartEpoch) && periodStartEpoch > filingEpoch) { + return false; + } + + return true; +} + +function candidatePeriodsForRows(input: CandidateSelectionInput) { + const usedPeriodIds = new Set(); + for (const row of input.rows) { + for (const periodId of Object.keys(row.values)) { + usedPeriodIds.add(periodId); + } + } + + return input.periods.filter((period) => usedPeriodIds.has(period.id)); +} + +function coverageScore(rows: TaxonomyStatementRow[], periodId: string) { + let count = 0; + + for (const row of rows) { + if (periodId in row.values) { + count += 1; + } + } + + return count; +} + +function compareBalancePeriods( + left: FinancialStatementPeriod, + right: FinancialStatementPeriod, + rowCoverage: Map +) { + const leftInstant = isInstantPeriod(left) ? 1 : 0; + const rightInstant = isInstantPeriod(right) ? 1 : 0; + if (leftInstant !== rightInstant) { + return rightInstant - leftInstant; + } + + const leftCoverage = rowCoverage.get(left.id) ?? 0; + const rightCoverage = rowCoverage.get(right.id) ?? 0; + if (leftCoverage !== rightCoverage) { + return rightCoverage - leftCoverage; + } + + const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); + const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); + if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { + return rightDate - leftDate; + } + + return left.id.localeCompare(right.id); +} + +function compareFlowPeriods( + left: FinancialStatementPeriod, + right: FinancialStatementPeriod, + rowCoverage: Map, + targetDays: number, + preferLaterBeforeDistance: boolean +) { + const leftDuration = isInstantPeriod(left) ? 0 : 1; + const rightDuration = isInstantPeriod(right) ? 0 : 1; + if (leftDuration !== rightDuration) { + return rightDuration - leftDuration; + } + + const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); + const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); + if (preferLaterBeforeDistance && Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { + return rightDate - leftDate; + } + + const leftCoverage = rowCoverage.get(left.id) ?? 0; + const rightCoverage = rowCoverage.get(right.id) ?? 0; + if (leftCoverage !== rightCoverage) { + return rightCoverage - leftCoverage; + } + + const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays); + const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays); + if (leftDistance !== rightDistance) { + return leftDistance - rightDistance; + } + + if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { + return rightDate - leftDate; + } + + return left.id.localeCompare(right.id); +} + +function chooseCandidates(snapshot: FilingTaxonomySnapshotRecord, rows: TaxonomyStatementRow[]) { + const periods = snapshot.periods ?? []; + const presentedRows = rows.filter(isPresentedStatementRow); + const plausiblePresented = candidatePeriodsForRows({ + rows: presentedRows, + periods + }).filter((period) => isPlausiblePrimaryPeriod(period, snapshot.filing_date)); + if (plausiblePresented.length > 0) { + return { + candidates: plausiblePresented, + coverageRows: presentedRows, + fallbackMode: false + }; + } + + const allCandidates = candidatePeriodsForRows({ + rows, + periods + }); + const plausibleAll = allCandidates.filter((period) => isPlausiblePrimaryPeriod(period, snapshot.filing_date)); + if (plausibleAll.length > 0) { + return { + candidates: plausibleAll, + coverageRows: rows, + fallbackMode: true + }; + } + + return { + candidates: allCandidates, + coverageRows: rows, + fallbackMode: true + }; +} + function selectPrimaryPeriodFromSnapshot( snapshot: FilingTaxonomySnapshotRecord, statement: FinancialStatementKind @@ -63,45 +209,21 @@ function selectPrimaryPeriodFromSnapshot( return null; } - const usedPeriodIds = new Set(); - for (const row of rows) { - for (const periodId of Object.keys(row.values)) { - usedPeriodIds.add(periodId); - } - } - - const candidates = (snapshot.periods ?? []).filter((period) => usedPeriodIds.has(period.id)); + const { candidates, coverageRows, fallbackMode } = chooseCandidates(snapshot, rows); if (candidates.length === 0) { return null; } - if (statement === 'balance') { - const instantCandidates = candidates.filter(isInstantPeriod); - return (instantCandidates.length > 0 ? instantCandidates : candidates) - .sort((left, right) => periodSorter(right, left))[0] ?? null; - } + const rowCoverage = new Map( + candidates.map((period) => [period.id, coverageScore(coverageRows, period.id)]) + ); - const durationCandidates = candidates.filter((period) => !isInstantPeriod(period)); - if (durationCandidates.length === 0) { - return candidates.sort((left, right) => periodSorter(right, left))[0] ?? null; + if (statement === 'balance') { + return [...candidates].sort((left, right) => compareBalancePeriods(left, right, rowCoverage))[0] ?? null; } const targetDays = preferredDurationDays(snapshot.filing_type); - return durationCandidates.sort((left, right) => { - const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); - const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); - if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { - return rightDate - leftDate; - } - - const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays); - const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays); - if (leftDistance !== rightDistance) { - return leftDistance - rightDistance; - } - - return left.id.localeCompare(right.id); - })[0] ?? null; + return [...candidates].sort((left, right) => compareFlowPeriods(left, right, rowCoverage, targetDays, fallbackMode))[0] ?? null; } function filingTypeForCadence(cadence: FinancialCadence) { diff --git a/lib/server/financials/standard-template.ts b/lib/server/financials/standard-template.ts new file mode 100644 index 0000000..e522c3c --- /dev/null +++ b/lib/server/financials/standard-template.ts @@ -0,0 +1,1471 @@ +import type { + FinancialStatementKind, + FinancialUnit +} from '@/lib/types'; + +export type TemplateSelectionPolicy = + | 'single_best_period_match' + | 'prefer_exact_local_name' + | 'prefer_primary_statement_concept' + | 'formula_only' + | 'aggregate_multiple_components' + | 'direct_or_formula_fallback'; + +export type TemplateFormula = + | { + kind: 'sum'; + sourceKeys: readonly string[]; + treatNullAsZero?: boolean; + } + | { + kind: 'subtract'; + left: string; + right: string; + } + | { + kind: 'divide'; + numerator: string; + denominator: string; + }; + +export type StandardTemplateRowDefinition = { + key: string; + label: string; + category: string; + includeInOutput?: boolean; + order: number; + statement: Extract; + unit: FinancialUnit; + selectionPolicy: TemplateSelectionPolicy; + matchers: { + exactLocalNames?: readonly string[]; + secondaryLocalNames?: readonly string[]; + allowedLabelPhrases?: readonly string[]; + excludeLabelPhrases?: readonly string[]; + excludeLocalNames?: readonly string[]; + }; + fallbackFormula?: TemplateFormula; + signTransform?: 'invert' | 'absolute'; +}; + +const income = ( + definition: Omit & { category?: string } +) => ({ + statement: 'income' as const, + category: definition.category ?? 'statement', + ...definition +}); + +const balance = ( + definition: Omit +) => ({ + statement: 'balance' as const, + ...definition +}); + +const cashFlow = ( + definition: Omit +) => ({ + statement: 'cash_flow' as const, + ...definition +}); + +export const STANDARD_FINANCIAL_TEMPLATES: Record< + Extract, + StandardTemplateRowDefinition[] +> = { + income: [ + income({ + key: 'revenue', + label: 'Total Revenues', + order: 10, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'Revenues', + 'RevenueFromContractWithCustomerExcludingAssessedTax', + 'SalesRevenueNet', + 'TotalRevenuesAndOtherIncome' + ], + secondaryLocalNames: ['RevenueFromContractWithCustomerIncludingAssessedTax'], + allowedLabelPhrases: ['total revenues', 'total revenue', 'revenue', 'net sales'], + excludeLabelPhrases: ['pro forma', 'acquiree'], + excludeLocalNames: [ + 'BusinessAcquisitionsProFormaRevenue', + 'BusinessCombinationProFormaInformationRevenueOfAcquireeSinceAcquisitionDateActual' + ] + } + }), + income({ + key: 'cost_of_revenue', + label: 'Cost of Sales', + order: 20, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'CostOfGoodsSoldExcludingEmployeeStockOptionPlanSpecialDividendCompensation', + 'CostOfRevenue', + 'CostOfGoodsSold', + 'CostOfSales', + 'CostOfGoodsAndServicesSold', + 'CostOfGoodsAndServiceExcludingDepreciationDepletionAndAmortization', + 'CostOfProductsSold', + 'CostOfServices' + ], + allowedLabelPhrases: ['cost of sales', 'cost of revenue', 'cost of goods sold'], + excludeLocalNames: ['CostOfGoodsAndServicesSoldDepreciationAndAmortization'] + } + }), + income({ + key: 'gross_profit', + label: 'Gross Profit', + order: 30, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['GrossProfit'], + allowedLabelPhrases: ['gross profit'] + }, + fallbackFormula: { + kind: 'subtract', + left: 'revenue', + right: 'cost_of_revenue' + } + }), + income({ + key: 'gross_margin', + label: 'Gross Profit Margin', + order: 40, + unit: 'percent', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'divide', + numerator: 'gross_profit', + denominator: 'revenue' + } + }), + income({ + key: 'selling_general_and_administrative', + label: 'Selling, General & Administrative Expenses', + order: 50, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: [ + 'SellingGeneralAndAdministrativeExpenseExcludingEmployeeStockOptionPlanSpecialDividendCompensation', + 'SellingGeneralAndAdministrativeExpense' + ], + secondaryLocalNames: ['OperatingExpenses'], + allowedLabelPhrases: ['selling general administrative', 'selling general and administrative'], + excludeLocalNames: ['SellingGeneralAndAdministrativeExpenseEmployeeStockOptionPlanSpecialDividendCompensation'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['sales_and_marketing', 'general_and_administrative'] + } + }), + income({ + key: 'research_and_development', + label: 'Research & Development Expenses', + order: 60, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ResearchAndDevelopmentExpense'], + allowedLabelPhrases: ['research development', 'research and development'] + } + }), + income({ + key: 'sales_and_marketing', + label: 'Sales & Marketing', + includeInOutput: false, + order: 61, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['SalesAndMarketingExpense', 'SellingAndMarketingExpense'], + allowedLabelPhrases: ['sales and marketing', 'selling and marketing'] + } + }), + income({ + key: 'general_and_administrative', + label: 'General & Administrative', + includeInOutput: false, + order: 62, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['GeneralAndAdministrativeExpense'], + allowedLabelPhrases: ['general and administrative'] + } + }), + income({ + key: 'operating_income', + label: 'Operating Profit', + order: 70, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['OperatingIncomeLoss', 'IncomeLossFromOperations'], + allowedLabelPhrases: ['operating profit', 'operating income', 'income from operations'], + excludeLocalNames: ['NonoperatingIncomeExpense', 'OtherNonoperatingIncomeExpense'] + } + }), + income({ + key: 'operating_margin', + label: 'Operating Margin', + order: 80, + unit: 'percent', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'divide', + numerator: 'operating_income', + denominator: 'revenue' + } + }), + income({ + key: 'pretax_income', + label: 'Pretax Income', + order: 90, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: [ + 'IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest', + 'IncomeBeforeTaxExpenseBenefit', + 'PretaxIncome' + ], + allowedLabelPhrases: ['pretax income', 'income before taxes', 'income before tax'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['net_income', 'income_tax_expense'] + } + }), + income({ + key: 'income_before_provision_for_income_taxes', + label: 'Income Before Provision for Income Taxes', + order: 95, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: [ + 'IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest', + 'IncomeBeforeTaxExpenseBenefit', + 'PretaxIncome' + ], + allowedLabelPhrases: ['income before provision for income taxes', 'income before taxes', 'pretax income'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['net_income', 'income_tax_expense'] + } + }), + income({ + key: 'income_tax_expense', + label: 'Income Tax Expense', + order: 100, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncomeTaxExpenseBenefit'], + secondaryLocalNames: ['IncomeTaxes'], + allowedLabelPhrases: ['income tax expense', 'income tax provision'], + excludeLocalNames: [ + 'CurrentIncomeTaxExpenseBenefit', + 'DeferredIncomeTaxExpenseBenefit', + 'DeferredFederalIncomeTaxExpenseBenefit', + 'DeferredForeignIncomeTaxExpenseBenefit', + 'DeferredStateAndLocalIncomeTaxExpenseBenefit' + ], + excludeLabelPhrases: ['current income tax', 'deferred income tax'] + } + }), + income({ + key: 'provision_for_income_taxes', + label: 'Provision for Income Taxes', + order: 105, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncomeTaxExpenseBenefit'], + secondaryLocalNames: ['IncomeTaxes'], + allowedLabelPhrases: ['provision for income taxes', 'income tax provision'] + } + }), + income({ + key: 'net_income', + label: 'Net Income', + order: 110, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetIncomeLossAvailableToCommonStockholdersBasic', 'NetIncomeLoss', 'ProfitLoss'], + allowedLabelPhrases: ['net income', 'net earnings'], + excludeLocalNames: ['BusinessAcquisitionsProFormaNetIncomeLoss'], + excludeLabelPhrases: ['pro forma'] + } + }), + income({ + key: 'consolidated_net_income', + label: 'Consolidated Net Income', + order: 112, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetIncomeLoss', 'ProfitLoss'], + allowedLabelPhrases: ['consolidated net income', 'net income'] + } + }), + income({ + key: 'net_income_attributable_to_common_shareholders', + label: 'Net Income Attributable to Common Shareholders', + order: 114, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetIncomeLossAvailableToCommonStockholdersBasic'], + allowedLabelPhrases: ['net income attributable to common shareholders', 'net income available to common stockholders'] + } + }), + income({ + key: 'depreciation_and_amortization', + label: 'Depreciation & Amortization', + order: 200, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'DepreciationDepletionAndAmortization', + 'DepreciationAmortizationAndAccretionNet', + 'DepreciationAndAmortization', + 'DepreciationAmortizationAndOther', + 'CostOfGoodsAndServicesSoldDepreciationAndAmortization' + ], + secondaryLocalNames: ['AmortizationOfIntangibleAssets'], + allowedLabelPhrases: ['depreciation amortization', 'depreciation and amortization'] + } + }), + income({ + key: 'depreciation_and_amortization_expenses', + label: 'Depreciation & Amortization Expense', + order: 205, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['CostOfGoodsAndServicesSoldDepreciationAndAmortization'], + allowedLabelPhrases: ['depreciation amortization expense', 'depreciation and amortization expense'] + } + }), + income({ + key: 'interest_income', + label: 'Interest Income', + order: 210, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['InterestIncomeOther', 'InvestmentIncomeInterest'], + secondaryLocalNames: ['InvestmentIncomeNet'], + allowedLabelPhrases: ['interest income'], + excludeLabelPhrases: ['effective income tax rate reconciliation', 'reconciliation'], + excludeLocalNames: ['InterestIncomeExpenseNet'] + } + }), + income({ + key: 'interest_expense', + label: 'Interest Expense', + order: 220, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['InterestIncomeExpenseNonoperatingNet', 'InterestExpense', 'InterestAndDebtExpense'], + allowedLabelPhrases: ['interest expense'] + }, + signTransform: 'absolute' + }), + income({ + key: 'other_non_operating_income', + label: 'Other Non-Operating Income', + order: 230, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['OtherNonoperatingIncomeExpense', 'NonoperatingIncomeExpense'], + allowedLabelPhrases: ['other non operating', 'other non-operating'] + } + }), + income({ + key: 'stock_based_compensation', + label: 'Share-Based Compensation Expense', + order: 240, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ShareBasedCompensation', 'AllocatedShareBasedCompensationExpense'], + allowedLabelPhrases: ['share based compensation', 'stock based compensation'] + } + }), + income({ + key: 'ebitda', + label: 'EBITDA', + order: 250, + unit: 'currency', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['operating_income', 'depreciation_and_amortization'] + } + }), + income({ + key: 'effective_tax_rate', + label: 'Effective Tax Rate', + order: 260, + unit: 'percent', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['EffectiveIncomeTaxRateContinuingOperations'], + allowedLabelPhrases: ['effective tax rate'] + }, + fallbackFormula: { + kind: 'divide', + numerator: 'income_tax_expense', + denominator: 'pretax_income' + } + }), + income({ + key: 'diluted_eps', + label: 'Diluted EPS', + order: 300, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['EarningsPerShareDiluted', 'DilutedEarningsPerShare'], + allowedLabelPhrases: ['diluted eps', 'diluted earnings per share'] + } + }), + income({ + key: 'basic_eps', + label: 'Basic EPS', + order: 310, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['EarningsPerShareBasic', 'BasicEarningsPerShare'], + allowedLabelPhrases: ['basic eps', 'basic earnings per share'] + } + }), + income({ + key: 'diluted_shares', + label: 'Diluted Shares', + order: 320, + unit: 'shares', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'WeightedAverageNumberOfDilutedSharesOutstanding', + 'WeightedAverageNumberOfShareOutstandingDiluted' + ], + allowedLabelPhrases: ['diluted shares', 'weighted average diluted'] + } + }), + income({ + key: 'basic_shares', + label: 'Basic Shares', + order: 330, + unit: 'shares', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'WeightedAverageNumberOfSharesOutstandingBasic', + 'WeightedAverageNumberOfShareOutstandingBasicAndDiluted' + ], + allowedLabelPhrases: ['basic shares', 'weighted average basic'] + } + }) + ], + balance: [ + balance({ + key: 'cash_and_equivalents', + label: 'Cash and Cash Equivalents', + category: 'assets', + order: 10, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['CashAndCashEquivalentsAtCarryingValue'], + secondaryLocalNames: ['Cash', 'CashCashEquivalentsAndFederalFundsSold'], + allowedLabelPhrases: ['cash and cash equivalents'] + } + }), + balance({ + key: 'short_term_investments', + label: 'Short-Term Investments', + category: 'assets', + order: 20, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['AvailableForSaleSecuritiesCurrent', 'ShortTermInvestments', 'MarketableSecuritiesCurrent'], + allowedLabelPhrases: ['short term investments', 'marketable securities'] + } + }), + balance({ + key: 'total_cash_and_equivalents', + label: 'Total Cash and Cash Equivalents', + category: 'assets', + order: 30, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: ['CashCashEquivalentsAndShortTermInvestments', 'CashAndShortTermInvestments'], + allowedLabelPhrases: ['total cash and cash equivalents'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['cash_and_equivalents', 'short_term_investments'], + treatNullAsZero: true + } + }), + balance({ + key: 'accounts_receivable', + label: 'Accounts Receivable', + category: 'assets', + order: 40, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['AccountsReceivableNetCurrent', 'ReceivablesNetCurrent'], + allowedLabelPhrases: ['accounts receivable'] + } + }), + balance({ + key: 'inventory', + label: 'Inventories', + category: 'assets', + order: 50, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['InventoryNet'], + allowedLabelPhrases: ['inventories', 'inventory'] + } + }), + balance({ + key: 'other_current_assets', + label: 'Other Current Assets', + category: 'assets', + order: 60, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['OtherAssetsCurrent'], + allowedLabelPhrases: ['other current assets'] + } + }), + balance({ + key: 'deferred_income_taxes_asset', + label: 'Deferred Income Taxes', + category: 'assets', + order: 70, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['DeferredIncomeTaxAssetsNet', 'DeferredTaxAssetsNet'], + allowedLabelPhrases: ['deferred income taxes', 'deferred tax assets'], + excludeLocalNames: ['DeferredIncomeTaxLiabilities', 'DeferredIncomeTaxLiabilitiesNet'] + } + }), + balance({ + key: 'current_assets', + label: 'Total Current Assets', + category: 'assets', + order: 80, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['AssetsCurrent'], + allowedLabelPhrases: ['total current assets', 'current assets'] + } + }), + balance({ + key: 'property_plant_equipment', + label: 'Net Property, Plant & Equipment', + category: 'assets', + order: 90, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['PropertyPlantAndEquipmentNet'], + secondaryLocalNames: ['PropertyPlantAndEquipmentAndFinanceLeaseRightOfUseAssetAfterAccumulatedDepreciationAndAmortization'], + allowedLabelPhrases: ['property plant equipment', 'property and equipment net'] + } + }), + balance({ + key: 'operating_lease_right_of_use_assets', + label: 'Operating Lease Right-of-Use Assets', + category: 'assets', + order: 100, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['OperatingLeaseRightOfUseAsset'], + allowedLabelPhrases: ['operating lease right of use assets', 'operating lease right of use asset'] + } + }), + balance({ + key: 'intangible_assets', + label: 'Net Intangible Assets', + category: 'assets', + order: 110, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'FiniteLivedIntangibleAssetsNet', + 'IntangibleAssetsNetExcludingGoodwill', + 'FiniteLivedIntangibleAssetsNetExcludingGoodwill' + ], + allowedLabelPhrases: ['net intangible assets', 'intangible assets net'] + } + }), + balance({ + key: 'goodwill', + label: 'Goodwill', + category: 'assets', + order: 120, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['Goodwill'], + allowedLabelPhrases: ['goodwill'], + excludeLabelPhrases: ['excluding goodwill'] + } + }), + balance({ + key: 'long_term_investments', + label: 'Long-Term Investments', + category: 'assets', + order: 130, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'AvailableForSaleSecuritiesDebtMaturitiesSingleMaturityDate', + 'AvailableForSaleSecuritiesNoncurrent', + 'LongTermInvestments' + ], + secondaryLocalNames: ['AvailableForSaleSecuritiesDebtSecurities'], + allowedLabelPhrases: ['long term investments'] + } + }), + balance({ + key: 'other_long_term_assets', + label: 'Other Long-Term Assets', + category: 'assets', + order: 140, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['OtherAssetsNoncurrent'], + allowedLabelPhrases: ['other long term assets', 'other noncurrent assets'], + excludeLocalNames: ['NoncurrentAssets'] + } + }), + balance({ + key: 'total_assets', + label: 'Total Assets', + category: 'assets', + order: 150, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['Assets'], + allowedLabelPhrases: ['total assets'] + } + }), + balance({ + key: 'accounts_payable', + label: 'Accounts Payable', + category: 'liabilities', + order: 160, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['AccountsPayableCurrent'], + allowedLabelPhrases: ['accounts payable'], + excludeLabelPhrases: ['business combination', 'liabilities assumed'] + } + }), + balance({ + key: 'accrued_liabilities', + label: 'Accrued Expenses', + category: 'liabilities', + order: 170, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'AccruedLiabilitiesCurrent', + 'OtherAccruedLiabilitiesCurrent', + 'AccruedCompensationCurrent', + 'EmployeeRelatedLiabilitiesCurrent', + 'OtherLiabilitiesCurrent', + 'AccruedPropertyTaxes' + ], + allowedLabelPhrases: ['accrued expenses', 'accrued liabilities', 'accrued compensation', 'accrued property taxes'] + } + }), + balance({ + key: 'short_term_debt', + label: 'Short-Term Debt', + category: 'liabilities', + order: 180, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ShortTermBorrowings', 'DebtCurrent', 'CommercialPaper'], + allowedLabelPhrases: ['short term debt', 'short-term debt'] + } + }), + balance({ + key: 'current_debt', + label: 'Current Portion of Long-Term Debt', + category: 'liabilities', + order: 190, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['LongTermDebtCurrent', 'CurrentPortionOfLongTermDebt', 'LongTermDebtAndCapitalLeaseObligationsCurrent'], + allowedLabelPhrases: ['current portion of long term debt', 'current portion of long-term debt'] + } + }), + balance({ + key: 'long_term_debt', + label: 'Long-Term Debt', + category: 'liabilities', + order: 200, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['LongTermDebtNoncurrent', 'DebtNoncurrent', 'LongTermDebtAndCapitalLeaseObligations'], + secondaryLocalNames: ['LongTermDebt', 'DebtInstrumentCarryingAmount'], + allowedLabelPhrases: ['long term debt', 'long-term debt'] + } + }), + balance({ + key: 'unearned_revenue', + label: 'Unearned Revenue', + category: 'liabilities', + order: 210, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['ContractWithCustomerLiabilityCurrent', 'DeferredRevenueCurrent'], + secondaryLocalNames: ['ContractWithCustomerLiability', 'DeferredRevenue'], + allowedLabelPhrases: ['unearned revenue', 'deferred revenue'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['deferred_revenue_current', 'deferred_revenue_noncurrent'], + treatNullAsZero: true + } + }), + balance({ + key: 'deferred_revenue_current', + label: 'Deferred Revenue, Current', + includeInOutput: false, + category: 'liabilities', + order: 211, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ContractWithCustomerLiabilityCurrent', 'DeferredRevenueCurrent'], + allowedLabelPhrases: ['deferred revenue current', 'current deferred revenue'] + } + }), + balance({ + key: 'deferred_revenue_noncurrent', + label: 'Deferred Revenue, Noncurrent', + includeInOutput: false, + category: 'liabilities', + order: 212, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ContractWithCustomerLiabilityNoncurrent', 'DeferredRevenueNoncurrent'], + allowedLabelPhrases: ['deferred revenue noncurrent', 'noncurrent deferred revenue'] + } + }), + balance({ + key: 'deferred_income_taxes_liability', + label: 'Deferred Income Taxes', + category: 'liabilities', + order: 220, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['DeferredIncomeTaxLiabilitiesNet', 'DeferredIncomeTaxLiabilities'], + allowedLabelPhrases: ['deferred income taxes', 'deferred tax liabilities'], + excludeLocalNames: ['DeferredIncomeTaxAssetsNet', 'DeferredTaxAssetsNet'] + } + }), + balance({ + key: 'other_long_term_liabilities', + label: 'Other Long-Term Liabilities', + category: 'liabilities', + order: 230, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'DeferredIncomeTaxLiabilitiesNet', + 'DeferredIncomeTaxLiabilities', + 'AssetRetirementObligationsNoncurrent', + 'OtherLiabilitiesNoncurrent' + ], + allowedLabelPhrases: ['other long term liabilities', 'other noncurrent liabilities'], + excludeLocalNames: [ + 'DeferredIncomeTaxLiabilitiesNet', + 'DeferredIncomeTaxLiabilities', + 'OperatingLeaseLiability', + 'OperatingLeaseLiabilityCurrent', + 'OperatingLeaseLiabilityNoncurrent', + 'FinanceLeaseLiability', + 'FinanceLeaseLiabilityCurrent', + 'FinanceLeaseLiabilityNoncurrent', + 'LesseeOperatingLeaseLiability' + ] + } + }), + balance({ + key: 'current_liabilities', + label: 'Current Liabilities', + category: 'liabilities', + order: 240, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['LiabilitiesCurrent'], + allowedLabelPhrases: ['current liabilities'] + } + }), + balance({ + key: 'total_current_liabilities', + label: 'Total Current Liabilities', + category: 'liabilities', + order: 241, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['LiabilitiesCurrent'], + allowedLabelPhrases: ['total current liabilities'] + } + }), + balance({ + key: 'lease_liabilities', + label: 'Lease Liabilities', + category: 'liabilities', + order: 250, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'OperatingLeaseLiabilityNoncurrent', + 'OperatingLeaseLiability', + 'FinanceLeaseLiability', + 'FinanceLeaseLiabilityNoncurrent', + 'LesseeOperatingLeaseLiability' + ], + allowedLabelPhrases: ['lease liabilities'] + } + }), + balance({ + key: 'leases', + label: 'Leases', + category: 'liabilities', + order: 255, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'OperatingLeaseLiability', + 'OperatingLeaseLiabilityCurrent', + 'OperatingLeaseLiabilityNoncurrent', + 'FinanceLeaseLiability', + 'FinanceLeaseLiabilityCurrent', + 'FinanceLeaseLiabilityNoncurrent', + 'LesseeOperatingLeaseLiability' + ], + allowedLabelPhrases: ['leases'] + } + }), + balance({ + key: 'total_liabilities', + label: 'Total Liabilities', + category: 'liabilities', + order: 260, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['Liabilities'], + allowedLabelPhrases: ['total liabilities'] + } + }), + balance({ + key: 'common_stock', + label: 'Common Stock', + category: 'equity', + order: 270, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['CommonStocksIncludingAdditionalPaidInCapital', 'CommonStockValue'], + secondaryLocalNames: ['AdditionalPaidInCapitalCommonStock', 'AdditionalPaidInCapital'], + allowedLabelPhrases: ['common stock'] + } + }), + balance({ + key: 'accumulated_other_comprehensive_income', + label: 'Accumulated Other Comprehensive Income', + category: 'equity', + order: 280, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['AccumulatedOtherComprehensiveIncomeLossNetOfTax'], + allowedLabelPhrases: ['accumulated other comprehensive income'] + } + }), + balance({ + key: 'retained_earnings', + label: 'Retained Earnings', + category: 'equity', + order: 290, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['RetainedEarningsAccumulatedDeficit'], + allowedLabelPhrases: ['retained earnings', 'accumulated deficit'] + } + }), + balance({ + key: 'total_equity', + label: "Total Shareholders' Equity", + category: 'equity', + order: 300, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['StockholdersEquity', 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', 'PartnersCapital'], + allowedLabelPhrases: ['total shareholders equity', 'total stockholders equity', 'total equity'] + }, + fallbackFormula: { + kind: 'subtract', + left: 'total_assets', + right: 'total_liabilities' + } + }), + balance({ + key: 'total_common_shareholders_equity', + label: "Total Common Shareholders' Equity", + category: 'equity', + order: 301, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['StockholdersEquity'], + allowedLabelPhrases: ['total common shareholders equity', 'total stockholders equity'] + } + }), + balance({ + key: 'total_liabilities_and_equity', + label: "Total Liabilities and Shareholders' Equity", + category: 'equity', + order: 310, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['LiabilitiesAndStockholdersEquity'], + allowedLabelPhrases: ['total liabilities and shareholders equity', 'total liabilities and stockholders equity'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['total_liabilities', 'total_equity'] + } + }), + balance({ + key: 'total_debt', + label: 'Total Debt', + category: 'liabilities', + order: 320, + unit: 'currency', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['short_term_debt', 'current_debt', 'long_term_debt', 'lease_liabilities'], + treatNullAsZero: true + } + }), + balance({ + key: 'net_cash_position', + label: 'Net Cash Position', + category: 'assets', + order: 330, + unit: 'currency', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'subtract', + left: 'total_cash_and_equivalents', + right: 'total_debt' + } + }) + ], + cash_flow: [ + cashFlow({ + key: 'net_income', + label: 'Net Income', + category: 'operating', + order: 10, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetIncomeLossAvailableToCommonStockholdersBasic', 'NetIncomeLoss', 'ProfitLoss'], + allowedLabelPhrases: ['net income'] + } + }), + cashFlow({ + key: 'depreciation_and_amortization', + label: 'Depreciation & Amortization', + category: 'operating', + order: 20, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'DepreciationDepletionAndAmortization', + 'DepreciationAmortizationAndAccretionNet', + 'DepreciationAndAmortization', + 'DepreciationAmortizationAndOther' + ], + secondaryLocalNames: ['AmortizationOfIntangibleAssets'], + allowedLabelPhrases: ['depreciation amortization', 'depreciation and amortization'] + } + }), + cashFlow({ + key: 'stock_based_compensation', + label: 'Share-Based Compensation Expense', + category: 'operating', + order: 30, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ShareBasedCompensation', 'AllocatedShareBasedCompensationExpense'], + allowedLabelPhrases: ['share based compensation', 'stock based compensation'] + } + }), + cashFlow({ + key: 'other_adjustments', + label: 'Other Adjustments', + category: 'operating', + order: 40, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'OtherAdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivities', + 'IncreaseDecreaseInDeferredIncomeTaxes', + 'OtherNoncashIncomeExpense' + ], + allowedLabelPhrases: ['other adjustments'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'changes_trade_receivables', + label: 'Changes in Trade Receivables', + category: 'operating', + order: 50, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInAccountsReceivable', 'IncreaseDecreaseInReceivables'], + allowedLabelPhrases: ['changes in trade receivables', 'accounts receivable'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'changes_inventories', + label: 'Changes in Inventories', + category: 'operating', + order: 60, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInInventories'], + allowedLabelPhrases: ['changes in inventories', 'inventories'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'changes_accounts_payable', + label: 'Changes in Accounts Payable', + category: 'operating', + order: 70, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInAccountsPayable'], + allowedLabelPhrases: ['changes in accounts payable', 'accounts payable'] + } + }), + cashFlow({ + key: 'changes_accrued_expenses', + label: 'Changes in Accrued Expenses', + category: 'operating', + order: 75, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'IncreaseDecreaseInAccruedLiabilities', + 'IncreaseDecreaseInEmployeeRelatedLiabilitiesCurrent', + 'IncreaseDecreaseInOtherLiabilitiesCurrent' + ], + allowedLabelPhrases: ['changes in accrued expenses', 'increase decrease in accrued liabilities'] + } + }), + cashFlow({ + key: 'changes_income_taxes_payable', + label: 'Changes in Income Taxes Payable', + category: 'operating', + order: 80, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInAccruedIncomeTaxesPayable', 'IncreaseDecreaseInIncomeTaxes'], + allowedLabelPhrases: ['changes in income taxes payable', 'income taxes'], + excludeLocalNames: ['IncreaseDecreaseInIncomeTaxesReceivable'] + } + }), + cashFlow({ + key: 'changes_unearned_revenue', + label: 'Changes in Unearned Revenue', + category: 'operating', + order: 90, + unit: 'currency', + selectionPolicy: 'direct_or_formula_fallback', + matchers: { + exactLocalNames: ['IncreaseDecreaseInDeferredRevenue'], + allowedLabelPhrases: ['changes in unearned revenue', 'deferred revenue'] + }, + fallbackFormula: { + kind: 'subtract', + left: 'contract_liability_incurred', + right: 'contract_liability_recognized' + } + }), + cashFlow({ + key: 'contract_liability_incurred', + label: 'Unearned Revenue Incurred', + includeInOutput: false, + category: 'operating', + order: 91, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ContractWithCustomerLiabilityIncurred'], + allowedLabelPhrases: ['deferral of unearned revenue'] + } + }), + cashFlow({ + key: 'contract_liability_recognized', + label: 'Unearned Revenue Recognized', + includeInOutput: false, + category: 'operating', + order: 92, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ContractWithCustomerLiabilityRevenueRecognized'], + allowedLabelPhrases: ['recognition of unearned revenue'] + } + }), + cashFlow({ + key: 'changes_other_operating_activities', + label: 'Changes in Other Operating Activities', + category: 'operating', + order: 100, + unit: 'currency', + selectionPolicy: 'aggregate_multiple_components', + matchers: { + exactLocalNames: [ + 'IncreaseDecreaseInOtherOperatingAssets', + 'IncreaseDecreaseInOtherOperatingLiabilities', + 'IncreaseDecreaseInDeferredIncomeTaxes', + 'IncreaseDecreaseInPrepaidExpense' + ], + allowedLabelPhrases: ['changes in other operating activities'] + }, + fallbackFormula: { + kind: 'sum', + sourceKeys: [ + 'changes_other_current_assets', + 'changes_other_current_liabilities', + 'changes_other_noncurrent_assets', + 'changes_other_noncurrent_liabilities' + ], + treatNullAsZero: true + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'changes_other_current_assets', + label: 'Other Current Assets', + includeInOutput: false, + category: 'operating', + order: 101, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInOtherCurrentAssets'] + } + }), + cashFlow({ + key: 'changes_other_current_liabilities', + label: 'Other Current Liabilities', + includeInOutput: false, + category: 'operating', + order: 102, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInOtherCurrentLiabilities'] + } + }), + cashFlow({ + key: 'changes_other_noncurrent_assets', + label: 'Other Long-Term Assets', + includeInOutput: false, + category: 'operating', + order: 103, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInOtherNoncurrentAssets'] + } + }), + cashFlow({ + key: 'changes_other_noncurrent_liabilities', + label: 'Other Long-Term Liabilities', + includeInOutput: false, + category: 'operating', + order: 104, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['IncreaseDecreaseInOtherNoncurrentLiabilities'] + } + }), + cashFlow({ + key: 'operating_cash_flow', + label: 'Cash from Operating Activities', + category: 'operating', + order: 110, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: [ + 'NetCashProvidedByUsedInOperatingActivities', + 'NetCashProvidedByUsedInOperatingActivitiesContinuingOperations' + ], + allowedLabelPhrases: ['cash from operating activities', 'net cash from operations'] + } + }), + cashFlow({ + key: 'capital_expenditures', + label: 'Capital Expenditures', + category: 'investing', + order: 120, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['PaymentsToAcquirePropertyPlantAndEquipment'], + secondaryLocalNames: ['CapitalExpendituresIncurredButNotYetPaid'], + allowedLabelPhrases: ['capital expenditures', 'capital expenditure'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'acquisitions', + label: 'Acquisitions', + category: 'investing', + order: 130, + unit: 'currency', + selectionPolicy: 'prefer_primary_statement_concept', + matchers: { + exactLocalNames: [ + 'PaymentsToAcquireBusinessesNetOfCashAcquired', + 'AcquisitionsNetOfCashAcquiredAndPurchasesOfIntangibleAndOtherAssets' + ], + allowedLabelPhrases: ['acquisitions'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'investments', + label: 'Investments', + category: 'investing', + order: 140, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['PaymentsForProceedsFromOtherInvestingActivities'], + allowedLabelPhrases: ['investments'] + } + }), + cashFlow({ + key: 'proceeds_from_sale_of_property_plant_and_equipment', + label: 'Proceeds from Sale of Property, Plant & Equipment', + category: 'investing', + order: 145, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ProceedsFromSaleOfPropertyPlantAndEquipment'], + allowedLabelPhrases: ['proceeds from sale of property plant equipment'] + } + }), + cashFlow({ + key: 'other_investing_activities', + label: 'Other Investing Activities', + category: 'investing', + order: 150, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['OtherInvestingActivitiesNet', 'OtherCashFlowFromInvestingActivities'], + allowedLabelPhrases: ['other investing activities'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'investing_cash_flow', + label: 'Cash from Investing Activities', + category: 'investing', + order: 160, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetCashProvidedByUsedInInvestingActivities'], + allowedLabelPhrases: ['cash from investing activities'] + } + }), + cashFlow({ + key: 'short_term_debt_issued', + label: 'Short-Term Debt Issued', + category: 'financing', + order: 170, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ProceedsFromShortTermDebt'], + allowedLabelPhrases: ['short term debt issued'] + } + }), + cashFlow({ + key: 'long_term_debt_issued', + label: 'Long-Term Debt Issued', + category: 'financing', + order: 180, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ProceedsFromIssuanceOfLongTermDebt'], + allowedLabelPhrases: ['long term debt issued'] + } + }), + cashFlow({ + key: 'debt_repaid', + label: 'Debt Repaid', + category: 'financing', + order: 190, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['RepaymentsOfDebt', 'RepaymentsOfLongTermDebt'], + allowedLabelPhrases: ['debt repaid'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'share_repurchases', + label: 'Repurchases of Common Shares', + category: 'financing', + order: 200, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['PaymentsForRepurchaseOfCommonStock', 'PaymentsForRepurchaseOfEquity'], + allowedLabelPhrases: ['repurchases of common shares', 'share repurchases'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'dividends_paid', + label: 'Common Share Dividends Paid', + category: 'financing', + order: 210, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['DividendsCommonStockCash', 'PaymentsOfDividendsCommonStock', 'PaymentsOfDividends'], + allowedLabelPhrases: ['common share dividends paid', 'dividends paid'] + }, + signTransform: 'invert' + }), + cashFlow({ + key: 'other_financing_activities', + label: 'Other Financing Activities', + category: 'financing', + order: 220, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['ProceedsFromPaymentsForOtherFinancingActivities'], + allowedLabelPhrases: ['other financing activities'] + } + }), + cashFlow({ + key: 'financing_cash_flow', + label: 'Cash from Financing Activities', + category: 'financing', + order: 230, + unit: 'currency', + selectionPolicy: 'prefer_exact_local_name', + matchers: { + exactLocalNames: ['NetCashProvidedByUsedInFinancingActivities'], + allowedLabelPhrases: ['cash from financing activities'] + } + }), + cashFlow({ + key: 'free_cash_flow', + label: 'Free Cash Flow', + category: 'free_cash_flow', + order: 240, + unit: 'currency', + selectionPolicy: 'formula_only', + matchers: {}, + fallbackFormula: { + kind: 'sum', + sourceKeys: ['operating_cash_flow', 'capital_expenditures'] + } + }) + ] +}; diff --git a/lib/server/financials/standardize.ts b/lib/server/financials/standardize.ts index 9dd1399..c9ba849 100644 --- a/lib/server/financials/standardize.ts +++ b/lib/server/financials/standardize.ts @@ -9,20 +9,30 @@ import type { TaxonomyStatementRow } from '@/lib/types'; import { - CANONICAL_ROW_DEFINITIONS, - type CanonicalRowDefinition -} from '@/lib/server/financials/canonical-definitions'; + STANDARD_FINANCIAL_TEMPLATES, + type StandardTemplateRowDefinition, + type TemplateFormula +} from '@/lib/server/financials/standard-template'; function normalizeToken(value: string) { return value.trim().toLowerCase(); } +function tokenizeLabel(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .split(/\s+/) + .filter((token) => token.length > 0); +} + function valueOrNull(values: Record, periodId: string) { return periodId in values ? values[periodId] : null; } -function sumValues(values: Array) { - if (values.some((value) => value === null)) { +function sumValues(values: Array, treatNullAsZero = false) { + if (!treatNullAsZero && values.some((value) => value === null)) { return null; } @@ -45,20 +55,102 @@ function divideValues(left: number | null, right: number | null) { return left / right; } -function matchesDefinition(row: TaxonomyStatementRow, definition: CanonicalRowDefinition) { - const rowLocalName = normalizeToken(row.localName); - if (definition.localNames?.some((localName) => normalizeToken(localName) === rowLocalName)) { - return true; +type CandidateMatchKind = 'exact_local_name' | 'secondary_local_name' | 'label_phrase'; + +type StatementRowCandidate = { + row: TaxonomyStatementRow; + matchKind: CandidateMatchKind; + aliasRank: number; + unit: FinancialUnit; + labelTokenCount: number; + matchedPhraseTokenCount: number; +}; + +type FactCandidate = { + fact: TaxonomyFactRow; + matchKind: Exclude; + aliasRank: number; + unit: FinancialUnit; +}; + +type ResolvedCandidate = + | { + sourceType: 'row'; + matchKind: CandidateMatchKind; + aliasRank: number; + unit: FinancialUnit; + labelTokenCount: number; + matchedPhraseTokenCount: number; + row: TaxonomyStatementRow; + } + | { + sourceType: 'fact'; + matchKind: Exclude; + aliasRank: number; + unit: FinancialUnit; + fact: TaxonomyFactRow; + }; + +type DerivedRole = 'expense' | 'addback'; + +type InternalRowMetadata = { + derivedRoleByPeriod: Record; +}; + +function resolvedCandidatesForPeriod(input: { + definition: StandardTemplateRowDefinition; + candidates: StatementRowCandidate[]; + factCandidates: FactCandidate[]; + period: FinancialStatementPeriod; +}) { + const rowCandidates = input.candidates + .filter((candidate) => input.period.id in candidate.row.values && candidate.row.values[input.period.id] !== null) + .map((candidate) => ({ + sourceType: 'row' as const, + ...candidate + })); + const factCandidates = input.factCandidates + .filter((candidate) => factMatchesPeriod(candidate.fact, input.period)) + .map((candidate) => ({ + sourceType: 'fact' as const, + ...candidate + })); + + if (input.definition.selectionPolicy === 'aggregate_multiple_components') { + const aggregateCandidates = [...rowCandidates, ...factCandidates] + .sort((left, right) => compareResolvedCandidates(left, right, input.definition)); + const dedupedCandidates: ResolvedCandidate[] = []; + const seenConcepts = new Set(); + + for (const candidate of aggregateCandidates) { + const conceptKey = candidate.sourceType === 'row' + ? candidate.row.key + : candidate.fact.conceptKey; + if (seenConcepts.has(conceptKey)) { + continue; + } + + seenConcepts.add(conceptKey); + dedupedCandidates.push(candidate); + } + + return dedupedCandidates; } - const label = normalizeToken(row.label); - return definition.labelIncludes?.some((token) => label.includes(normalizeToken(token))) ?? false; + const resolvedCandidate = [...rowCandidates, ...factCandidates] + .sort((left, right) => compareResolvedCandidates(left, right, input.definition))[0]; + + return resolvedCandidate ? [resolvedCandidate] : []; } -function matchesDefinitionFact(fact: TaxonomyFactRow, definition: CanonicalRowDefinition) { - const localName = normalizeToken(fact.localName); - return definition.localNames?.some((entry) => normalizeToken(entry) === localName) ?? false; -} +const GLOBAL_EXCLUDE_LABEL_PHRASES = [ + 'pro forma', + 'reconciliation', + 'acquiree', + 'business combination', + 'assets acquired', + 'liabilities assumed' +] as const; function inferUnit(rawUnit: string | null, fallback: FinancialUnit) { const normalized = (rawUnit ?? '').toLowerCase(); @@ -81,6 +173,192 @@ function inferUnit(rawUnit: string | null, fallback: FinancialUnit) { return fallback; } +function rowUnit(row: TaxonomyStatementRow, fallback: FinancialUnit) { + return inferUnit(Object.values(row.units)[0] ?? null, fallback); +} + +function isUnitCompatible(expected: FinancialUnit, actual: FinancialUnit) { + if (expected === actual) { + return true; + } + + if ((expected === 'percent' || expected === 'ratio') && (actual === 'percent' || actual === 'ratio')) { + return true; + } + + return false; +} + +function phraseTokens(phrase: string) { + return tokenizeLabel(phrase); +} + +function labelContainsPhrase(labelTokens: string[], phrase: string) { + const target = phraseTokens(phrase); + if (target.length === 0 || target.length > labelTokens.length) { + return false; + } + + for (let index = 0; index <= labelTokens.length - target.length; index += 1) { + let matched = true; + for (let offset = 0; offset < target.length; offset += 1) { + if (labelTokens[index + offset] !== target[offset]) { + matched = false; + break; + } + } + + if (matched) { + return true; + } + } + + return false; +} + +function matchRank(matchKind: CandidateMatchKind) { + switch (matchKind) { + case 'exact_local_name': + return 0; + case 'secondary_local_name': + return 1; + case 'label_phrase': + return 2; + } +} + +function aliasRank(localName: string, aliases: readonly string[] | undefined) { + const normalizedLocalName = normalizeToken(localName); + const matchIndex = (aliases ?? []).findIndex((alias) => normalizeToken(alias) === normalizedLocalName); + return matchIndex === -1 ? Number.MAX_SAFE_INTEGER : matchIndex; +} + +function applySignTransform(value: number | null, transform: StandardTemplateRowDefinition['signTransform']) { + if (value === null || !transform) { + return value; + } + + if (transform === 'invert') { + return value * -1; + } + + return Math.abs(value); +} + +function classifyStatementRowCandidate( + row: TaxonomyStatementRow, + definition: StandardTemplateRowDefinition +) { + if (definition.selectionPolicy === 'formula_only') { + return null; + } + + const rowLocalName = normalizeToken(row.localName); + if ((definition.matchers.excludeLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) { + return null; + } + + const labelTokens = tokenizeLabel(row.label); + const excludedLabelPhrases = [ + ...GLOBAL_EXCLUDE_LABEL_PHRASES, + ...(definition.matchers.excludeLabelPhrases ?? []) + ]; + if (excludedLabelPhrases.some((phrase) => labelContainsPhrase(labelTokens, phrase))) { + return null; + } + + const unit = rowUnit(row, definition.unit); + if (!isUnitCompatible(definition.unit, unit)) { + return null; + } + + if ((definition.matchers.exactLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) { + return { + row, + matchKind: 'exact_local_name', + aliasRank: aliasRank(row.localName, definition.matchers.exactLocalNames), + unit, + labelTokenCount: labelTokens.length, + matchedPhraseTokenCount: 0 + } satisfies StatementRowCandidate; + } + + if ((definition.matchers.secondaryLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) { + return { + row, + matchKind: 'secondary_local_name', + aliasRank: aliasRank(row.localName, definition.matchers.secondaryLocalNames), + unit, + labelTokenCount: labelTokens.length, + matchedPhraseTokenCount: 0 + } satisfies StatementRowCandidate; + } + + const matchedPhrase = (definition.matchers.allowedLabelPhrases ?? []) + .map((phrase) => ({ + phrase, + tokenCount: phraseTokens(phrase).length + })) + .filter(({ phrase }) => labelContainsPhrase(labelTokens, phrase)) + .sort((left, right) => right.tokenCount - left.tokenCount)[0]; + + if (!matchedPhrase) { + return null; + } + + if (row.hasDimensions) { + return null; + } + + return { + row, + matchKind: 'label_phrase', + aliasRank: Number.MAX_SAFE_INTEGER, + unit, + labelTokenCount: labelTokens.length, + matchedPhraseTokenCount: matchedPhrase.tokenCount + } satisfies StatementRowCandidate; +} + +function classifyFactCandidate( + fact: TaxonomyFactRow, + definition: StandardTemplateRowDefinition +) { + if (!fact.isDimensionless) { + return null; + } + + const localName = normalizeToken(fact.localName); + if ((definition.matchers.excludeLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) { + return null; + } + + const unit = inferUnit(fact.unit ?? null, definition.unit); + if (!isUnitCompatible(definition.unit, unit)) { + return null; + } + + if ((definition.matchers.exactLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) { + return { + fact, + matchKind: 'exact_local_name', + aliasRank: aliasRank(fact.localName, definition.matchers.exactLocalNames), + unit + } satisfies FactCandidate; + } + + if ((definition.matchers.secondaryLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) { + return { + fact, + matchKind: 'secondary_local_name', + aliasRank: aliasRank(fact.localName, definition.matchers.secondaryLocalNames), + unit + } satisfies FactCandidate; + } + + return null; +} + export function factMatchesPeriod(fact: TaxonomyFactRow, period: FinancialStatementPeriod) { if (period.periodStart) { return fact.periodStart === period.periodStart && fact.periodEnd === period.periodEnd; @@ -89,185 +367,317 @@ export function factMatchesPeriod(fact: TaxonomyFactRow, period: FinancialStatem return (fact.periodInstant ?? fact.periodEnd) === period.periodEnd; } -function buildCanonicalRow( - definition: CanonicalRowDefinition, - matches: TaxonomyStatementRow[], - facts: TaxonomyFactRow[], +function compareStatementRowCandidates( + left: StatementRowCandidate, + right: StatementRowCandidate, + definition: StandardTemplateRowDefinition +) { + const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind); + if (matchDelta !== 0) { + return matchDelta; + } + + if (left.aliasRank !== right.aliasRank) { + return left.aliasRank - right.aliasRank; + } + + if (left.row.hasDimensions !== right.row.hasDimensions) { + return left.row.hasDimensions ? 1 : -1; + } + + if (definition.selectionPolicy === 'prefer_primary_statement_concept' && left.row.isExtension !== right.row.isExtension) { + return left.row.isExtension ? 1 : -1; + } + + if (left.row.order !== right.row.order) { + return left.row.order - right.row.order; + } + + if (left.matchedPhraseTokenCount !== right.matchedPhraseTokenCount) { + return right.matchedPhraseTokenCount - left.matchedPhraseTokenCount; + } + + if (left.labelTokenCount !== right.labelTokenCount) { + return left.labelTokenCount - right.labelTokenCount; + } + + return left.row.label.localeCompare(right.row.label); +} + +function compareFactCandidates(left: FactCandidate, right: FactCandidate) { + const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind); + if (matchDelta !== 0) { + return matchDelta; + } + + if (left.aliasRank !== right.aliasRank) { + return left.aliasRank - right.aliasRank; + } + + return left.fact.qname.localeCompare(right.fact.qname); +} + +function compareResolvedCandidates( + left: ResolvedCandidate, + right: ResolvedCandidate, + definition: StandardTemplateRowDefinition +) { + const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind); + if (matchDelta !== 0) { + return matchDelta; + } + + if (left.aliasRank !== right.aliasRank) { + return left.aliasRank - right.aliasRank; + } + + if (left.sourceType === 'row' && right.sourceType === 'row') { + return compareStatementRowCandidates(left, right, definition); + } + + if (left.sourceType === 'fact' && right.sourceType === 'fact') { + return compareFactCandidates(left, right); + } + + if (left.sourceType === 'row' && right.sourceType === 'fact') { + return left.row.hasDimensions ? 1 : -1; + } + + if (left.sourceType === 'fact' && right.sourceType === 'row') { + return right.row.hasDimensions ? -1 : 1; + } + + return 0; +} + +function buildTemplateRow( + definition: StandardTemplateRowDefinition, + candidates: StatementRowCandidate[], + factCandidates: FactCandidate[], periods: FinancialStatementPeriod[] ) { - const sortedMatches = [...matches].sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); - const matchedFacts = facts.filter((fact) => matchesDefinitionFact(fact, definition) && fact.isDimensionless); - const sourceConcepts = new Set(); const sourceRowKeys = new Set(); const sourceFactIds = new Set(); - for (const row of sortedMatches) { - sourceConcepts.add(row.qname); - sourceRowKeys.add(row.key); - for (const factId of row.sourceFactIds) { - sourceFactIds.add(factId); - } - } - - const values: Record = {}; - const resolvedSourceRowKeys: Record = {}; + const matchedRowKeys = new Set(); + const values: Record = Object.fromEntries(periods.map((period) => [period.id, null])); + const resolvedSourceRowKeys: Record = Object.fromEntries(periods.map((period) => [period.id, null])); + const metadata: InternalRowMetadata = { + derivedRoleByPeriod: Object.fromEntries(periods.map((period) => [period.id, null])) + }; let unit = definition.unit; + let hasDimensions = false; for (const period of periods) { - const directMatch = sortedMatches.find((row) => period.id in row.values); - if (directMatch) { - values[period.id] = directMatch.values[period.id] ?? null; - unit = inferUnit(directMatch.units[period.id] ?? null, definition.unit); - resolvedSourceRowKeys[period.id] = directMatch.key; + const resolvedCandidates = resolvedCandidatesForPeriod({ + definition, + candidates, + factCandidates, + period + }); + + if (resolvedCandidates.length === 0) { continue; } - const factMatch = matchedFacts.find((fact) => factMatchesPeriod(fact, period)); - values[period.id] = factMatch?.value ?? null; - unit = inferUnit(factMatch?.unit ?? null, definition.unit); - resolvedSourceRowKeys[period.id] = factMatch?.conceptKey ?? null; + if (definition.key === 'depreciation_and_amortization') { + metadata.derivedRoleByPeriod[period.id] = resolvedCandidates.some((candidate) => { + const localName = candidate.sourceType === 'row' + ? candidate.row.localName + : candidate.fact.localName; + return normalizeToken(localName) === normalizeToken('CostOfGoodsAndServicesSoldDepreciationAndAmortization'); + }) + ? 'expense' + : 'addback'; + } - if (factMatch) { - sourceConcepts.add(factMatch.qname); - sourceRowKeys.add(factMatch.conceptKey); - sourceFactIds.add(factMatch.id); + values[period.id] = definition.selectionPolicy === 'aggregate_multiple_components' + ? sumValues(resolvedCandidates.map((candidate) => { + if (candidate.sourceType === 'row') { + return applySignTransform(candidate.row.values[period.id] ?? null, definition.signTransform); + } + + return applySignTransform(candidate.fact.value ?? null, definition.signTransform); + })) + : (() => { + const resolvedCandidate = resolvedCandidates[0]!; + if (resolvedCandidate.sourceType === 'row') { + return applySignTransform(resolvedCandidate.row.values[period.id] ?? null, definition.signTransform); + } + + return applySignTransform(resolvedCandidate.fact.value ?? null, definition.signTransform); + })(); + resolvedSourceRowKeys[period.id] = resolvedCandidates.length === 1 + ? (resolvedCandidates[0]!.sourceType === 'row' + ? resolvedCandidates[0]!.row.key + : resolvedCandidates[0]!.fact.conceptKey ?? null) + : null; + + for (const resolvedCandidate of resolvedCandidates) { + unit = resolvedCandidate.unit; + + if (resolvedCandidate.sourceType === 'row') { + hasDimensions = hasDimensions || resolvedCandidate.row.hasDimensions; + matchedRowKeys.add(resolvedCandidate.row.key); + sourceConcepts.add(resolvedCandidate.row.qname); + sourceRowKeys.add(resolvedCandidate.row.key); + for (const factId of resolvedCandidate.row.sourceFactIds) { + sourceFactIds.add(factId); + } + continue; + } + + sourceConcepts.add(resolvedCandidate.fact.qname); + sourceRowKeys.add(resolvedCandidate.fact.conceptKey); + sourceFactIds.add(resolvedCandidate.fact.id); } } return { - key: definition.key, - label: definition.label, - category: definition.category, - order: definition.order, - unit, - values, - sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), - sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), - sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), - formulaKey: null, - hasDimensions: sortedMatches.some((row) => row.hasDimensions), - resolvedSourceRowKeys - } satisfies StandardizedFinancialRow; + row: { + key: definition.key, + label: definition.label, + category: definition.category, + templateSection: definition.category, + order: definition.order, + unit, + values, + sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), + sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), + sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), + formulaKey: null, + hasDimensions, + resolvedSourceRowKeys + } satisfies StandardizedFinancialRow, + matchedRowKeys, + metadata + }; } -type FormulaDefinition = { - key: string; - formulaKey: string; - compute: (rowsByKey: Map, periodId: string) => number | null; -}; +function computeFormulaValue( + formula: TemplateFormula, + rowsByKey: Map, + periodId: string +) { + switch (formula.kind) { + case 'sum': + return sumValues( + formula.sourceKeys.map((key) => valueOrNull(rowsByKey.get(key)?.values ?? {}, periodId)), + formula.treatNullAsZero ?? false + ); + case 'subtract': + return subtractValues( + valueOrNull(rowsByKey.get(formula.left)?.values ?? {}, periodId), + valueOrNull(rowsByKey.get(formula.right)?.values ?? {}, periodId) + ); + case 'divide': + return divideValues( + valueOrNull(rowsByKey.get(formula.numerator)?.values ?? {}, periodId), + valueOrNull(rowsByKey.get(formula.denominator)?.values ?? {}, periodId) + ); + } +} -const FORMULAS: Record, FormulaDefinition[]> = { - income: [ - { - key: 'gross_profit', - formulaKey: 'gross_profit', - compute: (rowsByKey, periodId) => subtractValues( - valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('cost_of_revenue')?.values ?? {}, periodId) - ) - }, - { - key: 'gross_margin', - formulaKey: 'gross_margin', - compute: (rowsByKey, periodId) => divideValues( - valueOrNull(rowsByKey.get('gross_profit')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId) - ) - }, - { - key: 'operating_margin', - formulaKey: 'operating_margin', - compute: (rowsByKey, periodId) => divideValues( - valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId) - ) - }, - { - key: 'effective_tax_rate', - formulaKey: 'effective_tax_rate', - compute: (rowsByKey, periodId) => divideValues( - valueOrNull(rowsByKey.get('income_tax_expense')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('pretax_income')?.values ?? {}, periodId) - ) - }, - { - key: 'ebitda', - formulaKey: 'ebitda', - compute: (rowsByKey, periodId) => sumValues([ - valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('depreciation_and_amortization')?.values ?? {}, periodId) - ]) - } - ], - balance: [ - { - key: 'total_debt', - formulaKey: 'total_debt', - compute: (rowsByKey, periodId) => sumValues([ - valueOrNull(rowsByKey.get('long_term_debt')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('current_debt')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('lease_liabilities')?.values ?? {}, periodId) - ]) - }, - { - key: 'net_cash_position', - formulaKey: 'net_cash_position', - compute: (rowsByKey, periodId) => subtractValues( - sumValues([ - valueOrNull(rowsByKey.get('cash_and_equivalents')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('short_term_investments')?.values ?? {}, periodId) - ]), - valueOrNull(rowsByKey.get('total_debt')?.values ?? {}, periodId) - ) - } - ], - cash_flow: [ - { - key: 'free_cash_flow', - formulaKey: 'free_cash_flow', - compute: (rowsByKey, periodId) => subtractValues( - valueOrNull(rowsByKey.get('operating_cash_flow')?.values ?? {}, periodId), - valueOrNull(rowsByKey.get('capital_expenditures')?.values ?? {}, periodId) - ) - } - ] -}; +function rowValueForPeriod( + rowsByKey: Map, + key: string, + periodId: string +) { + return valueOrNull(rowsByKey.get(key)?.values ?? {}, periodId); +} + +function computeOperatingIncomeFallbackValue( + rowsByKey: Map, + rowMetadataByKey: Map, + periodId: string +) { + const grossProfit = rowValueForPeriod(rowsByKey, 'gross_profit', periodId); + const sellingGeneralAndAdministrative = rowValueForPeriod(rowsByKey, 'selling_general_and_administrative', periodId); + const researchAndDevelopment = rowValueForPeriod(rowsByKey, 'research_and_development', periodId) ?? 0; + const depreciationAndAmortization = rowValueForPeriod(rowsByKey, 'depreciation_and_amortization', periodId); + const depreciationRole = rowMetadataByKey.get('depreciation_and_amortization')?.derivedRoleByPeriod[periodId] ?? null; + + if ( + depreciationRole === 'expense' + && grossProfit !== null + && sellingGeneralAndAdministrative !== null + && depreciationAndAmortization !== null + ) { + return grossProfit - sellingGeneralAndAdministrative - researchAndDevelopment - depreciationAndAmortization; + } + + const pretaxIncome = rowValueForPeriod(rowsByKey, 'pretax_income', periodId); + if (pretaxIncome === null) { + return null; + } + + const interestExpense = rowValueForPeriod(rowsByKey, 'interest_expense', periodId) ?? 0; + const interestIncome = rowValueForPeriod(rowsByKey, 'interest_income', periodId) ?? 0; + const otherNonOperatingIncome = rowValueForPeriod(rowsByKey, 'other_non_operating_income', periodId) ?? 0; + + return pretaxIncome + interestExpense - interestIncome - otherNonOperatingIncome; +} + +function computeFallbackValueForDefinition( + definition: StandardTemplateRowDefinition, + rowsByKey: Map, + rowMetadataByKey: Map, + periodId: string +) { + if (definition.key === 'operating_income') { + return computeOperatingIncomeFallbackValue(rowsByKey, rowMetadataByKey, periodId); + } + + if (!definition.fallbackFormula) { + return null; + } + + return computeFormulaValue(definition.fallbackFormula, rowsByKey, periodId); +} function applyFormulas( rowsByKey: Map, - statement: Extract, + rowMetadataByKey: Map, + definitions: StandardTemplateRowDefinition[], periods: FinancialStatementPeriod[] ) { - for (const formula of FORMULAS[statement]) { - const target = rowsByKey.get(formula.key); - if (!target) { - continue; - } + for (let pass = 0; pass < definitions.length; pass += 1) { + let changed = false; - let usedFormula = target.formulaKey !== null; - for (const period of periods) { - if (target.values[period.id] !== null) { + for (const definition of definitions) { + if (!definition.fallbackFormula && definition.key !== 'operating_income') { continue; } - const computed = formula.compute(rowsByKey, period.id); - if (computed === null) { + const target = rowsByKey.get(definition.key); + if (!target) { continue; } - target.values[period.id] = computed; - target.resolvedSourceRowKeys[period.id] = null; - usedFormula = true; + let usedFormula = target.formulaKey !== null; + for (const period of periods) { + if (definition.selectionPolicy !== 'formula_only' && target.values[period.id] !== null) { + continue; + } + + const computed = computeFallbackValueForDefinition(definition, rowsByKey, rowMetadataByKey, period.id); + if (computed === null) { + continue; + } + + target.values[period.id] = applySignTransform(computed, definition.signTransform); + target.resolvedSourceRowKeys[period.id] = null; + usedFormula = true; + changed = true; + } + + if (usedFormula) { + target.formulaKey = definition.key; + } } - if (usedFormula) { - target.formulaKey = formula.formulaKey; + if (!changed) { + break; } } } @@ -278,31 +688,47 @@ export function buildStandardizedRows(input: { periods: FinancialStatementPeriod[]; facts: TaxonomyFactRow[]; }) { - const definitions = CANONICAL_ROW_DEFINITIONS[input.statement]; + const definitions = STANDARD_FINANCIAL_TEMPLATES[input.statement]; const rowsByKey = new Map(); + const rowMetadataByKey = new Map(); const matchedRowKeys = new Set(); for (const definition of definitions) { - const matches = input.rows.filter((row) => matchesDefinition(row, definition)); - for (const row of matches) { - matchedRowKeys.add(row.key); + const candidates = input.rows + .map((row) => classifyStatementRowCandidate(row, definition)) + .filter((candidate): candidate is StatementRowCandidate => candidate !== null); + const factCandidates = input.facts + .map((fact) => classifyFactCandidate(fact, definition)) + .filter((candidate): candidate is FactCandidate => candidate !== null); + const templateRow = buildTemplateRow(definition, candidates, factCandidates, input.periods); + + for (const rowKey of templateRow.matchedRowKeys) { + matchedRowKeys.add(rowKey); } - const canonical = buildCanonicalRow(definition, matches, input.facts, input.periods); - const hasAnyValue = Object.values(canonical.values).some((value) => value !== null); - if (hasAnyValue || definition.key.startsWith('gross_') || definition.key === 'operating_margin' || definition.key === 'effective_tax_rate' || definition.key === 'ebitda' || definition.key === 'total_debt' || definition.key === 'net_cash_position' || definition.key === 'free_cash_flow') { - rowsByKey.set(definition.key, canonical); + const hasAnyValue = Object.values(templateRow.row.values).some((value) => value !== null); + if (hasAnyValue || definition.fallbackFormula || definition.key === 'operating_income') { + rowsByKey.set(definition.key, templateRow.row); + rowMetadataByKey.set(definition.key, templateRow.metadata); } } - applyFormulas(rowsByKey, input.statement, input.periods); + applyFormulas(rowsByKey, rowMetadataByKey, definitions, input.periods); + + const templateRows = definitions + .filter((definition) => definition.includeInOutput !== false) + .map((definition) => rowsByKey.get(definition.key)) + .filter((row): row is StandardizedFinancialRow => row !== undefined); + const coveredTemplateSourceRowKeys = new Set(templateRows.flatMap((row) => row.sourceRowKeys)); const unmatchedRows = input.rows .filter((row) => !matchedRowKeys.has(row.key)) + .filter((row) => !(row.hasDimensions && coveredTemplateSourceRowKeys.has(row.key))) .map((row) => ({ key: `other:${row.key}`, label: row.label, category: 'other', + templateSection: 'other', order: 10_000 + row.order, unit: inferUnit(Object.values(row.units)[0] ?? null, 'currency'), values: { ...row.values }, @@ -316,7 +742,7 @@ export function buildStandardizedRows(input: { ) } satisfies StandardizedFinancialRow)); - return [...rowsByKey.values(), ...unmatchedRows].sort((left, right) => { + return [...templateRows, ...unmatchedRows].sort((left, right) => { if (left.order !== right.order) { return left.order - right.order; } diff --git a/lib/server/repos/company-financial-bundles.ts b/lib/server/repos/company-financial-bundles.ts index 9b58248..d93f3b9 100644 --- a/lib/server/repos/company-financial-bundles.ts +++ b/lib/server/repos/company-financial-bundles.ts @@ -6,7 +6,7 @@ import type { import { db } from '@/lib/server/db'; import { companyFinancialBundle } from '@/lib/server/db/schema'; -const BUNDLE_VERSION = 1; +export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14; export type CompanyFinancialBundleRecord = { id: number; @@ -70,7 +70,7 @@ export async function upsertCompanyFinancialBundle(input: { ticker: input.ticker.trim().toUpperCase(), surface_kind: input.surfaceKind, cadence: input.cadence, - bundle_version: BUNDLE_VERSION, + bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION, source_snapshot_ids: input.sourceSnapshotIds, source_signature: input.sourceSignature, payload: input.payload, @@ -84,7 +84,7 @@ export async function upsertCompanyFinancialBundle(input: { companyFinancialBundle.cadence ], set: { - bundle_version: BUNDLE_VERSION, + bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION, source_snapshot_ids: input.sourceSnapshotIds, source_signature: input.sourceSignature, payload: input.payload, @@ -103,5 +103,5 @@ export async function deleteCompanyFinancialBundlesForTicker(ticker: string) { } export const __companyFinancialBundlesInternals = { - BUNDLE_VERSION + BUNDLE_VERSION: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION }; diff --git a/lib/server/repos/filing-taxonomy.ts b/lib/server/repos/filing-taxonomy.ts index ec5ef37..561f7a4 100644 --- a/lib/server/repos/filing-taxonomy.ts +++ b/lib/server/repos/filing-taxonomy.ts @@ -582,7 +582,7 @@ export async function listTaxonomyFactsByTicker(input: { cursor?: string | null; limit?: number; }) { - const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 2000); + const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 10000); const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null; const conditions = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())]; diff --git a/lib/types.ts b/lib/types.ts index 90b5d0b..09bee72 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -398,6 +398,7 @@ export type DerivedFinancialRow = { key: string; label: string; category: FinancialCategory; + templateSection?: FinancialCategory; order: number; unit: FinancialUnit; values: Record;