From db01f207a5ae91fd5d8b9c6a4cfe3cf4b2745b01 Mon Sep 17 00:00:00 2001 From: francy51 Date: Sat, 7 Mar 2026 15:16:35 -0500 Subject: [PATCH] Expand financials surfaces with ratios, KPIs, and cadence support - Add bundled financial modeling pipeline (ratios, KPI dimensions/notes, trend series, standardization) - Introduce company financial bundles storage (Drizzle migration + repo wiring) - Refactor financials page/API/query flow to use surfaceKind + cadence and new response shapes --- app/financials/page.tsx | 1243 ++++++-------- bun.lock | 47 +- drizzle/0007_company_financial_bundles.sql | 14 + drizzle/meta/_journal.json | 7 + hooks/use-link-prefetch.ts | 4 +- lib/api.ts | 12 +- lib/query/keys.ts | 6 +- lib/query/options.ts | 16 +- lib/server/api/app.ts | 72 +- lib/server/db/index.ts | 4 + lib/server/db/schema.ts | 26 + lib/server/financial-statements.test.ts | 5 +- lib/server/financial-statements.ts | 16 +- lib/server/financial-taxonomy.test.ts | 32 +- lib/server/financial-taxonomy.ts | 1498 ++++++++--------- lib/server/financials/bundles.ts | 53 + lib/server/financials/cadence.ts | 303 ++++ .../financials/canonical-definitions.ts | 85 + lib/server/financials/kpi-dimensions.test.ts | 65 + lib/server/financials/kpi-dimensions.ts | 159 ++ lib/server/financials/kpi-notes.ts | 131 ++ lib/server/financials/kpi-registry.ts | 120 ++ lib/server/financials/ratios.test.ts | 81 + lib/server/financials/ratios.ts | 369 ++++ lib/server/financials/standardize.ts | 450 +++++ lib/server/financials/trend-series.ts | 82 + lib/server/prices.ts | 96 ++ lib/server/repos/company-financial-bundles.ts | 107 ++ lib/server/repos/filing-taxonomy.ts | 10 + lib/server/task-processors.ts | 5 + lib/types.ts | 102 +- package.json | 1 + scripts/dev.ts | 11 +- 33 files changed, 3589 insertions(+), 1643 deletions(-) create mode 100644 drizzle/0007_company_financial_bundles.sql create mode 100644 lib/server/financials/bundles.ts create mode 100644 lib/server/financials/cadence.ts create mode 100644 lib/server/financials/canonical-definitions.ts create mode 100644 lib/server/financials/kpi-dimensions.test.ts create mode 100644 lib/server/financials/kpi-dimensions.ts create mode 100644 lib/server/financials/kpi-notes.ts create mode 100644 lib/server/financials/kpi-registry.ts create mode 100644 lib/server/financials/ratios.test.ts create mode 100644 lib/server/financials/ratios.ts create mode 100644 lib/server/financials/standardize.ts create mode 100644 lib/server/financials/trend-series.ts create mode 100644 lib/server/repos/company-financial-bundles.ts diff --git a/app/financials/page.tsx b/app/financials/page.tsx index 86d1969..c209a93 100644 --- a/app/financials/page.tsx +++ b/app/financials/page.tsx @@ -2,16 +2,14 @@ import { useQueryClient } from '@tanstack/react-query'; import Link from 'next/link'; -import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { Fragment, Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { format } from 'date-fns'; import { useSearchParams } from 'next/navigation'; import { - Area, - AreaChart, Bar, - BarChart, CartesianGrid, Line, + LineChart, ResponsiveContainer, Tooltip, XAxis, @@ -19,18 +17,16 @@ import { } from 'recharts'; import { AlertTriangle, - ChartNoAxesCombined, ChevronDown, Download, - GitCompareArrows, RefreshCcw, Search } from 'lucide-react'; import { AppShell } from '@/components/shell/app-shell'; -import { MetricCard } from '@/components/dashboard/metric-card'; import { FinancialControlBar, type FinancialControlAction, + type FinancialControlOption, type FinancialControlSection } from '@/components/financials/control-bar'; import { Button } from '@/components/ui/button'; @@ -48,12 +44,16 @@ import { queryKeys } from '@/lib/query/keys'; import { companyFinancialStatementsQueryOptions } from '@/lib/query/options'; import type { CompanyFinancialStatementsResponse, - DimensionBreakdownRow, - FinancialHistoryWindow, - FinancialStatementSurfaceKind, - FinancialStatementKind, - StandardizedStatementRow, - TaxonomyStatementRow + DerivedFinancialRow, + FinancialCadence, + FinancialDisplayMode, + FinancialSurfaceKind, + FinancialUnit, + RatioRow, + StandardizedFinancialRow, + StructuredKpiRow, + TaxonomyStatementRow, + TrendSeries } from '@/lib/types'; type LoadOptions = { @@ -61,19 +61,7 @@ type LoadOptions = { append?: boolean; }; -type OverviewPoint = { - periodId: string; - filingDate: string; - periodEnd: string | null; - label: string; - revenue: number | null; - netIncome: number | null; - totalAssets: number | null; - cash: number | null; - debt: number | null; -}; - -type DisplayRow = TaxonomyStatementRow | StandardizedStatementRow; +type DisplayRow = TaxonomyStatementRow | StandardizedFinancialRow | RatioRow | StructuredKpiRow; const FINANCIAL_VALUE_SCALE_OPTIONS: Array<{ value: NumberScaleUnit; label: string }> = [ { value: 'thousands', label: 'Thousands (K)' }, @@ -81,20 +69,23 @@ const FINANCIAL_VALUE_SCALE_OPTIONS: Array<{ value: NumberScaleUnit; label: stri { value: 'billions', label: 'Billions (B)' } ]; -const STATEMENT_OPTIONS: Array<{ value: FinancialStatementKind; label: string }> = [ - { value: 'income', label: 'Income' }, - { value: 'balance', label: 'Balance Sheet' }, - { value: 'cash_flow', label: 'Cash Flow' }, - { value: 'equity', label: 'Equity' }, - { value: 'comprehensive_income', label: 'Comprehensive Income' } +const SURFACE_OPTIONS: FinancialControlOption[] = [ + { value: 'income_statement', label: 'Income Statement' }, + { value: 'balance_sheet', label: 'Balance Sheet' }, + { value: 'cash_flow_statement', label: 'Cash Flow Statement' }, + { value: 'ratios', label: 'Ratios' }, + { value: 'segments_kpis', label: 'Segments & KPIs' }, + { value: 'adjusted', label: 'Adjusted' }, + { value: 'custom_metrics', label: 'Custom Metrics' } ]; -const WINDOW_OPTIONS: Array<{ value: FinancialHistoryWindow; label: string }> = [ - { value: '10y', label: '10 Years' }, - { value: 'all', label: 'Full Available' } +const CADENCE_OPTIONS: FinancialControlOption[] = [ + { value: 'annual', label: 'Annual' }, + { value: 'quarterly', label: 'Quarterly' }, + { value: 'ltm', label: 'LTM' } ]; -const SURFACE_OPTIONS: Array<{ value: FinancialStatementSurfaceKind; label: string }> = [ +const DISPLAY_MODE_OPTIONS: FinancialControlOption[] = [ { value: 'standardized', label: 'Standardized' }, { value: 'faithful', label: 'Filing-faithful' } ]; @@ -113,113 +104,106 @@ function formatLongDate(value: string) { return format(parsed, 'MMM dd, yyyy'); } -function formatShortDate(value: string) { - const parsed = new Date(value); - if (Number.isNaN(parsed.getTime())) { - return 'Unknown'; - } - - return format(parsed, 'MMM yyyy'); -} - -function asDisplayCurrency(value: number | null, scale: NumberScaleUnit) { - return value === null ? 'n/a' : formatCurrencyByScale(value, scale); -} - -function asAxisCurrencyTick(value: number, scale: NumberScaleUnit) { - return formatCurrencyByScale(value, scale, { maximumFractionDigits: 1 }); -} - -function asTooltipCurrency(value: unknown, scale: NumberScaleUnit) { - const numeric = Number(value); - if (!Number.isFinite(numeric)) { - return 'n/a'; - } - - return formatCurrencyByScale(numeric, scale); -} - -function ratioPercent(numerator: number | null, denominator: number | null) { - if (numerator === null || denominator === null || denominator === 0) { - return null; - } - - return (numerator / denominator) * 100; -} - -function rowValue(row: { values: Record }, periodId: string) { - return periodId in row.values ? row.values[periodId] : null; -} - -function isFaithfulRow(row: DisplayRow): row is TaxonomyStatementRow { +function isTaxonomyRow(row: DisplayRow): row is TaxonomyStatementRow { return 'localName' in row; } -function isStandardizedRow(row: DisplayRow): row is StandardizedStatementRow { - return 'sourceRowKeys' in row; +function isDerivedRow(row: DisplayRow): row is DerivedFinancialRow { + return 'formulaKey' in row; } -function mergeSurfaceRows; hasDimensions: boolean }>( - rows: T[], - mergeRow: (existing: T, row: T) => void -) { - const rowMap = new Map(); - - for (const row of rows) { - const existing = rowMap.get(row.key); - if (!existing) { - rowMap.set(row.key, structuredClone(row)); - continue; - } - - mergeRow(existing, row); - } - - return [...rowMap.values()]; +function isKpiRow(row: DisplayRow): row is StructuredKpiRow { + return 'provenanceType' in row; } -function mergeOverviewMetrics( - base: CompanyFinancialStatementsResponse['overviewMetrics'], - next: CompanyFinancialStatementsResponse['overviewMetrics'] -): CompanyFinancialStatementsResponse['overviewMetrics'] { - const mergedSeriesMap = new Map(); +function rowValue(row: DisplayRow, periodId: string) { + return row.values[periodId] ?? null; +} - for (const source of [base.series, next.series]) { - for (const point of source) { - const existing = mergedSeriesMap.get(point.periodId); - if (!existing) { - mergedSeriesMap.set(point.periodId, { ...point }); - continue; - } - - existing.filingDate = existing.filingDate || point.filingDate; - existing.periodEnd = existing.periodEnd ?? point.periodEnd; - existing.label = existing.label || point.label; - existing.revenue = existing.revenue ?? point.revenue; - existing.netIncome = existing.netIncome ?? point.netIncome; - existing.totalAssets = existing.totalAssets ?? point.totalAssets; - existing.cash = existing.cash ?? point.cash; - existing.debt = existing.debt ?? point.debt; - } +function formatMetricValue(value: number | null, unit: FinancialUnit, scale: NumberScaleUnit) { + if (value === null) { + return 'n/a'; } - const series = [...mergedSeriesMap.values()].sort((left, right) => { - return Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate); - }); - const latest = series[series.length - 1] ?? null; + switch (unit) { + case 'currency': + return formatCurrencyByScale(value, scale); + case 'percent': + return formatPercent(value * 100); + case 'shares': + case 'count': + return new Intl.NumberFormat('en-US', { notation: 'compact', maximumFractionDigits: 2 }).format(value); + case 'ratio': + return `${value.toFixed(2)}x`; + default: + return String(value); + } +} - return { - referencePeriodId: next.referencePeriodId ?? base.referencePeriodId ?? latest?.periodId ?? null, - referenceDate: next.referenceDate ?? base.referenceDate ?? latest?.filingDate ?? null, - latest: { - revenue: next.latest.revenue ?? base.latest.revenue ?? latest?.revenue ?? null, - netIncome: next.latest.netIncome ?? base.latest.netIncome ?? latest?.netIncome ?? null, - totalAssets: next.latest.totalAssets ?? base.latest.totalAssets ?? latest?.totalAssets ?? null, - cash: next.latest.cash ?? base.latest.cash ?? latest?.cash ?? null, - debt: next.latest.debt ?? base.latest.debt ?? latest?.debt ?? null - }, - series - }; +function chartTickFormatter(value: number, unit: FinancialUnit, scale: NumberScaleUnit) { + if (!Number.isFinite(value)) { + return 'n/a'; + } + + return formatMetricValue(value, unit, scale); +} + +function buildDisplayValue(input: { + row: DisplayRow; + periodId: string; + previousPeriodId: string | null; + commonSizeRow: DisplayRow | null; + displayMode: FinancialDisplayMode; + showPercentChange: boolean; + showCommonSize: boolean; + scale: NumberScaleUnit; + surfaceKind: FinancialSurfaceKind; +}) { + const current = rowValue(input.row, input.periodId); + + if (input.showPercentChange && input.previousPeriodId) { + const previous = rowValue(input.row, input.previousPeriodId); + if (current === null || previous === null || previous === 0) { + return 'n/a'; + } + + return formatPercent(((current - previous) / previous) * 100); + } + + if (input.showCommonSize) { + if (input.surfaceKind === 'ratios' && isDerivedRow(input.row) && input.row.unit === 'percent') { + return formatPercent((current ?? 0) * 100); + } + + if (input.displayMode === 'faithful') { + return 'n/a'; + } + + const denominator = input.commonSizeRow ? rowValue(input.commonSizeRow, input.periodId) : null; + if (current === null || denominator === null || denominator === 0) { + return 'n/a'; + } + + return formatPercent((current / denominator) * 100); + } + + const unit = isTaxonomyRow(input.row) + ? 'currency' + : input.row.unit; + return formatMetricValue(current, unit, input.scale); +} + +function groupRows(rows: DisplayRow[], categories: CompanyFinancialStatementsResponse['categories']) { + if (categories.length === 0) { + return [{ label: null, rows }]; + } + + return categories + .map((category) => ({ + label: category.label, + rows: rows.filter((row) => !isTaxonomyRow(row) && row.category === category.key) + })) + .filter((group) => group.rows.length > 0); } function mergeFinancialPages( @@ -232,253 +216,43 @@ function mergeFinancialPages( const periods = [...base.periods, ...next.periods] .filter((period, index, list) => list.findIndex((item) => item.id === period.id) === index) - .sort((a, b) => Date.parse(a.filingDate) - Date.parse(b.filingDate)); + .sort((left, right) => Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate)); - const faithfulRows = mergeSurfaceRows( - [...base.surfaces.faithful.rows, ...next.surfaces.faithful.rows], - (existing, row) => { - existing.hasDimensions = existing.hasDimensions || row.hasDimensions; - existing.order = Math.min(existing.order, row.order); - existing.depth = Math.min(existing.depth, row.depth); - if (!existing.parentKey && row.parentKey) { - existing.parentKey = row.parentKey; - } - - for (const [periodId, value] of Object.entries(row.values)) { - if (!(periodId in existing.values)) { - existing.values[periodId] = value; - } - } - - for (const [periodId, unit] of Object.entries(row.units)) { - if (!(periodId in existing.units)) { - existing.units[periodId] = unit; - } - } - - for (const factId of row.sourceFactIds) { - if (!existing.sourceFactIds.includes(factId)) { - existing.sourceFactIds.push(factId); - } - } - } - ).sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); - - const standardizedRows = mergeSurfaceRows( - [...base.surfaces.standardized.rows, ...next.surfaces.standardized.rows], - (existing, row) => { - existing.hasDimensions = existing.hasDimensions || row.hasDimensions; - existing.order = Math.min(existing.order, row.order); - - for (const [periodId, value] of Object.entries(row.values)) { - if (!(periodId in existing.values)) { - existing.values[periodId] = value; - } - } - - for (const [periodId, sourceRowKey] of Object.entries(row.resolvedSourceRowKeys)) { - if (!(periodId in existing.resolvedSourceRowKeys)) { - existing.resolvedSourceRowKeys[periodId] = sourceRowKey; - } - } - - for (const concept of row.sourceConcepts) { - if (!existing.sourceConcepts.includes(concept)) { - existing.sourceConcepts.push(concept); - } - } - - for (const rowKey of row.sourceRowKeys) { - if (!existing.sourceRowKeys.includes(rowKey)) { - existing.sourceRowKeys.push(rowKey); - } - } - - for (const factId of row.sourceFactIds) { - if (!existing.sourceFactIds.includes(factId)) { - existing.sourceFactIds.push(factId); - } - } - } - ).sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); - - const dimensionBreakdown = (() => { - if (!base.dimensionBreakdown && !next.dimensionBreakdown) { - return null; - } - - const map = new Map(); - for (const source of [base.dimensionBreakdown, next.dimensionBreakdown]) { - if (!source) { + const mergeRows = }>(rows: T[]) => { + const map = new Map(); + for (const row of rows) { + const existing = map.get(row.key); + if (!existing) { + map.set(row.key, structuredClone(row)); continue; } - for (const [key, rows] of Object.entries(source)) { - const existing = map.get(key); - if (existing) { - existing.push(...rows); - } else { - map.set(key, [...rows]); - } - } + existing.values = { + ...existing.values, + ...row.values + }; } - return Object.fromEntries(map.entries()); - })(); + return [...map.values()]; + }; return { ...next, periods, - surfaces: { - faithful: { - kind: 'faithful' as const, - rows: faithfulRows - }, - standardized: { - kind: 'standardized' as const, - rows: standardizedRows + statementRows: next.statementRows && base.statementRows + ? { + faithful: mergeRows([...base.statementRows.faithful, ...next.statementRows.faithful]), + standardized: mergeRows([...base.statementRows.standardized, ...next.statementRows.standardized]) } - }, - nextCursor: next.nextCursor, - coverage: { - ...next.coverage, - filings: periods.length, - rows: faithfulRows.length - }, - dataSourceStatus: { - ...next.dataSourceStatus, - queuedSync: base.dataSourceStatus.queuedSync || next.dataSourceStatus.queuedSync - }, - overviewMetrics: mergeOverviewMetrics(base.overviewMetrics, next.overviewMetrics), - dimensionBreakdown + : next.statementRows, + ratioRows: next.ratioRows && base.ratioRows ? mergeRows([...base.ratioRows, ...next.ratioRows]) : next.ratioRows, + kpiRows: next.kpiRows && base.kpiRows ? mergeRows([...base.kpiRows, ...next.kpiRows]) : next.kpiRows, + trendSeries: next.trendSeries, + categories: next.categories, + dimensionBreakdown: next.dimensionBreakdown ?? base.dimensionBreakdown }; } -function findFaithfulRowByLocalNames(rows: TaxonomyStatementRow[], localNames: string[]) { - const normalizedNames = localNames.map((name) => name.toLowerCase()); - const exact = rows.find((row) => normalizedNames.includes(row.localName.toLowerCase())); - if (exact) { - return exact; - } - - return rows.find((row) => { - const haystack = `${row.key} ${row.label} ${row.localName} ${row.qname}`.toLowerCase(); - return normalizedNames.some((name) => haystack.includes(name)); - }) ?? null; -} - -function findStandardizedRow(rows: StandardizedStatementRow[], key: string) { - return rows.find((row) => row.key === key) ?? null; -} - -function buildOverviewSeries( - incomeData: CompanyFinancialStatementsResponse | null, - balanceData: CompanyFinancialStatementsResponse | null -): OverviewPoint[] { - const overviewSeriesMap = new Map(); - - for (const source of [incomeData?.overviewMetrics.series ?? [], balanceData?.overviewMetrics.series ?? []]) { - for (const point of source) { - const existing = overviewSeriesMap.get(point.periodId); - if (!existing) { - overviewSeriesMap.set(point.periodId, { ...point }); - continue; - } - - existing.filingDate = existing.filingDate || point.filingDate; - existing.periodEnd = existing.periodEnd ?? point.periodEnd; - existing.label = existing.label || point.label; - existing.revenue = existing.revenue ?? point.revenue; - existing.netIncome = existing.netIncome ?? point.netIncome; - existing.totalAssets = existing.totalAssets ?? point.totalAssets; - existing.cash = existing.cash ?? point.cash; - existing.debt = existing.debt ?? point.debt; - } - } - - if (overviewSeriesMap.size > 0) { - return [...overviewSeriesMap.values()].sort((left, right) => { - return Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate); - }); - } - - const periodMap = new Map(); - - for (const source of [incomeData, balanceData]) { - for (const period of source?.periods ?? []) { - periodMap.set(period.id, { - filingDate: period.filingDate, - periodEnd: period.periodEnd - }); - } - } - - const periods = [...periodMap.entries()] - .map(([periodId, data]) => ({ - periodId, - filingDate: data.filingDate, - periodEnd: data.periodEnd - })) - .sort((a, b) => Date.parse(a.periodEnd ?? a.filingDate) - Date.parse(b.periodEnd ?? b.filingDate)); - - const incomeStandardized = incomeData?.surfaces.standardized.rows ?? []; - const balanceStandardized = balanceData?.surfaces.standardized.rows ?? []; - const incomeFaithful = incomeData?.surfaces.faithful.rows ?? []; - const balanceFaithful = balanceData?.surfaces.faithful.rows ?? []; - - const revenueRow = findStandardizedRow(incomeStandardized, 'revenue') - ?? findFaithfulRowByLocalNames(incomeFaithful, ['RevenueFromContractWithCustomerExcludingAssessedTax', 'Revenues', 'SalesRevenueNet', 'Revenue']); - const netIncomeRow = findStandardizedRow(incomeStandardized, 'net-income') - ?? findFaithfulRowByLocalNames(incomeFaithful, ['NetIncomeLoss', 'ProfitLoss']); - const assetsRow = findStandardizedRow(balanceStandardized, 'total-assets') - ?? findFaithfulRowByLocalNames(balanceFaithful, ['Assets']); - const cashRow = findStandardizedRow(balanceStandardized, 'cash-and-equivalents') - ?? findFaithfulRowByLocalNames(balanceFaithful, ['CashAndCashEquivalentsAtCarryingValue', 'CashAndShortTermInvestments', 'Cash']); - const debtRow = findStandardizedRow(balanceStandardized, 'total-debt') - ?? findFaithfulRowByLocalNames(balanceFaithful, ['LongTermDebt', 'Debt', 'LongTermDebtNoncurrent']); - - return periods.map((period) => ({ - periodId: period.periodId, - filingDate: period.filingDate, - periodEnd: period.periodEnd, - label: formatShortDate(period.periodEnd ?? period.filingDate), - revenue: revenueRow ? rowValue(revenueRow, period.periodId) : null, - netIncome: netIncomeRow ? rowValue(netIncomeRow, period.periodId) : null, - totalAssets: assetsRow ? rowValue(assetsRow, period.periodId) : null, - cash: cashRow ? rowValue(cashRow, period.periodId) : null, - debt: debtRow ? rowValue(debtRow, period.periodId) : null - })); -} - -function groupDimensionRows( - rows: DimensionBreakdownRow[], - surface: FinancialStatementSurfaceKind -) { - if (surface === 'faithful') { - return rows; - } - - return [...rows].sort((left, right) => { - if (left.periodId !== right.periodId) { - return left.periodId.localeCompare(right.periodId); - } - - return `${left.sourceLabel ?? ''}${left.concept ?? ''}`.localeCompare(`${right.sourceLabel ?? ''}${right.concept ?? ''}`); - }); -} - function ChartFrame({ children }: { children: React.ReactNode }) { const containerRef = useRef(null); const [ready, setReady] = useState(false); @@ -503,19 +277,7 @@ function ChartFrame({ children }: { children: React.ReactNode }) { return () => observer.disconnect(); }, []); - return ( -
- {ready ? children : null} -
- ); -} - -export default function FinancialsPage() { - return ( - Loading financial terminal...}> - - - ); + return
{ready ? children : null}
; } function FinancialsPageContent() { @@ -526,15 +288,15 @@ function FinancialsPageContent() { const [tickerInput, setTickerInput] = useState('MSFT'); const [ticker, setTicker] = useState('MSFT'); - const [statement, setStatement] = useState('income'); - const [window, setWindow] = useState('10y'); - const [surface, setSurface] = useState('standardized'); + const [surfaceKind, setSurfaceKind] = useState('income_statement'); + const [cadence, setCadence] = useState('annual'); + const [displayMode, setDisplayMode] = useState('standardized'); const [valueScale, setValueScale] = useState('millions'); + const [rowSearch, setRowSearch] = useState(''); + const [showPercentChange, setShowPercentChange] = useState(false); + const [showCommonSize, setShowCommonSize] = useState(false); const [financials, setFinancials] = useState(null); - const [overviewIncome, setOverviewIncome] = useState(null); - const [overviewBalance, setOverviewBalance] = useState(null); const [selectedRowKey, setSelectedRowKey] = useState(null); - const [dimensionsEnabled, setDimensionsEnabled] = useState(false); const [loading, setLoading] = useState(true); const [loadingMore, setLoadingMore] = useState(false); const [syncingFinancials, setSyncingFinancials] = useState(false); @@ -555,34 +317,27 @@ function FinancialsPageContent() { setTicker(normalized); }, [searchParams]); - const loadOverview = useCallback(async (symbol: string, selectedWindow: FinancialHistoryWindow) => { - const [incomeResponse, balanceResponse] = await Promise.all([ - queryClient.ensureQueryData(companyFinancialStatementsQueryOptions({ - ticker: symbol, - statement: 'income', - window: selectedWindow, - includeDimensions: false, - includeFacts: false, - limit: selectedWindow === 'all' ? 120 : 80 - })), - queryClient.ensureQueryData(companyFinancialStatementsQueryOptions({ - ticker: symbol, - statement: 'balance', - window: selectedWindow, - includeDimensions: false, - includeFacts: false, - limit: selectedWindow === 'all' ? 120 : 80 - })) - ]); + useEffect(() => { + const statementSurface = surfaceKind === 'income_statement' || surfaceKind === 'balance_sheet' || surfaceKind === 'cash_flow_statement'; + if (!statementSurface && displayMode !== 'standardized') { + setDisplayMode('standardized'); + } - setOverviewIncome(incomeResponse.financials); - setOverviewBalance(balanceResponse.financials); - }, [queryClient]); + if (displayMode === 'faithful') { + setShowPercentChange(false); + setShowCommonSize(false); + } + + if (surfaceKind === 'adjusted' || surfaceKind === 'custom_metrics') { + setShowPercentChange(false); + setShowCommonSize(false); + } + }, [displayMode, surfaceKind]); const loadFinancials = useCallback(async (symbol: string, options?: LoadOptions) => { const normalizedTicker = symbol.trim().toUpperCase(); const nextCursor = options?.cursor ?? null; - const includeDimensions = dimensionsEnabled || selectedRowKey !== null; + const includeDimensions = selectedRowKey !== null && (surfaceKind === 'segments_kpis' || surfaceKind === 'income_statement' || surfaceKind === 'balance_sheet' || surfaceKind === 'cash_flow_statement'); if (!options?.append) { setLoading(true); @@ -595,23 +350,15 @@ function FinancialsPageContent() { try { const response = await queryClient.ensureQueryData(companyFinancialStatementsQueryOptions({ ticker: normalizedTicker, - statement, - window, + surfaceKind, + cadence, includeDimensions, includeFacts: false, cursor: nextCursor, - limit: window === 'all' ? 60 : 80 + limit: 12 })); - setFinancials((current) => { - if (options?.append) { - return mergeFinancialPages(current, response.financials); - } - - return response.financials; - }); - - await loadOverview(normalizedTicker, window); + setFinancials((current) => options?.append ? mergeFinancialPages(current, response.financials) : response.financials); } catch (err) { setError(err instanceof Error ? err.message : 'Unable to load financial history'); if (!options?.append) { @@ -621,14 +368,7 @@ function FinancialsPageContent() { setLoading(false); setLoadingMore(false); } - }, [ - dimensionsEnabled, - loadOverview, - queryClient, - selectedRowKey, - statement, - window - ]); + }, [cadence, queryClient, selectedRowKey, surfaceKind]); const syncFinancials = useCallback(async () => { const targetTicker = (financials?.company.ticker ?? ticker).trim().toUpperCase(); @@ -660,128 +400,162 @@ function FinancialsPageContent() { const periods = useMemo(() => { return [...(financials?.periods ?? [])] - .sort((a, b) => Date.parse(a.filingDate) - Date.parse(b.filingDate)); + .sort((left, right) => Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate)); }, [financials?.periods]); - const faithfulRows = useMemo(() => financials?.surfaces.faithful.rows ?? [], [financials?.surfaces.faithful.rows]); - const standardizedRows = useMemo(() => financials?.surfaces.standardized.rows ?? [], [financials?.surfaces.standardized.rows]); - const statementRows = useMemo(() => { - return surface === 'standardized' ? standardizedRows : faithfulRows; - }, [faithfulRows, standardizedRows, surface]); + const activeRows = useMemo(() => { + if (!financials) { + return []; + } - const overviewSeries = useMemo(() => { - return buildOverviewSeries(overviewIncome, overviewBalance); - }, [overviewIncome, overviewBalance]); + switch (surfaceKind) { + case 'income_statement': + case 'balance_sheet': + case 'cash_flow_statement': + return displayMode === 'faithful' + ? financials.statementRows?.faithful ?? [] + : financials.statementRows?.standardized ?? []; + case 'ratios': + return financials.ratioRows ?? []; + case 'segments_kpis': + return financials.kpiRows ?? []; + default: + return []; + } + }, [displayMode, financials, surfaceKind]); - const latestOverview = overviewSeries[overviewSeries.length - 1] ?? null; - const latestTaxonomyMetrics = financials?.metrics.taxonomy - ?? overviewIncome?.metrics.taxonomy - ?? overviewBalance?.metrics.taxonomy - ?? null; + const filteredRows = useMemo(() => { + const normalizedSearch = rowSearch.trim().toLowerCase(); + if (!normalizedSearch) { + return activeRows; + } - const latestStandardizedIncome = overviewIncome?.surfaces.standardized.rows ?? []; - const latestStandardizedBalance = overviewBalance?.surfaces.standardized.rows ?? []; - const latestRevenue = overviewIncome?.overviewMetrics.latest.revenue - ?? latestOverview?.revenue - ?? findStandardizedRow(latestStandardizedIncome, 'revenue')?.values[periods[periods.length - 1]?.id ?? ''] - ?? latestTaxonomyMetrics?.revenue - ?? null; - const latestNetIncome = overviewIncome?.overviewMetrics.latest.netIncome - ?? latestOverview?.netIncome - ?? findStandardizedRow(latestStandardizedIncome, 'net-income')?.values[periods[periods.length - 1]?.id ?? ''] - ?? latestTaxonomyMetrics?.netIncome - ?? null; - const latestTotalAssets = overviewBalance?.overviewMetrics.latest.totalAssets - ?? latestOverview?.totalAssets - ?? findStandardizedRow(latestStandardizedBalance, 'total-assets')?.values[periods[periods.length - 1]?.id ?? ''] - ?? latestTaxonomyMetrics?.totalAssets - ?? null; - const latestCash = overviewBalance?.overviewMetrics.latest.cash - ?? latestOverview?.cash - ?? findStandardizedRow(latestStandardizedBalance, 'cash-and-equivalents')?.values[periods[periods.length - 1]?.id ?? ''] - ?? latestTaxonomyMetrics?.cash - ?? null; - const latestDebt = overviewBalance?.overviewMetrics.latest.debt - ?? latestOverview?.debt - ?? findStandardizedRow(latestStandardizedBalance, 'total-debt')?.values[periods[periods.length - 1]?.id ?? ''] - ?? latestTaxonomyMetrics?.debt - ?? null; - const latestReferenceDate = overviewIncome?.overviewMetrics.referenceDate - ?? overviewBalance?.overviewMetrics.referenceDate - ?? latestOverview?.filingDate - ?? periods[periods.length - 1]?.filingDate - ?? null; + return activeRows.filter((row) => row.label.toLowerCase().includes(normalizedSearch)); + }, [activeRows, rowSearch]); + + const groupedRows = useMemo(() => { + return groupRows(filteredRows, financials?.categories ?? []); + }, [filteredRows, financials?.categories]); const selectedRow = useMemo(() => { if (!selectedRowKey) { return null; } - return statementRows.find((row) => row.key === selectedRowKey) ?? null; - }, [selectedRowKey, statementRows]); + return activeRows.find((row) => row.key === selectedRowKey) ?? null; + }, [activeRows, selectedRowKey]); const dimensionRows = useMemo(() => { if (!selectedRow || !financials?.dimensionBreakdown) { return []; } - const direct = financials.dimensionBreakdown[selectedRow.key] ?? []; - return groupDimensionRows(direct, surface); - }, [financials?.dimensionBreakdown, selectedRow, surface]); + return financials.dimensionBreakdown[selectedRow.key] ?? []; + }, [financials?.dimensionBreakdown, selectedRow]); - const selectedScaleLabel = useMemo(() => { - return FINANCIAL_VALUE_SCALE_OPTIONS.find((option) => option.value === valueScale)?.label ?? 'Millions (M)'; - }, [valueScale]); - - const controlSections = useMemo(() => [ - { - id: 'statement', - label: 'Statement', - value: statement, - options: STATEMENT_OPTIONS, - onChange: (nextValue) => { - setStatement(nextValue as FinancialStatementKind); - setSelectedRowKey(null); - } - }, - { - id: 'history', - label: 'Window', - value: window, - options: WINDOW_OPTIONS, - onChange: (nextValue) => { - setWindow(nextValue as FinancialHistoryWindow); - setSelectedRowKey(null); - } - }, - { - id: 'scale', - label: 'Scale', - value: valueScale, - options: FINANCIAL_VALUE_SCALE_OPTIONS, - onChange: (nextValue) => setValueScale(nextValue as NumberScaleUnit) + const commonSizeRow = useMemo(() => { + if (displayMode === 'faithful' || !financials?.statementRows) { + return null; } - ], [statement, valueScale, window]); - const controlActions = useMemo(() => { - const actions: FinancialControlAction[] = []; + const standardizedRows = financials.statementRows.standardized; + if (surfaceKind === 'income_statement' || surfaceKind === 'cash_flow_statement') { + return standardizedRows.find((row) => row.key === 'revenue') ?? null; + } - if (window === '10y') { - actions.push({ - id: 'load-full-history', - label: 'Load Full History', - variant: 'secondary', - onClick: () => { - setWindow('all'); + if (surfaceKind === 'balance_sheet') { + return standardizedRows.find((row) => row.key === 'total_assets') ?? null; + } + + return null; + }, [displayMode, financials?.statementRows, surfaceKind]); + + const trendSeries = financials?.trendSeries ?? []; + const chartData = useMemo(() => { + return periods.map((period) => ({ + label: formatLongDate(period.periodEnd ?? period.filingDate), + ...Object.fromEntries(trendSeries.map((series) => [series.key, series.values[period.id] ?? null])) + })); + }, [periods, trendSeries]); + + const controlSections = useMemo(() => { + const sections: FinancialControlSection[] = [ + { + id: 'surface', + label: 'Surface', + value: surfaceKind, + options: SURFACE_OPTIONS, + onChange: (value) => { + setSurfaceKind(value as FinancialSurfaceKind); + setSelectedRowKey(null); + } + }, + { + id: 'cadence', + label: 'Cadence', + value: cadence, + options: CADENCE_OPTIONS, + onChange: (value) => { + setCadence(value as FinancialCadence); + setSelectedRowKey(null); + } + } + ]; + + if (surfaceKind === 'income_statement' || surfaceKind === 'balance_sheet' || surfaceKind === 'cash_flow_statement') { + sections.push({ + id: 'display', + label: 'Display', + value: displayMode, + options: DISPLAY_MODE_OPTIONS, + onChange: (value) => { + setDisplayMode(value as FinancialDisplayMode); setSelectedRowKey(null); } }); } - if (window === 'all' && financials?.nextCursor) { + sections.push({ + id: 'scale', + label: 'Scale', + value: valueScale, + options: FINANCIAL_VALUE_SCALE_OPTIONS, + onChange: (value) => setValueScale(value as NumberScaleUnit) + }); + + return sections; + }, [cadence, displayMode, surfaceKind, valueScale]); + + const controlActions = useMemo(() => { + const actions: FinancialControlAction[] = []; + + if ((surfaceKind === 'income_statement' || surfaceKind === 'balance_sheet' || surfaceKind === 'cash_flow_statement' || surfaceKind === 'ratios') && displayMode !== 'faithful') { actions.push({ - id: 'load-older-periods', - label: loadingMore ? 'Loading Older...' : 'Load Older Periods', + id: 'percent-change', + label: showPercentChange ? '% Change On' : '% Change', + variant: showPercentChange ? 'primary' : 'secondary', + onClick: () => { + setShowPercentChange((current) => !current); + setShowCommonSize(false); + } + }); + + actions.push({ + id: 'common-size', + label: showCommonSize ? 'Common Size On' : 'Common Size', + variant: showCommonSize ? 'primary' : 'secondary', + onClick: () => { + setShowCommonSize((current) => !current); + setShowPercentChange(false); + }, + disabled: surfaceKind === 'ratios' && selectedRow !== null && isDerivedRow(selectedRow) && selectedRow.unit !== 'percent' + }); + } + + if (financials?.nextCursor) { + actions.push({ + id: 'load-older', + label: loadingMore ? 'Loading Older...' : 'Load Older', variant: 'secondary', disabled: loadingMore, onClick: () => { @@ -798,7 +572,7 @@ function FinancialsPageContent() { } return actions; - }, [financials?.nextCursor, loadFinancials, loadingMore, ticker, window]); + }, [displayMode, financials?.nextCursor, loadFinancials, loadingMore, selectedRow, showCommonSize, showPercentChange, surfaceKind, ticker]); if (isPending || !isAuthenticated) { return
Loading financial terminal...
; @@ -807,7 +581,7 @@ function FinancialsPageContent() { return ( @@ -833,7 +607,7 @@ function FinancialsPageContent() { )} > - +
{ @@ -874,126 +648,82 @@ function FinancialsPageContent() { ) : null} -
- - = 0} - /> - - -
- - -
- {SURFACE_OPTIONS.map((option) => ( - - ))} + +
+ setRowSearch(event.target.value)} + placeholder="Search rows by label" + className="max-w-sm" + /> + + {filteredRows.length} of {activeRows.length} rows +
-
- - {loading ? ( -

Loading overview chart...

- ) : overviewSeries.length === 0 ? ( -

No income history available yet.

- ) : ( - - - - - - asAxisCurrencyTick(value, valueScale)} /> - asTooltipCurrency(value, valueScale)} - contentStyle={{ - backgroundColor: CHART_TOOLTIP_BG, - border: `1px solid ${CHART_TOOLTIP_BORDER}`, - borderRadius: '0.75rem' - }} - /> - - - - - - )} - - - - {loading ? ( -

Loading balance chart...

- ) : overviewSeries.length === 0 ? ( -

No balance history available yet.

- ) : ( - - - - - - asAxisCurrencyTick(value, valueScale)} /> - asTooltipCurrency(value, valueScale)} - contentStyle={{ - backgroundColor: CHART_TOOLTIP_BG, - border: `1px solid ${CHART_TOOLTIP_BORDER}`, - borderRadius: '0.75rem' - }} - /> - - - - - - - )} - -
- - + {loading ? ( -

Loading statement matrix...

- ) : periods.length === 0 || statementRows.length === 0 ? ( -

No statement rows available for the selected filters yet.

+

Loading trend chart...

+ ) : chartData.length === 0 || trendSeries.length === 0 ? ( +

No trend data available for the selected surface.

+ ) : ( + + + + + + chartTickFormatter(value, trendSeries[0]?.unit ?? 'currency', valueScale)} + /> + { + const numeric = Number(value); + const unit = trendSeries.find((series) => series.key === entry.dataKey)?.unit ?? 'currency'; + return formatMetricValue(Number.isFinite(numeric) ? numeric : null, unit, valueScale); + }} + contentStyle={{ + backgroundColor: CHART_TOOLTIP_BG, + border: `1px solid ${CHART_TOOLTIP_BORDER}`, + borderRadius: '0.75rem' + }} + /> + {trendSeries.map((series, index) => ( + + ))} + + + + )} + + + + {loading ? ( +

Loading financial matrix...

+ ) : surfaceKind === 'adjusted' || surfaceKind === 'custom_metrics' ? ( +
+

This surface is not yet available in v1.

+

Adjusted and custom metrics are API-visible placeholders only. No edits or derived rows are available yet.

+
+ ) : periods.length === 0 || filteredRows.length === 0 ? ( +

No rows available for the selected filters yet.

) : (
@@ -1003,7 +733,7 @@ function FinancialsPageContent() { {periods.map((period) => ( @@ -1011,48 +741,51 @@ function FinancialsPageContent() { - {statementRows.map((row) => ( - { - setSelectedRowKey(row.key); - if (row.hasDimensions && !dimensionsEnabled) { - setDimensionsEnabled(true); - } - }} - > - + + + ) : null} + {group.rows.map((row) => ( + setSelectedRowKey(row.key)} + > + - {periods.map((period) => ( - + {isDerivedRow(row) && row.formulaKey ? ( + Formula: {row.formulaKey} + ) : null} + {isKpiRow(row) ? ( + Provenance: {row.provenanceType} + ) : null} + + + {periods.map((period, index) => ( + + ))} + ))} - + ))}
- {formatLongDate(period.filingDate)} + {formatLongDate(period.periodEnd ?? period.filingDate)} {period.filingType} ยท {period.periodLabel}
-
-
- {row.label} - {isFaithfulRow(row) && row.isExtension ? ( - Ext - ) : null} - {row.hasDimensions ? : null} -
- {isStandardizedRow(row) ? ( -
- - Mapped from {row.sourceConcepts.length} concept{row.sourceConcepts.length === 1 ? '' : 's'} - -
- {row.sourceConcepts.map((concept) => ( - - {concept} - - ))} + {groupedRows.map((group) => ( + + {group.label ? ( +
{group.label}
+
+
+ {row.label} + {'hasDimensions' in row && row.hasDimensions ? : null}
- - ) : null} -
- 		- {asDisplayCurrency(rowValue(row, period.id), valueScale)} - + {buildDisplayValue({ + row, + periodId: period.id, + previousPeriodId: index > 0 ? periods[index - 1]?.id ?? null : null, + commonSizeRow, + displayMode, + showPercentChange, + showCommonSize, + scale: valueScale, + surfaceKind + })} +
@@ -1060,53 +793,85 @@ function FinancialsPageContent() { )} - + {!selectedRow ? ( -

Select a statement row to inspect dimensional facts.

- ) : !selectedRow.hasDimensions ? ( -

No dimensional data is available for {selectedRow.label}.

- ) : !dimensionsEnabled ? ( -

Enable dimensions by selecting the row again.

- ) : dimensionRows.length === 0 ? ( -

Dimensions are still loading or unavailable for this row.

+

Select a row to inspect details.

) : ( -
- - - - - {surface === 'standardized' ? : null} - - - - - - - {dimensionRows.map((row, index) => { - const period = periods.find((item) => item.id === row.periodId); - return ( - - - {surface === 'standardized' ? : null} - - - +
+
+
+

Label

+

{selectedRow.label}

+
+
+

Key

+

{selectedRow.key}

+
+
+ + {isTaxonomyRow(selectedRow) ? ( +
+

Taxonomy Concept

+

{selectedRow.qname}

+
+ ) : ( +
+
+

Category

+

{selectedRow.category}

+
+
+

Unit

+

{selectedRow.unit}

+
+
+ )} + + {isDerivedRow(selectedRow) ? ( +
+

Source Row Keys

+

{selectedRow.sourceRowKeys.join(', ') || 'n/a'}

+

Source Concepts

+

{selectedRow.sourceConcepts.join(', ') || 'n/a'}

+

Source Fact IDs

+

{selectedRow.sourceFactIds.join(', ') || 'n/a'}

+
+ ) : null} + + {!selectedRow || !('hasDimensions' in selectedRow) || !selectedRow.hasDimensions ? ( +

No dimensional drill-down is available for this row.

+ ) : dimensionRows.length === 0 ? ( +

No dimensional facts were returned for the selected row.

+ ) : ( +
+
PeriodSourceAxisMemberValue
{period ? formatLongDate(period.filingDate) : row.periodId}{row.sourceLabel ?? row.concept ?? 'Unknown source'}{row.axis}{row.member}{asDisplayCurrency(row.value, valueScale)}
+ + + + + + - ); - })} - -
PeriodAxisMemberValue
+ + + {dimensionRows.map((row, index) => ( + + {periods.find((period) => period.id === row.periodId)?.periodLabel ?? row.periodId} + {row.axis} + {row.member} + {formatMetricValue(row.value, 'currency', valueScale)} + + ))} + + +
+ )}
)} - {financials ? ( - + {(surfaceKind === 'income_statement' || surfaceKind === 'balance_sheet' || surfaceKind === 'cash_flow_statement') && financials ? ( +
Overall status: {financials.metrics.validation?.status ?? 'not_run'} @@ -1129,8 +894,8 @@ function FinancialsPageContent() { {financials.metrics.validation?.checks.map((check) => ( {check.metricKey} - {asDisplayCurrency(check.taxonomyValue, valueScale)} - {asDisplayCurrency(check.llmValue, valueScale)} + {formatMetricValue(check.taxonomyValue, 'currency', valueScale)} + {formatMetricValue(check.llmValue, 'currency', valueScale)} {check.status} {check.evidencePages.join(', ') || 'n/a'} @@ -1141,40 +906,14 @@ function FinancialsPageContent() { )} ) : null} - - {financials ? ( - -
-
-

Hydrated

-

{financials.dataSourceStatus.hydratedFilings}

-
-
-

Partial

-

{financials.dataSourceStatus.partialFilings}

-
-
-

Failed

-

{financials.dataSourceStatus.failedFilings}

-
-
-

Pending

-

{financials.dataSourceStatus.pendingFilings}

-
-
-

Background Sync

-

{financials.dataSourceStatus.queuedSync ? 'Queued' : 'Idle'}

-
-
- - ) : null} - - -
- - Financial Statements V3: faithful filing reconstruction + standardized taxonomy comparison -
- ); } + +export default function FinancialsPage() { + return ( + Loading financial terminal...
}> + + + ); +} diff --git a/bun.lock b/bun.lock index a28e74a..4a86664 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,7 @@ "@workflow/world-postgres": "^4.1.0-beta.34", "ai": "^6.0.104", "better-auth": "^1.4.19", + "cheerio": "^1.1.2", "clsx": "^2.1.1", "date-fns": "^4.1.0", "drizzle-orm": "^0.41.0", @@ -692,6 +693,8 @@ "body-parser": ["body-parser@1.20.4", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.14.0", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], "boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="], @@ -738,6 +741,10 @@ "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], + "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], + + "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], "citty": ["citty@0.1.6", "", { "dependencies": { "consola": "^3.2.3" } }, "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ=="], @@ -782,6 +789,10 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], "d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="], @@ -840,6 +851,14 @@ "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + "dotenv": ["dotenv@17.3.1", "", {}, "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA=="], "drizzle-kit": ["drizzle-kit@0.31.9", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "esbuild-register": "^3.5.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-GViD3IgsXn7trFyBUUHyTFBpH/FsHTxYJ66qdbVggxef4UBPHRYxQaRzYLTuekYnk9i5FIEL9pbBIwMqX/Uwrg=="], @@ -862,8 +881,12 @@ "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + "enhanced-resolve": ["enhanced-resolve@5.19.0", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.0" } }, "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg=="], + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="], "environment": ["environment@1.1.0", "", {}, "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q=="], @@ -1000,6 +1023,8 @@ "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], + "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], + "http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], @@ -1008,7 +1033,7 @@ "human-signals": ["human-signals@2.1.0", "", {}, "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw=="], - "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], @@ -1190,6 +1215,8 @@ "npm-run-path": ["npm-run-path@4.0.1", "", { "dependencies": { "path-key": "^3.0.0" } }, "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw=="], + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "nypm": ["nypm@0.6.5", "", { "dependencies": { "citty": "^0.2.0", "pathe": "^2.0.3", "tinyexec": "^1.0.2" }, "bin": { "nypm": "dist/cli.mjs" } }, "sha512-K6AJy1GMVyfyMXRVB88700BJqNUkByijGJM8kEHpLdcAt+vSQAVfkWWHYzuRXHSY6xA2sNc5RjTj0p9rE2izVQ=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -1218,6 +1245,12 @@ "parse-json": ["parse-json@5.2.0", "", { "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", "json-parse-even-better-errors": "^2.3.0", "lines-and-columns": "^1.1.6" } }, "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg=="], + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="], + + "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="], + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], "path-exists": ["path-exists@5.0.0", "", {}, "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ=="], @@ -1498,6 +1531,10 @@ "webpack-virtual-modules": ["webpack-virtual-modules@0.6.2", "", {}, "sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ=="], + "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + + "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], + "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], @@ -1792,6 +1829,8 @@ "body-parser/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "body-parser/iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], + "boxen/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], "boxen/widest-line": ["widest-line@5.0.0", "", { "dependencies": { "string-width": "^7.0.0" } }, "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA=="], @@ -1828,6 +1867,8 @@ "graphile-config/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], + "is-inside-container/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], "lightningcss/detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -1846,6 +1887,10 @@ "ora/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + + "raw-body/iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], + "seek-bzip/commander": ["commander@6.2.1", "", {}, "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA=="], "send/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], diff --git a/drizzle/0007_company_financial_bundles.sql b/drizzle/0007_company_financial_bundles.sql new file mode 100644 index 0000000..2b24079 --- /dev/null +++ b/drizzle/0007_company_financial_bundles.sql @@ -0,0 +1,14 @@ +CREATE TABLE `company_financial_bundle` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `ticker` text NOT NULL, + `surface_kind` text NOT NULL, + `cadence` text NOT NULL, + `bundle_version` integer NOT NULL, + `source_snapshot_ids` text NOT NULL, + `source_signature` text NOT NULL, + `payload` text NOT NULL, + `created_at` text NOT NULL, + `updated_at` text NOT NULL +); +CREATE UNIQUE INDEX `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`); +CREATE INDEX `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`); diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9c03668..ea5d4c2 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1772830800000, "tag": "0006_coverage_journal_tracking", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1772863200000, + "tag": "0007_company_financial_bundles", + "breakpoints": true } ] } diff --git a/hooks/use-link-prefetch.ts b/hooks/use-link-prefetch.ts index 0ce3a3c..c44f992 100644 --- a/hooks/use-link-prefetch.ts +++ b/hooks/use-link-prefetch.ts @@ -40,8 +40,8 @@ export function useLinkPrefetch() { void queryClient.prefetchQuery(companyAnalysisQueryOptions(normalizedTicker)); void queryClient.prefetchQuery(companyFinancialStatementsQueryOptions({ ticker: normalizedTicker, - statement: 'income', - window: '10y', + surfaceKind: 'income_statement', + cadence: 'annual', includeDimensions: false })); void queryClient.prefetchQuery(filingsQueryOptions({ ticker: normalizedTicker, limit: 120 })); diff --git a/lib/api.ts b/lib/api.ts index e3f718f..334b524 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -7,9 +7,9 @@ import type { CoveragePriority, CoverageStatus, Filing, + FinancialCadence, + FinancialSurfaceKind, Holding, - FinancialHistoryWindow, - FinancialStatementKind, PortfolioInsight, PortfolioSummary, ResearchJournalEntry, @@ -307,8 +307,8 @@ export async function getCompanyAnalysis(ticker: string) { export async function getCompanyFinancialStatements(input: { ticker: string; - statement: FinancialStatementKind; - window: FinancialHistoryWindow; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; includeDimensions?: boolean; includeFacts?: boolean; factsCursor?: string | null; @@ -318,8 +318,8 @@ export async function getCompanyFinancialStatements(input: { }) { const query = { ticker: input.ticker.trim().toUpperCase(), - statement: input.statement, - window: input.window, + surface: input.surfaceKind, + cadence: input.cadence, includeDimensions: input.includeDimensions ? 'true' : 'false', includeFacts: input.includeFacts ? 'true' : 'false', ...(typeof input.cursor === 'string' && input.cursor.trim().length > 0 diff --git a/lib/query/keys.ts b/lib/query/keys.ts index a2c9957..331bec3 100644 --- a/lib/query/keys.ts +++ b/lib/query/keys.ts @@ -2,15 +2,15 @@ export const queryKeys = { companyAnalysis: (ticker: string) => ['analysis', ticker] as const, companyFinancialStatements: ( ticker: string, - statement: string, - window: string, + surfaceKind: string, + cadence: string, includeDimensions: boolean, includeFacts: boolean, factsCursor: string | null, factsLimit: number, cursor: string | null, limit: number - ) => ['financials-v3', ticker, statement, window, includeDimensions ? 'dims' : 'no-dims', includeFacts ? 'facts' : 'rows', factsCursor ?? '', factsLimit, cursor ?? '', limit] as const, + ) => ['financials-v3', ticker, surfaceKind, cadence, includeDimensions ? 'dims' : 'no-dims', includeFacts ? 'facts' : 'rows', factsCursor ?? '', factsLimit, cursor ?? '', limit] as const, filings: (ticker: string | null, limit: number) => ['filings', ticker ?? '', limit] as const, report: (accessionNumber: string) => ['report', accessionNumber] as const, watchlist: () => ['watchlist'] as const, diff --git a/lib/query/options.ts b/lib/query/options.ts index 6c6258f..de861e0 100644 --- a/lib/query/options.ts +++ b/lib/query/options.ts @@ -15,8 +15,8 @@ import { } from '@/lib/api'; import { queryKeys } from '@/lib/query/keys'; import type { - FinancialHistoryWindow, - FinancialStatementKind + FinancialCadence, + FinancialSurfaceKind } from '@/lib/types'; export function companyAnalysisQueryOptions(ticker: string) { @@ -31,8 +31,8 @@ export function companyAnalysisQueryOptions(ticker: string) { export function companyFinancialStatementsQueryOptions(input: { ticker: string; - statement: FinancialStatementKind; - window: FinancialHistoryWindow; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; includeDimensions?: boolean; includeFacts?: boolean; factsCursor?: string | null; @@ -51,8 +51,8 @@ export function companyFinancialStatementsQueryOptions(input: { return queryOptions({ queryKey: queryKeys.companyFinancialStatements( normalizedTicker, - input.statement, - input.window, + input.surfaceKind, + input.cadence, includeDimensions, includeFacts, factsCursor, @@ -62,8 +62,8 @@ export function companyFinancialStatementsQueryOptions(input: { ), queryFn: () => getCompanyFinancialStatements({ ticker: normalizedTicker, - statement: input.statement, - window: input.window, + surfaceKind: input.surfaceKind, + cadence: input.cadence, includeDimensions, includeFacts, factsCursor, diff --git a/lib/server/api/app.ts b/lib/server/api/app.ts index 04a105b..7dbd73b 100644 --- a/lib/server/api/app.ts +++ b/lib/server/api/app.ts @@ -4,8 +4,9 @@ import type { CoveragePriority, CoverageStatus, Filing, - FinancialHistoryWindow, + FinancialCadence, FinancialStatementKind, + FinancialSurfaceKind, ResearchJournalEntryType, TaskStatus } from '@/lib/types'; @@ -15,7 +16,7 @@ import { asErrorMessage, jsonError } from '@/lib/server/http'; import { buildPortfolioSummary } from '@/lib/server/portfolio'; import { defaultFinancialSyncLimit, - getCompanyFinancialTaxonomy + getCompanyFinancials } from '@/lib/server/financial-taxonomy'; import { redactInternalFilingAnalysisFields } from '@/lib/server/api/filing-redaction'; import { @@ -68,7 +69,16 @@ const FINANCIAL_STATEMENT_KINDS: FinancialStatementKind[] = [ 'equity', 'comprehensive_income' ]; -const FINANCIAL_HISTORY_WINDOWS: FinancialHistoryWindow[] = ['10y', 'all']; +const FINANCIAL_CADENCES: FinancialCadence[] = ['annual', 'quarterly', 'ltm']; +const FINANCIAL_SURFACES: FinancialSurfaceKind[] = [ + 'income_statement', + 'balance_sheet', + 'cash_flow_statement', + 'ratios', + 'segments_kpis', + 'adjusted', + 'custom_metrics' +]; const COVERAGE_STATUSES: CoverageStatus[] = ['backlog', 'active', 'watch', 'archive']; const COVERAGE_PRIORITIES: CoveragePriority[] = ['low', 'medium', 'high']; const JOURNAL_ENTRY_TYPES: ResearchJournalEntryType[] = ['note', 'filing_note', 'status_change']; @@ -152,10 +162,29 @@ function asStatementKind(value: unknown): FinancialStatementKind { : 'income'; } -function asHistoryWindow(value: unknown): FinancialHistoryWindow { - return FINANCIAL_HISTORY_WINDOWS.includes(value as FinancialHistoryWindow) - ? value as FinancialHistoryWindow - : '10y'; +function asCadence(value: unknown): FinancialCadence { + return FINANCIAL_CADENCES.includes(value as FinancialCadence) + ? value as FinancialCadence + : 'annual'; +} + +function surfaceFromLegacyStatement(statement: FinancialStatementKind): FinancialSurfaceKind { + switch (statement) { + case 'balance': + return 'balance_sheet'; + case 'cash_flow': + return 'cash_flow_statement'; + default: + return 'income_statement'; + } +} + +function asSurfaceKind(surface: unknown, statement: unknown): FinancialSurfaceKind { + if (FINANCIAL_SURFACES.includes(surface as FinancialSurfaceKind)) { + return surface as FinancialSurfaceKind; + } + + return surfaceFromLegacyStatement(asStatementKind(statement)); } function asCoverageStatus(value: unknown) { @@ -926,8 +955,8 @@ export const app = new Elysia({ prefix: '/api' }) return jsonError('ticker is required'); } - const statement = asStatementKind(query.statement); - const window = asHistoryWindow(query.window); + const surfaceKind = asSurfaceKind(query.surface, query.statement); + const cadence = asCadence(query.cadence); const includeDimensions = asBoolean(query.includeDimensions, false); const includeFacts = asBoolean(query.includeFacts, false); const cursor = typeof query.cursor === 'string' && query.cursor.trim().length > 0 @@ -943,10 +972,10 @@ export const app = new Elysia({ prefix: '/api' }) ? Number(query.factsLimit) : undefined; - let payload = await getCompanyFinancialTaxonomy({ + let payload = await getCompanyFinancials({ ticker, - statement, - window, + surfaceKind, + cadence, includeDimensions, includeFacts, factsCursor, @@ -961,7 +990,7 @@ export const app = new Elysia({ prefix: '/api' }) const shouldQueueSync = cursor === null && ( payload.dataSourceStatus.pendingFilings > 0 || payload.coverage.filings === 0 - || (window === 'all' && payload.nextCursor !== null) + || payload.nextCursor !== null ); if (shouldQueueSync) { @@ -972,7 +1001,7 @@ export const app = new Elysia({ prefix: '/api' }) taskType: 'sync_filings', payload: buildSyncFilingsPayload({ ticker, - limit: defaultFinancialSyncLimit(window), + limit: defaultFinancialSyncLimit(), category: watchlistItem?.category, tags: watchlistItem?.tags }), @@ -999,6 +1028,20 @@ export const app = new Elysia({ prefix: '/api' }) }, { query: t.Object({ ticker: t.String({ minLength: 1 }), + surface: t.Optional(t.Union([ + t.Literal('income_statement'), + t.Literal('balance_sheet'), + t.Literal('cash_flow_statement'), + t.Literal('ratios'), + t.Literal('segments_kpis'), + t.Literal('adjusted'), + t.Literal('custom_metrics') + ])), + cadence: t.Optional(t.Union([ + t.Literal('annual'), + t.Literal('quarterly'), + t.Literal('ltm') + ])), statement: t.Optional(t.Union([ t.Literal('income'), t.Literal('balance'), @@ -1006,7 +1049,6 @@ export const app = new Elysia({ prefix: '/api' }) t.Literal('equity'), t.Literal('comprehensive_income') ])), - window: t.Optional(t.Union([t.Literal('10y'), t.Literal('all')])), includeDimensions: t.Optional(t.Union([t.String(), t.Boolean()])), includeFacts: t.Optional(t.Union([t.String(), t.Boolean()])), cursor: t.Optional(t.String()), diff --git a/lib/server/db/index.ts b/lib/server/db/index.ts index 11ee26f..ed5722f 100644 --- a/lib/server/db/index.ts +++ b/lib/server/db/index.ts @@ -119,6 +119,10 @@ function ensureLocalSqliteSchema(client: Database) { applySqlFile(client, '0005_financial_taxonomy_v3.sql'); } + if (!hasTable(client, 'company_financial_bundle')) { + applySqlFile(client, '0007_company_financial_bundles.sql'); + } + if (!hasTable(client, 'research_journal_entry')) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_journal_entry\` ( diff --git a/lib/server/db/schema.ts b/lib/server/db/schema.ts index b906a6a..95158ec 100644 --- a/lib/server/db/schema.ts +++ b/lib/server/db/schema.ts @@ -30,6 +30,15 @@ type TaxonomyMetricValidationStatus = 'not_run' | 'matched' | 'mismatch' | 'erro type CoverageStatus = 'backlog' | 'active' | 'watch' | 'archive'; type CoveragePriority = 'low' | 'medium' | 'high'; type ResearchJournalEntryType = 'note' | 'filing_note' | 'status_change'; +type FinancialCadence = 'annual' | 'quarterly' | 'ltm'; +type FinancialSurfaceKind = + | 'income_statement' + | 'balance_sheet' + | 'cash_flow_statement' + | 'ratios' + | 'segments_kpis' + | 'adjusted' + | 'custom_metrics'; type FilingAnalysis = { provider?: string; @@ -460,6 +469,22 @@ export const filingTaxonomyMetricValidation = sqliteTable('filing_taxonomy_metri filingTaxonomyMetricValidationUnique: uniqueIndex('filing_taxonomy_metric_validation_uidx').on(table.snapshot_id, table.metric_key) })); +export const companyFinancialBundle = sqliteTable('company_financial_bundle', { + id: integer('id').primaryKey({ autoIncrement: true }), + ticker: text('ticker').notNull(), + surface_kind: text('surface_kind').$type().notNull(), + cadence: text('cadence').$type().notNull(), + bundle_version: integer('bundle_version').notNull(), + source_snapshot_ids: text('source_snapshot_ids', { mode: 'json' }).$type().notNull(), + source_signature: text('source_signature').notNull(), + payload: text('payload', { mode: 'json' }).$type>().notNull(), + created_at: text('created_at').notNull(), + updated_at: text('updated_at').notNull() +}, (table) => ({ + companyFinancialBundleUnique: uniqueIndex('company_financial_bundle_uidx').on(table.ticker, table.surface_kind, table.cadence), + companyFinancialBundleTickerIndex: index('company_financial_bundle_ticker_idx').on(table.ticker, table.updated_at) +})); + export const filingLink = sqliteTable('filing_link', { id: integer('id').primaryKey({ autoIncrement: true }), filing_id: integer('filing_id').notNull().references(() => filing.id, { onDelete: 'cascade' }), @@ -565,6 +590,7 @@ export const appSchema = { filingTaxonomyConcept, filingTaxonomyFact, filingTaxonomyMetricValidation, + companyFinancialBundle, filingLink, taskRun, taskStageEvent, diff --git a/lib/server/financial-statements.test.ts b/lib/server/financial-statements.test.ts index 28cf508..6c2d4d5 100644 --- a/lib/server/financial-statements.test.ts +++ b/lib/server/financial-statements.test.ts @@ -2,8 +2,7 @@ import { describe, expect, it } from 'bun:test'; import { __financialStatementsInternals } from './financial-statements'; describe('financial statements service internals', () => { - it('returns default sync limits by window', () => { - expect(__financialStatementsInternals.defaultFinancialSyncLimit('10y')).toBe(60); - expect(__financialStatementsInternals.defaultFinancialSyncLimit('all')).toBe(120); + it('returns the default sync limit', () => { + expect(__financialStatementsInternals.defaultFinancialSyncLimit()).toBe(60); }); }); diff --git a/lib/server/financial-statements.ts b/lib/server/financial-statements.ts index fd73084..ceb1446 100644 --- a/lib/server/financial-statements.ts +++ b/lib/server/financial-statements.ts @@ -1,17 +1,17 @@ import type { CompanyFinancialStatementsResponse, - FinancialHistoryWindow, - FinancialStatementKind + FinancialCadence, + FinancialSurfaceKind } from '@/lib/types'; import { defaultFinancialSyncLimit, - getCompanyFinancialTaxonomy + getCompanyFinancials } from '@/lib/server/financial-taxonomy'; type GetCompanyFinancialStatementsInput = { ticker: string; - statement: FinancialStatementKind; - window: FinancialHistoryWindow; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; includeDimensions: boolean; includeFacts?: boolean; factsCursor?: string | null; @@ -26,10 +26,10 @@ type GetCompanyFinancialStatementsInput = { export async function getCompanyFinancialStatements( input: GetCompanyFinancialStatementsInput ): Promise { - return await getCompanyFinancialTaxonomy({ + return await getCompanyFinancials({ ticker: input.ticker, - statement: input.statement, - window: input.window, + surfaceKind: input.surfaceKind, + cadence: input.cadence, includeDimensions: input.includeDimensions, includeFacts: input.includeFacts ?? false, factsCursor: input.factsCursor, diff --git a/lib/server/financial-taxonomy.test.ts b/lib/server/financial-taxonomy.test.ts index fe4e395..a7911d2 100644 --- a/lib/server/financial-taxonomy.test.ts +++ b/lib/server/financial-taxonomy.test.ts @@ -171,7 +171,7 @@ describe('financial taxonomy internals', () => { ] }); - const selection = __financialTaxonomyInternals.selectPrimaryPeriods([snapshot], 'income'); + const selection = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([snapshot], 'income', 'quarterly'); expect(selection.periods).toHaveLength(1); expect(selection.periods[0]?.id).toBe('quarter'); @@ -189,7 +189,7 @@ describe('financial taxonomy internals', () => { ] }); - const selection = __financialTaxonomyInternals.selectPrimaryPeriods([snapshot], 'balance'); + const selection = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([snapshot], 'balance', 'annual'); expect(selection.periods).toHaveLength(1); expect(selection.periods[0]?.id).toBe('current'); @@ -218,9 +218,11 @@ describe('financial taxonomy internals', () => { ] }); - const periods = __financialTaxonomyInternals.buildPeriods([annual, quarterly], 'income'); + const annualPeriods = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([annual, quarterly], 'income', 'annual').periods; + const quarterlyPeriods = __financialTaxonomyInternals.selectPrimaryPeriodsByCadence([annual, quarterly], 'income', 'quarterly').periods; - expect(periods.map((period) => period.id)).toEqual(['annual', 'quarter']); + expect(annualPeriods.map((period) => period.id)).toEqual(['annual']); + expect(quarterlyPeriods.map((period) => period.id)).toEqual(['quarter']); }); it('maps overlapping GAAP aliases into one standardized COGS row while preserving faithful rows', () => { @@ -281,14 +283,17 @@ describe('financial taxonomy internals', () => { ], 'income', new Set(['2024-q4', '2025-q4'])); const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows( - faithfulRows, - 'income', - [period2024, period2025] + { + rows: faithfulRows, + statement: 'income', + periods: [period2024, period2025], + facts: [] + } ); expect(faithfulRows).toHaveLength(2); - const cogs = standardizedRows.find((row) => row.key === 'cost-of-revenue'); + const cogs = standardizedRows.find((row) => row.key === 'cost_of_revenue'); expect(cogs).toBeDefined(); expect(cogs?.values['2024-q4']).toBe(45_000); expect(cogs?.values['2025-q4']).toBe(48_000); @@ -327,9 +332,12 @@ describe('financial taxonomy internals', () => { }) ]; const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows( - faithfulRows, - 'income', - [period2024, period2025] + { + rows: faithfulRows, + statement: 'income', + periods: [period2024, period2025], + facts: [] + } ); const breakdown = __financialTaxonomyInternals.buildDimensionBreakdown([ @@ -355,7 +363,7 @@ describe('financial taxonomy internals', () => { }) ], [period2024, period2025], faithfulRows, standardizedRows); - const cogs = breakdown?.['cost-of-revenue'] ?? []; + const cogs = breakdown?.['cost_of_revenue'] ?? []; expect(cogs).toHaveLength(2); expect(cogs.map((row) => row.sourceLabel)).toEqual([ 'Cost of Revenue', diff --git a/lib/server/financial-taxonomy.ts b/lib/server/financial-taxonomy.ts index cc2a225..5b79294 100644 --- a/lib/server/financial-taxonomy.ts +++ b/lib/server/financial-taxonomy.ts @@ -1,10 +1,13 @@ import type { CompanyFinancialStatementsResponse, - DimensionBreakdownRow, - FinancialHistoryWindow, + FinancialCadence, + FinancialDisplayMode, FinancialStatementKind, FinancialStatementPeriod, - StandardizedStatementRow, + FinancialSurfaceKind, + StandardizedFinancialRow, + StructuredKpiRow, + TaxonomyFactRow, TaxonomyStatementRow } from '@/lib/types'; import { listFilingsRecords } from '@/lib/server/repos/filings'; @@ -14,32 +17,58 @@ import { listTaxonomyFactsByTicker, type FilingTaxonomySnapshotRecord } from '@/lib/server/repos/filing-taxonomy'; +import { + buildLtmFaithfulRows, + buildLtmPeriods, + buildRows, + isStatementSurface, + periodSorter, + selectPrimaryPeriodsByCadence, + surfaceToStatementKind +} from '@/lib/server/financials/cadence'; +import { + readCachedFinancialBundle, + writeFinancialBundle +} from '@/lib/server/financials/bundles'; +import { + buildDimensionBreakdown, + buildLtmStandardizedRows, + buildStandardizedRows +} from '@/lib/server/financials/standardize'; +import { buildRatioRows } from '@/lib/server/financials/ratios'; +import { buildFinancialCategories, buildTrendSeries } from '@/lib/server/financials/trend-series'; +import { getHistoricalClosingPrices } from '@/lib/server/prices'; +import { resolveKpiDefinitions } from '@/lib/server/financials/kpi-registry'; +import { extractStructuredKpisFromDimensions } from '@/lib/server/financials/kpi-dimensions'; +import { extractStructuredKpisFromNotes } from '@/lib/server/financials/kpi-notes'; -type GetCompanyFinancialTaxonomyInput = { +type DimensionBreakdownMap = Record[string]>; + +type GetCompanyFinancialsInput = { ticker: string; - statement: FinancialStatementKind; - window: FinancialHistoryWindow; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; includeDimensions: boolean; includeFacts: boolean; factsCursor?: string | null; factsLimit?: number; cursor?: string | null; limit?: number; - v3Enabled: boolean; queuedSync: boolean; + v3Enabled: boolean; }; -type CanonicalRowDefinition = { - key: string; - label: string; - category: string; - order: number; - localNames?: readonly string[]; - labelIncludes?: readonly string[]; - formula?: ( - rowsByKey: Map, - periodIds: string[] - ) => Pick | null; +type StandardizedStatementBundlePayload = { + rows: StandardizedFinancialRow[]; + trendSeries: CompanyFinancialStatementsResponse['trendSeries']; +}; + +type FilingDocumentRef = { + filingId: number; + cik: string; + accessionNumber: string; + filingUrl: string | null; + primaryDocument: string | null; }; function safeTicker(input: string) { @@ -50,645 +79,10 @@ function isFinancialForm(type: string): type is '10-K' | '10-Q' { return type === '10-K' || type === '10-Q'; } -function parseEpoch(value: string | null) { - if (!value) { - return Number.NaN; - } - - return Date.parse(value); -} - -function periodSorter(left: FinancialStatementPeriod, right: FinancialStatementPeriod) { - const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); - const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); - if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { - return leftDate - rightDate; - } - - return left.id.localeCompare(right.id); -} - -function isInstantPeriod(period: FinancialStatementPeriod) { - return period.periodStart === null; -} - -function periodDurationDays(period: FinancialStatementPeriod) { - if (!period.periodStart || !period.periodEnd) { - return null; - } - - const start = Date.parse(period.periodStart); - const end = Date.parse(period.periodEnd); - if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { - return null; - } - - return Math.round((end - start) / 86_400_000) + 1; -} - -function preferredDurationDays(filingType: FinancialStatementPeriod['filingType']) { - return filingType === '10-K' ? 365 : 90; -} - -function selectPrimaryPeriods( - snapshots: FilingTaxonomySnapshotRecord[], - statement: FinancialStatementKind -) { - const selectedByFilingId = new Map(); - - for (const snapshot of snapshots) { - const rows = snapshot.statement_rows?.[statement] ?? []; - if (rows.length === 0) { - continue; - } - - const usedPeriodIds = new Set(); - for (const row of rows) { - for (const periodId of Object.keys(row.values)) { - usedPeriodIds.add(periodId); - } - } - - const candidates = (snapshot.periods ?? []).filter((period) => usedPeriodIds.has(period.id)); - if (candidates.length === 0) { - continue; - } - - const selected = (() => { - if (statement === 'balance') { - const instantCandidates = candidates.filter(isInstantPeriod); - return (instantCandidates.length > 0 ? instantCandidates : candidates) - .sort((left, right) => periodSorter(right, left))[0] ?? null; - } - - const durationCandidates = candidates.filter((period) => !isInstantPeriod(period)); - if (durationCandidates.length === 0) { - return candidates.sort((left, right) => periodSorter(right, left))[0] ?? null; - } - - const targetDays = preferredDurationDays(snapshot.filing_type); - return durationCandidates.sort((left, right) => { - const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); - const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); - if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { - return rightDate - leftDate; - } - - const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays); - const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays); - if (leftDistance !== rightDistance) { - return leftDistance - rightDistance; - } - - return left.id.localeCompare(right.id); - })[0] ?? null; - })(); - - if (selected) { - selectedByFilingId.set(selected.filingId, selected); - } - } - - const periods = [...selectedByFilingId.values()].sort(periodSorter); - return { - periods, - selectedPeriodIds: new Set(periods.map((period) => period.id)), - periodByFilingId: new Map(periods.map((period) => [period.filingId, period])) - }; -} - -function buildPeriods( - snapshots: FilingTaxonomySnapshotRecord[], - statement: FinancialStatementKind -) { - return selectPrimaryPeriods(snapshots, statement).periods; -} - -function buildRows( - snapshots: FilingTaxonomySnapshotRecord[], - statement: FinancialStatementKind, - selectedPeriodIds: Set -) { - const rowMap = new Map(); - - for (const snapshot of snapshots) { - const rows = snapshot.statement_rows?.[statement] ?? []; - - for (const row of rows) { - const existing = rowMap.get(row.key); - if (!existing) { - rowMap.set(row.key, { - ...row, - values: Object.fromEntries( - Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId)) - ), - units: Object.fromEntries( - Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId)) - ), - sourceFactIds: [...row.sourceFactIds] - }); - if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) { - rowMap.delete(row.key); - } - continue; - } - - existing.hasDimensions = existing.hasDimensions || row.hasDimensions; - existing.order = Math.min(existing.order, row.order); - existing.depth = Math.min(existing.depth, row.depth); - if (!existing.parentKey && row.parentKey) { - existing.parentKey = row.parentKey; - } - - for (const [periodId, value] of Object.entries(row.values)) { - if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) { - existing.values[periodId] = value; - } - } - - for (const [periodId, unit] of Object.entries(row.units)) { - if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) { - existing.units[periodId] = unit; - } - } - - for (const factId of row.sourceFactIds) { - if (!existing.sourceFactIds.includes(factId)) { - existing.sourceFactIds.push(factId); - } - } - } - } - - return [...rowMap.values()].sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); -} - -function normalizeToken(value: string) { - return value.trim().toLowerCase(); -} - -function sumValues(left: number | null, right: number | null) { - if (left === null || right === null) { - return null; - } - - return left + right; -} - -function subtractValues(left: number | null, right: number | null) { - if (left === null || right === null) { - return null; - } - - return left - right; -} - -const STANDARDIZED_ROW_DEFINITIONS: Record = { - income: [ - { - key: 'revenue', - label: 'Revenue', - category: 'revenue', - order: 10, - localNames: [ - 'RevenueFromContractWithCustomerExcludingAssessedTax', - 'Revenues', - 'SalesRevenueNet', - 'TotalRevenuesAndOtherIncome' - ] - }, - { - key: 'cost-of-revenue', - label: 'Cost of Revenue', - category: 'expense', - order: 20, - localNames: [ - 'CostOfRevenue', - 'CostOfGoodsSold', - 'CostOfSales', - 'CostOfProductsSold', - 'CostOfServices' - ] - }, - { - key: 'gross-profit', - label: 'Gross Profit', - category: 'profit', - order: 30, - localNames: ['GrossProfit'], - formula: (rowsByKey, periodIds) => { - const revenue = rowsByKey.get('revenue'); - const cogs = rowsByKey.get('cost-of-revenue'); - if (!revenue || !cogs) { - return null; - } - - return { - values: Object.fromEntries(periodIds.map((periodId) => [ - periodId, - subtractValues(revenue.values[periodId] ?? null, cogs.values[periodId] ?? null) - ])), - resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) - }; - } - }, - { - key: 'research-and-development', - label: 'Research & Development', - category: 'opex', - order: 40, - localNames: ['ResearchAndDevelopmentExpense'] - }, - { - key: 'selling-general-and-administrative', - label: 'Selling, General & Administrative', - category: 'opex', - order: 50, - localNames: [ - 'SellingGeneralAndAdministrativeExpense', - 'SellingAndMarketingExpense', - 'GeneralAndAdministrativeExpense' - ], - labelIncludes: ['selling, general', 'selling general', 'general and administrative'] - }, - { - key: 'operating-income', - label: 'Operating Income', - category: 'profit', - order: 60, - localNames: ['OperatingIncomeLoss', 'IncomeLossFromOperations'] - }, - { - key: 'net-income', - label: 'Net Income', - category: 'profit', - order: 70, - localNames: ['NetIncomeLoss', 'ProfitLoss'] - } - ], - balance: [ - { - key: 'cash-and-equivalents', - label: 'Cash & Equivalents', - category: 'asset', - order: 10, - localNames: [ - 'CashAndCashEquivalentsAtCarryingValue', - 'CashCashEquivalentsAndShortTermInvestments', - 'CashAndShortTermInvestments' - ] - }, - { - key: 'accounts-receivable', - label: 'Accounts Receivable', - category: 'asset', - order: 20, - localNames: [ - 'AccountsReceivableNetCurrent', - 'ReceivablesNetCurrent' - ] - }, - { - key: 'inventory', - label: 'Inventory', - category: 'asset', - order: 30, - localNames: ['InventoryNet'] - }, - { - key: 'total-assets', - label: 'Total Assets', - category: 'asset', - order: 40, - localNames: ['Assets'] - }, - { - key: 'current-liabilities', - label: 'Current Liabilities', - category: 'liability', - order: 50, - localNames: ['LiabilitiesCurrent'] - }, - { - key: 'long-term-debt', - label: 'Long-Term Debt', - category: 'liability', - order: 60, - localNames: [ - 'LongTermDebtNoncurrent', - 'LongTermDebt', - 'DebtNoncurrent', - 'LongTermDebtAndCapitalLeaseObligations' - ] - }, - { - key: 'current-debt', - label: 'Current Debt', - category: 'liability', - order: 70, - localNames: ['DebtCurrent', 'ShortTermBorrowings', 'LongTermDebtCurrent'] - }, - { - key: 'total-debt', - label: 'Total Debt', - category: 'liability', - order: 80, - localNames: ['DebtAndFinanceLeaseLiabilities', 'Debt'], - formula: (rowsByKey, periodIds) => { - const longTermDebt = rowsByKey.get('long-term-debt'); - const currentDebt = rowsByKey.get('current-debt'); - if (!longTermDebt || !currentDebt) { - return null; - } - - return { - values: Object.fromEntries(periodIds.map((periodId) => [ - periodId, - sumValues(longTermDebt.values[periodId] ?? null, currentDebt.values[periodId] ?? null) - ])), - resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) - }; - } - }, - { - key: 'total-equity', - label: 'Total Equity', - category: 'equity', - order: 90, - localNames: [ - 'StockholdersEquity', - 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', - 'PartnersCapital' - ] - } - ], - cash_flow: [ - { - key: 'operating-cash-flow', - label: 'Operating Cash Flow', - category: 'cash-flow', - order: 10, - localNames: [ - 'NetCashProvidedByUsedInOperatingActivities', - 'NetCashProvidedByUsedInOperatingActivitiesContinuingOperations' - ] - }, - { - key: 'capital-expenditures', - label: 'Capital Expenditures', - category: 'cash-flow', - order: 20, - localNames: ['PaymentsToAcquirePropertyPlantAndEquipment', 'CapitalExpendituresIncurredButNotYetPaid'] - }, - { - key: 'free-cash-flow', - label: 'Free Cash Flow', - category: 'cash-flow', - order: 30, - formula: (rowsByKey, periodIds) => { - const operatingCashFlow = rowsByKey.get('operating-cash-flow'); - const capex = rowsByKey.get('capital-expenditures'); - if (!operatingCashFlow || !capex) { - return null; - } - - return { - values: Object.fromEntries(periodIds.map((periodId) => [ - periodId, - subtractValues(operatingCashFlow.values[periodId] ?? null, capex.values[periodId] ?? null) - ])), - resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) - }; - } - } - ], - equity: [ - { - key: 'total-equity', - label: 'Total Equity', - category: 'equity', - order: 10, - localNames: [ - 'StockholdersEquity', - 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', - 'PartnersCapital' - ] - } - ], - comprehensive_income: [ - { - key: 'comprehensive-income', - label: 'Comprehensive Income', - category: 'profit', - order: 10, - localNames: ['ComprehensiveIncomeNetOfTax', 'ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest'] - } - ] -}; - -function matchesDefinition(row: TaxonomyStatementRow, definition: CanonicalRowDefinition) { - const rowLocalName = normalizeToken(row.localName); - if (definition.localNames?.some((localName) => normalizeToken(localName) === rowLocalName)) { - return true; - } - - const label = normalizeToken(row.label); - return definition.labelIncludes?.some((token) => label.includes(normalizeToken(token))) ?? false; -} - -function buildCanonicalRow( - definition: CanonicalRowDefinition, - matches: TaxonomyStatementRow[], - periodIds: string[] -) { - const sortedMatches = [...matches].sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); - - const sourceConcepts = new Set(); - const sourceRowKeys = new Set(); - const sourceFactIds = new Set(); - - for (const row of sortedMatches) { - sourceConcepts.add(row.qname); - sourceRowKeys.add(row.key); - for (const factId of row.sourceFactIds) { - sourceFactIds.add(factId); - } - } - - const values: Record = {}; - const resolvedSourceRowKeys: Record = {}; - - for (const periodId of periodIds) { - const match = sortedMatches.find((row) => periodId in row.values); - values[periodId] = match?.values[periodId] ?? null; - resolvedSourceRowKeys[periodId] = match?.key ?? null; - } - - return { - key: definition.key, - label: definition.label, - category: definition.category, - order: definition.order, - values, - hasDimensions: sortedMatches.some((row) => row.hasDimensions), - sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), - sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), - sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), - resolvedSourceRowKeys - } satisfies StandardizedStatementRow; -} - -function buildStandardizedRows( - rows: TaxonomyStatementRow[], - statement: FinancialStatementKind, - periods: FinancialStatementPeriod[] -) { - const definitions = STANDARDIZED_ROW_DEFINITIONS[statement] ?? []; - const periodIds = periods.map((period) => period.id); - const rowsByKey = new Map(); - const matchedRowKeys = new Set(); - - for (const definition of definitions) { - const matches = rows.filter((row) => matchesDefinition(row, definition)); - if (matches.length === 0 && !definition.formula) { - continue; - } - - for (const row of matches) { - matchedRowKeys.add(row.key); - } - - const canonicalRow = buildCanonicalRow(definition, matches, periodIds); - rowsByKey.set(definition.key, canonicalRow); - - const derived = definition.formula?.(rowsByKey, periodIds) ?? null; - if (derived) { - rowsByKey.set(definition.key, { - ...canonicalRow, - values: derived.values, - resolvedSourceRowKeys: derived.resolvedSourceRowKeys - }); - } - } - - const unmatchedRows = rows - .filter((row) => !matchedRowKeys.has(row.key)) - .map((row) => ({ - key: `other:${row.key}`, - label: row.label, - category: 'other', - order: 10_000 + row.order, - values: { ...row.values }, - hasDimensions: row.hasDimensions, - sourceConcepts: [row.qname], - sourceRowKeys: [row.key], - sourceFactIds: [...row.sourceFactIds], - resolvedSourceRowKeys: Object.fromEntries( - periodIds.map((periodId) => [periodId, periodId in row.values ? row.key : null]) - ) - } satisfies StandardizedStatementRow)); - - return [...rowsByKey.values(), ...unmatchedRows].sort((left, right) => { - if (left.order !== right.order) { - return left.order - right.order; - } - - return left.label.localeCompare(right.label); - }); -} - -function buildDimensionBreakdown( - facts: Awaited>['facts'], - periods: FinancialStatementPeriod[], - faithfulRows: TaxonomyStatementRow[], - standardizedRows: StandardizedStatementRow[] -) { - const periodByFilingId = new Map(); - for (const period of periods) { - periodByFilingId.set(period.filingId, period); - } - - const faithfulRowByKey = new Map(faithfulRows.map((row) => [row.key, row])); - const standardizedRowsBySource = new Map(); - for (const row of standardizedRows) { - for (const sourceRowKey of row.sourceRowKeys) { - const existing = standardizedRowsBySource.get(sourceRowKey); - if (existing) { - existing.push(row); - } else { - standardizedRowsBySource.set(sourceRowKey, [row]); - } - } - } - - const map = new Map(); - const pushRow = (key: string, row: DimensionBreakdownRow) => { - const existing = map.get(key); - if (existing) { - existing.push(row); - } else { - map.set(key, [row]); - } - }; - - for (const fact of facts) { - if (fact.dimensions.length === 0) { - continue; - } - - const period = periodByFilingId.get(fact.filingId) ?? null; - if (!period) { - continue; - } - - const matchesPeriod = period.periodStart - ? fact.periodStart === period.periodStart && fact.periodEnd === period.periodEnd - : (fact.periodInstant ?? fact.periodEnd) === period.periodEnd; - - if (!matchesPeriod) { - continue; - } - - const faithfulRow = faithfulRowByKey.get(fact.conceptKey) ?? null; - const standardizedMatches = standardizedRowsBySource.get(fact.conceptKey) ?? []; - - for (const dimension of fact.dimensions) { - const faithfulDimensionRow: DimensionBreakdownRow = { - rowKey: fact.conceptKey, - concept: fact.qname, - sourceRowKey: fact.conceptKey, - sourceLabel: faithfulRow?.label ?? null, - periodId: period.id, - axis: dimension.axis, - member: dimension.member, - value: fact.value, - unit: fact.unit - }; - - pushRow(fact.conceptKey, faithfulDimensionRow); - - for (const standardizedRow of standardizedMatches) { - pushRow(standardizedRow.key, { - ...faithfulDimensionRow, - rowKey: standardizedRow.key - }); - } - } - } - - return map.size > 0 ? Object.fromEntries(map.entries()) : null; +function cadenceFilingTypes(cadence: FinancialCadence) { + return cadence === 'annual' + ? ['10-K'] as Array<'10-K' | '10-Q'> + : ['10-Q'] as Array<'10-K' | '10-Q'>; } function latestMetrics(snapshots: FilingTaxonomySnapshotRecord[]) { @@ -707,201 +101,711 @@ function latestMetrics(snapshots: FilingTaxonomySnapshotRecord[]) { }; } -function rowValue(row: { values: Record }, periodId: string) { - return periodId in row.values ? row.values[periodId] : null; +function defaultDisplayModes(surfaceKind: FinancialSurfaceKind): FinancialDisplayMode[] { + return isStatementSurface(surfaceKind) + ? ['standardized', 'faithful'] + : ['standardized']; } -function findStandardizedRow(rows: StandardizedStatementRow[], key: string) { - return rows.find((row) => row.key === key) ?? null; +function rekeyRowsByFilingId; + resolvedSourceRowKeys?: Record; +}>(rows: T[], sourcePeriods: FinancialStatementPeriod[], targetPeriods: FinancialStatementPeriod[]) { + const targetPeriodByFilingId = new Map(targetPeriods.map((period) => [period.filingId, period])); + + return rows.map((row) => { + const nextValues: Record = {}; + const nextResolvedSourceRowKeys: Record = {}; + + for (const sourcePeriod of sourcePeriods) { + const targetPeriod = targetPeriodByFilingId.get(sourcePeriod.filingId); + if (!targetPeriod) { + continue; + } + + nextValues[targetPeriod.id] = row.values[sourcePeriod.id] ?? null; + if (row.resolvedSourceRowKeys) { + nextResolvedSourceRowKeys[targetPeriod.id] = row.resolvedSourceRowKeys[sourcePeriod.id] ?? null; + } + } + + return { + ...row, + values: nextValues, + ...(row.resolvedSourceRowKeys ? { resolvedSourceRowKeys: nextResolvedSourceRowKeys } : {}) + }; + }); } -function findFaithfulRowByLocalNames(rows: TaxonomyStatementRow[], localNames: string[]) { - const normalizedNames = localNames.map((name) => name.toLowerCase()); - const exact = rows.find((row) => normalizedNames.includes(row.localName.toLowerCase())); - if (exact) { - return exact; +function mergeDimensionBreakdownMaps(...maps: Array) { + const merged = new Map[string]>(); + + for (const map of maps) { + if (!map) { + continue; + } + + for (const [key, rows] of Object.entries(map)) { + const existing = merged.get(key); + if (existing) { + existing.push(...rows); + } else { + merged.set(key, [...rows]); + } + } } - return rows.find((row) => { - const haystack = `${row.key} ${row.label} ${row.localName} ${row.qname}`.toLowerCase(); - return normalizedNames.some((name) => haystack.includes(name)); - }) ?? null; + return merged.size > 0 ? Object.fromEntries(merged.entries()) : null; } -function asOverviewLabel(period: FinancialStatementPeriod) { - const source = period.periodEnd ?? period.filingDate; - const parsed = Date.parse(source); - if (!Number.isFinite(parsed)) { - return source; +function buildKpiDimensionBreakdown(input: { + rows: StructuredKpiRow[]; + periods: FinancialStatementPeriod[]; + facts: TaxonomyFactRow[]; +}) { + const map = new Map[string]>(); + + for (const row of input.rows) { + if (row.provenanceType !== 'taxonomy') { + continue; + } + + const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id)); + if (matchedFacts.length === 0) { + continue; + } + + map.set(row.key, matchedFacts.flatMap((fact) => { + const matchedPeriod = input.periods.find((period) => period.filingId === fact.filingId); + if (!matchedPeriod) { + return []; + } + + return fact.dimensions.map((dimension) => ({ + rowKey: row.key, + concept: fact.qname, + sourceRowKey: fact.conceptKey, + sourceLabel: row.label, + periodId: matchedPeriod.id, + axis: dimension.axis, + member: dimension.member, + value: fact.value, + unit: fact.unit, + provenanceType: 'taxonomy' as const + })); + })); } - return new Intl.DateTimeFormat('en-US', { - month: 'short', - year: 'numeric' - }).format(new Date(parsed)); + return map.size > 0 ? Object.fromEntries(map.entries()) : null; } -function buildOverviewMetrics(input: { +function latestPeriodDate(period: FinancialStatementPeriod) { + return period.periodEnd ?? period.filingDate; +} + +function buildEmptyResponse(input: { + ticker: string; + companyName: string; + cik: string | null; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; + queuedSync: boolean; + enabled: boolean; + metrics: CompanyFinancialStatementsResponse['metrics']; + nextCursor: string | null; + coverageFacts: number; +}) { + return { + company: { + ticker: input.ticker, + companyName: input.companyName, + cik: input.cik + }, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + displayModes: defaultDisplayModes(input.surfaceKind), + defaultDisplayMode: 'standardized', + periods: [], + statementRows: isStatementSurface(input.surfaceKind) + ? { faithful: [], standardized: [] } + : null, + ratioRows: input.surfaceKind === 'ratios' ? [] : null, + kpiRows: input.surfaceKind === 'segments_kpis' ? [] : null, + trendSeries: [], + categories: [], + availability: { + adjusted: false, + customMetrics: false + }, + nextCursor: input.nextCursor, + facts: null, + coverage: { + filings: 0, + rows: 0, + dimensions: 0, + facts: input.coverageFacts + }, + dataSourceStatus: { + enabled: input.enabled, + hydratedFilings: 0, + partialFilings: 0, + failedFilings: 0, + pendingFilings: 0, + queuedSync: input.queuedSync + }, + metrics: input.metrics, + dimensionBreakdown: null + } satisfies CompanyFinancialStatementsResponse; +} + +async function buildStatementSurfaceBundle(input: { + surfaceKind: Extract; + cadence: FinancialCadence; periods: FinancialStatementPeriod[]; faithfulRows: TaxonomyStatementRow[]; - standardizedRows: StandardizedStatementRow[]; -}): CompanyFinancialStatementsResponse['overviewMetrics'] { - const periods = [...input.periods].sort(periodSorter); - const revenueRow = findStandardizedRow(input.standardizedRows, 'revenue') - ?? findFaithfulRowByLocalNames(input.faithfulRows, ['RevenueFromContractWithCustomerExcludingAssessedTax', 'Revenues', 'SalesRevenueNet', 'Revenue']); - const netIncomeRow = findStandardizedRow(input.standardizedRows, 'net-income') - ?? findFaithfulRowByLocalNames(input.faithfulRows, ['NetIncomeLoss', 'ProfitLoss']); - const assetsRow = findStandardizedRow(input.standardizedRows, 'total-assets') - ?? findFaithfulRowByLocalNames(input.faithfulRows, ['Assets']); - const cashRow = findStandardizedRow(input.standardizedRows, 'cash-and-equivalents') - ?? findFaithfulRowByLocalNames(input.faithfulRows, ['CashAndCashEquivalentsAtCarryingValue', 'CashAndShortTermInvestments', 'Cash']); - const debtRow = findStandardizedRow(input.standardizedRows, 'total-debt') - ?? findFaithfulRowByLocalNames(input.faithfulRows, ['LongTermDebt', 'Debt', 'LongTermDebtNoncurrent']); + facts: TaxonomyFactRow[]; + snapshots: FilingTaxonomySnapshotRecord[]; +}) { + const cached = await readCachedFinancialBundle({ + ticker: input.snapshots[0]?.ticker ?? '', + surfaceKind: input.surfaceKind, + cadence: input.cadence, + snapshots: input.snapshots + }); - const series = periods.map((period) => ({ - periodId: period.id, - filingDate: period.filingDate, - periodEnd: period.periodEnd, - label: asOverviewLabel(period), - revenue: revenueRow ? rowValue(revenueRow, period.id) : null, - netIncome: netIncomeRow ? rowValue(netIncomeRow, period.id) : null, - totalAssets: assetsRow ? rowValue(assetsRow, period.id) : null, - cash: cashRow ? rowValue(cashRow, period.id) : null, - debt: debtRow ? rowValue(debtRow, period.id) : null - })); + if (cached) { + return cached as StandardizedStatementBundlePayload; + } - const latest = series[series.length - 1] ?? null; + const statement = surfaceToStatementKind(input.surfaceKind); + if (!statement || (statement !== 'income' && statement !== 'balance' && statement !== 'cash_flow')) { + return { + rows: [], + trendSeries: [] + } satisfies StandardizedStatementBundlePayload; + } - return { - referencePeriodId: latest?.periodId ?? null, - referenceDate: latest?.filingDate ?? null, - latest: { - revenue: latest?.revenue ?? null, - netIncome: latest?.netIncome ?? null, - totalAssets: latest?.totalAssets ?? null, - cash: latest?.cash ?? null, - debt: latest?.debt ?? null + const standardizedRows = buildStandardizedRows({ + rows: input.faithfulRows, + statement, + periods: input.periods, + facts: input.facts.filter((fact) => fact.statement === statement) + }); + + const payload = { + rows: standardizedRows, + trendSeries: buildTrendSeries({ + surfaceKind: input.surfaceKind, + statementRows: standardizedRows + }) + } satisfies StandardizedStatementBundlePayload; + + await writeFinancialBundle({ + ticker: input.snapshots[0]?.ticker ?? '', + surfaceKind: input.surfaceKind, + cadence: input.cadence, + snapshots: input.snapshots, + payload: payload as unknown as Record + }); + + return payload; +} + +async function buildRatioSurfaceBundle(input: { + ticker: string; + cadence: FinancialCadence; + periods: FinancialStatementPeriod[]; + snapshots: FilingTaxonomySnapshotRecord[]; + incomeRows: StandardizedFinancialRow[]; + balanceRows: StandardizedFinancialRow[]; + cashFlowRows: StandardizedFinancialRow[]; +}) { + const cached = await readCachedFinancialBundle({ + ticker: input.ticker, + surfaceKind: 'ratios', + cadence: input.cadence, + snapshots: input.snapshots + }); + + if (cached) { + return cached as Pick; + } + + const pricesByDate = await getHistoricalClosingPrices(input.ticker, input.periods.map((period) => latestPeriodDate(period))); + const pricesByPeriodId = Object.fromEntries(input.periods.map((period) => [period.id, pricesByDate[latestPeriodDate(period)] ?? null])); + const ratioRows = buildRatioRows({ + periods: input.periods, + cadence: input.cadence, + rows: { + income: input.incomeRows, + balance: input.balanceRows, + cashFlow: input.cashFlowRows }, - series - }; + pricesByPeriodId + }); + + const payload = { + ratioRows, + trendSeries: buildTrendSeries({ + surfaceKind: 'ratios', + ratioRows + }), + categories: buildFinancialCategories(ratioRows, 'ratios') + } satisfies Pick; + + await writeFinancialBundle({ + ticker: input.ticker, + surfaceKind: 'ratios', + cadence: input.cadence, + snapshots: input.snapshots, + payload: payload as unknown as Record + }); + + return payload; } -export function defaultFinancialSyncLimit(window: FinancialHistoryWindow) { - return window === 'all' ? 120 : 60; +async function buildKpiSurfaceBundle(input: { + ticker: string; + cadence: FinancialCadence; + periods: FinancialStatementPeriod[]; + snapshots: FilingTaxonomySnapshotRecord[]; + facts: TaxonomyFactRow[]; + filings: FilingDocumentRef[]; +}) { + const cached = await readCachedFinancialBundle({ + ticker: input.ticker, + surfaceKind: 'segments_kpis', + cadence: input.cadence, + snapshots: input.snapshots + }); + + if (cached) { + return cached as Pick; + } + + const resolved = resolveKpiDefinitions(input.ticker); + if (!resolved.template) { + return { + kpiRows: [], + trendSeries: [], + categories: [] + }; + } + + const taxonomyRows = extractStructuredKpisFromDimensions({ + facts: input.facts, + periods: input.periods, + definitions: resolved.definitions + }); + + const noteRows = await extractStructuredKpisFromNotes({ + ticker: input.ticker, + periods: input.periods, + filings: input.filings, + definitions: resolved.definitions + }); + + const rowsByKey = new Map(); + for (const row of [...taxonomyRows, ...noteRows]) { + const existing = rowsByKey.get(row.key); + if (existing) { + existing.values = { + ...existing.values, + ...row.values + }; + continue; + } + + rowsByKey.set(row.key, row); + } + + const kpiRows = [...rowsByKey.values()].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); + + const payload = { + kpiRows, + trendSeries: buildTrendSeries({ + surfaceKind: 'segments_kpis', + kpiRows + }), + categories: buildFinancialCategories(kpiRows, 'segments_kpis') + } satisfies Pick; + + await writeFinancialBundle({ + ticker: input.ticker, + surfaceKind: 'segments_kpis', + cadence: input.cadence, + snapshots: input.snapshots, + payload: payload as unknown as Record + }); + + return payload; } -export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxonomyInput): Promise { +export function defaultFinancialSyncLimit() { + return 60; +} + +export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Promise { const ticker = safeTicker(input.ticker); - const snapshotResult = await listFilingTaxonomySnapshotsByTicker({ - ticker, - window: input.window, - limit: input.limit, - cursor: input.cursor - }); + const filingTypes = cadenceFilingTypes(input.cadence); + const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 12), 1), 40); + const snapshotLimit = input.cadence === 'ltm' ? safeLimit + 3 : safeLimit; - const statuses = await countFilingTaxonomySnapshotStatuses(ticker); - const filings = await listFilingsRecords({ - ticker, - limit: input.window === 'all' ? 250 : 120 - }); - - const financialFilings = filings.filter((filing) => isFinancialForm(filing.filing_type)); - const selection = selectPrimaryPeriods(snapshotResult.snapshots, input.statement); - const periods = selection.periods; - const faithfulRows = buildRows(snapshotResult.snapshots, input.statement, selection.selectedPeriodIds); - const standardizedRows = buildStandardizedRows(faithfulRows, input.statement, periods); - - const factsResult = input.includeFacts - ? await listTaxonomyFactsByTicker({ + const [snapshotResult, statuses, filings] = await Promise.all([ + listFilingTaxonomySnapshotsByTicker({ ticker, - window: input.window, - statement: input.statement, - cursor: input.factsCursor, - limit: input.factsLimit - }) - : { facts: [], nextCursor: null }; - - const dimensionFacts = input.includeDimensions - ? await listTaxonomyFactsByTicker({ + window: 'all', + filingTypes: [...filingTypes], + limit: snapshotLimit, + cursor: input.cursor + }), + countFilingTaxonomySnapshotStatuses(ticker), + listFilingsRecords({ ticker, - window: input.window, - statement: input.statement, - limit: 1200 + limit: 250 }) - : { facts: [], nextCursor: null }; + ]); const latestFiling = filings[0] ?? null; + const financialFilings = filings.filter((filing) => isFinancialForm(filing.filing_type)); const metrics = latestMetrics(snapshotResult.snapshots); - const dimensionBreakdown = input.includeDimensions - ? buildDimensionBreakdown(dimensionFacts.facts, periods, faithfulRows, standardizedRows) + + if (snapshotResult.snapshots.length === 0) { + return buildEmptyResponse({ + ticker, + companyName: latestFiling?.company_name ?? ticker, + cik: latestFiling?.cik ?? null, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + queuedSync: input.queuedSync, + enabled: input.v3Enabled, + metrics, + nextCursor: snapshotResult.nextCursor, + coverageFacts: 0 + }); + } + + if (input.surfaceKind === 'adjusted' || input.surfaceKind === 'custom_metrics') { + return { + ...buildEmptyResponse({ + ticker, + companyName: latestFiling?.company_name ?? ticker, + cik: latestFiling?.cik ?? null, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + queuedSync: input.queuedSync, + enabled: input.v3Enabled, + metrics, + nextCursor: snapshotResult.nextCursor, + coverageFacts: 0 + }), + dataSourceStatus: { + enabled: input.v3Enabled, + hydratedFilings: statuses.ready, + partialFilings: statuses.partial, + failedFilings: statuses.failed, + pendingFilings: Math.max(0, financialFilings.filter((filing) => filingTypes.includes(filing.filing_type as '10-K' | '10-Q')).length - statuses.ready - statuses.partial - statuses.failed), + queuedSync: input.queuedSync + } + }; + } + + const allFacts = await listTaxonomyFactsByTicker({ + ticker, + window: 'all', + filingTypes: [...filingTypes], + limit: 2000 + }); + + if (isStatementSurface(input.surfaceKind)) { + const statement = surfaceToStatementKind(input.surfaceKind); + if (!statement) { + throw new Error(`Unsupported statement surface ${input.surfaceKind}`); + } + + const selection = input.cadence === 'ltm' + ? selectPrimaryPeriodsByCadence(snapshotResult.snapshots, statement, 'quarterly') + : selectPrimaryPeriodsByCadence(snapshotResult.snapshots, statement, input.cadence); + + const periods = input.cadence === 'ltm' + ? buildLtmPeriods(selection.periods) + : selection.periods; + const faithfulRows = input.cadence === 'ltm' + ? buildLtmFaithfulRows( + buildRows(selection.snapshots, statement, selection.selectedPeriodIds), + selection.periods, + periods, + statement + ) + : buildRows(selection.snapshots, statement, selection.selectedPeriodIds); + + const factsForStatement = allFacts.facts.filter((fact) => fact.statement === statement); + const standardizedPayload = await buildStatementSurfaceBundle({ + surfaceKind: input.surfaceKind as Extract, + cadence: input.cadence, + periods, + faithfulRows, + facts: factsForStatement, + snapshots: selection.snapshots + }); + + const standardizedRows = input.cadence === 'ltm' + ? buildLtmStandardizedRows( + buildStandardizedRows({ + rows: buildRows(selection.snapshots, statement, selection.selectedPeriodIds), + statement: statement as Extract, + periods: selection.periods, + facts: factsForStatement + }), + selection.periods, + periods, + statement as Extract + ) + : standardizedPayload.rows; + + const rawFacts = input.includeFacts + ? await listTaxonomyFactsByTicker({ + ticker, + window: 'all', + filingTypes: [...filingTypes], + statement, + cursor: input.factsCursor, + limit: input.factsLimit + }) + : { facts: [], nextCursor: null }; + + const dimensionBreakdown = input.includeDimensions + ? buildDimensionBreakdown(factsForStatement, periods, faithfulRows, standardizedRows) + : null; + + return { + company: { + ticker, + companyName: latestFiling?.company_name ?? ticker, + cik: latestFiling?.cik ?? null + }, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + displayModes: defaultDisplayModes(input.surfaceKind), + defaultDisplayMode: 'standardized', + periods, + statementRows: { + faithful: faithfulRows, + standardized: standardizedRows + }, + ratioRows: null, + kpiRows: null, + trendSeries: buildTrendSeries({ + surfaceKind: input.surfaceKind, + statementRows: standardizedRows + }), + categories: [], + availability: { + adjusted: false, + customMetrics: false + }, + nextCursor: snapshotResult.nextCursor, + facts: input.includeFacts + ? { + rows: rawFacts.facts, + nextCursor: rawFacts.nextCursor + } + : null, + coverage: { + filings: periods.length, + rows: standardizedRows.length, + dimensions: dimensionBreakdown ? Object.values(dimensionBreakdown).reduce((sum, rows) => sum + rows.length, 0) : 0, + facts: input.includeFacts ? rawFacts.facts.length : allFacts.facts.length + }, + dataSourceStatus: { + enabled: input.v3Enabled, + hydratedFilings: statuses.ready, + partialFilings: statuses.partial, + failedFilings: statuses.failed, + pendingFilings: Math.max(0, financialFilings.filter((filing) => filingTypes.includes(filing.filing_type as '10-K' | '10-Q')).length - statuses.ready - statuses.partial - statuses.failed), + queuedSync: input.queuedSync + }, + metrics, + dimensionBreakdown + }; + } + + const incomeSelection = input.cadence === 'ltm' + ? selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'income', 'quarterly') + : selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'income', input.cadence); + const balanceSelection = input.cadence === 'ltm' + ? selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'balance', 'quarterly') + : selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'balance', input.cadence); + const cashFlowSelection = input.cadence === 'ltm' + ? selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'cash_flow', 'quarterly') + : selectPrimaryPeriodsByCadence(snapshotResult.snapshots, 'cash_flow', input.cadence); + + const basePeriods = input.cadence === 'ltm' + ? buildLtmPeriods(incomeSelection.periods) + : incomeSelection.periods; + + const incomeQuarterlyRows = buildStandardizedRows({ + rows: buildRows(incomeSelection.snapshots, 'income', incomeSelection.selectedPeriodIds), + statement: 'income', + periods: incomeSelection.periods, + facts: allFacts.facts.filter((fact) => fact.statement === 'income') + }); + const balanceQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({ + rows: buildRows(balanceSelection.snapshots, 'balance', balanceSelection.selectedPeriodIds), + statement: 'balance', + periods: balanceSelection.periods, + facts: allFacts.facts.filter((fact) => fact.statement === 'balance') + }), balanceSelection.periods, incomeSelection.periods); + const cashFlowQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({ + rows: buildRows(cashFlowSelection.snapshots, 'cash_flow', cashFlowSelection.selectedPeriodIds), + statement: 'cash_flow', + periods: cashFlowSelection.periods, + facts: allFacts.facts.filter((fact) => fact.statement === 'cash_flow') + }), cashFlowSelection.periods, incomeSelection.periods); + + const incomeRows = input.cadence === 'ltm' + ? buildLtmStandardizedRows(incomeQuarterlyRows, incomeSelection.periods, basePeriods, 'income') + : incomeQuarterlyRows; + const balanceRows = input.cadence === 'ltm' + ? buildLtmStandardizedRows(balanceQuarterlyRows, incomeSelection.periods, basePeriods, 'balance') + : balanceQuarterlyRows; + const cashFlowRows = input.cadence === 'ltm' + ? buildLtmStandardizedRows(cashFlowQuarterlyRows, incomeSelection.periods, basePeriods, 'cash_flow') + : cashFlowQuarterlyRows; + + if (input.surfaceKind === 'ratios') { + const ratioBundle = await buildRatioSurfaceBundle({ + ticker, + cadence: input.cadence, + periods: basePeriods, + snapshots: incomeSelection.snapshots, + incomeRows, + balanceRows, + cashFlowRows + }); + + return { + company: { + ticker, + companyName: latestFiling?.company_name ?? ticker, + cik: latestFiling?.cik ?? null + }, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + displayModes: defaultDisplayModes(input.surfaceKind), + defaultDisplayMode: 'standardized', + periods: basePeriods, + statementRows: null, + ratioRows: ratioBundle.ratioRows, + kpiRows: null, + trendSeries: ratioBundle.trendSeries, + categories: ratioBundle.categories, + availability: { + adjusted: false, + customMetrics: false + }, + nextCursor: snapshotResult.nextCursor, + facts: null, + coverage: { + filings: basePeriods.length, + rows: ratioBundle.ratioRows?.length ?? 0, + dimensions: 0, + facts: allFacts.facts.length + }, + dataSourceStatus: { + enabled: input.v3Enabled, + hydratedFilings: statuses.ready, + partialFilings: statuses.partial, + failedFilings: statuses.failed, + pendingFilings: Math.max(0, financialFilings.filter((filing) => filingTypes.includes(filing.filing_type as '10-K' | '10-Q')).length - statuses.ready - statuses.partial - statuses.failed), + queuedSync: input.queuedSync + }, + metrics, + dimensionBreakdown: null + }; + } + + const filingRefs: FilingDocumentRef[] = filings.map((filing) => ({ + filingId: filing.id, + cik: filing.cik, + accessionNumber: filing.accession_number, + filingUrl: filing.filing_url ?? null, + primaryDocument: filing.primary_document ?? null + })); + const kpiBundle = await buildKpiSurfaceBundle({ + ticker, + cadence: input.cadence, + periods: basePeriods, + snapshots: incomeSelection.snapshots, + facts: allFacts.facts, + filings: filingRefs + }); + const kpiBreakdown = input.includeDimensions + ? buildKpiDimensionBreakdown({ + rows: kpiBundle.kpiRows ?? [], + periods: basePeriods, + facts: allFacts.facts + }) : null; - const dimensionsCount = input.includeDimensions - ? dimensionFacts.facts.reduce((total, fact) => total + fact.dimensions.length, 0) - : 0; - - const factsCoverage = input.includeFacts - ? factsResult.facts.length - : snapshotResult.snapshots.reduce((total, snapshot) => total + snapshot.facts_count, 0); - return { company: { ticker, companyName: latestFiling?.company_name ?? ticker, cik: latestFiling?.cik ?? null }, - statement: input.statement, - window: input.window, - defaultSurface: 'standardized', - periods, - surfaces: { - faithful: { - kind: 'faithful', - rows: faithfulRows - }, - standardized: { - kind: 'standardized', - rows: standardizedRows - } + surfaceKind: input.surfaceKind, + cadence: input.cadence, + displayModes: defaultDisplayModes(input.surfaceKind), + defaultDisplayMode: 'standardized', + periods: basePeriods, + statementRows: null, + ratioRows: null, + kpiRows: kpiBundle.kpiRows, + trendSeries: kpiBundle.trendSeries, + categories: kpiBundle.categories, + availability: { + adjusted: false, + customMetrics: false }, nextCursor: snapshotResult.nextCursor, - facts: input.includeFacts - ? { - rows: factsResult.facts, - nextCursor: factsResult.nextCursor - } - : null, + facts: null, coverage: { - filings: periods.length, - rows: faithfulRows.length, - dimensions: dimensionsCount, - facts: factsCoverage + filings: basePeriods.length, + rows: kpiBundle.kpiRows?.length ?? 0, + dimensions: kpiBreakdown ? Object.values(kpiBreakdown).reduce((sum, rows) => sum + rows.length, 0) : 0, + facts: allFacts.facts.length }, dataSourceStatus: { enabled: input.v3Enabled, hydratedFilings: statuses.ready, partialFilings: statuses.partial, failedFilings: statuses.failed, - pendingFilings: Math.max(0, financialFilings.length - statuses.ready - statuses.partial - statuses.failed), + pendingFilings: Math.max(0, financialFilings.filter((filing) => filingTypes.includes(filing.filing_type as '10-K' | '10-Q')).length - statuses.ready - statuses.partial - statuses.failed), queuedSync: input.queuedSync }, - overviewMetrics: buildOverviewMetrics({ - periods, - faithfulRows, - standardizedRows - }), metrics, - dimensionBreakdown + dimensionBreakdown: mergeDimensionBreakdownMaps(kpiBreakdown) }; } +export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialsInput) { + return await getCompanyFinancials(input); +} + export const __financialTaxonomyInternals = { - buildPeriods, buildRows, buildStandardizedRows, buildDimensionBreakdown, - isInstantPeriod, - matchesDefinition, - periodDurationDays, - selectPrimaryPeriods + periodSorter, + selectPrimaryPeriodsByCadence, + buildLtmPeriods, + buildLtmFaithfulRows, + buildLtmStandardizedRows }; diff --git a/lib/server/financials/bundles.ts b/lib/server/financials/bundles.ts new file mode 100644 index 0000000..b3d0d17 --- /dev/null +++ b/lib/server/financials/bundles.ts @@ -0,0 +1,53 @@ +import type { + FinancialCadence, + FinancialSurfaceKind +} from '@/lib/types'; +import { + getCompanyFinancialBundle, + upsertCompanyFinancialBundle +} from '@/lib/server/repos/company-financial-bundles'; +import type { FilingTaxonomySnapshotRecord } from '@/lib/server/repos/filing-taxonomy'; + +export function computeSourceSignature(snapshots: FilingTaxonomySnapshotRecord[]) { + return snapshots + .map((snapshot) => `${snapshot.id}:${snapshot.updated_at}`) + .sort((left, right) => left.localeCompare(right)) + .join('|'); +} + +export async function readCachedFinancialBundle(input: { + ticker: string; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; + snapshots: FilingTaxonomySnapshotRecord[]; +}) { + const sourceSignature = computeSourceSignature(input.snapshots); + const cached = await getCompanyFinancialBundle({ + ticker: input.ticker, + surfaceKind: input.surfaceKind, + cadence: input.cadence + }); + + if (!cached || cached.source_signature !== sourceSignature) { + return null; + } + + return cached.payload; +} + +export async function writeFinancialBundle(input: { + ticker: string; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; + snapshots: FilingTaxonomySnapshotRecord[]; + payload: Record; +}) { + return await upsertCompanyFinancialBundle({ + ticker: input.ticker, + surfaceKind: input.surfaceKind, + cadence: input.cadence, + sourceSnapshotIds: input.snapshots.map((snapshot) => snapshot.id), + sourceSignature: computeSourceSignature(input.snapshots), + payload: input.payload + }); +} diff --git a/lib/server/financials/cadence.ts b/lib/server/financials/cadence.ts new file mode 100644 index 0000000..8045607 --- /dev/null +++ b/lib/server/financials/cadence.ts @@ -0,0 +1,303 @@ +import type { + FinancialCadence, + FinancialStatementKind, + FinancialStatementPeriod, + FinancialSurfaceKind, + TaxonomyStatementRow +} from '@/lib/types'; +import type { FilingTaxonomySnapshotRecord } from '@/lib/server/repos/filing-taxonomy'; + +type PrimaryPeriodSelection = { + periods: FinancialStatementPeriod[]; + selectedPeriodIds: Set; + snapshots: FilingTaxonomySnapshotRecord[]; +}; + +function parseEpoch(value: string | null) { + if (!value) { + return Number.NaN; + } + + return Date.parse(value); +} + +export function periodSorter(left: FinancialStatementPeriod, right: FinancialStatementPeriod) { + const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); + const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); + + if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { + return leftDate - rightDate; + } + + return left.id.localeCompare(right.id); +} + +export function isInstantPeriod(period: FinancialStatementPeriod) { + return period.periodStart === null; +} + +function periodDurationDays(period: FinancialStatementPeriod) { + if (!period.periodStart || !period.periodEnd) { + return null; + } + + const start = Date.parse(period.periodStart); + const end = Date.parse(period.periodEnd); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { + return null; + } + + return Math.round((end - start) / 86_400_000) + 1; +} + +function preferredDurationDays(filingType: FinancialStatementPeriod['filingType']) { + return filingType === '10-K' ? 365 : 90; +} + +function selectPrimaryPeriodFromSnapshot( + snapshot: FilingTaxonomySnapshotRecord, + statement: FinancialStatementKind +) { + const rows = snapshot.statement_rows?.[statement] ?? []; + if (rows.length === 0) { + return null; + } + + const usedPeriodIds = new Set(); + for (const row of rows) { + for (const periodId of Object.keys(row.values)) { + usedPeriodIds.add(periodId); + } + } + + const candidates = (snapshot.periods ?? []).filter((period) => usedPeriodIds.has(period.id)); + if (candidates.length === 0) { + return null; + } + + if (statement === 'balance') { + const instantCandidates = candidates.filter(isInstantPeriod); + return (instantCandidates.length > 0 ? instantCandidates : candidates) + .sort((left, right) => periodSorter(right, left))[0] ?? null; + } + + const durationCandidates = candidates.filter((period) => !isInstantPeriod(period)); + if (durationCandidates.length === 0) { + return candidates.sort((left, right) => periodSorter(right, left))[0] ?? null; + } + + const targetDays = preferredDurationDays(snapshot.filing_type); + return durationCandidates.sort((left, right) => { + const leftDate = parseEpoch(left.periodEnd ?? left.filingDate); + const rightDate = parseEpoch(right.periodEnd ?? right.filingDate); + if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) { + return rightDate - leftDate; + } + + const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays); + const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays); + if (leftDistance !== rightDistance) { + return leftDistance - rightDistance; + } + + return left.id.localeCompare(right.id); + })[0] ?? null; +} + +function filingTypeForCadence(cadence: FinancialCadence) { + return cadence === 'annual' ? '10-K' : '10-Q'; +} + +export function surfaceToStatementKind(surfaceKind: FinancialSurfaceKind): FinancialStatementKind | null { + switch (surfaceKind) { + case 'income_statement': + return 'income'; + case 'balance_sheet': + return 'balance'; + case 'cash_flow_statement': + return 'cash_flow'; + default: + return null; + } +} + +export function isStatementSurface(surfaceKind: FinancialSurfaceKind) { + return surfaceToStatementKind(surfaceKind) !== null; +} + +export function selectPrimaryPeriodsByCadence( + snapshots: FilingTaxonomySnapshotRecord[], + statement: FinancialStatementKind, + cadence: FinancialCadence +): PrimaryPeriodSelection { + const filingType = filingTypeForCadence(cadence); + const filteredSnapshots = snapshots.filter((snapshot) => snapshot.filing_type === filingType); + + const selected = filteredSnapshots + .map((snapshot) => ({ + snapshot, + period: selectPrimaryPeriodFromSnapshot(snapshot, statement) + })) + .filter((entry): entry is { snapshot: FilingTaxonomySnapshotRecord; period: FinancialStatementPeriod } => entry.period !== null) + .sort((left, right) => periodSorter(left.period, right.period)); + + const periods = selected.map((entry) => entry.period); + return { + periods, + selectedPeriodIds: new Set(periods.map((period) => period.id)), + snapshots: selected.map((entry) => entry.snapshot) + }; +} + +export function buildRows( + snapshots: FilingTaxonomySnapshotRecord[], + statement: FinancialStatementKind, + selectedPeriodIds: Set +) { + const rowMap = new Map(); + + for (const snapshot of snapshots) { + const rows = snapshot.statement_rows?.[statement] ?? []; + + for (const row of rows) { + const existing = rowMap.get(row.key); + if (!existing) { + rowMap.set(row.key, { + ...row, + values: Object.fromEntries( + Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId)) + ), + units: Object.fromEntries( + Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId)) + ), + sourceFactIds: [...row.sourceFactIds] + }); + + if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) { + rowMap.delete(row.key); + } + continue; + } + + existing.hasDimensions = existing.hasDimensions || row.hasDimensions; + existing.order = Math.min(existing.order, row.order); + existing.depth = Math.min(existing.depth, row.depth); + if (!existing.parentKey && row.parentKey) { + existing.parentKey = row.parentKey; + } + + for (const [periodId, value] of Object.entries(row.values)) { + if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) { + existing.values[periodId] = value; + } + } + + for (const [periodId, unit] of Object.entries(row.units)) { + if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) { + existing.units[periodId] = unit; + } + } + + for (const factId of row.sourceFactIds) { + if (!existing.sourceFactIds.includes(factId)) { + existing.sourceFactIds.push(factId); + } + } + } + } + + return [...rowMap.values()].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); +} + +function canBuildRollingFour(periods: FinancialStatementPeriod[]) { + if (periods.length < 4) { + return false; + } + + const sorted = [...periods].sort(periodSorter); + const first = sorted[0]; + const last = sorted[sorted.length - 1]; + const spanDays = Math.round((Date.parse(last.periodEnd ?? last.filingDate) - Date.parse(first.periodEnd ?? first.filingDate)) / 86_400_000); + return spanDays >= 250 && spanDays <= 460; +} + +function aggregateValues(values: Array) { + if (values.some((value) => value === null)) { + return null; + } + + return values.reduce((sum, value) => sum + (value ?? 0), 0); +} + +export function buildLtmPeriods(periods: FinancialStatementPeriod[]) { + const sorted = [...periods].sort(periodSorter); + const windows: FinancialStatementPeriod[] = []; + + for (let index = 3; index < sorted.length; index += 1) { + const slice = sorted.slice(index - 3, index + 1); + if (!canBuildRollingFour(slice)) { + continue; + } + + const last = slice[slice.length - 1]; + windows.push({ + ...last, + id: `ltm:${last.id}`, + periodStart: slice[0]?.periodStart ?? null, + periodEnd: last.periodEnd, + periodLabel: `LTM ending ${last.periodEnd ?? last.filingDate}` + }); + } + + return windows; +} + +export function buildLtmFaithfulRows( + quarterlyRows: TaxonomyStatementRow[], + quarterlyPeriods: FinancialStatementPeriod[], + ltmPeriods: FinancialStatementPeriod[], + statement: FinancialStatementKind +) { + const sourceRows = new Map(quarterlyRows.map((row) => [row.key, row])); + const rowMap = new Map(); + const sortedQuarterlyPeriods = [...quarterlyPeriods].sort(periodSorter); + + for (const row of quarterlyRows) { + rowMap.set(row.key, { + ...row, + values: {}, + units: {} + }); + } + + for (const ltmPeriod of ltmPeriods) { + const anchorIndex = sortedQuarterlyPeriods.findIndex((period) => `ltm:${period.id}` === ltmPeriod.id); + if (anchorIndex < 3) { + continue; + } + + const slice = sortedQuarterlyPeriods.slice(anchorIndex - 3, anchorIndex + 1); + for (const row of rowMap.values()) { + const sourceRow = sourceRows.get(row.key); + if (!sourceRow) { + continue; + } + + const sourceValues = slice.map((period) => sourceRow.values[period.id] ?? null); + const sourceUnits = slice.map((period) => sourceRow.units[period.id] ?? null).filter((unit): unit is string => unit !== null); + + row.values[ltmPeriod.id] = statement === 'balance' + ? sourceValues[sourceValues.length - 1] ?? null + : aggregateValues(sourceValues); + row.units[ltmPeriod.id] = sourceUnits[sourceUnits.length - 1] ?? null; + } + } + + return [...rowMap.values()].filter((row) => Object.keys(row.values).length > 0); +} diff --git a/lib/server/financials/canonical-definitions.ts b/lib/server/financials/canonical-definitions.ts new file mode 100644 index 0000000..6c603ae --- /dev/null +++ b/lib/server/financials/canonical-definitions.ts @@ -0,0 +1,85 @@ +import type { + FinancialStatementKind, + FinancialUnit +} from '@/lib/types'; + +export type CanonicalRowDefinition = { + key: string; + label: string; + category: string; + order: number; + unit: FinancialUnit; + localNames?: readonly string[]; + labelIncludes?: readonly string[]; +}; + +const INCOME_DEFINITIONS: CanonicalRowDefinition[] = [ + { key: 'revenue', label: 'Revenue', category: 'revenue', order: 10, unit: 'currency', localNames: ['RevenueFromContractWithCustomerExcludingAssessedTax', 'Revenues', 'SalesRevenueNet', 'TotalRevenuesAndOtherIncome'], labelIncludes: ['revenue', 'sales'] }, + { key: 'cost_of_revenue', label: 'Cost of Revenue', category: 'expense', order: 20, unit: 'currency', localNames: ['CostOfRevenue', 'CostOfGoodsSold', 'CostOfSales', 'CostOfProductsSold', 'CostOfServices'], labelIncludes: ['cost of revenue', 'cost of sales', 'cost of goods sold'] }, + { key: 'gross_profit', label: 'Gross Profit', category: 'profit', order: 30, unit: 'currency', localNames: ['GrossProfit'] }, + { key: 'gross_margin', label: 'Gross Margin', category: 'margin', order: 40, unit: 'percent', labelIncludes: ['gross margin'] }, + { key: 'research_and_development', label: 'Research & Development', category: 'opex', order: 50, unit: 'currency', localNames: ['ResearchAndDevelopmentExpense'], labelIncludes: ['research and development'] }, + { key: 'sales_and_marketing', label: 'Sales & Marketing', category: 'opex', order: 60, unit: 'currency', localNames: ['SellingAndMarketingExpense'], labelIncludes: ['sales and marketing', 'selling and marketing'] }, + { key: 'general_and_administrative', label: 'General & Administrative', category: 'opex', order: 70, unit: 'currency', localNames: ['GeneralAndAdministrativeExpense'], labelIncludes: ['general and administrative'] }, + { key: 'selling_general_and_administrative', label: 'Selling, General & Administrative', category: 'opex', order: 80, unit: 'currency', localNames: ['SellingGeneralAndAdministrativeExpense'], labelIncludes: ['selling, general', 'selling general'] }, + { key: 'operating_income', label: 'Operating Income', category: 'profit', order: 90, unit: 'currency', localNames: ['OperatingIncomeLoss', 'IncomeLossFromOperations'], labelIncludes: ['operating income'] }, + { key: 'operating_margin', label: 'Operating Margin', category: 'margin', order: 100, unit: 'percent', labelIncludes: ['operating margin'] }, + { key: 'interest_income', label: 'Interest Income', category: 'non_operating', order: 110, unit: 'currency', localNames: ['InterestIncomeExpenseNonoperatingNet', 'InterestIncomeOther', 'InvestmentIncomeInterest'], labelIncludes: ['interest income'] }, + { key: 'interest_expense', label: 'Interest Expense', category: 'non_operating', order: 120, unit: 'currency', localNames: ['InterestExpense', 'InterestAndDebtExpense'], labelIncludes: ['interest expense'] }, + { key: 'other_non_operating_income', label: 'Other Non-Operating Income', category: 'non_operating', order: 130, unit: 'currency', localNames: ['OtherNonoperatingIncomeExpense', 'NonoperatingIncomeExpense'], labelIncludes: ['other non-operating', 'non-operating income'] }, + { key: 'pretax_income', label: 'Pretax Income', category: 'profit', order: 140, unit: 'currency', localNames: ['IncomeBeforeTaxExpenseBenefit', 'PretaxIncome'], labelIncludes: ['income before taxes', 'pretax income'] }, + { key: 'income_tax_expense', label: 'Income Tax Expense', category: 'tax', order: 150, unit: 'currency', localNames: ['IncomeTaxExpenseBenefit'], labelIncludes: ['income tax'] }, + { key: 'effective_tax_rate', label: 'Effective Tax Rate', category: 'tax', order: 160, unit: 'percent', labelIncludes: ['effective tax rate'] }, + { key: 'net_income', label: 'Net Income', category: 'profit', order: 170, unit: 'currency', localNames: ['NetIncomeLoss', 'ProfitLoss'], labelIncludes: ['net income'] }, + { key: 'diluted_eps', label: 'Diluted EPS', category: 'per_share', order: 180, unit: 'currency', localNames: ['EarningsPerShareDiluted', 'DilutedEarningsPerShare'], labelIncludes: ['diluted eps', 'diluted earnings per share'] }, + { key: 'basic_eps', label: 'Basic EPS', category: 'per_share', order: 190, unit: 'currency', localNames: ['EarningsPerShareBasic', 'BasicEarningsPerShare'], labelIncludes: ['basic eps', 'basic earnings per share'] }, + { key: 'diluted_shares', label: 'Diluted Shares', category: 'shares', order: 200, unit: 'shares', localNames: ['WeightedAverageNumberOfDilutedSharesOutstanding', 'WeightedAverageNumberOfShareOutstandingDiluted'], labelIncludes: ['diluted shares', 'weighted average diluted'] }, + { key: 'basic_shares', label: 'Basic Shares', category: 'shares', order: 210, unit: 'shares', localNames: ['WeightedAverageNumberOfSharesOutstandingBasic', 'WeightedAverageNumberOfShareOutstandingBasicAndDiluted'], labelIncludes: ['basic shares', 'weighted average basic'] }, + { key: 'depreciation_and_amortization', label: 'Depreciation & Amortization', category: 'non_cash', order: 220, unit: 'currency', localNames: ['DepreciationDepletionAndAmortization', 'DepreciationAmortizationAndAccretionNet', 'DepreciationAndAmortization'], labelIncludes: ['depreciation', 'amortization'] }, + { key: 'ebitda', label: 'EBITDA', category: 'profit', order: 230, unit: 'currency', localNames: ['OperatingIncomeLoss'], labelIncludes: ['ebitda'] }, + { key: 'stock_based_compensation', label: 'Stock-Based Compensation', category: 'non_cash', order: 240, unit: 'currency', localNames: ['ShareBasedCompensation', 'AllocatedShareBasedCompensationExpense'], labelIncludes: ['stock-based compensation', 'share-based compensation'] } +]; + +const BALANCE_DEFINITIONS: CanonicalRowDefinition[] = [ + { key: 'cash_and_equivalents', label: 'Cash & Equivalents', category: 'asset', order: 10, unit: 'currency', localNames: ['CashAndCashEquivalentsAtCarryingValue', 'CashCashEquivalentsAndShortTermInvestments', 'CashAndShortTermInvestments'], labelIncludes: ['cash and cash equivalents'] }, + { key: 'short_term_investments', label: 'Short-Term Investments', category: 'asset', order: 20, unit: 'currency', localNames: ['AvailableForSaleSecuritiesCurrent', 'ShortTermInvestments'], labelIncludes: ['short-term investments', 'marketable securities'] }, + { key: 'accounts_receivable', label: 'Accounts Receivable', category: 'asset', order: 30, unit: 'currency', localNames: ['AccountsReceivableNetCurrent', 'ReceivablesNetCurrent'], labelIncludes: ['accounts receivable'] }, + { key: 'inventory', label: 'Inventory', category: 'asset', order: 40, unit: 'currency', localNames: ['InventoryNet'], labelIncludes: ['inventory'] }, + { key: 'other_current_assets', label: 'Other Current Assets', category: 'asset', order: 50, unit: 'currency', localNames: ['OtherAssetsCurrent'], labelIncludes: ['other current assets'] }, + { key: 'current_assets', label: 'Current Assets', category: 'asset', order: 60, unit: 'currency', localNames: ['AssetsCurrent'], labelIncludes: ['current assets'] }, + { key: 'property_plant_equipment', label: 'Property, Plant & Equipment', category: 'asset', order: 70, unit: 'currency', localNames: ['PropertyPlantAndEquipmentNet'], labelIncludes: ['property, plant and equipment', 'property and equipment'] }, + { key: 'goodwill', label: 'Goodwill', category: 'asset', order: 80, unit: 'currency', localNames: ['Goodwill'], labelIncludes: ['goodwill'] }, + { key: 'intangible_assets', label: 'Intangible Assets', category: 'asset', order: 90, unit: 'currency', localNames: ['FiniteLivedIntangibleAssetsNet', 'IndefiniteLivedIntangibleAssetsExcludingGoodwill', 'IntangibleAssetsNetExcludingGoodwill'], labelIncludes: ['intangible assets'] }, + { key: 'total_assets', label: 'Total Assets', category: 'asset', order: 100, unit: 'currency', localNames: ['Assets'], labelIncludes: ['total assets'] }, + { key: 'accounts_payable', label: 'Accounts Payable', category: 'liability', order: 110, unit: 'currency', localNames: ['AccountsPayableCurrent'], labelIncludes: ['accounts payable'] }, + { key: 'accrued_liabilities', label: 'Accrued Liabilities', category: 'liability', order: 120, unit: 'currency', localNames: ['AccruedLiabilitiesCurrent'], labelIncludes: ['accrued liabilities'] }, + { key: 'deferred_revenue_current', label: 'Deferred Revenue, Current', category: 'liability', order: 130, unit: 'currency', localNames: ['ContractWithCustomerLiabilityCurrent', 'DeferredRevenueCurrent'], labelIncludes: ['deferred revenue current', 'current deferred revenue'] }, + { key: 'current_liabilities', label: 'Current Liabilities', category: 'liability', order: 140, unit: 'currency', localNames: ['LiabilitiesCurrent'], labelIncludes: ['current liabilities'] }, + { key: 'long_term_debt', label: 'Long-Term Debt', category: 'liability', order: 150, unit: 'currency', localNames: ['LongTermDebtNoncurrent', 'LongTermDebt', 'DebtNoncurrent', 'LongTermDebtAndCapitalLeaseObligations'], labelIncludes: ['long-term debt'] }, + { key: 'current_debt', label: 'Current Debt', category: 'liability', order: 160, unit: 'currency', localNames: ['DebtCurrent', 'ShortTermBorrowings', 'LongTermDebtCurrent'], labelIncludes: ['current debt', 'short-term debt'] }, + { key: 'lease_liabilities', label: 'Lease Liabilities', category: 'liability', order: 170, unit: 'currency', localNames: ['OperatingLeaseLiabilityNoncurrent', 'FinanceLeaseLiabilityNoncurrent', 'LesseeOperatingLeaseLiability'], labelIncludes: ['lease liabilities'] }, + { key: 'total_debt', label: 'Total Debt', category: 'liability', order: 180, unit: 'currency', localNames: ['DebtAndFinanceLeaseLiabilities', 'Debt'], labelIncludes: ['total debt'] }, + { key: 'deferred_revenue_noncurrent', label: 'Deferred Revenue, Noncurrent', category: 'liability', order: 190, unit: 'currency', localNames: ['ContractWithCustomerLiabilityNoncurrent', 'DeferredRevenueNoncurrent'], labelIncludes: ['deferred revenue noncurrent'] }, + { key: 'total_liabilities', label: 'Total Liabilities', category: 'liability', order: 200, unit: 'currency', localNames: ['Liabilities'], labelIncludes: ['total liabilities'] }, + { key: 'retained_earnings', label: 'Retained Earnings', category: 'equity', order: 210, unit: 'currency', localNames: ['RetainedEarningsAccumulatedDeficit'], labelIncludes: ['retained earnings', 'accumulated deficit'] }, + { key: 'total_equity', label: 'Total Equity', category: 'equity', order: 220, unit: 'currency', localNames: ['StockholdersEquity', 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', 'PartnersCapital'], labelIncludes: ['total equity', 'stockholdersโ€™ equity', 'stockholders equity'] }, + { key: 'net_cash_position', label: 'Net Cash Position', category: 'liquidity', order: 230, unit: 'currency', labelIncludes: ['net cash position'] } +]; + +const CASH_FLOW_DEFINITIONS: CanonicalRowDefinition[] = [ + { key: 'operating_cash_flow', label: 'Operating Cash Flow', category: 'cash_flow', order: 10, unit: 'currency', localNames: ['NetCashProvidedByUsedInOperatingActivities', 'NetCashProvidedByUsedInOperatingActivitiesContinuingOperations'], labelIncludes: ['operating cash flow'] }, + { key: 'capital_expenditures', label: 'Capital Expenditures', category: 'cash_flow', order: 20, unit: 'currency', localNames: ['PaymentsToAcquirePropertyPlantAndEquipment', 'CapitalExpendituresIncurredButNotYetPaid'], labelIncludes: ['capital expenditures', 'capital expenditure'] }, + { key: 'free_cash_flow', label: 'Free Cash Flow', category: 'cash_flow', order: 30, unit: 'currency', labelIncludes: ['free cash flow'] }, + { key: 'stock_based_compensation', label: 'Stock-Based Compensation', category: 'non_cash', order: 40, unit: 'currency', localNames: ['ShareBasedCompensation', 'AllocatedShareBasedCompensationExpense'], labelIncludes: ['stock-based compensation', 'share-based compensation'] }, + { key: 'acquisitions', label: 'Acquisitions', category: 'investing', order: 50, unit: 'currency', localNames: ['PaymentsToAcquireBusinessesNetOfCashAcquired'], labelIncludes: ['acquisitions'] }, + { key: 'share_repurchases', label: 'Share Repurchases', category: 'financing', order: 60, unit: 'currency', localNames: ['PaymentsForRepurchaseOfCommonStock', 'PaymentsForRepurchaseOfEquity'], labelIncludes: ['share repurchases', 'repurchase of common stock'] }, + { key: 'dividends_paid', label: 'Dividends Paid', category: 'financing', order: 70, unit: 'currency', localNames: ['PaymentsOfDividends', 'PaymentsOfDividendsCommonStock'], labelIncludes: ['dividends paid'] }, + { key: 'debt_issued', label: 'Debt Issued', category: 'financing', order: 80, unit: 'currency', localNames: ['ProceedsFromIssuanceOfLongTermDebt'], labelIncludes: ['debt issued'] }, + { key: 'debt_repaid', label: 'Debt Repaid', category: 'financing', order: 90, unit: 'currency', localNames: ['RepaymentsOfLongTermDebt', 'RepaymentsOfDebt'], labelIncludes: ['debt repaid', 'repayment of debt'] } +]; + +export const CANONICAL_ROW_DEFINITIONS: Record, CanonicalRowDefinition[]> = { + income: INCOME_DEFINITIONS, + balance: BALANCE_DEFINITIONS, + cash_flow: CASH_FLOW_DEFINITIONS +}; diff --git a/lib/server/financials/kpi-dimensions.test.ts b/lib/server/financials/kpi-dimensions.test.ts new file mode 100644 index 0000000..645b03d --- /dev/null +++ b/lib/server/financials/kpi-dimensions.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'bun:test'; +import type { + FinancialStatementPeriod, + TaxonomyFactRow +} from '@/lib/types'; +import { extractStructuredKpisFromDimensions } from './kpi-dimensions'; + +const PERIOD: FinancialStatementPeriod = { + id: '2025-q4', + filingId: 1, + accessionNumber: '0000-1', + filingDate: '2026-01-31', + periodStart: '2025-10-01', + periodEnd: '2025-12-31', + filingType: '10-Q', + periodLabel: 'Q4 2025' +}; + +const FACT: TaxonomyFactRow = { + id: 10, + snapshotId: 5, + filingId: 1, + filingDate: '2026-01-31', + statement: 'income', + roleUri: 'income', + conceptKey: 'us-gaap:Revenues', + qname: 'us-gaap:Revenues', + namespaceUri: 'http://fasb.org/us-gaap/2024', + localName: 'Revenues', + value: 50000, + contextId: 'ctx-1', + unit: 'iso4217:USD', + decimals: null, + periodStart: '2025-10-01', + periodEnd: '2025-12-31', + periodInstant: null, + dimensions: [{ + axis: 'srt:ProductOrServiceAxis', + member: 'msft:CloudMember' + }], + isDimensionless: false, + sourceFile: null +}; + +describe('dimension KPI extraction', () => { + it('builds stable taxonomy KPI keys and provenance', () => { + const rows = extractStructuredKpisFromDimensions({ + facts: [FACT], + periods: [PERIOD], + definitions: [{ + key: 'segment_revenue', + label: 'Segment Revenue', + category: 'segment_revenue', + unit: 'currency', + preferredConceptNames: ['Revenues'] + }] + }); + + expect(rows).toHaveLength(1); + expect(rows[0]?.key).toBe('segment_revenue__srt_productorserviceaxis__msft_cloudmember'); + expect(rows[0]?.provenanceType).toBe('taxonomy'); + expect(rows[0]?.values['2025-q4']).toBe(50000); + expect(rows[0]?.sourceFactIds).toEqual([10]); + }); +}); diff --git a/lib/server/financials/kpi-dimensions.ts b/lib/server/financials/kpi-dimensions.ts new file mode 100644 index 0000000..dbf0ed2 --- /dev/null +++ b/lib/server/financials/kpi-dimensions.ts @@ -0,0 +1,159 @@ +import type { + FinancialStatementPeriod, + StructuredKpiRow, + TaxonomyFactRow +} from '@/lib/types'; +import type { KpiDefinition } from '@/lib/server/financials/kpi-registry'; +import { factMatchesPeriod } from '@/lib/server/financials/standardize'; + +function normalizeSegmentToken(value: string) { + return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, ''); +} + +function humanizeMember(value: string) { + const source = value.split(':').pop() ?? value; + return source + .replace(/Member$/i, '') + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/_/g, ' ') + .trim(); +} + +function factMatchesDefinition(fact: TaxonomyFactRow, definition: KpiDefinition) { + if (definition.preferredConceptNames && !definition.preferredConceptNames.includes(fact.localName)) { + return false; + } + + if (!definition.preferredAxisIncludes || definition.preferredAxisIncludes.length === 0) { + return fact.dimensions.length > 0; + } + + return fact.dimensions.some((dimension) => { + const axisMatch = definition.preferredAxisIncludes?.some((token) => dimension.axis.toLowerCase().includes(token.toLowerCase())) ?? false; + const memberMatch = definition.preferredMemberIncludes && definition.preferredMemberIncludes.length > 0 + ? definition.preferredMemberIncludes.some((token) => dimension.member.toLowerCase().includes(token.toLowerCase())) + : true; + return axisMatch && memberMatch; + }); +} + +function categoryForDefinition(definition: KpiDefinition, axis: string) { + if (definition.key === 'segment_revenue' && /geo|country|region|area/i.test(axis)) { + return 'geographic_mix'; + } + + return definition.category; +} + +export function extractStructuredKpisFromDimensions(input: { + facts: TaxonomyFactRow[]; + periods: FinancialStatementPeriod[]; + definitions: KpiDefinition[]; +}) { + const rowMap = new Map(); + const orderByKey = new Map(); + + input.definitions.forEach((definition, index) => { + orderByKey.set(definition.key, (index + 1) * 10); + }); + + for (const definition of input.definitions) { + for (const fact of input.facts) { + if (fact.dimensions.length === 0 || !factMatchesDefinition(fact, definition)) { + continue; + } + + const matchedPeriod = input.periods.find((period) => period.filingId === fact.filingId && factMatchesPeriod(fact, period)); + if (!matchedPeriod) { + continue; + } + + for (const dimension of fact.dimensions) { + const axis = dimension.axis; + const member = dimension.member; + const normalizedAxis = normalizeSegmentToken(axis); + const normalizedMember = normalizeSegmentToken(member); + const key = `${definition.key}__${normalizedAxis}__${normalizedMember}`; + const labelSuffix = humanizeMember(member); + const existing = rowMap.get(key); + + if (existing) { + existing.values[matchedPeriod.id] = fact.value; + if (!existing.sourceConcepts.includes(fact.qname)) { + existing.sourceConcepts.push(fact.qname); + } + if (!existing.sourceFactIds.includes(fact.id)) { + existing.sourceFactIds.push(fact.id); + } + continue; + } + + rowMap.set(key, { + key, + label: `${definition.label} - ${labelSuffix}`, + category: categoryForDefinition(definition, axis), + unit: definition.unit, + order: orderByKey.get(definition.key) ?? 999, + segment: labelSuffix || null, + axis, + member, + values: { [matchedPeriod.id]: fact.value }, + sourceConcepts: [fact.qname], + sourceFactIds: [fact.id], + provenanceType: 'taxonomy', + hasDimensions: true + }); + } + } + } + + const rows = [...rowMap.values()].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); + + const marginRows = new Map(); + for (const row of rows.filter((entry) => entry.category === 'segment_profit')) { + const revenueKey = row.key.replace(/^segment_profit__/, 'segment_revenue__'); + const revenueRow = rowMap.get(revenueKey); + if (!revenueRow) { + continue; + } + + const values: Record = {}; + for (const period of input.periods) { + const revenue = revenueRow.values[period.id] ?? null; + const profit = row.values[period.id] ?? null; + values[period.id] = revenue === null || profit === null || revenue === 0 + ? null + : profit / revenue; + } + + marginRows.set(row.key.replace(/^segment_profit__/, 'segment_margin__'), { + key: row.key.replace(/^segment_profit__/, 'segment_margin__'), + label: row.label.replace(/^Segment Profit/, 'Segment Margin'), + category: 'segment_margin', + unit: 'percent', + order: 25, + segment: row.segment, + axis: row.axis, + member: row.member, + values, + sourceConcepts: [...new Set([...row.sourceConcepts, ...revenueRow.sourceConcepts])], + sourceFactIds: [...new Set([...row.sourceFactIds, ...revenueRow.sourceFactIds])], + provenanceType: 'taxonomy', + hasDimensions: true + }); + } + + return [...rows, ...marginRows.values()].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); +} diff --git a/lib/server/financials/kpi-notes.ts b/lib/server/financials/kpi-notes.ts new file mode 100644 index 0000000..d081cf3 --- /dev/null +++ b/lib/server/financials/kpi-notes.ts @@ -0,0 +1,131 @@ +import { load } from 'cheerio'; +import type { + FinancialStatementPeriod, + StructuredKpiRow +} from '@/lib/types'; +import { resolvePrimaryFilingUrl } from '@/lib/server/sec'; +import type { KpiDefinition } from '@/lib/server/financials/kpi-registry'; + +type FilingDocumentRef = { + filingId: number; + cik: string; + accessionNumber: string; + filingUrl: string | null; + primaryDocument: string | null; +}; + +function parseNumericCell(value: string) { + const normalized = value.replace(/[$,%]/g, '').replace(/[(),]/g, '').trim(); + if (!normalized) { + return null; + } + + const numeric = Number(normalized); + return Number.isFinite(numeric) ? numeric : null; +} + +function buildRowKey(definition: KpiDefinition, label: string) { + const normalized = label.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, ''); + return normalized ? `${definition.key}__note__${normalized}` : definition.key; +} + +async function fetchHtml(ref: FilingDocumentRef) { + const url = resolvePrimaryFilingUrl({ + filingUrl: ref.filingUrl, + cik: ref.cik, + accessionNumber: ref.accessionNumber, + primaryDocument: ref.primaryDocument + }); + + if (!url) { + return null; + } + + try { + const response = await fetch(url, { + headers: { + 'User-Agent': process.env.SEC_USER_AGENT || 'Fiscal Clone ' + }, + cache: 'no-store' + }); + + if (!response.ok) { + return null; + } + + return await response.text(); + } catch { + return null; + } +} + +export async function extractStructuredKpisFromNotes(input: { + ticker: string; + periods: FinancialStatementPeriod[]; + filings: FilingDocumentRef[]; + definitions: KpiDefinition[]; +}) { + const rows = new Map(); + + for (const definition of input.definitions) { + if (!definition.noteLabelIncludes || definition.noteLabelIncludes.length === 0) { + continue; + } + + for (const period of input.periods) { + const filing = input.filings.find((entry) => entry.filingId === period.filingId); + if (!filing) { + continue; + } + + const html = await fetchHtml(filing); + if (!html) { + continue; + } + + const $ = load(html); + $('table tr').each((_index, element) => { + const cells = $(element).find('th,td').toArray().map((node) => $(node).text().replace(/\s+/g, ' ').trim()).filter(Boolean); + if (cells.length < 2) { + return; + } + + const label = cells[0] ?? ''; + const normalizedLabel = label.toLowerCase(); + if (!definition.noteLabelIncludes?.some((token) => normalizedLabel.includes(token.toLowerCase()))) { + return; + } + + const numericCell = cells.slice(1).map(parseNumericCell).find((value) => value !== null) ?? null; + if (numericCell === null) { + return; + } + + const key = buildRowKey(definition, label === definition.label ? '' : label); + const existing = rows.get(key); + if (existing) { + existing.values[period.id] = numericCell; + return; + } + + rows.set(key, { + key, + label: label || definition.label, + category: definition.category, + unit: definition.unit, + order: 500, + segment: null, + axis: null, + member: null, + values: { [period.id]: numericCell }, + sourceConcepts: [], + sourceFactIds: [], + provenanceType: 'structured_note', + hasDimensions: false + }); + }); + } + } + + return [...rows.values()].sort((left, right) => left.label.localeCompare(right.label)); +} diff --git a/lib/server/financials/kpi-registry.ts b/lib/server/financials/kpi-registry.ts new file mode 100644 index 0000000..3f92c17 --- /dev/null +++ b/lib/server/financials/kpi-registry.ts @@ -0,0 +1,120 @@ +import type { + FinancialCategory, + FinancialUnit +} from '@/lib/types'; + +export type IndustryTemplate = + | 'internet_platforms' + | 'software_saas' + | 'semiconductors_industrial_auto'; + +export type KpiDefinition = { + key: string; + label: string; + category: FinancialCategory; + unit: FinancialUnit; + preferredConceptNames?: string[]; + preferredAxisIncludes?: string[]; + preferredMemberIncludes?: string[]; + noteLabelIncludes?: string[]; +}; + +type RegistryBundle = { + tickerTemplates: Record; + globalDefinitions: KpiDefinition[]; + industryDefinitions: Record; + tickerDefinitions: Record; +}; + +const KPI_REGISTRY: RegistryBundle = { + tickerTemplates: { + GOOG: 'internet_platforms', + META: 'internet_platforms', + NFLX: 'internet_platforms', + MSFT: 'software_saas', + CRM: 'software_saas', + NOW: 'software_saas', + NVDA: 'semiconductors_industrial_auto', + TSLA: 'semiconductors_industrial_auto', + CAT: 'semiconductors_industrial_auto' + }, + globalDefinitions: [ + { + key: 'segment_revenue', + label: 'Segment Revenue', + category: 'segment_revenue', + unit: 'currency', + preferredConceptNames: ['Revenues', 'RevenueFromContractWithCustomerExcludingAssessedTax', 'SalesRevenueNet'] + }, + { + key: 'segment_profit', + label: 'Segment Profit', + category: 'segment_profit', + unit: 'currency', + preferredConceptNames: ['OperatingIncomeLoss', 'SegmentProfitLoss'] + } + ], + industryDefinitions: { + internet_platforms: [ + { key: 'tac', label: 'TAC', category: 'operating_kpi', unit: 'currency', preferredConceptNames: ['TrafficAcquisitionCosts'], noteLabelIncludes: ['traffic acquisition costs', 'tac'] }, + { key: 'paid_clicks', label: 'Paid Clicks', category: 'operating_kpi', unit: 'count', noteLabelIncludes: ['paid clicks'] }, + { key: 'cpc', label: 'CPC', category: 'operating_kpi', unit: 'currency', noteLabelIncludes: ['cost per click', 'cpc'] }, + { key: 'dau', label: 'DAU', category: 'user_metric', unit: 'count', preferredConceptNames: ['DailyActiveUsers'], noteLabelIncludes: ['daily active users', 'dau'] }, + { key: 'mau', label: 'MAU', category: 'user_metric', unit: 'count', preferredConceptNames: ['MonthlyActiveUsers'], noteLabelIncludes: ['monthly active users', 'mau'] }, + { key: 'arpu', label: 'ARPU', category: 'operating_kpi', unit: 'currency', noteLabelIncludes: ['average revenue per user', 'arpu'] }, + { key: 'arpp', label: 'ARPP', category: 'operating_kpi', unit: 'currency', noteLabelIncludes: ['average revenue per paying user', 'arpp'] }, + { key: 'backlog', label: 'Backlog', category: 'backlog', unit: 'currency', noteLabelIncludes: ['backlog'] } + ], + software_saas: [ + { key: 'arr', label: 'ARR', category: 'operating_kpi', unit: 'currency', noteLabelIncludes: ['annual recurring revenue', 'arr'] }, + { key: 'rpo', label: 'RPO', category: 'backlog', unit: 'currency', preferredConceptNames: ['RemainingPerformanceObligation'], noteLabelIncludes: ['remaining performance obligations', 'rpo'] }, + { key: 'remaining_performance_obligations', label: 'Remaining Performance Obligations', category: 'backlog', unit: 'currency', preferredConceptNames: ['RemainingPerformanceObligation'], noteLabelIncludes: ['remaining performance obligations'] }, + { key: 'large_customer_count', label: 'Large Customer Count', category: 'operating_kpi', unit: 'count', noteLabelIncludes: ['customers contributing', 'large customers'] } + ], + semiconductors_industrial_auto: [ + { key: 'deliveries', label: 'Deliveries', category: 'operating_kpi', unit: 'count', noteLabelIncludes: ['deliveries'] }, + { key: 'utilization', label: 'Utilization', category: 'operating_kpi', unit: 'percent', noteLabelIncludes: ['utilization'] }, + { key: 'backlog', label: 'Backlog', category: 'backlog', unit: 'currency', noteLabelIncludes: ['backlog'] } + ] + }, + tickerDefinitions: {} +}; + +export function getTickerIndustryTemplate(ticker: string): IndustryTemplate | null { + return KPI_REGISTRY.tickerTemplates[ticker.trim().toUpperCase()] ?? null; +} + +export function resolveKpiDefinitions(ticker: string) { + const normalizedTicker = ticker.trim().toUpperCase(); + const template = getTickerIndustryTemplate(normalizedTicker); + + const definitionsByKey = new Map(); + for (const definition of KPI_REGISTRY.globalDefinitions) { + definitionsByKey.set(definition.key, definition); + } + if (template) { + for (const definition of KPI_REGISTRY.industryDefinitions[template]) { + definitionsByKey.set(definition.key, definition); + } + } + for (const definition of KPI_REGISTRY.tickerDefinitions[normalizedTicker] ?? []) { + definitionsByKey.set(definition.key, definition); + } + + return { + template, + definitions: [...definitionsByKey.values()] + }; +} + +export const KPI_CATEGORY_ORDER = [ + 'segment_revenue', + 'segment_profit', + 'segment_margin', + 'operating_kpi', + 'geographic_mix', + 'capital_returns', + 'backlog', + 'user_metric', + 'other' +] as const; diff --git a/lib/server/financials/ratios.test.ts b/lib/server/financials/ratios.test.ts new file mode 100644 index 0000000..6870321 --- /dev/null +++ b/lib/server/financials/ratios.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'bun:test'; +import type { + FinancialCadence, + FinancialStatementPeriod, + StandardizedFinancialRow +} from '@/lib/types'; +import { buildRatioRows } from './ratios'; + +function createPeriod(id: string, filingId: number, filingDate: string, periodEnd: string): FinancialStatementPeriod { + return { + id, + filingId, + accessionNumber: `0000-${filingId}`, + filingDate, + periodStart: '2025-01-01', + periodEnd, + filingType: '10-Q', + periodLabel: id + }; +} + +function createRow(key: string, values: Record): StandardizedFinancialRow { + return { + key, + label: key, + category: 'test', + order: 10, + unit: 'currency', + values, + sourceConcepts: [`us-gaap:${key}`], + sourceRowKeys: [key], + sourceFactIds: [1], + formulaKey: null, + hasDimensions: false, + resolvedSourceRowKeys: Object.fromEntries(Object.keys(values).map((periodId) => [periodId, key])) + }; +} + +describe('ratio engine', () => { + it('nulls valuation ratios when price data is unavailable', () => { + const periods = [createPeriod('2025-q4', 1, '2026-01-31', '2025-12-31')]; + const rows = { + income: [createRow('revenue', { '2025-q4': 100 }), createRow('diluted_eps', { '2025-q4': 2 }), createRow('ebitda', { '2025-q4': 20 }), createRow('net_income', { '2025-q4': 10 })], + balance: [createRow('total_equity', { '2025-q4': 50 }), createRow('total_debt', { '2025-q4': 30 }), createRow('cash_and_equivalents', { '2025-q4': 5 }), createRow('short_term_investments', { '2025-q4': 5 }), createRow('diluted_shares', { '2025-q4': 10 })], + cashFlow: [createRow('free_cash_flow', { '2025-q4': 12 })] + }; + + const ratioRows = buildRatioRows({ + periods, + cadence: 'quarterly' satisfies FinancialCadence, + rows, + pricesByPeriodId: { '2025-q4': null } + }); + + expect(ratioRows.find((row) => row.key === 'market_cap')?.values['2025-q4']).toBeNull(); + expect(ratioRows.find((row) => row.key === 'price_to_earnings')?.values['2025-q4']).toBeNull(); + expect(ratioRows.find((row) => row.key === 'ev_to_sales')?.values['2025-q4']).toBeNull(); + }); + + it('nulls ratios on zero denominators', () => { + const periods = [ + createPeriod('2024-q4', 1, '2025-01-31', '2024-12-31'), + createPeriod('2025-q4', 2, '2026-01-31', '2025-12-31') + ]; + const rows = { + income: [createRow('net_income', { '2024-q4': 5, '2025-q4': 10 }), createRow('revenue', { '2024-q4': 100, '2025-q4': 120 }), createRow('diluted_eps', { '2024-q4': 1, '2025-q4': 2 }), createRow('ebitda', { '2024-q4': 10, '2025-q4': 12 })], + balance: [createRow('total_equity', { '2024-q4': 0, '2025-q4': 0 }), createRow('total_assets', { '2024-q4': 50, '2025-q4': 60 }), createRow('total_debt', { '2024-q4': 20, '2025-q4': 25 }), createRow('cash_and_equivalents', { '2024-q4': 2, '2025-q4': 3 }), createRow('short_term_investments', { '2024-q4': 1, '2025-q4': 1 }), createRow('current_assets', { '2024-q4': 10, '2025-q4': 12 }), createRow('current_liabilities', { '2024-q4': 0, '2025-q4': 0 }), createRow('diluted_shares', { '2024-q4': 10, '2025-q4': 10 })], + cashFlow: [createRow('free_cash_flow', { '2024-q4': 6, '2025-q4': 7 })] + }; + + const ratioRows = buildRatioRows({ + periods, + cadence: 'quarterly', + rows, + pricesByPeriodId: { '2024-q4': 10, '2025-q4': 12 } + }); + + expect(ratioRows.find((row) => row.key === 'debt_to_equity')?.values['2025-q4']).toBeNull(); + expect(ratioRows.find((row) => row.key === 'current_ratio')?.values['2025-q4']).toBeNull(); + }); +}); diff --git a/lib/server/financials/ratios.ts b/lib/server/financials/ratios.ts new file mode 100644 index 0000000..1e85a08 --- /dev/null +++ b/lib/server/financials/ratios.ts @@ -0,0 +1,369 @@ +import type { + FinancialCadence, + FinancialStatementPeriod, + RatioRow, + StandardizedFinancialRow +} from '@/lib/types'; + +type StatementRowMap = { + income: StandardizedFinancialRow[]; + balance: StandardizedFinancialRow[]; + cashFlow: StandardizedFinancialRow[]; +}; + +type RatioDefinition = { + key: string; + label: string; + category: string; + order: number; + unit: RatioRow['unit']; + denominatorKey: string | null; +}; + +const RATIO_DEFINITIONS: RatioDefinition[] = [ + { key: 'gross_margin', label: 'Gross Margin', category: 'margins', order: 10, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'operating_margin', label: 'Operating Margin', category: 'margins', order: 20, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'ebitda_margin', label: 'EBITDA Margin', category: 'margins', order: 30, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'net_margin', label: 'Net Margin', category: 'margins', order: 40, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'fcf_margin', label: 'FCF Margin', category: 'margins', order: 50, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'roa', label: 'ROA', category: 'returns', order: 60, unit: 'percent', denominatorKey: 'total_assets' }, + { key: 'roe', label: 'ROE', category: 'returns', order: 70, unit: 'percent', denominatorKey: 'total_equity' }, + { key: 'roic', label: 'ROIC', category: 'returns', order: 80, unit: 'percent', denominatorKey: 'average_invested_capital' }, + { key: 'roce', label: 'ROCE', category: 'returns', order: 90, unit: 'percent', denominatorKey: 'average_capital_employed' }, + { key: 'debt_to_equity', label: 'Debt to Equity', category: 'financial_health', order: 100, unit: 'ratio', denominatorKey: 'total_equity' }, + { key: 'net_debt_to_ebitda', label: 'Net Debt to EBITDA', category: 'financial_health', order: 110, unit: 'ratio', denominatorKey: 'ebitda' }, + { key: 'cash_to_debt', label: 'Cash to Debt', category: 'financial_health', order: 120, unit: 'ratio', denominatorKey: 'total_debt' }, + { key: 'current_ratio', label: 'Current Ratio', category: 'financial_health', order: 130, unit: 'ratio', denominatorKey: 'current_liabilities' }, + { key: 'revenue_per_share', label: 'Revenue per Share', category: 'per_share', order: 140, unit: 'currency', denominatorKey: 'diluted_shares' }, + { key: 'fcf_per_share', label: 'FCF per Share', category: 'per_share', order: 150, unit: 'currency', denominatorKey: 'diluted_shares' }, + { key: 'book_value_per_share', label: 'Book Value per Share', category: 'per_share', order: 160, unit: 'currency', denominatorKey: 'diluted_shares' }, + { key: 'revenue_yoy', label: 'Revenue YoY', category: 'growth', order: 170, unit: 'percent', denominatorKey: 'revenue' }, + { key: 'net_income_yoy', label: 'Net Income YoY', category: 'growth', order: 180, unit: 'percent', denominatorKey: 'net_income' }, + { key: 'eps_yoy', label: 'EPS YoY', category: 'growth', order: 190, unit: 'percent', denominatorKey: 'diluted_eps' }, + { key: 'fcf_yoy', label: 'FCF YoY', category: 'growth', order: 200, unit: 'percent', denominatorKey: 'free_cash_flow' }, + { key: '3y_revenue_cagr', label: '3Y Revenue CAGR', category: 'growth', order: 210, unit: 'percent', denominatorKey: 'revenue' }, + { key: '5y_revenue_cagr', label: '5Y Revenue CAGR', category: 'growth', order: 220, unit: 'percent', denominatorKey: 'revenue' }, + { key: '3y_eps_cagr', label: '3Y EPS CAGR', category: 'growth', order: 230, unit: 'percent', denominatorKey: 'diluted_eps' }, + { key: '5y_eps_cagr', label: '5Y EPS CAGR', category: 'growth', order: 240, unit: 'percent', denominatorKey: 'diluted_eps' }, + { key: 'market_cap', label: 'Market Cap', category: 'valuation', order: 250, unit: 'currency', denominatorKey: null }, + { key: 'enterprise_value', label: 'Enterprise Value', category: 'valuation', order: 260, unit: 'currency', denominatorKey: null }, + { key: 'price_to_earnings', label: 'Price to Earnings', category: 'valuation', order: 270, unit: 'ratio', denominatorKey: 'diluted_eps' }, + { key: 'price_to_fcf', label: 'Price to FCF', category: 'valuation', order: 280, unit: 'ratio', denominatorKey: 'free_cash_flow' }, + { key: 'price_to_book', label: 'Price to Book', category: 'valuation', order: 290, unit: 'ratio', denominatorKey: 'total_equity' }, + { key: 'ev_to_sales', label: 'EV to Sales', category: 'valuation', order: 300, unit: 'ratio', denominatorKey: 'revenue' }, + { key: 'ev_to_ebitda', label: 'EV to EBITDA', category: 'valuation', order: 310, unit: 'ratio', denominatorKey: 'ebitda' }, + { key: 'ev_to_fcf', label: 'EV to FCF', category: 'valuation', order: 320, unit: 'ratio', denominatorKey: 'free_cash_flow' } +]; + +function valueFor(row: StandardizedFinancialRow | undefined, periodId: string) { + return row?.values[periodId] ?? null; +} + +function divideValues(left: number | null, right: number | null) { + if (left === null || right === null || right === 0) { + return null; + } + + return left / right; +} + +function averageValues(left: number | null, right: number | null) { + if (left === null || right === null) { + return null; + } + + return (left + right) / 2; +} + +function sumValues(values: Array) { + if (values.some((value) => value === null)) { + return null; + } + + return values.reduce((sum, value) => sum + (value ?? 0), 0); +} + +function subtractValues(left: number | null, right: number | null) { + if (left === null || right === null) { + return null; + } + + return left - right; +} + +function buildRowMap(rows: StatementRowMap) { + return new Map([ + ...rows.income.map((row) => [row.key, row] as const), + ...rows.balance.map((row) => [row.key, row] as const), + ...rows.cashFlow.map((row) => [row.key, row] as const) + ]); +} + +function collectSourceRowData(rowsByKey: Map, keys: string[]) { + const sourceConcepts = new Set(); + const sourceRowKeys = new Set(); + const sourceFactIds = new Set(); + let hasDimensions = false; + + for (const key of keys) { + const row = rowsByKey.get(key); + if (!row) { + continue; + } + + hasDimensions = hasDimensions || row.hasDimensions; + for (const concept of row.sourceConcepts) { + sourceConcepts.add(concept); + } + for (const sourceRowKey of row.sourceRowKeys) { + sourceRowKeys.add(sourceRowKey); + } + for (const sourceFactId of row.sourceFactIds) { + sourceFactIds.add(sourceFactId); + } + } + + return { + sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), + sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), + sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), + hasDimensions + }; +} + +function previousPeriodId(periods: FinancialStatementPeriod[], periodId: string) { + const index = periods.findIndex((period) => period.id === periodId); + return index > 0 ? periods[index - 1]?.id ?? null : null; +} + +function cagr(current: number | null, previous: number | null, years: number) { + if (current === null || previous === null || previous <= 0 || years <= 0) { + return null; + } + + return Math.pow(current / previous, 1 / years) - 1; +} + +export function buildRatioRows(input: { + periods: FinancialStatementPeriod[]; + cadence: FinancialCadence; + rows: StatementRowMap; + pricesByPeriodId: Record; +}) { + const periods = [...input.periods].sort((left, right) => { + return Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate); + }); + const rowsByKey = buildRowMap(input.rows); + + const valuesByKey = new Map>(); + const setValue = (key: string, periodId: string, value: number | null) => { + const existing = valuesByKey.get(key); + if (existing) { + existing[periodId] = value; + } else { + valuesByKey.set(key, { [periodId]: value }); + } + }; + + for (const period of periods) { + const periodId = period.id; + const priorId = previousPeriodId(periods, periodId); + + const revenue = valueFor(rowsByKey.get('revenue'), periodId); + const grossProfit = valueFor(rowsByKey.get('gross_profit'), periodId); + const operatingIncome = valueFor(rowsByKey.get('operating_income'), periodId); + const ebitda = valueFor(rowsByKey.get('ebitda'), periodId); + const netIncome = valueFor(rowsByKey.get('net_income'), periodId); + const freeCashFlow = valueFor(rowsByKey.get('free_cash_flow'), periodId); + const totalAssets = valueFor(rowsByKey.get('total_assets'), periodId); + const priorAssets = priorId ? valueFor(rowsByKey.get('total_assets'), priorId) : null; + const totalEquity = valueFor(rowsByKey.get('total_equity'), periodId); + const priorEquity = priorId ? valueFor(rowsByKey.get('total_equity'), priorId) : null; + const totalDebt = valueFor(rowsByKey.get('total_debt'), periodId); + const cash = valueFor(rowsByKey.get('cash_and_equivalents'), periodId); + const shortTermInvestments = valueFor(rowsByKey.get('short_term_investments'), periodId); + const currentAssets = valueFor(rowsByKey.get('current_assets'), periodId); + const currentLiabilities = valueFor(rowsByKey.get('current_liabilities'), periodId); + const dilutedShares = valueFor(rowsByKey.get('diluted_shares'), periodId); + const dilutedEps = valueFor(rowsByKey.get('diluted_eps'), periodId); + const effectiveTaxRate = valueFor(rowsByKey.get('effective_tax_rate'), periodId); + const pretaxIncome = valueFor(rowsByKey.get('pretax_income'), periodId); + const incomeTaxExpense = valueFor(rowsByKey.get('income_tax_expense'), periodId); + const priorRevenue = priorId ? valueFor(rowsByKey.get('revenue'), priorId) : null; + const priorNetIncome = priorId ? valueFor(rowsByKey.get('net_income'), priorId) : null; + const priorDilutedEps = priorId ? valueFor(rowsByKey.get('diluted_eps'), priorId) : null; + const priorFcf = priorId ? valueFor(rowsByKey.get('free_cash_flow'), priorId) : null; + const price = input.pricesByPeriodId[periodId] ?? null; + + const fallbackTaxRate = divideValues(incomeTaxExpense, pretaxIncome); + const nopat = operatingIncome === null + ? null + : (effectiveTaxRate ?? fallbackTaxRate) === null + ? null + : operatingIncome * (1 - ((effectiveTaxRate ?? fallbackTaxRate) ?? 0)); + const investedCapital = subtractValues(sumValues([totalDebt, totalEquity]), sumValues([cash, shortTermInvestments])); + const priorInvestedCapital = priorId + ? subtractValues( + sumValues([ + valueFor(rowsByKey.get('total_debt'), priorId), + valueFor(rowsByKey.get('total_equity'), priorId) + ]), + sumValues([ + valueFor(rowsByKey.get('cash_and_equivalents'), priorId), + valueFor(rowsByKey.get('short_term_investments'), priorId) + ]) + ) + : null; + const averageInvestedCapital = averageValues(investedCapital, priorInvestedCapital); + const capitalEmployed = subtractValues(totalAssets, currentLiabilities); + const priorCapitalEmployed = priorId + ? subtractValues(valueFor(rowsByKey.get('total_assets'), priorId), valueFor(rowsByKey.get('current_liabilities'), priorId)) + : null; + const averageCapitalEmployed = averageValues(capitalEmployed, priorCapitalEmployed); + const marketCap = price === null || dilutedShares === null ? null : price * dilutedShares; + const enterpriseValue = marketCap === null ? null : subtractValues(sumValues([marketCap, totalDebt]), sumValues([cash, shortTermInvestments])); + + setValue('gross_margin', periodId, divideValues(grossProfit, revenue)); + setValue('operating_margin', periodId, divideValues(operatingIncome, revenue)); + setValue('ebitda_margin', periodId, divideValues(ebitda, revenue)); + setValue('net_margin', periodId, divideValues(netIncome, revenue)); + setValue('fcf_margin', periodId, divideValues(freeCashFlow, revenue)); + setValue('roa', periodId, divideValues(netIncome, averageValues(totalAssets, priorAssets))); + setValue('roe', periodId, divideValues(netIncome, averageValues(totalEquity, priorEquity))); + setValue('roic', periodId, divideValues(nopat, averageInvestedCapital)); + setValue('roce', periodId, divideValues(operatingIncome, averageCapitalEmployed)); + setValue('debt_to_equity', periodId, divideValues(totalDebt, totalEquity)); + setValue('net_debt_to_ebitda', periodId, divideValues(subtractValues(totalDebt, sumValues([cash, shortTermInvestments])), ebitda)); + setValue('cash_to_debt', periodId, divideValues(sumValues([cash, shortTermInvestments]), totalDebt)); + setValue('current_ratio', periodId, divideValues(currentAssets, currentLiabilities)); + setValue('revenue_per_share', periodId, divideValues(revenue, dilutedShares)); + setValue('fcf_per_share', periodId, divideValues(freeCashFlow, dilutedShares)); + setValue('book_value_per_share', periodId, divideValues(totalEquity, dilutedShares)); + setValue('revenue_yoy', periodId, priorId ? divideValues(subtractValues(revenue, priorRevenue), priorRevenue) : null); + setValue('net_income_yoy', periodId, priorId ? divideValues(subtractValues(netIncome, priorNetIncome), priorNetIncome) : null); + setValue('eps_yoy', periodId, priorId ? divideValues(subtractValues(dilutedEps, priorDilutedEps), priorDilutedEps) : null); + setValue('fcf_yoy', periodId, priorId ? divideValues(subtractValues(freeCashFlow, priorFcf), priorFcf) : null); + setValue('market_cap', periodId, marketCap); + setValue('enterprise_value', periodId, enterpriseValue); + setValue('price_to_earnings', periodId, divideValues(price, dilutedEps)); + setValue('price_to_fcf', periodId, divideValues(marketCap, freeCashFlow)); + setValue('price_to_book', periodId, divideValues(marketCap, totalEquity)); + setValue('ev_to_sales', periodId, divideValues(enterpriseValue, revenue)); + setValue('ev_to_ebitda', periodId, divideValues(enterpriseValue, ebitda)); + setValue('ev_to_fcf', periodId, divideValues(enterpriseValue, freeCashFlow)); + } + + if (input.cadence === 'annual') { + for (let index = 0; index < periods.length; index += 1) { + const period = periods[index]; + const periodId = period.id; + const revenue = valueFor(rowsByKey.get('revenue'), periodId); + const eps = valueFor(rowsByKey.get('diluted_eps'), periodId); + setValue('3y_revenue_cagr', periodId, index >= 3 ? cagr(revenue, valueFor(rowsByKey.get('revenue'), periods[index - 3]?.id ?? ''), 3) : null); + setValue('5y_revenue_cagr', periodId, index >= 5 ? cagr(revenue, valueFor(rowsByKey.get('revenue'), periods[index - 5]?.id ?? ''), 5) : null); + setValue('3y_eps_cagr', periodId, index >= 3 ? cagr(eps, valueFor(rowsByKey.get('diluted_eps'), periods[index - 3]?.id ?? ''), 3) : null); + setValue('5y_eps_cagr', periodId, index >= 5 ? cagr(eps, valueFor(rowsByKey.get('diluted_eps'), periods[index - 5]?.id ?? ''), 5) : null); + } + } + + return RATIO_DEFINITIONS.map((definition) => { + const dependencyKeys = (() => { + switch (definition.key) { + case 'gross_margin': + return ['gross_profit', 'revenue']; + case 'operating_margin': + return ['operating_income', 'revenue']; + case 'ebitda_margin': + return ['ebitda', 'revenue']; + case 'net_margin': + return ['net_income', 'revenue']; + case 'fcf_margin': + return ['free_cash_flow', 'revenue']; + case 'roa': + return ['net_income', 'total_assets']; + case 'roe': + return ['net_income', 'total_equity']; + case 'roic': + return ['operating_income', 'effective_tax_rate', 'income_tax_expense', 'pretax_income', 'total_debt', 'total_equity', 'cash_and_equivalents', 'short_term_investments']; + case 'roce': + return ['operating_income', 'total_assets', 'current_liabilities']; + case 'debt_to_equity': + return ['total_debt', 'total_equity']; + case 'net_debt_to_ebitda': + return ['total_debt', 'cash_and_equivalents', 'short_term_investments', 'ebitda']; + case 'cash_to_debt': + return ['cash_and_equivalents', 'short_term_investments', 'total_debt']; + case 'current_ratio': + return ['current_assets', 'current_liabilities']; + case 'revenue_per_share': + return ['revenue', 'diluted_shares']; + case 'fcf_per_share': + return ['free_cash_flow', 'diluted_shares']; + case 'book_value_per_share': + return ['total_equity', 'diluted_shares']; + case 'revenue_yoy': + case '3y_revenue_cagr': + case '5y_revenue_cagr': + return ['revenue']; + case 'net_income_yoy': + return ['net_income']; + case 'eps_yoy': + case '3y_eps_cagr': + case '5y_eps_cagr': + return ['diluted_eps']; + case 'fcf_yoy': + return ['free_cash_flow']; + case 'market_cap': + return ['diluted_shares']; + case 'enterprise_value': + return ['diluted_shares', 'total_debt', 'cash_and_equivalents', 'short_term_investments']; + case 'price_to_earnings': + return ['diluted_eps']; + case 'price_to_fcf': + return ['diluted_shares', 'free_cash_flow']; + case 'price_to_book': + return ['diluted_shares', 'total_equity']; + case 'ev_to_sales': + return ['diluted_shares', 'total_debt', 'cash_and_equivalents', 'short_term_investments', 'revenue']; + case 'ev_to_ebitda': + return ['diluted_shares', 'total_debt', 'cash_and_equivalents', 'short_term_investments', 'ebitda']; + case 'ev_to_fcf': + return ['diluted_shares', 'total_debt', 'cash_and_equivalents', 'short_term_investments', 'free_cash_flow']; + default: + return []; + } + })(); + const sources = collectSourceRowData(rowsByKey, dependencyKeys); + + return { + key: definition.key, + label: definition.label, + category: definition.category, + order: definition.order, + unit: definition.unit, + values: valuesByKey.get(definition.key) ?? {}, + sourceConcepts: sources.sourceConcepts, + sourceRowKeys: sources.sourceRowKeys, + sourceFactIds: sources.sourceFactIds, + formulaKey: definition.key, + hasDimensions: false, + resolvedSourceRowKeys: Object.fromEntries(periods.map((period) => [period.id, null])), + denominatorKey: definition.denominatorKey + } satisfies RatioRow; + }).filter((row) => { + if (row.key.includes('_cagr')) { + return Object.values(row.values).some((value) => value !== null); + } + + return true; + }); +} + +export const RATIO_CATEGORY_ORDER = [ + 'margins', + 'returns', + 'financial_health', + 'per_share', + 'growth', + 'valuation' +] as const; diff --git a/lib/server/financials/standardize.ts b/lib/server/financials/standardize.ts new file mode 100644 index 0000000..9dd1399 --- /dev/null +++ b/lib/server/financials/standardize.ts @@ -0,0 +1,450 @@ +import type { + DerivedFinancialRow, + DimensionBreakdownRow, + FinancialStatementKind, + FinancialStatementPeriod, + FinancialUnit, + StandardizedFinancialRow, + TaxonomyFactRow, + TaxonomyStatementRow +} from '@/lib/types'; +import { + CANONICAL_ROW_DEFINITIONS, + type CanonicalRowDefinition +} from '@/lib/server/financials/canonical-definitions'; + +function normalizeToken(value: string) { + return value.trim().toLowerCase(); +} + +function valueOrNull(values: Record, periodId: string) { + return periodId in values ? values[periodId] : null; +} + +function sumValues(values: Array) { + if (values.some((value) => value === null)) { + return null; + } + + return values.reduce((sum, value) => sum + (value ?? 0), 0); +} + +function subtractValues(left: number | null, right: number | null) { + if (left === null || right === null) { + return null; + } + + return left - right; +} + +function divideValues(left: number | null, right: number | null) { + if (left === null || right === null || right === 0) { + return null; + } + + return left / right; +} + +function matchesDefinition(row: TaxonomyStatementRow, definition: CanonicalRowDefinition) { + const rowLocalName = normalizeToken(row.localName); + if (definition.localNames?.some((localName) => normalizeToken(localName) === rowLocalName)) { + return true; + } + + const label = normalizeToken(row.label); + return definition.labelIncludes?.some((token) => label.includes(normalizeToken(token))) ?? false; +} + +function matchesDefinitionFact(fact: TaxonomyFactRow, definition: CanonicalRowDefinition) { + const localName = normalizeToken(fact.localName); + return definition.localNames?.some((entry) => normalizeToken(entry) === localName) ?? false; +} + +function inferUnit(rawUnit: string | null, fallback: FinancialUnit) { + const normalized = (rawUnit ?? '').toLowerCase(); + if (!normalized) { + return fallback; + } + + if (normalized.includes('usd') || normalized.includes('iso4217')) { + return 'currency'; + } + + if (normalized.includes('shares')) { + return 'shares'; + } + + if (normalized.includes('pure') || normalized.includes('percent')) { + return fallback === 'percent' ? 'percent' : 'ratio'; + } + + return fallback; +} + +export function factMatchesPeriod(fact: TaxonomyFactRow, period: FinancialStatementPeriod) { + if (period.periodStart) { + return fact.periodStart === period.periodStart && fact.periodEnd === period.periodEnd; + } + + return (fact.periodInstant ?? fact.periodEnd) === period.periodEnd; +} + +function buildCanonicalRow( + definition: CanonicalRowDefinition, + matches: TaxonomyStatementRow[], + facts: TaxonomyFactRow[], + periods: FinancialStatementPeriod[] +) { + const sortedMatches = [...matches].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); + const matchedFacts = facts.filter((fact) => matchesDefinitionFact(fact, definition) && fact.isDimensionless); + + const sourceConcepts = new Set(); + const sourceRowKeys = new Set(); + const sourceFactIds = new Set(); + for (const row of sortedMatches) { + sourceConcepts.add(row.qname); + sourceRowKeys.add(row.key); + for (const factId of row.sourceFactIds) { + sourceFactIds.add(factId); + } + } + + const values: Record = {}; + const resolvedSourceRowKeys: Record = {}; + let unit = definition.unit; + + for (const period of periods) { + const directMatch = sortedMatches.find((row) => period.id in row.values); + if (directMatch) { + values[period.id] = directMatch.values[period.id] ?? null; + unit = inferUnit(directMatch.units[period.id] ?? null, definition.unit); + resolvedSourceRowKeys[period.id] = directMatch.key; + continue; + } + + const factMatch = matchedFacts.find((fact) => factMatchesPeriod(fact, period)); + values[period.id] = factMatch?.value ?? null; + unit = inferUnit(factMatch?.unit ?? null, definition.unit); + resolvedSourceRowKeys[period.id] = factMatch?.conceptKey ?? null; + + if (factMatch) { + sourceConcepts.add(factMatch.qname); + sourceRowKeys.add(factMatch.conceptKey); + sourceFactIds.add(factMatch.id); + } + } + + return { + key: definition.key, + label: definition.label, + category: definition.category, + order: definition.order, + unit, + values, + sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), + sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), + sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), + formulaKey: null, + hasDimensions: sortedMatches.some((row) => row.hasDimensions), + resolvedSourceRowKeys + } satisfies StandardizedFinancialRow; +} + +type FormulaDefinition = { + key: string; + formulaKey: string; + compute: (rowsByKey: Map, periodId: string) => number | null; +}; + +const FORMULAS: Record, FormulaDefinition[]> = { + income: [ + { + key: 'gross_profit', + formulaKey: 'gross_profit', + compute: (rowsByKey, periodId) => subtractValues( + valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('cost_of_revenue')?.values ?? {}, periodId) + ) + }, + { + key: 'gross_margin', + formulaKey: 'gross_margin', + compute: (rowsByKey, periodId) => divideValues( + valueOrNull(rowsByKey.get('gross_profit')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId) + ) + }, + { + key: 'operating_margin', + formulaKey: 'operating_margin', + compute: (rowsByKey, periodId) => divideValues( + valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId) + ) + }, + { + key: 'effective_tax_rate', + formulaKey: 'effective_tax_rate', + compute: (rowsByKey, periodId) => divideValues( + valueOrNull(rowsByKey.get('income_tax_expense')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('pretax_income')?.values ?? {}, periodId) + ) + }, + { + key: 'ebitda', + formulaKey: 'ebitda', + compute: (rowsByKey, periodId) => sumValues([ + valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('depreciation_and_amortization')?.values ?? {}, periodId) + ]) + } + ], + balance: [ + { + key: 'total_debt', + formulaKey: 'total_debt', + compute: (rowsByKey, periodId) => sumValues([ + valueOrNull(rowsByKey.get('long_term_debt')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('current_debt')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('lease_liabilities')?.values ?? {}, periodId) + ]) + }, + { + key: 'net_cash_position', + formulaKey: 'net_cash_position', + compute: (rowsByKey, periodId) => subtractValues( + sumValues([ + valueOrNull(rowsByKey.get('cash_and_equivalents')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('short_term_investments')?.values ?? {}, periodId) + ]), + valueOrNull(rowsByKey.get('total_debt')?.values ?? {}, periodId) + ) + } + ], + cash_flow: [ + { + key: 'free_cash_flow', + formulaKey: 'free_cash_flow', + compute: (rowsByKey, periodId) => subtractValues( + valueOrNull(rowsByKey.get('operating_cash_flow')?.values ?? {}, periodId), + valueOrNull(rowsByKey.get('capital_expenditures')?.values ?? {}, periodId) + ) + } + ] +}; + +function applyFormulas( + rowsByKey: Map, + statement: Extract, + periods: FinancialStatementPeriod[] +) { + for (const formula of FORMULAS[statement]) { + const target = rowsByKey.get(formula.key); + if (!target) { + continue; + } + + let usedFormula = target.formulaKey !== null; + for (const period of periods) { + if (target.values[period.id] !== null) { + continue; + } + + const computed = formula.compute(rowsByKey, period.id); + if (computed === null) { + continue; + } + + target.values[period.id] = computed; + target.resolvedSourceRowKeys[period.id] = null; + usedFormula = true; + } + + if (usedFormula) { + target.formulaKey = formula.formulaKey; + } + } +} + +export function buildStandardizedRows(input: { + rows: TaxonomyStatementRow[]; + statement: Extract; + periods: FinancialStatementPeriod[]; + facts: TaxonomyFactRow[]; +}) { + const definitions = CANONICAL_ROW_DEFINITIONS[input.statement]; + const rowsByKey = new Map(); + const matchedRowKeys = new Set(); + + for (const definition of definitions) { + const matches = input.rows.filter((row) => matchesDefinition(row, definition)); + for (const row of matches) { + matchedRowKeys.add(row.key); + } + + const canonical = buildCanonicalRow(definition, matches, input.facts, input.periods); + const hasAnyValue = Object.values(canonical.values).some((value) => value !== null); + if (hasAnyValue || definition.key.startsWith('gross_') || definition.key === 'operating_margin' || definition.key === 'effective_tax_rate' || definition.key === 'ebitda' || definition.key === 'total_debt' || definition.key === 'net_cash_position' || definition.key === 'free_cash_flow') { + rowsByKey.set(definition.key, canonical); + } + } + + applyFormulas(rowsByKey, input.statement, input.periods); + + const unmatchedRows = input.rows + .filter((row) => !matchedRowKeys.has(row.key)) + .map((row) => ({ + key: `other:${row.key}`, + label: row.label, + category: 'other', + order: 10_000 + row.order, + unit: inferUnit(Object.values(row.units)[0] ?? null, 'currency'), + values: { ...row.values }, + sourceConcepts: [row.qname], + sourceRowKeys: [row.key], + sourceFactIds: [...row.sourceFactIds], + formulaKey: null, + hasDimensions: row.hasDimensions, + resolvedSourceRowKeys: Object.fromEntries( + input.periods.map((period) => [period.id, period.id in row.values ? row.key : null]) + ) + } satisfies StandardizedFinancialRow)); + + return [...rowsByKey.values(), ...unmatchedRows].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); +} + +export function buildDimensionBreakdown( + facts: TaxonomyFactRow[], + periods: FinancialStatementPeriod[], + faithfulRows: TaxonomyStatementRow[], + standardizedRows: StandardizedFinancialRow[] +) { + const periodByFilingId = new Map(); + for (const period of periods) { + periodByFilingId.set(period.filingId, period); + } + + const faithfulRowByKey = new Map(faithfulRows.map((row) => [row.key, row])); + const standardizedRowsBySource = new Map(); + for (const row of standardizedRows) { + for (const sourceRowKey of row.sourceRowKeys) { + const existing = standardizedRowsBySource.get(sourceRowKey); + if (existing) { + existing.push(row); + } else { + standardizedRowsBySource.set(sourceRowKey, [row]); + } + } + } + + const map = new Map(); + const pushRow = (key: string, row: DimensionBreakdownRow) => { + const existing = map.get(key); + if (existing) { + existing.push(row); + } else { + map.set(key, [row]); + } + }; + + for (const fact of facts) { + if (fact.dimensions.length === 0) { + continue; + } + + const period = periodByFilingId.get(fact.filingId) ?? null; + if (!period || !factMatchesPeriod(fact, period)) { + continue; + } + + const faithfulRow = faithfulRowByKey.get(fact.conceptKey) ?? null; + const standardizedMatches = standardizedRowsBySource.get(fact.conceptKey) ?? []; + + for (const dimension of fact.dimensions) { + const faithfulDimensionRow: DimensionBreakdownRow = { + rowKey: fact.conceptKey, + concept: fact.qname, + sourceRowKey: fact.conceptKey, + sourceLabel: faithfulRow?.label ?? null, + periodId: period.id, + axis: dimension.axis, + member: dimension.member, + value: fact.value, + unit: fact.unit, + provenanceType: 'taxonomy' + }; + + pushRow(fact.conceptKey, faithfulDimensionRow); + for (const standardizedRow of standardizedMatches) { + pushRow(standardizedRow.key, { + ...faithfulDimensionRow, + rowKey: standardizedRow.key + }); + } + } + } + + return map.size > 0 ? Object.fromEntries(map.entries()) : null; +} + +export function cloneStandardizedRows(rows: StandardizedFinancialRow[]) { + return rows.map((row) => ({ + ...row, + values: { ...row.values }, + sourceConcepts: [...row.sourceConcepts], + sourceRowKeys: [...row.sourceRowKeys], + sourceFactIds: [...row.sourceFactIds], + resolvedSourceRowKeys: { ...row.resolvedSourceRowKeys } + })); +} + +export function buildLtmStandardizedRows( + quarterlyRows: StandardizedFinancialRow[], + quarterlyPeriods: FinancialStatementPeriod[], + ltmPeriods: FinancialStatementPeriod[], + statement: Extract +) { + const sortedQuarterlyPeriods = [...quarterlyPeriods].sort((left, right) => { + return Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate); + }); + const result = cloneStandardizedRows(quarterlyRows).map((row) => ({ + ...row, + values: {} as Record, + resolvedSourceRowKeys: {} as Record + })); + + for (const row of result) { + const source = quarterlyRows.find((entry) => entry.key === row.key); + if (!source) { + continue; + } + + for (const ltmPeriod of ltmPeriods) { + const anchorIndex = sortedQuarterlyPeriods.findIndex((period) => `ltm:${period.id}` === ltmPeriod.id); + if (anchorIndex < 3) { + continue; + } + + const slice = sortedQuarterlyPeriods.slice(anchorIndex - 3, anchorIndex + 1); + const sourceValues = slice.map((period) => source.values[period.id] ?? null); + row.values[ltmPeriod.id] = statement === 'balance' + ? sourceValues[sourceValues.length - 1] ?? null + : sumValues(sourceValues); + row.resolvedSourceRowKeys[ltmPeriod.id] = source.formulaKey ? null : source.resolvedSourceRowKeys[slice[slice.length - 1]?.id ?? ''] ?? null; + } + } + + return result; +} diff --git a/lib/server/financials/trend-series.ts b/lib/server/financials/trend-series.ts new file mode 100644 index 0000000..8b966eb --- /dev/null +++ b/lib/server/financials/trend-series.ts @@ -0,0 +1,82 @@ +import type { + FinancialSurfaceKind, + RatioRow, + StandardizedFinancialRow, + StructuredKpiRow, + TrendSeries +} from '@/lib/types'; +import { KPI_CATEGORY_ORDER } from '@/lib/server/financials/kpi-registry'; +import { RATIO_CATEGORY_ORDER } from '@/lib/server/financials/ratios'; + +function toTrendSeriesRow(row: { + key: string; + label: string; + category: string; + unit: TrendSeries['unit']; + values: Record; +}) { + return { + key: row.key, + label: row.label, + category: row.category, + unit: row.unit, + values: row.values + } satisfies TrendSeries; +} + +export function buildFinancialCategories(rows: Array<{ category: string }>, surfaceKind: FinancialSurfaceKind) { + const counts = new Map(); + for (const row of rows) { + counts.set(row.category, (counts.get(row.category) ?? 0) + 1); + } + + const order = surfaceKind === 'ratios' + ? [...RATIO_CATEGORY_ORDER] + : surfaceKind === 'segments_kpis' + ? [...KPI_CATEGORY_ORDER] + : [...counts.keys()]; + + return order + .filter((key) => (counts.get(key) ?? 0) > 0) + .map((key) => ({ + key, + label: key.replace(/_/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase()), + count: counts.get(key) ?? 0 + })); +} + +export function buildTrendSeries(input: { + surfaceKind: FinancialSurfaceKind; + statementRows?: StandardizedFinancialRow[]; + ratioRows?: RatioRow[]; + kpiRows?: StructuredKpiRow[]; +}) { + switch (input.surfaceKind) { + case 'income_statement': + return (input.statementRows ?? []) + .filter((row) => row.key === 'revenue' || row.key === 'net_income') + .map(toTrendSeriesRow); + case 'balance_sheet': + return (input.statementRows ?? []) + .filter((row) => row.key === 'total_assets' || row.key === 'cash_and_equivalents' || row.key === 'total_debt') + .map(toTrendSeriesRow); + case 'cash_flow_statement': + return (input.statementRows ?? []) + .filter((row) => row.key === 'operating_cash_flow' || row.key === 'free_cash_flow' || row.key === 'capital_expenditures') + .map(toTrendSeriesRow); + case 'ratios': + return (input.ratioRows ?? []) + .filter((row) => row.category === 'margins') + .map(toTrendSeriesRow); + case 'segments_kpis': { + const rows = input.kpiRows ?? []; + const firstCategory = buildFinancialCategories(rows, 'segments_kpis')[0]?.key ?? null; + return rows + .filter((row) => row.category === firstCategory) + .slice(0, 4) + .map(toTrendSeriesRow); + } + default: + return []; + } +} diff --git a/lib/server/prices.ts b/lib/server/prices.ts index 2217c6c..ac8ec69 100644 --- a/lib/server/prices.ts +++ b/lib/server/prices.ts @@ -43,6 +43,102 @@ export async function getQuote(ticker: string): Promise { } } +export async function getQuoteOrNull(ticker: string): Promise { + const normalizedTicker = ticker.trim().toUpperCase(); + + try { + const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)' + }, + cache: 'no-store' + }); + + if (!response.ok) { + return null; + } + + const payload = await response.json() as { + chart?: { + result?: Array<{ meta?: { regularMarketPrice?: number } }>; + }; + }; + + const price = payload.chart?.result?.[0]?.meta?.regularMarketPrice; + return typeof price === 'number' && Number.isFinite(price) ? price : null; + } catch { + return null; + } +} + +export async function getHistoricalClosingPrices(ticker: string, dates: string[]) { + const normalizedTicker = ticker.trim().toUpperCase(); + const normalizedDates = dates + .map((value) => { + const parsed = Date.parse(value); + return Number.isFinite(parsed) + ? { raw: value, iso: new Date(parsed).toISOString().slice(0, 10), epoch: parsed } + : null; + }) + .filter((entry): entry is { raw: string; iso: string; epoch: number } => entry !== null); + + if (normalizedDates.length === 0) { + return {} as Record; + } + + try { + const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)' + }, + cache: 'no-store' + }); + + if (!response.ok) { + return Object.fromEntries(normalizedDates.map((entry) => [entry.raw, null])); + } + + const payload = await response.json() as { + chart?: { + result?: Array<{ + timestamp?: number[]; + indicators?: { + quote?: Array<{ + close?: Array; + }>; + }; + }>; + }; + }; + + const result = payload.chart?.result?.[0]; + const timestamps = result?.timestamp ?? []; + const closes = result?.indicators?.quote?.[0]?.close ?? []; + const points = timestamps + .map((timestamp, index) => { + const close = closes[index]; + if (typeof close !== 'number' || !Number.isFinite(close)) { + return null; + } + + return { + epoch: timestamp * 1000, + close + }; + }) + .filter((entry): entry is { epoch: number; close: number } => entry !== null); + + return Object.fromEntries(normalizedDates.map((entry) => { + const point = [...points] + .reverse() + .find((candidate) => candidate.epoch <= entry.epoch) ?? null; + return [entry.raw, point?.close ?? null]; + })); + } catch { + return Object.fromEntries(normalizedDates.map((entry) => [entry.raw, null])); + } +} + export async function getPriceHistory(ticker: string): Promise> { const normalizedTicker = ticker.trim().toUpperCase(); diff --git a/lib/server/repos/company-financial-bundles.ts b/lib/server/repos/company-financial-bundles.ts new file mode 100644 index 0000000..9b58248 --- /dev/null +++ b/lib/server/repos/company-financial-bundles.ts @@ -0,0 +1,107 @@ +import { and, eq } from 'drizzle-orm'; +import type { + FinancialCadence, + FinancialSurfaceKind +} from '@/lib/types'; +import { db } from '@/lib/server/db'; +import { companyFinancialBundle } from '@/lib/server/db/schema'; + +const BUNDLE_VERSION = 1; + +export type CompanyFinancialBundleRecord = { + id: number; + ticker: string; + surface_kind: FinancialSurfaceKind; + cadence: FinancialCadence; + bundle_version: number; + source_snapshot_ids: number[]; + source_signature: string; + payload: Record; + created_at: string; + updated_at: string; +}; + +function toBundleRecord(row: typeof companyFinancialBundle.$inferSelect): CompanyFinancialBundleRecord { + return { + id: row.id, + ticker: row.ticker, + surface_kind: row.surface_kind, + cadence: row.cadence, + bundle_version: row.bundle_version, + source_snapshot_ids: row.source_snapshot_ids ?? [], + source_signature: row.source_signature, + payload: row.payload, + created_at: row.created_at, + updated_at: row.updated_at + }; +} + +export async function getCompanyFinancialBundle(input: { + ticker: string; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; +}) { + const [row] = await db + .select() + .from(companyFinancialBundle) + .where(and( + eq(companyFinancialBundle.ticker, input.ticker.trim().toUpperCase()), + eq(companyFinancialBundle.surface_kind, input.surfaceKind), + eq(companyFinancialBundle.cadence, input.cadence) + )) + .limit(1); + + return row ? toBundleRecord(row) : null; +} + +export async function upsertCompanyFinancialBundle(input: { + ticker: string; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; + sourceSnapshotIds: number[]; + sourceSignature: string; + payload: Record; +}) { + const now = new Date().toISOString(); + + const [saved] = await db + .insert(companyFinancialBundle) + .values({ + ticker: input.ticker.trim().toUpperCase(), + surface_kind: input.surfaceKind, + cadence: input.cadence, + bundle_version: BUNDLE_VERSION, + source_snapshot_ids: input.sourceSnapshotIds, + source_signature: input.sourceSignature, + payload: input.payload, + created_at: now, + updated_at: now + }) + .onConflictDoUpdate({ + target: [ + companyFinancialBundle.ticker, + companyFinancialBundle.surface_kind, + companyFinancialBundle.cadence + ], + set: { + bundle_version: BUNDLE_VERSION, + source_snapshot_ids: input.sourceSnapshotIds, + source_signature: input.sourceSignature, + payload: input.payload, + updated_at: now + } + }) + .returning(); + + return toBundleRecord(saved); +} + +export async function deleteCompanyFinancialBundlesForTicker(ticker: string) { + return await db + .delete(companyFinancialBundle) + .where(eq(companyFinancialBundle.ticker, ticker.trim().toUpperCase())); +} + +export const __companyFinancialBundlesInternals = { + BUNDLE_VERSION +}; diff --git a/lib/server/repos/filing-taxonomy.ts b/lib/server/repos/filing-taxonomy.ts index 600eace..ec5ef37 100644 --- a/lib/server/repos/filing-taxonomy.ts +++ b/lib/server/repos/filing-taxonomy.ts @@ -515,6 +515,7 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn export async function listFilingTaxonomySnapshotsByTicker(input: { ticker: string; window: '10y' | 'all'; + filingTypes?: Array<'10-K' | '10-Q'>; limit?: number; cursor?: string | null; }) { @@ -530,6 +531,10 @@ export async function listFilingTaxonomySnapshotsByTicker(input: { constraints.push(lt(filingTaxonomySnapshot.id, cursorId)); } + if (input.filingTypes && input.filingTypes.length > 0) { + constraints.push(inArray(filingTaxonomySnapshot.filing_type, input.filingTypes)); + } + const rows = await db .select() .from(filingTaxonomySnapshot) @@ -573,6 +578,7 @@ export async function listTaxonomyFactsByTicker(input: { ticker: string; window: '10y' | 'all'; statement?: FinancialStatementKind; + filingTypes?: Array<'10-K' | '10-Q'>; cursor?: string | null; limit?: number; }) { @@ -588,6 +594,10 @@ export async function listTaxonomyFactsByTicker(input: { conditions.push(eq(filingTaxonomyFact.statement_kind, input.statement)); } + if (input.filingTypes && input.filingTypes.length > 0) { + conditions.push(inArray(filingTaxonomySnapshot.filing_type, input.filingTypes)); + } + if (cursorId && Number.isFinite(cursorId) && cursorId > 0) { conditions.push(lt(filingTaxonomyFact.id, cursorId)); } diff --git a/lib/server/task-processors.ts b/lib/server/task-processors.ts index 4a1c76e..4991365 100644 --- a/lib/server/task-processors.ts +++ b/lib/server/task-processors.ts @@ -16,6 +16,9 @@ import { updateFilingMetricsById, upsertFilingsRecords } from '@/lib/server/repos/filings'; +import { + deleteCompanyFinancialBundlesForTicker +} from '@/lib/server/repos/company-financial-bundles'; import { getFilingTaxonomySnapshotByFilingId, upsertFilingTaxonomySnapshot @@ -623,6 +626,7 @@ async function processSyncFilings(task: Task) { await upsertFilingTaxonomySnapshot(snapshot); await updateFilingMetricsById(filing.id, snapshot.derived_metrics); + await deleteCompanyFinancialBundlesForTicker(filing.ticker); taxonomySnapshotsHydrated += 1; } catch (error) { const now = new Date().toISOString(); @@ -656,6 +660,7 @@ async function processSyncFilings(task: Task) { facts: [], metric_validations: [] }); + await deleteCompanyFinancialBundlesForTicker(filing.ticker); taxonomySnapshotsFailed += 1; } diff --git a/lib/types.ts b/lib/types.ts index 735fc77..3bea929 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -200,6 +200,18 @@ export type CompanyFinancialPoint = { export type FinancialStatementKind = 'income' | 'balance' | 'cash_flow' | 'equity' | 'comprehensive_income'; export type FinancialHistoryWindow = '10y' | 'all'; +export type FinancialCadence = 'annual' | 'quarterly' | 'ltm'; +export type FinancialDisplayMode = 'faithful' | 'standardized'; +export type FinancialSurfaceKind = + | 'income_statement' + | 'balance_sheet' + | 'cash_flow_statement' + | 'ratios' + | 'segments_kpis' + | 'adjusted' + | 'custom_metrics'; +export type FinancialUnit = 'currency' | 'count' | 'shares' | 'percent' | 'ratio'; +export type FinancialCategory = string; export type FinancialStatementPeriod = { id: string; @@ -236,21 +248,46 @@ export type TaxonomyStatementRow = { sourceFactIds: number[]; }; -export type FinancialStatementSurfaceKind = 'faithful' | 'standardized'; +export type FinancialStatementSurfaceKind = FinancialDisplayMode; -export type StandardizedStatementRow = { +export type DerivedFinancialRow = { key: string; label: string; - category: string; + category: FinancialCategory; order: number; + unit: FinancialUnit; values: Record; - hasDimensions: boolean; sourceConcepts: string[]; sourceRowKeys: string[]; sourceFactIds: number[]; + formulaKey: string | null; + hasDimensions: boolean; resolvedSourceRowKeys: Record; }; +export type StandardizedFinancialRow = DerivedFinancialRow; +export type StandardizedStatementRow = StandardizedFinancialRow; + +export type RatioRow = DerivedFinancialRow & { + denominatorKey: string | null; +}; + +export type StructuredKpiRow = { + key: string; + label: string; + category: FinancialCategory; + unit: FinancialUnit; + order: number; + segment: string | null; + axis: string | null; + member: string | null; + values: Record; + sourceConcepts: string[]; + sourceFactIds: number[]; + provenanceType: 'taxonomy' | 'structured_note'; + hasDimensions: boolean; +}; + export type TaxonomyFactRow = { id: number; snapshotId: number; @@ -315,11 +352,15 @@ export type DimensionBreakdownRow = { member: string; value: number | null; unit: string | null; + provenanceType?: 'taxonomy' | 'structured_note'; }; -export type FinancialStatementSurface = { - kind: FinancialStatementSurfaceKind; - rows: Row[]; +export type TrendSeries = { + key: string; + label: string; + category: FinancialCategory; + unit: FinancialUnit; + values: Record; }; export type CompanyFinancialStatementsResponse = { @@ -328,13 +369,26 @@ export type CompanyFinancialStatementsResponse = { companyName: string; cik: string | null; }; - statement: FinancialStatementKind; - window: FinancialHistoryWindow; - defaultSurface: FinancialStatementSurfaceKind; + surfaceKind: FinancialSurfaceKind; + cadence: FinancialCadence; + displayModes: FinancialDisplayMode[]; + defaultDisplayMode: FinancialDisplayMode; periods: FinancialStatementPeriod[]; - surfaces: { - faithful: FinancialStatementSurface; - standardized: FinancialStatementSurface; + statementRows: { + faithful: TaxonomyStatementRow[]; + standardized: StandardizedFinancialRow[]; + } | null; + ratioRows: RatioRow[] | null; + kpiRows: StructuredKpiRow[] | null; + trendSeries: TrendSeries[]; + categories: Array<{ + key: FinancialCategory; + label: string; + count: number; + }>; + availability: { + adjusted: boolean; + customMetrics: boolean; }; nextCursor: string | null; facts: { @@ -355,28 +409,6 @@ export type CompanyFinancialStatementsResponse = { pendingFilings: number; queuedSync: boolean; }; - overviewMetrics: { - referencePeriodId: string | null; - referenceDate: string | null; - latest: { - revenue: number | null; - netIncome: number | null; - totalAssets: number | null; - cash: number | null; - debt: number | null; - }; - series: Array<{ - periodId: string; - filingDate: string; - periodEnd: string | null; - label: string; - revenue: number | null; - netIncome: number | null; - totalAssets: number | null; - cash: number | null; - debt: number | null; - }>; - }; metrics: { taxonomy: Filing['metrics']; validation: MetricValidationResult | null; diff --git a/package.json b/package.json index 1b6e187..0813079 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@workflow/world-postgres": "^4.1.0-beta.34", "ai": "^6.0.104", "better-auth": "^1.4.19", + "cheerio": "^1.1.2", "clsx": "^2.1.1", "date-fns": "^4.1.0", "drizzle-orm": "^0.41.0", diff --git a/scripts/dev.ts b/scripts/dev.ts index f4b996c..a65a937 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -71,7 +71,16 @@ function bootstrapFreshDatabase(databaseUrl: string) { try { database.exec('PRAGMA foreign_keys = ON;'); - if (hasTable(database, 'user')) { + const existingCoreTables = [ + 'user', + 'filing', + 'watchlist_item', + 'filing_statement_snapshot', + 'filing_taxonomy_snapshot', + 'task_run' + ]; + + if (existingCoreTables.some((tableName) => hasTable(database, tableName))) { return false; }