From 8b1fff4130212cb8bd5254fda61b7c9aa8b91ece Mon Sep 17 00:00:00 2001 From: francy51 Date: Fri, 6 Mar 2026 16:24:56 -0500 Subject: [PATCH] Implement dual-surface financials and db bootstrap --- app/financials/page.tsx | 444 ++++++++++++++++------- hooks/use-task-notifications-center.ts | 65 +++- lib/server/db/index.test.ts | 34 ++ lib/server/db/index.ts | 77 +++- lib/server/financial-taxonomy.test.ts | 263 ++++++++++++-- lib/server/financial-taxonomy.ts | 465 ++++++++++++++++++++++++- lib/types.ts | 38 +- 7 files changed, 1207 insertions(+), 179 deletions(-) create mode 100644 lib/server/db/index.test.ts diff --git a/app/financials/page.tsx b/app/financials/page.tsx index 96f7666..012a595 100644 --- a/app/financials/page.tsx +++ b/app/financials/page.tsx @@ -2,7 +2,7 @@ import { useQueryClient } from '@tanstack/react-query'; import Link from 'next/link'; -import { Suspense, useCallback, useEffect, useMemo, useState } from 'react'; +import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { format } from 'date-fns'; import { useSearchParams } from 'next/navigation'; import { @@ -17,7 +17,15 @@ import { XAxis, YAxis } from 'recharts'; -import { AlertTriangle, ChartNoAxesCombined, ChevronDown, Download, RefreshCcw, Search } from 'lucide-react'; +import { + AlertTriangle, + ChartNoAxesCombined, + ChevronDown, + Download, + GitCompareArrows, + RefreshCcw, + Search +} from 'lucide-react'; import { AppShell } from '@/components/shell/app-shell'; import { MetricCard } from '@/components/dashboard/metric-card'; import { @@ -42,7 +50,9 @@ import type { CompanyFinancialStatementsResponse, DimensionBreakdownRow, FinancialHistoryWindow, + FinancialStatementSurfaceKind, FinancialStatementKind, + StandardizedStatementRow, TaxonomyStatementRow } from '@/lib/types'; @@ -51,6 +61,20 @@ type LoadOptions = { append?: boolean; }; +type OverviewPoint = { + periodId: string; + filingDate: string; + periodEnd: string | null; + label: string; + revenue: number | null; + netIncome: number | null; + totalAssets: number | null; + cash: number | null; + debt: number | null; +}; + +type DisplayRow = TaxonomyStatementRow | StandardizedStatementRow; + const FINANCIAL_VALUE_SCALE_OPTIONS: Array<{ value: NumberScaleUnit; label: string }> = [ { value: 'thousands', label: 'Thousands (K)' }, { value: 'millions', label: 'Millions (M)' }, @@ -70,23 +94,16 @@ const WINDOW_OPTIONS: Array<{ value: FinancialHistoryWindow; label: string }> = { value: 'all', label: 'Full Available' } ]; +const SURFACE_OPTIONS: Array<{ value: FinancialStatementSurfaceKind; label: string }> = [ + { value: 'standardized', label: 'Standardized' }, + { value: 'faithful', label: 'Filing-faithful' } +]; + const CHART_MUTED = '#b4ced9'; const CHART_GRID = 'rgba(126, 217, 255, 0.24)'; const CHART_TOOLTIP_BG = 'rgba(6, 17, 24, 0.95)'; const CHART_TOOLTIP_BORDER = 'rgba(123, 255, 217, 0.45)'; -type OverviewPoint = { - periodId: string; - filingDate: string; - periodEnd: string | null; - label: string; - revenue: number | null; - netIncome: number | null; - totalAssets: number | null; - cash: number | null; - debt: number | null; -}; - function formatLongDate(value: string) { const parsed = new Date(value); if (Number.isNaN(parsed.getTime())) { @@ -134,6 +151,33 @@ function rowValue(row: { values: Record }, periodId: stri return periodId in row.values ? row.values[periodId] : null; } +function isFaithfulRow(row: DisplayRow): row is TaxonomyStatementRow { + return 'localName' in row; +} + +function isStandardizedRow(row: DisplayRow): row is StandardizedStatementRow { + return 'sourceRowKeys' in row; +} + +function mergeSurfaceRows; hasDimensions: boolean }>( + rows: T[], + mergeRow: (existing: T, row: T) => void +) { + const rowMap = new Map(); + + for (const row of rows) { + const existing = rowMap.get(row.key); + if (!existing) { + rowMap.set(row.key, structuredClone(row)); + continue; + } + + mergeRow(existing, row); + } + + return [...rowMap.values()]; +} + function mergeFinancialPages( base: CompanyFinancialStatementsResponse | null, next: CompanyFinancialStatementsResponse @@ -146,42 +190,85 @@ function mergeFinancialPages( .filter((period, index, list) => list.findIndex((item) => item.id === period.id) === index) .sort((a, b) => Date.parse(a.filingDate) - Date.parse(b.filingDate)); - const rowMap = new Map(); + const faithfulRows = mergeSurfaceRows( + [...base.surfaces.faithful.rows, ...next.surfaces.faithful.rows], + (existing, row) => { + existing.hasDimensions = existing.hasDimensions || row.hasDimensions; + existing.order = Math.min(existing.order, row.order); + existing.depth = Math.min(existing.depth, row.depth); + if (!existing.parentKey && row.parentKey) { + existing.parentKey = row.parentKey; + } - for (const row of [...base.rows, ...next.rows]) { - const existing = rowMap.get(row.key); - if (!existing) { - rowMap.set(row.key, { - ...row, - values: { ...row.values }, - units: { ...row.units }, - sourceFactIds: [...row.sourceFactIds] - }); - continue; - } + for (const [periodId, value] of Object.entries(row.values)) { + if (!(periodId in existing.values)) { + existing.values[periodId] = value; + } + } - existing.hasDimensions = existing.hasDimensions || row.hasDimensions; - existing.order = Math.min(existing.order, row.order); - existing.depth = Math.min(existing.depth, row.depth); + for (const [periodId, unit] of Object.entries(row.units)) { + if (!(periodId in existing.units)) { + existing.units[periodId] = unit; + } + } - for (const [periodId, value] of Object.entries(row.values)) { - if (!(periodId in existing.values)) { - existing.values[periodId] = value; + for (const factId of row.sourceFactIds) { + if (!existing.sourceFactIds.includes(factId)) { + existing.sourceFactIds.push(factId); + } } } - - for (const [periodId, unit] of Object.entries(row.units)) { - if (!(periodId in existing.units)) { - existing.units[periodId] = unit; - } + ).sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; } - for (const factId of row.sourceFactIds) { - if (!existing.sourceFactIds.includes(factId)) { - existing.sourceFactIds.push(factId); + return left.label.localeCompare(right.label); + }); + + const standardizedRows = mergeSurfaceRows( + [...base.surfaces.standardized.rows, ...next.surfaces.standardized.rows], + (existing, row) => { + existing.hasDimensions = existing.hasDimensions || row.hasDimensions; + existing.order = Math.min(existing.order, row.order); + + for (const [periodId, value] of Object.entries(row.values)) { + if (!(periodId in existing.values)) { + existing.values[periodId] = value; + } + } + + for (const [periodId, sourceRowKey] of Object.entries(row.resolvedSourceRowKeys)) { + if (!(periodId in existing.resolvedSourceRowKeys)) { + existing.resolvedSourceRowKeys[periodId] = sourceRowKey; + } + } + + for (const concept of row.sourceConcepts) { + if (!existing.sourceConcepts.includes(concept)) { + existing.sourceConcepts.push(concept); + } + } + + for (const rowKey of row.sourceRowKeys) { + if (!existing.sourceRowKeys.includes(rowKey)) { + existing.sourceRowKeys.push(rowKey); + } + } + + for (const factId of row.sourceFactIds) { + if (!existing.sourceFactIds.includes(factId)) { + existing.sourceFactIds.push(factId); + } } } - } + ).sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); const dimensionBreakdown = (() => { if (!base.dimensionBreakdown && !next.dimensionBreakdown) { @@ -210,16 +297,21 @@ function mergeFinancialPages( return { ...next, periods, - rows: [...rowMap.values()], + surfaces: { + faithful: { + kind: 'faithful' as const, + rows: faithfulRows + }, + standardized: { + kind: 'standardized' as const, + rows: standardizedRows + } + }, nextCursor: next.nextCursor, coverage: { ...next.coverage, filings: periods.length, - rows: rowMap.size, - dimensions: dimensionBreakdown - ? Object.values(dimensionBreakdown).reduce((total, rows) => total + rows.length, 0) - : 0, - facts: next.coverage.facts + rows: faithfulRows.length }, dataSourceStatus: { ...next.dataSourceStatus, @@ -229,23 +321,23 @@ function mergeFinancialPages( }; } -function findRowByLocalNames(rows: TaxonomyStatementRow[], localNames: string[]) { - const exact = rows.find((row) => localNames.some((name) => row.localName.toLowerCase() === name.toLowerCase())); +function findFaithfulRowByLocalNames(rows: TaxonomyStatementRow[], localNames: string[]) { + const normalizedNames = localNames.map((name) => name.toLowerCase()); + const exact = rows.find((row) => normalizedNames.includes(row.localName.toLowerCase())); if (exact) { return exact; } - const exactLabel = rows.find((row) => localNames.some((name) => row.label.toLowerCase() === name.toLowerCase())); - if (exactLabel) { - return exactLabel; - } - return rows.find((row) => { const haystack = `${row.key} ${row.label} ${row.localName} ${row.qname}`.toLowerCase(); - return localNames.some((name) => haystack.includes(name.toLowerCase())); + return normalizedNames.some((name) => haystack.includes(name)); }) ?? null; } +function findStandardizedRow(rows: StandardizedStatementRow[], key: string) { + return rows.find((row) => row.key === key) ?? null; +} + function buildOverviewSeries( incomeData: CompanyFinancialStatementsResponse | null, balanceData: CompanyFinancialStatementsResponse | null @@ -269,24 +361,21 @@ function buildOverviewSeries( })) .sort((a, b) => Date.parse(a.periodEnd ?? a.filingDate) - Date.parse(b.periodEnd ?? b.filingDate)); - const incomeRows = incomeData?.rows ?? []; - const balanceRows = balanceData?.rows ?? []; + const incomeStandardized = incomeData?.surfaces.standardized.rows ?? []; + const balanceStandardized = balanceData?.surfaces.standardized.rows ?? []; + const incomeFaithful = incomeData?.surfaces.faithful.rows ?? []; + const balanceFaithful = balanceData?.surfaces.faithful.rows ?? []; - const revenueRow = findRowByLocalNames(incomeRows, [ - 'RevenueFromContractWithCustomerExcludingAssessedTax', - 'Revenues', - 'SalesRevenueNet', - 'Revenue' - ]); - const netIncomeRow = findRowByLocalNames(incomeRows, ['NetIncomeLoss', 'ProfitLoss']); - const assetsRow = findRowByLocalNames(balanceRows, ['Assets']); - const cashRow = findRowByLocalNames(balanceRows, [ - 'CashAndCashEquivalentsAtCarryingValue', - 'CashCashEquivalentsAndShortTermInvestments', - 'CashAndShortTermInvestments', - 'Cash' - ]); - const debtRow = findRowByLocalNames(balanceRows, ['LongTermDebt', 'Debt', 'LongTermDebtNoncurrent']); + const revenueRow = findStandardizedRow(incomeStandardized, 'revenue') + ?? findFaithfulRowByLocalNames(incomeFaithful, ['RevenueFromContractWithCustomerExcludingAssessedTax', 'Revenues', 'SalesRevenueNet', 'Revenue']); + const netIncomeRow = findStandardizedRow(incomeStandardized, 'net-income') + ?? findFaithfulRowByLocalNames(incomeFaithful, ['NetIncomeLoss', 'ProfitLoss']); + const assetsRow = findStandardizedRow(balanceStandardized, 'total-assets') + ?? findFaithfulRowByLocalNames(balanceFaithful, ['Assets']); + const cashRow = findStandardizedRow(balanceStandardized, 'cash-and-equivalents') + ?? findFaithfulRowByLocalNames(balanceFaithful, ['CashAndCashEquivalentsAtCarryingValue', 'CashAndShortTermInvestments', 'Cash']); + const debtRow = findStandardizedRow(balanceStandardized, 'total-debt') + ?? findFaithfulRowByLocalNames(balanceFaithful, ['LongTermDebt', 'Debt', 'LongTermDebtNoncurrent']); return periods.map((period) => ({ periodId: period.periodId, @@ -301,6 +390,54 @@ function buildOverviewSeries( })); } +function groupDimensionRows( + rows: DimensionBreakdownRow[], + surface: FinancialStatementSurfaceKind +) { + if (surface === 'faithful') { + return rows; + } + + return [...rows].sort((left, right) => { + if (left.periodId !== right.periodId) { + return left.periodId.localeCompare(right.periodId); + } + + return `${left.sourceLabel ?? ''}${left.concept ?? ''}`.localeCompare(`${right.sourceLabel ?? ''}${right.concept ?? ''}`); + }); +} + +function ChartFrame({ children }: { children: React.ReactNode }) { + const containerRef = useRef(null); + const [ready, setReady] = useState(false); + + useEffect(() => { + const element = containerRef.current; + if (!element) { + return; + } + + const observer = new ResizeObserver((entries) => { + const next = entries[0]; + if (!next) { + return; + } + + const { width, height } = next.contentRect; + setReady(width > 0 && height > 0); + }); + + observer.observe(element); + return () => observer.disconnect(); + }, []); + + return ( +
+ {ready ? children : null} +
+ ); +} + export default function FinancialsPage() { return ( Loading financial terminal...}> @@ -319,6 +456,7 @@ function FinancialsPageContent() { const [ticker, setTicker] = useState('MSFT'); const [statement, setStatement] = useState('income'); const [window, setWindow] = useState('10y'); + const [surface, setSurface] = useState('standardized'); const [valueScale, setValueScale] = useState('millions'); const [financials, setFinancials] = useState(null); const [overviewIncome, setOverviewIncome] = useState(null); @@ -412,12 +550,12 @@ function FinancialsPageContent() { setLoadingMore(false); } }, [ - queryClient, - statement, - window, dimensionsEnabled, + loadOverview, + queryClient, selectedRowKey, - loadOverview + statement, + window ]); const syncFinancials = useCallback(async () => { @@ -440,20 +578,24 @@ function FinancialsPageContent() { } finally { setSyncingFinancials(false); } - }, [financials?.company.ticker, ticker, queryClient, loadFinancials]); + }, [financials?.company.ticker, loadFinancials, queryClient, ticker]); useEffect(() => { if (!isPending && isAuthenticated) { void loadFinancials(ticker); } - }, [isPending, isAuthenticated, ticker, statement, window, dimensionsEnabled, loadFinancials]); + }, [isPending, isAuthenticated, loadFinancials, ticker]); const periods = useMemo(() => { return [...(financials?.periods ?? [])] .sort((a, b) => Date.parse(a.filingDate) - Date.parse(b.filingDate)); }, [financials?.periods]); - const statementRows = useMemo(() => financials?.rows ?? [], [financials?.rows]); + const faithfulRows = useMemo(() => financials?.surfaces.faithful.rows ?? [], [financials?.surfaces.faithful.rows]); + const standardizedRows = useMemo(() => financials?.surfaces.standardized.rows ?? [], [financials?.surfaces.standardized.rows]); + const statementRows = useMemo(() => { + return surface === 'standardized' ? standardizedRows : faithfulRows; + }, [faithfulRows, standardizedRows, surface]); const overviewSeries = useMemo(() => { return buildOverviewSeries(overviewIncome, overviewBalance); @@ -464,11 +606,29 @@ function FinancialsPageContent() { ?? overviewIncome?.metrics.taxonomy ?? overviewBalance?.metrics.taxonomy ?? null; - const latestRevenue = latestOverview?.revenue ?? latestTaxonomyMetrics?.revenue ?? null; - const latestNetIncome = latestOverview?.netIncome ?? latestTaxonomyMetrics?.netIncome ?? null; - const latestTotalAssets = latestOverview?.totalAssets ?? latestTaxonomyMetrics?.totalAssets ?? null; - const latestCash = latestOverview?.cash ?? latestTaxonomyMetrics?.cash ?? null; - const latestDebt = latestOverview?.debt ?? latestTaxonomyMetrics?.debt ?? null; + + const latestStandardizedIncome = overviewIncome?.surfaces.standardized.rows ?? []; + const latestStandardizedBalance = overviewBalance?.surfaces.standardized.rows ?? []; + const latestRevenue = latestOverview?.revenue + ?? findStandardizedRow(latestStandardizedIncome, 'revenue')?.values[periods[periods.length - 1]?.id ?? ''] + ?? latestTaxonomyMetrics?.revenue + ?? null; + const latestNetIncome = latestOverview?.netIncome + ?? findStandardizedRow(latestStandardizedIncome, 'net-income')?.values[periods[periods.length - 1]?.id ?? ''] + ?? latestTaxonomyMetrics?.netIncome + ?? null; + const latestTotalAssets = latestOverview?.totalAssets + ?? findStandardizedRow(latestStandardizedBalance, 'total-assets')?.values[periods[periods.length - 1]?.id ?? ''] + ?? latestTaxonomyMetrics?.totalAssets + ?? null; + const latestCash = latestOverview?.cash + ?? findStandardizedRow(latestStandardizedBalance, 'cash-and-equivalents')?.values[periods[periods.length - 1]?.id ?? ''] + ?? latestTaxonomyMetrics?.cash + ?? null; + const latestDebt = latestOverview?.debt + ?? findStandardizedRow(latestStandardizedBalance, 'total-debt')?.values[periods[periods.length - 1]?.id ?? ''] + ?? latestTaxonomyMetrics?.debt + ?? null; const latestReferenceDate = latestOverview?.filingDate ?? periods[periods.length - 1]?.filingDate ?? null; const selectedRow = useMemo(() => { @@ -485,20 +645,8 @@ function FinancialsPageContent() { } const direct = financials.dimensionBreakdown[selectedRow.key] ?? []; - if (direct.length > 0) { - return direct; - } - - const conceptKey = selectedRow.qname.toLowerCase(); - for (const rows of Object.values(financials.dimensionBreakdown)) { - const matched = rows.filter((row) => (row.concept ?? '').toLowerCase() === conceptKey); - if (matched.length > 0) { - return matched; - } - } - - return []; - }, [selectedRow, financials?.dimensionBreakdown]); + return groupDimensionRows(direct, surface); + }, [financials?.dimensionBreakdown, selectedRow, surface]); const selectedScaleLabel = useMemo(() => { return FINANCIAL_VALUE_SCALE_OPTIONS.find((option) => option.value === valueScale)?.label ?? 'Millions (M)'; @@ -532,7 +680,7 @@ function FinancialsPageContent() { options: FINANCIAL_VALUE_SCALE_OPTIONS, onChange: (nextValue) => setValueScale(nextValue as NumberScaleUnit) } - ], [statement, window, valueScale]); + ], [statement, valueScale, window]); const controlActions = useMemo(() => { const actions: FinancialControlAction[] = []; @@ -569,7 +717,7 @@ function FinancialsPageContent() { } return actions; - }, [window, financials?.nextCursor, loadingMore, loadFinancials, ticker]); + }, [financials?.nextCursor, loadFinancials, loadingMore, ticker, window]); if (isPending || !isAuthenticated) { return
Loading financial terminal...
; @@ -578,7 +726,7 @@ function FinancialsPageContent() { return ( @@ -604,7 +752,7 @@ function FinancialsPageContent() { )} > - +
{ @@ -673,28 +821,42 @@ function FinancialsPageContent() { + +
+ {SURFACE_OPTIONS.map((option) => ( + + ))} +
+ +
- + {loading ? (

Loading overview chart...

) : overviewSeries.length === 0 ? (

No income history available yet.

) : ( -
+ - asAxisCurrencyTick(value, valueScale)} - /> + asAxisCurrencyTick(value, valueScale)} /> asTooltipCurrency(value, valueScale)} contentStyle={{ @@ -707,26 +869,22 @@ function FinancialsPageContent() { -
+ )} - + {loading ? (

Loading balance chart...

) : overviewSeries.length === 0 ? (

No balance history available yet.

) : ( -
+ - asAxisCurrencyTick(value, valueScale)} - /> + asAxisCurrencyTick(value, valueScale)} /> asTooltipCurrency(value, valueScale)} contentStyle={{ @@ -740,14 +898,16 @@ function FinancialsPageContent() { -
+ )}
{loading ? (

Loading statement matrix...

@@ -782,10 +942,28 @@ function FinancialsPageContent() { }} > -
- {row.label} - {row.isExtension ? Ext : null} - {row.hasDimensions ? : null} +
+
+ {row.label} + {isFaithfulRow(row) && row.isExtension ? ( + Ext + ) : null} + {row.hasDimensions ? : null} +
+ {isStandardizedRow(row) ? ( +
+ + Mapped from {row.sourceConcepts.length} concept{row.sourceConcepts.length === 1 ? '' : 's'} + +
+ {row.sourceConcepts.map((concept) => ( + + {concept} + + ))} +
+
+ ) : null}
{periods.map((period) => ( @@ -799,10 +977,14 @@ function FinancialsPageContent() {
)} - - + {!selectedRow ? (

Select a statement row to inspect dimensional facts.

) : !selectedRow.hasDimensions ? ( @@ -813,10 +995,11 @@ function FinancialsPageContent() {

Dimensions are still loading or unavailable for this row.

) : (
- +
+ {surface === 'standardized' ? : null} @@ -828,6 +1011,7 @@ function FinancialsPageContent() { return ( + {surface === 'standardized' ? : null} @@ -841,7 +1025,7 @@ function FinancialsPageContent() { {financials ? ( - +
Overall status: {financials.metrics.validation?.status ?? 'not_run'} @@ -878,7 +1062,7 @@ function FinancialsPageContent() { ) : null} {financials ? ( - +

Hydrated

@@ -907,7 +1091,7 @@ function FinancialsPageContent() {
- Financial Statements V3: taxonomy + PDF LLM validation + Financial Statements V3: faithful filing reconstruction + standardized taxonomy comparison
diff --git a/hooks/use-task-notifications-center.ts b/hooks/use-task-notifications-center.ts index f1449cb..85a0bbe 100644 --- a/hooks/use-task-notifications-center.ts +++ b/hooks/use-task-notifications-center.ts @@ -104,6 +104,13 @@ export function useTaskNotificationsCenter(): UseTaskNotificationsCenterResult { const invalidatedTerminalRef = useRef(new Set()); const activeSnapshotRef = useRef([]); const finishedSnapshotRef = useRef([]); + const [isDocumentVisible, setIsDocumentVisible] = useState(() => { + if (typeof document === 'undefined') { + return true; + } + + return document.visibilityState === 'visible'; + }); const applyTaskLocally = useCallback((task: Task) => { setActiveTasks((prev) => prev.map((entry) => (entry.id === task.id ? task : entry))); @@ -285,10 +292,54 @@ export function useTaskNotificationsCenter(): UseTaskNotificationsCenterResult { } }, [processSnapshots]); + useEffect(() => { + if (typeof document === 'undefined') { + return; + } + + const onVisibilityChange = () => { + setIsDocumentVisible(document.visibilityState === 'visible'); + }; + + document.addEventListener('visibilitychange', onVisibilityChange); + return () => document.removeEventListener('visibilitychange', onVisibilityChange); + }, []); + useEffect(() => { let cancelled = false; let activeTimer: ReturnType | null = null; let terminalTimer: ReturnType | null = null; + let stableTerminalPolls = 0; + let previousTerminalSignature = ''; + + const nextActiveDelay = () => { + if (!isDocumentVisible) { + return 30_000; + } + + const hasActiveTasks = activeSnapshotRef.current.length > 0; + if (isPopoverOpen || isDetailOpen || hasActiveTasks) { + return 2_000; + } + + return 12_000; + }; + + const nextTerminalDelay = () => { + if (!isDocumentVisible) { + return 60_000; + } + + if (isPopoverOpen || isDetailOpen) { + return 4_000; + } + + if (finishedSnapshotRef.current.some((task) => isUnread(task))) { + return 15_000; + } + + return stableTerminalPolls >= 2 ? 45_000 : 20_000; + }; const runActiveLoop = async () => { if (cancelled) { @@ -314,7 +365,7 @@ export function useTaskNotificationsCenter(): UseTaskNotificationsCenterResult { // ignore transient polling failures } - activeTimer = setTimeout(runActiveLoop, 2_000); + activeTimer = setTimeout(runActiveLoop, nextActiveDelay()); }; const runTerminalLoop = async () => { @@ -337,11 +388,19 @@ export function useTaskNotificationsCenter(): UseTaskNotificationsCenterResult { setHasLoadedFinished(true); setFinishedTasks(response.tasks); processSnapshots(); + + const signature = response.tasks.map((task) => taskSignature(task)).join('||'); + if (signature === previousTerminalSignature) { + stableTerminalPolls += 1; + } else { + stableTerminalPolls = 0; + previousTerminalSignature = signature; + } } catch { // ignore transient polling failures } - terminalTimer = setTimeout(runTerminalLoop, 4_000); + terminalTimer = setTimeout(runTerminalLoop, nextTerminalDelay()); }; void runActiveLoop(); @@ -356,7 +415,7 @@ export function useTaskNotificationsCenter(): UseTaskNotificationsCenterResult { clearTimeout(terminalTimer); } }; - }, [processSnapshots]); + }, [isDetailOpen, isDocumentVisible, isPopoverOpen, processSnapshots]); const normalizedActiveTasks = useMemo(() => { return activeTasks.filter((task) => ACTIVE_STATUSES.includes(task.status)); diff --git a/lib/server/db/index.test.ts b/lib/server/db/index.test.ts new file mode 100644 index 0000000..c12dac4 --- /dev/null +++ b/lib/server/db/index.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { Database } from 'bun:sqlite'; +import { __dbInternals } from './index'; + +function applyMigration(client: Database, fileName: string) { + const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); + client.exec(sql); +} + +describe('sqlite schema compatibility bootstrap', () => { + it('adds missing watchlist columns and taxonomy tables for older local databases', () => { + const client = new Database(':memory:'); + client.exec('PRAGMA foreign_keys = ON;'); + + applyMigration(client, '0000_cold_silver_centurion.sql'); + applyMigration(client, '0001_glossy_statement_snapshots.sql'); + applyMigration(client, '0002_workflow_task_projection_metadata.sql'); + applyMigration(client, '0003_task_stage_event_timeline.sql'); + + expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(false); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(false); + + __dbInternals.ensureLocalSqliteSchema(client); + + expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(true); + expect(__dbInternals.hasColumn(client, 'watchlist_item', 'tags')).toBe(true); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true); + + client.close(); + }); +}); diff --git a/lib/server/db/index.ts b/lib/server/db/index.ts index 7828041..0b83337 100644 --- a/lib/server/db/index.ts +++ b/lib/server/db/index.ts @@ -1,5 +1,5 @@ -import { mkdirSync } from 'node:fs'; -import { dirname } from 'node:path'; +import { mkdirSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import { Database } from 'bun:sqlite'; import { drizzle } from 'drizzle-orm/bun-sqlite'; import { schema } from './schema'; @@ -28,6 +28,71 @@ function getDatabasePath() { return databasePath; } +function hasTable(client: Database, tableName: string) { + const row = client + .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') + .get('table', tableName) as { name: string } | null; + + return row !== null; +} + +function hasColumn(client: Database, tableName: string, columnName: string) { + if (!hasTable(client, tableName)) { + return false; + } + + const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; + return rows.some((row) => row.name === columnName); +} + +function applySqlFile(client: Database, fileName: string) { + const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); + client.exec(sql); +} + +function ensureLocalSqliteSchema(client: Database) { + if (!hasTable(client, 'filing_statement_snapshot')) { + applySqlFile(client, '0001_glossy_statement_snapshots.sql'); + } + + if (hasTable(client, 'task_run')) { + const missingTaskColumns: Array<{ name: string; sql: string }> = [ + { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" }, + { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' }, + { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' }, + { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' }, + { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' } + ]; + + for (const column of missingTaskColumns) { + if (!hasColumn(client, 'task_run', column.name)) { + client.exec(column.sql); + } + } + } + + if (!hasTable(client, 'task_stage_event')) { + applySqlFile(client, '0003_task_stage_event_timeline.sql'); + } + + if (hasTable(client, 'watchlist_item')) { + const missingWatchlistColumns: Array<{ name: string; sql: string }> = [ + { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' }, + { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' } + ]; + + for (const column of missingWatchlistColumns) { + if (!hasColumn(client, 'watchlist_item', column.name)) { + client.exec(column.sql); + } + } + } + + if (!hasTable(client, 'filing_taxonomy_snapshot')) { + applySqlFile(client, '0005_financial_taxonomy_v3.sql'); + } +} + export function getSqliteClient() { if (!globalThis.__fiscalSqliteClient) { const databasePath = getDatabasePath(); @@ -40,6 +105,7 @@ export function getSqliteClient() { client.exec('PRAGMA foreign_keys = ON;'); client.exec('PRAGMA journal_mode = WAL;'); client.exec('PRAGMA busy_timeout = 5000;'); + ensureLocalSqliteSchema(client); globalThis.__fiscalSqliteClient = client; } @@ -56,3 +122,10 @@ export const db = globalThis.__fiscalDrizzleDb ?? createDb(); if (!globalThis.__fiscalDrizzleDb) { globalThis.__fiscalDrizzleDb = db; } + +export const __dbInternals = { + ensureLocalSqliteSchema, + getDatabasePath, + hasColumn, + hasTable +}; diff --git a/lib/server/financial-taxonomy.test.ts b/lib/server/financial-taxonomy.test.ts index 29f284a..fe4e395 100644 --- a/lib/server/financial-taxonomy.test.ts +++ b/lib/server/financial-taxonomy.test.ts @@ -1,26 +1,47 @@ import { describe, expect, it } from 'bun:test'; import { __financialTaxonomyInternals } from './financial-taxonomy'; import type { FilingTaxonomySnapshotRecord } from './repos/filing-taxonomy'; -import type { FinancialStatementKind, TaxonomyStatementRow } from '@/lib/types'; +import type { + FinancialStatementKind, + FinancialStatementPeriod, + TaxonomyFactRow, + TaxonomyStatementRow +} from '@/lib/types'; + +function createRow(input: { + key?: string; + label?: string; + conceptKey?: string; + qname?: string; + localName?: string; + statement?: FinancialStatementKind; + order?: number; + depth?: number; + hasDimensions?: boolean; + values: Record; + sourceFactIds?: number[]; +}): TaxonomyStatementRow { + const localName = input.localName ?? 'RevenueFromContractWithCustomerExcludingAssessedTax'; + const conceptKey = input.conceptKey ?? `http://fasb.org/us-gaap/2024#${localName}`; + const qname = input.qname ?? `us-gaap:${localName}`; -function createRow(periodIds: string[]): TaxonomyStatementRow { return { - key: 'us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax', - label: 'Revenue From Contract With Customer Excluding Assessed Tax', - conceptKey: 'us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax', - qname: 'us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax', - namespaceUri: 'http://fasb.org/us-gaap/2021-01-31', - localName: 'RevenueFromContractWithCustomerExcludingAssessedTax', + key: input.key ?? conceptKey, + label: input.label ?? localName, + conceptKey, + qname, + namespaceUri: 'http://fasb.org/us-gaap/2024', + localName, isExtension: false, - statement: 'income', - roleUri: 'income', - order: 1, - depth: 0, + statement: input.statement ?? 'income', + roleUri: input.statement ?? 'income', + order: input.order ?? 1, + depth: input.depth ?? 0, parentKey: null, - values: Object.fromEntries(periodIds.map((periodId, index) => [periodId, 100 + index])), - units: Object.fromEntries(periodIds.map((periodId) => [periodId, 'iso4217:USD'])), - hasDimensions: false, - sourceFactIds: periodIds.map((_, index) => index + 1) + values: input.values, + units: Object.fromEntries(Object.keys(input.values).map((periodId) => [periodId, 'iso4217:USD'])), + hasDimensions: input.hasDimensions ?? false, + sourceFactIds: input.sourceFactIds ?? [1] }; } @@ -35,8 +56,12 @@ function createSnapshot(input: { periodLabel: string; }>; statement: FinancialStatementKind; + rows?: TaxonomyStatementRow[]; }) { - const row = createRow(input.periods.map((period) => period.id)); + const defaultRow = createRow({ + statement: input.statement, + values: Object.fromEntries(input.periods.map((period, index) => [period.id, 100 + index])) + }); return { id: input.filingId, @@ -58,9 +83,9 @@ function createSnapshot(input: { periodLabel: period.periodLabel })), statement_rows: { - income: input.statement === 'income' ? [row] : [], - balance: input.statement === 'balance' ? [{ ...row, statement: 'balance' }] : [], - cash_flow: [], + income: input.statement === 'income' ? (input.rows ?? [defaultRow]) : [], + balance: input.statement === 'balance' ? (input.rows ?? [{ ...defaultRow, statement: 'balance' }]) : [], + cash_flow: input.statement === 'cash_flow' ? (input.rows ?? [{ ...defaultRow, statement: 'cash_flow' }]) : [], equity: [], comprehensive_income: [] }, @@ -74,6 +99,64 @@ function createSnapshot(input: { } satisfies FilingTaxonomySnapshotRecord; } +function createPeriod(input: { + id: string; + filingId: number; + filingDate: string; + periodEnd: string; + periodStart?: string | null; + filingType?: '10-K' | '10-Q'; +}): FinancialStatementPeriod { + return { + id: input.id, + filingId: input.filingId, + accessionNumber: `0000-${input.filingId}`, + filingDate: input.filingDate, + periodStart: input.periodStart ?? null, + periodEnd: input.periodEnd, + filingType: input.filingType ?? '10-Q', + periodLabel: 'Test period' + }; +} + +function createDimensionFact(input: { + filingId: number; + filingDate: string; + conceptKey: string; + qname: string; + localName: string; + periodEnd: string; + value: number; + axis?: string; + member?: string; +}): TaxonomyFactRow { + return { + id: input.filingId, + snapshotId: input.filingId, + filingId: input.filingId, + filingDate: input.filingDate, + statement: 'income', + roleUri: 'income', + conceptKey: input.conceptKey, + qname: input.qname, + namespaceUri: 'http://fasb.org/us-gaap/2024', + localName: input.localName, + value: input.value, + contextId: `ctx-${input.filingId}`, + unit: 'iso4217:USD', + decimals: null, + periodStart: '2025-01-01', + periodEnd: input.periodEnd, + periodInstant: null, + dimensions: [{ + axis: input.axis ?? 'srt:ProductOrServiceAxis', + member: input.member ?? 'msft:CloudMember' + }], + isDimensionless: false, + sourceFile: null + }; +} + describe('financial taxonomy internals', () => { it('selects the primary quarter duration for 10-Q income statements', () => { const snapshot = createSnapshot({ @@ -139,4 +222,144 @@ describe('financial taxonomy internals', () => { expect(periods.map((period) => period.id)).toEqual(['annual', 'quarter']); }); + + it('maps overlapping GAAP aliases into one standardized COGS row while preserving faithful rows', () => { + const period2024 = createPeriod({ + id: '2024-q4', + filingId: 30, + filingDate: '2025-01-29', + periodEnd: '2024-12-31' + }); + const period2025 = createPeriod({ + id: '2025-q4', + filingId: 31, + filingDate: '2026-01-28', + periodEnd: '2025-12-31' + }); + + const faithfulRows = __financialTaxonomyInternals.buildRows([ + createSnapshot({ + filingId: 30, + filingType: '10-Q', + filingDate: '2025-01-29', + statement: 'income', + periods: [{ + id: '2024-q4', + periodStart: '2024-10-01', + periodEnd: '2024-12-31', + periodLabel: '2024-10-01 to 2024-12-31' + }], + rows: [ + createRow({ + localName: 'CostOfRevenue', + label: 'Cost of Revenue', + values: { '2024-q4': 45_000 }, + sourceFactIds: [101] + }) + ] + }), + createSnapshot({ + filingId: 31, + filingType: '10-Q', + filingDate: '2026-01-28', + statement: 'income', + periods: [{ + id: '2025-q4', + periodStart: '2025-10-01', + periodEnd: '2025-12-31', + periodLabel: '2025-10-01 to 2025-12-31' + }], + rows: [ + createRow({ + localName: 'CostOfGoodsSold', + label: 'Cost of Goods Sold', + values: { '2025-q4': 48_000 }, + sourceFactIds: [202] + }) + ] + }) + ], 'income', new Set(['2024-q4', '2025-q4'])); + + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows( + faithfulRows, + 'income', + [period2024, period2025] + ); + + expect(faithfulRows).toHaveLength(2); + + const cogs = standardizedRows.find((row) => row.key === 'cost-of-revenue'); + expect(cogs).toBeDefined(); + expect(cogs?.values['2024-q4']).toBe(45_000); + expect(cogs?.values['2025-q4']).toBe(48_000); + expect(cogs?.sourceConcepts).toEqual([ + 'us-gaap:CostOfGoodsSold', + 'us-gaap:CostOfRevenue' + ]); + expect(cogs?.sourceRowKeys).toHaveLength(2); + }); + + it('aggregates standardized dimension drill-down across mapped source concepts', () => { + const period2024 = createPeriod({ + id: '2024-q4', + filingId: 40, + filingDate: '2025-01-29', + periodEnd: '2024-12-31' + }); + const period2025 = createPeriod({ + id: '2025-q4', + filingId: 41, + filingDate: '2026-01-28', + periodEnd: '2025-12-31' + }); + const faithfulRows = [ + createRow({ + localName: 'CostOfRevenue', + label: 'Cost of Revenue', + values: { '2024-q4': 45_000 }, + hasDimensions: true + }), + createRow({ + localName: 'CostOfGoodsSold', + label: 'Cost of Goods Sold', + values: { '2025-q4': 48_000 }, + hasDimensions: true + }) + ]; + const standardizedRows = __financialTaxonomyInternals.buildStandardizedRows( + faithfulRows, + 'income', + [period2024, period2025] + ); + + const breakdown = __financialTaxonomyInternals.buildDimensionBreakdown([ + createDimensionFact({ + filingId: 40, + filingDate: '2025-01-29', + conceptKey: faithfulRows[0].key, + qname: faithfulRows[0].qname, + localName: faithfulRows[0].localName, + periodEnd: '2024-12-31', + value: 20_000, + member: 'msft:ProductivityMember' + }), + createDimensionFact({ + filingId: 41, + filingDate: '2026-01-28', + conceptKey: faithfulRows[1].key, + qname: faithfulRows[1].qname, + localName: faithfulRows[1].localName, + periodEnd: '2025-12-31', + value: 28_000, + member: 'msft:IntelligentCloudMember' + }) + ], [period2024, period2025], faithfulRows, standardizedRows); + + const cogs = breakdown?.['cost-of-revenue'] ?? []; + expect(cogs).toHaveLength(2); + expect(cogs.map((row) => row.sourceLabel)).toEqual([ + 'Cost of Revenue', + 'Cost of Goods Sold' + ]); + }); }); diff --git a/lib/server/financial-taxonomy.ts b/lib/server/financial-taxonomy.ts index f9bd8b2..6efb922 100644 --- a/lib/server/financial-taxonomy.ts +++ b/lib/server/financial-taxonomy.ts @@ -4,6 +4,7 @@ import type { FinancialHistoryWindow, FinancialStatementKind, FinancialStatementPeriod, + StandardizedStatementRow, TaxonomyStatementRow } from '@/lib/types'; import { listFilingsRecords } from '@/lib/server/repos/filings'; @@ -28,6 +29,19 @@ type GetCompanyFinancialTaxonomyInput = { queuedSync: boolean; }; +type CanonicalRowDefinition = { + key: string; + label: string; + category: string; + order: number; + localNames?: readonly string[]; + labelIncludes?: readonly string[]; + formula?: ( + rowsByKey: Map, + periodIds: string[] + ) => Pick | null; +}; + function safeTicker(input: string) { return input.trim().toUpperCase(); } @@ -215,16 +229,419 @@ function buildRows( }); } +function normalizeToken(value: string) { + return value.trim().toLowerCase(); +} + +function sumValues(left: number | null, right: number | null) { + if (left === null || right === null) { + return null; + } + + return left + right; +} + +function subtractValues(left: number | null, right: number | null) { + if (left === null || right === null) { + return null; + } + + return left - right; +} + +const STANDARDIZED_ROW_DEFINITIONS: Record = { + income: [ + { + key: 'revenue', + label: 'Revenue', + category: 'revenue', + order: 10, + localNames: [ + 'RevenueFromContractWithCustomerExcludingAssessedTax', + 'Revenues', + 'SalesRevenueNet', + 'TotalRevenuesAndOtherIncome' + ] + }, + { + key: 'cost-of-revenue', + label: 'Cost of Revenue', + category: 'expense', + order: 20, + localNames: [ + 'CostOfRevenue', + 'CostOfGoodsSold', + 'CostOfSales', + 'CostOfProductsSold', + 'CostOfServices' + ] + }, + { + key: 'gross-profit', + label: 'Gross Profit', + category: 'profit', + order: 30, + localNames: ['GrossProfit'], + formula: (rowsByKey, periodIds) => { + const revenue = rowsByKey.get('revenue'); + const cogs = rowsByKey.get('cost-of-revenue'); + if (!revenue || !cogs) { + return null; + } + + return { + values: Object.fromEntries(periodIds.map((periodId) => [ + periodId, + subtractValues(revenue.values[periodId] ?? null, cogs.values[periodId] ?? null) + ])), + resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) + }; + } + }, + { + key: 'research-and-development', + label: 'Research & Development', + category: 'opex', + order: 40, + localNames: ['ResearchAndDevelopmentExpense'] + }, + { + key: 'selling-general-and-administrative', + label: 'Selling, General & Administrative', + category: 'opex', + order: 50, + localNames: [ + 'SellingGeneralAndAdministrativeExpense', + 'SellingAndMarketingExpense', + 'GeneralAndAdministrativeExpense' + ], + labelIncludes: ['selling, general', 'selling general', 'general and administrative'] + }, + { + key: 'operating-income', + label: 'Operating Income', + category: 'profit', + order: 60, + localNames: ['OperatingIncomeLoss', 'IncomeLossFromOperations'] + }, + { + key: 'net-income', + label: 'Net Income', + category: 'profit', + order: 70, + localNames: ['NetIncomeLoss', 'ProfitLoss'] + } + ], + balance: [ + { + key: 'cash-and-equivalents', + label: 'Cash & Equivalents', + category: 'asset', + order: 10, + localNames: [ + 'CashAndCashEquivalentsAtCarryingValue', + 'CashCashEquivalentsAndShortTermInvestments', + 'CashAndShortTermInvestments' + ] + }, + { + key: 'accounts-receivable', + label: 'Accounts Receivable', + category: 'asset', + order: 20, + localNames: [ + 'AccountsReceivableNetCurrent', + 'ReceivablesNetCurrent' + ] + }, + { + key: 'inventory', + label: 'Inventory', + category: 'asset', + order: 30, + localNames: ['InventoryNet'] + }, + { + key: 'total-assets', + label: 'Total Assets', + category: 'asset', + order: 40, + localNames: ['Assets'] + }, + { + key: 'current-liabilities', + label: 'Current Liabilities', + category: 'liability', + order: 50, + localNames: ['LiabilitiesCurrent'] + }, + { + key: 'long-term-debt', + label: 'Long-Term Debt', + category: 'liability', + order: 60, + localNames: [ + 'LongTermDebtNoncurrent', + 'LongTermDebt', + 'DebtNoncurrent', + 'LongTermDebtAndCapitalLeaseObligations' + ] + }, + { + key: 'current-debt', + label: 'Current Debt', + category: 'liability', + order: 70, + localNames: ['DebtCurrent', 'ShortTermBorrowings', 'LongTermDebtCurrent'] + }, + { + key: 'total-debt', + label: 'Total Debt', + category: 'liability', + order: 80, + localNames: ['DebtAndFinanceLeaseLiabilities', 'Debt'], + formula: (rowsByKey, periodIds) => { + const longTermDebt = rowsByKey.get('long-term-debt'); + const currentDebt = rowsByKey.get('current-debt'); + if (!longTermDebt || !currentDebt) { + return null; + } + + return { + values: Object.fromEntries(periodIds.map((periodId) => [ + periodId, + sumValues(longTermDebt.values[periodId] ?? null, currentDebt.values[periodId] ?? null) + ])), + resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) + }; + } + }, + { + key: 'total-equity', + label: 'Total Equity', + category: 'equity', + order: 90, + localNames: [ + 'StockholdersEquity', + 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', + 'PartnersCapital' + ] + } + ], + cash_flow: [ + { + key: 'operating-cash-flow', + label: 'Operating Cash Flow', + category: 'cash-flow', + order: 10, + localNames: [ + 'NetCashProvidedByUsedInOperatingActivities', + 'NetCashProvidedByUsedInOperatingActivitiesContinuingOperations' + ] + }, + { + key: 'capital-expenditures', + label: 'Capital Expenditures', + category: 'cash-flow', + order: 20, + localNames: ['PaymentsToAcquirePropertyPlantAndEquipment', 'CapitalExpendituresIncurredButNotYetPaid'] + }, + { + key: 'free-cash-flow', + label: 'Free Cash Flow', + category: 'cash-flow', + order: 30, + formula: (rowsByKey, periodIds) => { + const operatingCashFlow = rowsByKey.get('operating-cash-flow'); + const capex = rowsByKey.get('capital-expenditures'); + if (!operatingCashFlow || !capex) { + return null; + } + + return { + values: Object.fromEntries(periodIds.map((periodId) => [ + periodId, + subtractValues(operatingCashFlow.values[periodId] ?? null, capex.values[periodId] ?? null) + ])), + resolvedSourceRowKeys: Object.fromEntries(periodIds.map((periodId) => [periodId, null])) + }; + } + } + ], + equity: [ + { + key: 'total-equity', + label: 'Total Equity', + category: 'equity', + order: 10, + localNames: [ + 'StockholdersEquity', + 'StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest', + 'PartnersCapital' + ] + } + ], + comprehensive_income: [ + { + key: 'comprehensive-income', + label: 'Comprehensive Income', + category: 'profit', + order: 10, + localNames: ['ComprehensiveIncomeNetOfTax', 'ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest'] + } + ] +}; + +function matchesDefinition(row: TaxonomyStatementRow, definition: CanonicalRowDefinition) { + const rowLocalName = normalizeToken(row.localName); + if (definition.localNames?.some((localName) => normalizeToken(localName) === rowLocalName)) { + return true; + } + + const label = normalizeToken(row.label); + return definition.labelIncludes?.some((token) => label.includes(normalizeToken(token))) ?? false; +} + +function buildCanonicalRow( + definition: CanonicalRowDefinition, + matches: TaxonomyStatementRow[], + periodIds: string[] +) { + const sortedMatches = [...matches].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); + + const sourceConcepts = new Set(); + const sourceRowKeys = new Set(); + const sourceFactIds = new Set(); + + for (const row of sortedMatches) { + sourceConcepts.add(row.qname); + sourceRowKeys.add(row.key); + for (const factId of row.sourceFactIds) { + sourceFactIds.add(factId); + } + } + + const values: Record = {}; + const resolvedSourceRowKeys: Record = {}; + + for (const periodId of periodIds) { + const match = sortedMatches.find((row) => periodId in row.values); + values[periodId] = match?.values[periodId] ?? null; + resolvedSourceRowKeys[periodId] = match?.key ?? null; + } + + return { + key: definition.key, + label: definition.label, + category: definition.category, + order: definition.order, + values, + hasDimensions: sortedMatches.some((row) => row.hasDimensions), + sourceConcepts: [...sourceConcepts].sort((left, right) => left.localeCompare(right)), + sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)), + sourceFactIds: [...sourceFactIds].sort((left, right) => left - right), + resolvedSourceRowKeys + } satisfies StandardizedStatementRow; +} + +function buildStandardizedRows( + rows: TaxonomyStatementRow[], + statement: FinancialStatementKind, + periods: FinancialStatementPeriod[] +) { + const definitions = STANDARDIZED_ROW_DEFINITIONS[statement] ?? []; + const periodIds = periods.map((period) => period.id); + const rowsByKey = new Map(); + const matchedRowKeys = new Set(); + + for (const definition of definitions) { + const matches = rows.filter((row) => matchesDefinition(row, definition)); + if (matches.length === 0 && !definition.formula) { + continue; + } + + for (const row of matches) { + matchedRowKeys.add(row.key); + } + + const canonicalRow = buildCanonicalRow(definition, matches, periodIds); + rowsByKey.set(definition.key, canonicalRow); + + const derived = definition.formula?.(rowsByKey, periodIds) ?? null; + if (derived) { + rowsByKey.set(definition.key, { + ...canonicalRow, + values: derived.values, + resolvedSourceRowKeys: derived.resolvedSourceRowKeys + }); + } + } + + const unmatchedRows = rows + .filter((row) => !matchedRowKeys.has(row.key)) + .map((row) => ({ + key: `other:${row.key}`, + label: row.label, + category: 'other', + order: 10_000 + row.order, + values: { ...row.values }, + hasDimensions: row.hasDimensions, + sourceConcepts: [row.qname], + sourceRowKeys: [row.key], + sourceFactIds: [...row.sourceFactIds], + resolvedSourceRowKeys: Object.fromEntries( + periodIds.map((periodId) => [periodId, periodId in row.values ? row.key : null]) + ) + } satisfies StandardizedStatementRow)); + + return [...rowsByKey.values(), ...unmatchedRows].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + + return left.label.localeCompare(right.label); + }); +} + function buildDimensionBreakdown( facts: Awaited>['facts'], - periods: FinancialStatementPeriod[] + periods: FinancialStatementPeriod[], + faithfulRows: TaxonomyStatementRow[], + standardizedRows: StandardizedStatementRow[] ) { const periodByFilingId = new Map(); for (const period of periods) { periodByFilingId.set(period.filingId, period); } + const faithfulRowByKey = new Map(faithfulRows.map((row) => [row.key, row])); + const standardizedRowsBySource = new Map(); + for (const row of standardizedRows) { + for (const sourceRowKey of row.sourceRowKeys) { + const existing = standardizedRowsBySource.get(sourceRowKey); + if (existing) { + existing.push(row); + } else { + standardizedRowsBySource.set(sourceRowKey, [row]); + } + } + } + const map = new Map(); + const pushRow = (key: string, row: DimensionBreakdownRow) => { + const existing = map.get(key); + if (existing) { + existing.push(row); + } else { + map.set(key, [row]); + } + }; for (const fact of facts) { if (fact.dimensions.length === 0) { @@ -244,10 +661,15 @@ function buildDimensionBreakdown( continue; } + const faithfulRow = faithfulRowByKey.get(fact.conceptKey) ?? null; + const standardizedMatches = standardizedRowsBySource.get(fact.conceptKey) ?? []; + for (const dimension of fact.dimensions) { - const row: DimensionBreakdownRow = { + const faithfulDimensionRow: DimensionBreakdownRow = { rowKey: fact.conceptKey, concept: fact.qname, + sourceRowKey: fact.conceptKey, + sourceLabel: faithfulRow?.label ?? null, periodId: period.id, axis: dimension.axis, member: dimension.member, @@ -255,11 +677,13 @@ function buildDimensionBreakdown( unit: fact.unit }; - const existing = map.get(fact.conceptKey); - if (existing) { - existing.push(row); - } else { - map.set(fact.conceptKey, [row]); + pushRow(fact.conceptKey, faithfulDimensionRow); + + for (const standardizedRow of standardizedMatches) { + pushRow(standardizedRow.key, { + ...faithfulDimensionRow, + rowKey: standardizedRow.key + }); } } } @@ -305,7 +729,8 @@ export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxo const financialFilings = filings.filter((filing) => isFinancialForm(filing.filing_type)); const selection = selectPrimaryPeriods(snapshotResult.snapshots, input.statement); const periods = selection.periods; - const rows = buildRows(snapshotResult.snapshots, input.statement, selection.selectedPeriodIds); + const faithfulRows = buildRows(snapshotResult.snapshots, input.statement, selection.selectedPeriodIds); + const standardizedRows = buildStandardizedRows(faithfulRows, input.statement, periods); const factsResult = input.includeFacts ? await listTaxonomyFactsByTicker({ @@ -329,11 +754,11 @@ export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxo const latestFiling = filings[0] ?? null; const metrics = latestMetrics(snapshotResult.snapshots); const dimensionBreakdown = input.includeDimensions - ? buildDimensionBreakdown(dimensionFacts.facts, periods) + ? buildDimensionBreakdown(dimensionFacts.facts, periods, faithfulRows, standardizedRows) : null; - const dimensionsCount = dimensionBreakdown - ? Object.values(dimensionBreakdown).reduce((total, entries) => total + entries.length, 0) + const dimensionsCount = input.includeDimensions + ? dimensionFacts.facts.reduce((total, fact) => total + fact.dimensions.length, 0) : 0; const factsCoverage = input.includeFacts @@ -348,8 +773,18 @@ export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxo }, statement: input.statement, window: input.window, + defaultSurface: 'standardized', periods, - rows, + surfaces: { + faithful: { + kind: 'faithful', + rows: faithfulRows + }, + standardized: { + kind: 'standardized', + rows: standardizedRows + } + }, nextCursor: snapshotResult.nextCursor, facts: input.includeFacts ? { @@ -359,7 +794,7 @@ export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxo : null, coverage: { filings: periods.length, - rows: rows.length, + rows: faithfulRows.length, dimensions: dimensionsCount, facts: factsCoverage }, @@ -378,7 +813,11 @@ export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxo export const __financialTaxonomyInternals = { buildPeriods, + buildRows, + buildStandardizedRows, + buildDimensionBreakdown, isInstantPeriod, + matchesDefinition, periodDurationDays, selectPrimaryPeriods }; diff --git a/lib/types.ts b/lib/types.ts index 1117dd7..1b4c3b5 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -213,6 +213,21 @@ export type TaxonomyStatementRow = { sourceFactIds: number[]; }; +export type FinancialStatementSurfaceKind = 'faithful' | 'standardized'; + +export type StandardizedStatementRow = { + key: string; + label: string; + category: string; + order: number; + values: Record; + hasDimensions: boolean; + sourceConcepts: string[]; + sourceRowKeys: string[]; + sourceFactIds: number[]; + resolvedSourceRowKeys: Record; +}; + export type TaxonomyFactRow = { id: number; snapshotId: number; @@ -256,16 +271,6 @@ export type MetricValidationResult = { validatedAt: string | null; }; -export type StandardizedStatementRow = { - key: string; - label: string; - concept: string; - category: string; - sourceConcepts: string[]; - values: Record; - hasDimensions: boolean; -}; - export type FilingFaithfulStatementRow = { key: string; label: string; @@ -280,6 +285,8 @@ export type FilingFaithfulStatementRow = { export type DimensionBreakdownRow = { rowKey: string; concept: string | null; + sourceRowKey: string | null; + sourceLabel: string | null; periodId: string; axis: string; member: string; @@ -287,6 +294,11 @@ export type DimensionBreakdownRow = { unit: string | null; }; +export type FinancialStatementSurface = { + kind: FinancialStatementSurfaceKind; + rows: Row[]; +}; + export type CompanyFinancialStatementsResponse = { company: { ticker: string; @@ -295,8 +307,12 @@ export type CompanyFinancialStatementsResponse = { }; statement: FinancialStatementKind; window: FinancialHistoryWindow; + defaultSurface: FinancialStatementSurfaceKind; periods: FinancialStatementPeriod[]; - rows: TaxonomyStatementRow[]; + surfaces: { + faithful: FinancialStatementSurface; + standardized: FinancialStatementSurface; + }; nextCursor: string | null; facts: { rows: TaxonomyFactRow[];
PeriodSourceAxis Member Value
{period ? formatLongDate(period.filingDate) : row.periodId}{row.sourceLabel ?? row.concept ?? 'Unknown source'}{row.axis} {row.member} {asDisplayCurrency(row.value, valueScale)}