WIP main worktree changes before merge

This commit is contained in:
2026-03-13 00:20:22 -04:00
parent 58bf80189d
commit e5141238fb
25 changed files with 940 additions and 208 deletions

View File

@@ -137,7 +137,9 @@ function AnalysisPageContent() {
<PriceHistoryCard
loading={loading}
priceHistory={analysis.priceHistory}
benchmarkHistory={analysis.benchmarkHistory}
quote={analysis.quote}
position={analysis.position}
/>
</section>

View File

@@ -700,6 +700,10 @@ function FinancialsPageContent() {
return null;
}, [displayMode, financials?.statementRows, surfaceKind]);
const hasUnmappedResidualRows = useMemo(() => {
return (financials?.statementDetails?.unmapped?.length ?? 0) > 0;
}, [financials?.statementDetails]);
const trendSeries = financials?.trendSeries ?? [];
const chartData = useMemo(() => {
return periods.map((period) => ({
@@ -1042,9 +1046,16 @@ function FinancialsPageContent() {
) : (
<div className="space-y-3">
{isStatementSurfaceKind(surfaceKind) ? (
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
USD · {valueScaleLabel}
</p>
<div className="space-y-2">
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
USD · {valueScaleLabel}
</p>
{isTreeStatementMode && hasUnmappedResidualRows ? (
<p className="text-sm text-[color:var(--terminal-muted)]">
Parser residual rows are available under the <span className="text-[color:var(--terminal-bright)]">Unmapped / Residual</span> section.
</p>
) : null}
</div>
) : null}
{isTreeStatementMode && treeModel ? (
<StatementMatrix

View File

@@ -17,6 +17,7 @@
"date-fns": "^4.1.0",
"drizzle-orm": "^0.41.0",
"elysia": "latest",
"html-to-image": "^1.11.13",
"lucide-react": "^0.575.0",
"next": "^16.1.6",
"react": "^19.2.4",
@@ -1068,6 +1069,8 @@
"hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="],
"html-to-image": ["html-to-image@1.11.13", "", {}, "sha512-cuOPoI7WApyhBElTTb9oqsawRvZ0rHhaHwghRLlTuffoD1B2aDemlCruLeZrUIIdvG7gs9xeELEPm6PhuASqrg=="],
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],

View File

@@ -1,120 +1,127 @@
'use client';
import { format } from 'date-fns';
import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts';
import { Panel } from '@/components/ui/panel';
import { formatCurrency } from '@/lib/format';
import { InteractivePriceChart } from '@/components/charts/interactive-price-chart';
import type { DataSeries, Holding } from '@/lib/types';
type PriceHistoryCardProps = {
loading: boolean;
priceHistory: Array<{ date: string; close: number }>;
benchmarkHistory: Array<{ date: string; close: number }>;
quote: number;
position: Holding | null;
};
const CHART_TEXT = '#f3f5f7';
const CHART_MUTED = '#a1a9b3';
const CHART_GRID = 'rgba(196, 202, 211, 0.18)';
const CHART_TOOLTIP_BG = 'rgba(31, 34, 39, 0.96)';
const CHART_TOOLTIP_BORDER = 'rgba(220, 226, 234, 0.24)';
function formatShortDate(value: string) {
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM yy');
}
function formatLongDate(value: string) {
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM dd, yyyy');
}
function asFiniteNumber(value: string | number | null | undefined) {
if (value === null || value === undefined) {
return null;
}
const parsed = typeof value === 'number' ? value : Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
export function PriceHistoryCard(props: PriceHistoryCardProps) {
const series = props.priceHistory.map((point) => ({
...point,
label: formatShortDate(point.date)
}));
const firstPoint = props.priceHistory[0] ?? null;
const lastPoint = props.priceHistory[props.priceHistory.length - 1] ?? null;
const change = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
const changePct = firstPoint && lastPoint && firstPoint.close !== 0
? ((lastPoint.close - firstPoint.close) / firstPoint.close) * 100
const firstPoint = props.priceHistory[0];
const lastPoint = props.priceHistory[props.priceHistory.length - 1];
const inPortfolio = props.position !== null;
const defaultChange = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
const defaultChangePct = firstPoint && firstPoint.close > 0 && defaultChange !== null
? (defaultChange / firstPoint.close) * 100
: null;
const holdingChange = asFiniteNumber(props.position?.gain_loss);
const holdingChangePct = asFiniteNumber(props.position?.gain_loss_pct);
const change = inPortfolio ? holdingChange : defaultChange;
const changePct = inPortfolio ? holdingChangePct : defaultChangePct;
const rangeLabel = inPortfolio ? 'Holding return' : '1Y return';
const changeLabel = inPortfolio ? 'Holding P/L' : '1Y change';
const dateRange = inPortfolio
? props.position?.created_at && (props.position.last_price_at ?? lastPoint?.date)
? `${formatLongDate(props.position.created_at)} to ${formatLongDate(props.position.last_price_at ?? lastPoint?.date ?? '')}`
: 'No holding period available'
: firstPoint && lastPoint
? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}`
: 'No comparison range';
const statusToneClass = inPortfolio ? 'text-[#96f5bf]' : 'text-[color:var(--terminal-bright)]';
const performanceToneClass = change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]';
const chartData = props.priceHistory.map(point => ({
date: point.date,
price: point.close
}));
const comparisonSeries: DataSeries[] = [
{
id: 'stock',
label: 'Stock',
data: chartData,
color: '#96f5bf',
type: 'line'
},
{
id: 'sp500',
label: 'S&P 500',
data: props.benchmarkHistory.map((point) => ({
date: point.date,
price: point.close
})),
color: '#8ba0b8',
type: 'line'
}
];
const helperText = Number.isFinite(props.quote)
? `Spot price ${formatCurrency(props.quote)}`
: 'Spot price unavailable';
return (
<Panel
title="Price chart"
subtitle="One-year weekly close with current spot price and trailing move."
className="h-full pt-2"
>
<div className="grid gap-4 xl:grid-cols-[minmax(0,1fr)_220px]">
<div>
{props.loading ? (
<p className="text-sm text-[color:var(--terminal-muted)]">Loading price history...</p>
) : series.length === 0 ? (
<p className="text-sm text-[color:var(--terminal-muted)]">No price history available.</p>
) : (
<div className="h-[320px]">
<ResponsiveContainer width="100%" height="100%">
<LineChart data={series}>
<CartesianGrid strokeDasharray="2 2" stroke={CHART_GRID} />
<XAxis
dataKey="label"
minTickGap={32}
stroke={CHART_MUTED}
fontSize={12}
axisLine={{ stroke: CHART_MUTED }}
tickLine={{ stroke: CHART_MUTED }}
tick={{ fill: CHART_MUTED }}
/>
<YAxis
stroke={CHART_MUTED}
fontSize={12}
axisLine={{ stroke: CHART_MUTED }}
tickLine={{ stroke: CHART_MUTED }}
tick={{ fill: CHART_MUTED }}
tickFormatter={(value: number) => `$${value.toFixed(0)}`}
domain={[(dataMin) => dataMin * 0.05, (dataMax) => dataMax * 1.05]}
allowDataOverflow
/>
<Tooltip
formatter={(value) => formatCurrency(Array.isArray(value) ? value[0] : value)}
labelFormatter={(value) => String(value)}
contentStyle={{
backgroundColor: CHART_TOOLTIP_BG,
border: `1px solid ${CHART_TOOLTIP_BORDER}`,
borderRadius: '0.75rem'
}}
labelStyle={{ color: CHART_TEXT }}
itemStyle={{ color: CHART_TEXT }}
cursor={{ stroke: 'rgba(220, 226, 234, 0.28)', strokeWidth: 1 }}
/>
<Line type="monotone" dataKey="close" stroke="#d9dee5" strokeWidth={2.5} dot={false} />
</LineChart>
</ResponsiveContainer>
</div>
)}
</div>
<div className="grid gap-0 border-t border-[color:var(--line-weak)] sm:grid-cols-3 xl:grid-cols-1 xl:border-l">
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b xl:border-l-0">
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Current price</p>
<p className="mt-2 text-3xl font-semibold text-[color:var(--terminal-bright)]">{formatCurrency(props.quote)}</p>
<Panel title="Price chart" subtitle="Interactive chart with historical data">
<div className="space-y-4">
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Portfolio status</p>
<p className={`mt-2 text-2xl font-semibold ${statusToneClass}`}>{inPortfolio ? 'In portfolio' : 'Not in portfolio'}</p>
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">{helperText}</p>
</div>
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b">
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y change</p>
<p className={`mt-2 text-2xl font-semibold ${change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{changeLabel}</p>
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
{change === null ? 'n/a' : formatCurrency(change)}
</p>
</div>
<div className="px-4 py-4">
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y return</p>
<p className={`mt-2 text-2xl font-semibold ${changePct !== null && changePct < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{rangeLabel}</p>
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
{changePct === null ? 'n/a' : `${changePct.toFixed(2)}%`}
</p>
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">
{firstPoint && lastPoint ? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}` : 'No comparison range'}
{dateRange}
</p>
</div>
</div>
<InteractivePriceChart
data={chartData}
dataSeries={comparisonSeries}
defaultChartType="line"
defaultTimeRange="1Y"
showVolume={false}
showToolbar={true}
height={320}
loading={props.loading}
formatters={{
price: formatCurrency,
date: (date: string) => format(new Date(date), 'MMM dd, yyyy')
}}
/>
</div>
</Panel>
);

View File

@@ -29,6 +29,7 @@ function SummaryCard(props: {
export function NormalizationSummary({ normalization }: NormalizationSummaryProps) {
const hasMaterialUnmapped = normalization.materialUnmappedRowCount > 0;
const hasWarnings = normalization.warnings.length > 0;
return (
<Panel
@@ -36,10 +37,13 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
subtitle="Pack, parser, and residual mapping health for the compact statement surface."
variant="surface"
>
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-5">
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-8">
<SummaryCard label="Pack" value={normalization.fiscalPack ?? 'unknown'} />
<SummaryCard label="Regime" value={normalization.regime} />
<SummaryCard label="Parser" value={`fiscal-xbrl ${normalization.parserVersion}`} />
<SummaryCard label="Parser" value={`${normalization.parserEngine} ${normalization.parserVersion}`} />
<SummaryCard label="Surface Rows" value={String(normalization.surfaceRowCount)} />
<SummaryCard label="Detail Rows" value={String(normalization.detailRowCount)} />
<SummaryCard label="KPI Rows" value={String(normalization.kpiRowCount)} />
<SummaryCard label="Unmapped Rows" value={String(normalization.unmappedRowCount)} />
<SummaryCard
label="Material Unmapped"
@@ -47,6 +51,23 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
tone={hasMaterialUnmapped ? 'warning' : 'default'}
/>
</div>
{hasWarnings ? (
<div className="mt-3 rounded-xl border border-[color:var(--line-weak)] bg-[color:var(--panel-soft)] px-3 py-3">
<p className="panel-heading text-[10px] uppercase tracking-[0.16em] text-[color:var(--terminal-muted)]">
Parser Warnings
</p>
<div className="mt-2 flex flex-wrap gap-2">
{normalization.warnings.map((warning) => (
<span
key={warning}
className="rounded-full border border-[color:var(--line-weak)] bg-[rgba(88,102,122,0.16)] px-3 py-1 text-xs text-[color:var(--terminal-bright)]"
>
{warning}
</span>
))}
</div>
</div>
) : null}
{hasMaterialUnmapped ? (
<div className="mt-3 flex items-start gap-2 rounded-xl border border-[#7f6250] bg-[rgba(91,66,46,0.18)] px-3 py-3 text-sm text-[#f5d5c0]">
<AlertTriangle className="mt-0.5 size-4 shrink-0" />

View File

@@ -37,6 +37,9 @@ function renderList(values: string[]) {
export function StatementRowInspector(props: StatementRowInspectorProps) {
const selection = props.selection;
const parentSurfaceLabel = selection?.kind === 'detail'
? selection.parentSurfaceRow?.label ?? (selection.row.parentSurfaceKey === 'unmapped' ? 'Unmapped / Residual' : selection.row.parentSurfaceKey)
: null;
return (
<Panel
@@ -122,7 +125,7 @@ export function StatementRowInspector(props: StatementRowInspectorProps) {
<div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-4">
<InspectorCard label="Label" value={selection.row.label} />
<InspectorCard label="Key" value={selection.row.key} />
<InspectorCard label="Parent Surface" value={selection.parentSurfaceRow?.label ?? selection.row.parentSurfaceKey} />
<InspectorCard label="Parent Surface" value={parentSurfaceLabel ?? selection.row.parentSurfaceKey} />
<InspectorCard label="Residual" value={selection.row.residualFlag ? 'Yes' : 'No'} />
</div>

View File

@@ -242,6 +242,23 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
dimensionsSummary: [],
residualFlag: false
}
],
unmapped: [
{
key: 'other_income_unmapped',
parentSurfaceKey: 'unmapped',
label: 'Other Income Residual',
conceptKey: 'other_income_unmapped',
qname: 'us-gaap:OtherNonoperatingIncomeExpense',
namespaceUri: 'http://fasb.org/us-gaap/2024',
localName: 'OtherNonoperatingIncomeExpense',
unit: 'USD',
values: { [`${prefix}-fy24`]: 1_200, [`${prefix}-fy25`]: 1_450 },
sourceFactIds: [107],
isExtension: false,
dimensionsSummary: [],
residualFlag: true
}
]
},
ratioRows: [],
@@ -285,11 +302,16 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
validation: null
},
normalization: {
parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: isBank ? 'bank_lender' : 'core',
parserVersion: '0.1.0',
unmappedRowCount: 0,
materialUnmappedRowCount: 0
surfaceRowCount: 8,
detailRowCount: isBank ? 0 : 4,
kpiRowCount: 0,
unmappedRowCount: isBank ? 0 : 1,
materialUnmappedRowCount: 0,
warnings: isBank ? [] : ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
},
dimensionBreakdown: null
}
@@ -316,6 +338,11 @@ test('renders the standardized operating expense tree and inspector details', as
await page.goto('/financials?ticker=MSFT');
await expect(page.getByText('Normalization Summary')).toBeVisible();
await expect(page.getByText('fiscal-xbrl 0.1.0')).toBeVisible();
await expect(page.getByText('Parser Warnings')).toBeVisible();
await expect(page.getByText('income_sparse_mapping')).toBeVisible();
await expect(page.getByText('unmapped_cash_flow_bridge')).toBeVisible();
await expect(page.getByText('Parser residual rows are available under the Unmapped / Residual section.')).toBeVisible();
await expect(page.getByRole('button', { name: 'Expand Operating Expenses details' })).toBeVisible();
await page.getByRole('button', { name: 'Expand Operating Expenses details' }).click();
@@ -327,6 +354,11 @@ test('renders the standardized operating expense tree and inspector details', as
await expect(page.getByText('Row Details')).toBeVisible();
await expect(page.getByText('selling_general_and_administrative', { exact: true }).first()).toBeVisible();
await expect(page.getByText('Corporate SG&A')).toBeVisible();
await expect(page.getByText('Unmapped / Residual')).toBeVisible();
await page.getByRole('button', { name: /^Other Income Residual/ }).click();
await expect(page.getByText('other_income_unmapped', { exact: true })).toBeVisible();
await expect(page.getByText('Unmapped / Residual', { exact: true }).last()).toBeVisible();
});
test('shows not meaningful expense breakdown rows for bank pack filings', async ({ page }, testInfo) => {

View File

@@ -230,11 +230,16 @@ function createFinancialsPayload(input: {
validation: null
},
normalization: {
parserEngine: 'fiscal-xbrl',
regime: 'unknown',
fiscalPack,
parserVersion: '0.0.0',
surfaceRowCount: 0,
detailRowCount: 0,
kpiRowCount: 0,
unmappedRowCount: 0,
materialUnmappedRowCount: 0
materialUnmappedRowCount: 0,
warnings: []
},
dimensionBreakdown: null
}

View File

@@ -210,4 +210,86 @@ describe('statement view model', () => {
row: { key: 'gp_unmapped', residualFlag: true }
});
});
it('renders unmapped detail rows in a dedicated residual section and counts them', () => {
const model = buildStatementTree({
surfaceKind: 'income_statement',
rows: [
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
],
statementDetails: {
unmapped: [
createDetailRow({
key: 'unmapped_other_income',
label: 'Other income residual',
parentSurfaceKey: 'unmapped',
values: { p1: 5 },
residualFlag: true
})
]
},
categories: [],
searchQuery: '',
expandedRowKeys: new Set()
});
expect(model.sections).toHaveLength(2);
expect(model.sections[1]).toMatchObject({
key: 'unmapped_residual',
label: 'Unmapped / Residual'
});
expect(model.sections[1]?.nodes[0]).toMatchObject({
kind: 'detail',
row: { key: 'unmapped_other_income', parentSurfaceKey: 'unmapped' }
});
expect(model.visibleNodeCount).toBe(2);
expect(model.totalNodeCount).toBe(2);
});
it('matches search and resolves selection for unmapped detail rows without a real parent surface', () => {
const rows = [
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
];
const statementDetails = {
unmapped: [
createDetailRow({
key: 'unmapped_fx_gain',
label: 'FX gain residual',
parentSurfaceKey: 'unmapped',
values: { p1: 2 },
residualFlag: true
})
]
};
const model = buildStatementTree({
surfaceKind: 'income_statement',
rows,
statementDetails,
categories: [],
searchQuery: 'fx gain',
expandedRowKeys: new Set()
});
expect(model.sections).toHaveLength(1);
expect(model.sections[0]).toMatchObject({
key: 'unmapped_residual',
label: 'Unmapped / Residual'
});
expect(model.visibleNodeCount).toBe(1);
expect(model.totalNodeCount).toBe(2);
const selection = resolveStatementSelection({
surfaceKind: 'income_statement',
rows,
statementDetails,
selection: { kind: 'detail', key: 'unmapped_fx_gain', parentKey: 'unmapped' }
});
expect(selection).toMatchObject({
kind: 'detail',
row: { key: 'unmapped_fx_gain', parentSurfaceKey: 'unmapped' },
parentSurfaceRow: null
});
});
});

View File

@@ -79,6 +79,10 @@ type Categories = Array<{
count: number;
}>;
const UNMAPPED_DETAIL_GROUP_KEY = 'unmapped';
const UNMAPPED_SECTION_KEY = 'unmapped_residual';
const UNMAPPED_SECTION_LABEL = 'Unmapped / Residual';
function surfaceConfigForKind(surfaceKind: Extract<FinancialSurfaceKind, 'income_statement' | 'balance_sheet' | 'cash_flow_statement'>) {
return SURFACE_CHILDREN[surfaceKind] ?? {};
}
@@ -129,6 +133,25 @@ function sortDetailRows(left: DetailFinancialRow, right: DetailFinancialRow) {
return left.label.localeCompare(right.label);
}
function buildUnmappedDetailNodes(input: {
statementDetails: SurfaceDetailMap | null;
searchQuery: string;
}) {
const normalizedSearch = normalize(input.searchQuery);
return [...(input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? [])]
.sort(sortDetailRows)
.filter((detail) => normalizedSearch.length === 0 || searchTextForDetail(detail).includes(normalizedSearch))
.map((detail) => ({
kind: 'detail',
id: detailNodeId(UNMAPPED_DETAIL_GROUP_KEY, detail),
level: 0,
row: detail,
parentSurfaceKey: UNMAPPED_DETAIL_GROUP_KEY,
matchesSearch: normalizedSearch.length > 0 && searchTextForDetail(detail).includes(normalizedSearch)
}) satisfies StatementTreeDetailNode);
}
function countNodes(nodes: StatementTreeNode[]) {
let count = 0;
@@ -223,10 +246,26 @@ export function buildStatementTree(input: {
.filter((node): node is StatementTreeSurfaceNode => Boolean(node));
if (input.categories.length === 0) {
const sections: StatementTreeSection[] = rootNodes.length > 0
? [{ key: 'ungrouped', label: null, nodes: rootNodes }]
: [];
const unmappedNodes = buildUnmappedDetailNodes({
statementDetails: input.statementDetails,
searchQuery: input.searchQuery
});
if (unmappedNodes.length > 0) {
sections.push({
key: UNMAPPED_SECTION_KEY,
label: UNMAPPED_SECTION_LABEL,
nodes: unmappedNodes
});
}
return {
sections: [{ key: 'ungrouped', label: null, nodes: rootNodes }],
sections,
autoExpandedKeys,
visibleNodeCount: countNodes(rootNodes),
visibleNodeCount: sections.reduce((sum, section) => sum + countNodes(section.nodes), 0),
totalNodeCount: input.rows.length + Object.values(input.statementDetails ?? {}).reduce((sum, rows) => sum + rows.length, 0)
};
}
@@ -254,6 +293,18 @@ export function buildStatementTree(input: {
});
}
const unmappedNodes = buildUnmappedDetailNodes({
statementDetails: input.statementDetails,
searchQuery: input.searchQuery
});
if (unmappedNodes.length > 0) {
sections.push({
key: UNMAPPED_SECTION_KEY,
label: UNMAPPED_SECTION_LABEL,
nodes: unmappedNodes
});
}
return {
sections,
autoExpandedKeys,
@@ -297,9 +348,11 @@ export function resolveStatementSelection(input: {
}
const parentSurfaceKey = selection.parentKey ?? null;
const detailRows = parentSurfaceKey
? input.statementDetails?.[parentSurfaceKey] ?? []
: Object.values(input.statementDetails ?? {}).flat();
const detailRows = parentSurfaceKey === UNMAPPED_DETAIL_GROUP_KEY
? input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? []
: parentSurfaceKey
? input.statementDetails?.[parentSurfaceKey] ?? []
: Object.values(input.statementDetails ?? {}).flat();
const row = detailRows.find((candidate) => candidate.key === selection.key) ?? null;
if (!row) {

View File

@@ -104,11 +104,16 @@ function createFinancials(input: {
validation: null
},
normalization: {
parserEngine: 'fiscal-xbrl',
regime: 'unknown',
fiscalPack: input.fiscalPack ?? null,
parserVersion: '0.0.0',
surfaceRowCount: 0,
detailRowCount: 0,
kpiRowCount: 0,
unmappedRowCount: 0,
materialUnmappedRowCount: 0
materialUnmappedRowCount: 0,
warnings: []
},
dimensionBreakdown: null
} satisfies CompanyFinancialStatementsResponse;

View File

@@ -18,6 +18,7 @@ import type {
} from '@/lib/types';
import { auth } from '@/lib/auth';
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
import { getLatestFinancialIngestionSchemaStatus } from '@/lib/server/db/financial-ingestion-schema';
import { asErrorMessage, jsonError } from '@/lib/server/http';
import { buildPortfolioSummary } from '@/lib/server/portfolio';
import {
@@ -391,16 +392,36 @@ export const app = new Elysia({ prefix: '/api' })
getTaskQueueSnapshot(),
checkWorkflowBackend()
]);
const ingestionSchema = getLatestFinancialIngestionSchemaStatus();
const ingestionSchemaPayload = ingestionSchema
? {
ok: ingestionSchema.ok,
mode: ingestionSchema.mode,
missingIndexes: ingestionSchema.missingIndexes,
duplicateGroups: ingestionSchema.duplicateGroups,
lastCheckedAt: ingestionSchema.lastCheckedAt
}
: {
ok: false,
mode: 'failed' as const,
missingIndexes: [],
duplicateGroups: 0,
lastCheckedAt: new Date().toISOString()
};
const schemaHealthy = ingestionSchema?.ok ?? false;
if (!workflowBackend.ok) {
if (!workflowBackend.ok || !schemaHealthy) {
return Response.json({
status: 'degraded',
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
database: {
ingestionSchema: ingestionSchemaPayload
},
workflow: {
ok: false,
reason: workflowBackend.reason
ok: workflowBackend.ok,
...(workflowBackend.ok ? {} : { reason: workflowBackend.reason })
}
}, { status: 503 });
}
@@ -410,6 +431,9 @@ export const app = new Elysia({ prefix: '/api' })
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
database: {
ingestionSchema: ingestionSchemaPayload
},
workflow: {
ok: true
}
@@ -1366,12 +1390,13 @@ export const app = new Elysia({ prefix: '/api' })
return jsonError('ticker is required');
}
const [filings, holding, watchlistItem, liveQuote, priceHistory, journalPreview, memo, secProfile] = await Promise.all([
const [filings, holding, watchlistItem, liveQuote, priceHistory, benchmarkHistory, journalPreview, memo, secProfile] = await Promise.all([
listFilingsRecords({ ticker, limit: 40 }),
getHoldingByTicker(session.user.id, ticker),
getWatchlistItemByTicker(session.user.id, ticker),
getQuote(ticker),
getPriceHistory(ticker),
getPriceHistory('^GSPC'),
listResearchJournalEntries(session.user.id, ticker, 6),
getResearchMemoByTicker(session.user.id, ticker),
getSecCompanyProfile(ticker)
@@ -1478,6 +1503,7 @@ export const app = new Elysia({ prefix: '/api' })
quote: liveQuote,
position: holding,
priceHistory,
benchmarkHistory,
financials,
filings: redactedFilings.slice(0, 20),
aiReports,

View File

@@ -74,11 +74,44 @@ function resetDbSingletons() {
const globalState = globalThis as typeof globalThis & {
__fiscalSqliteClient?: { close?: () => void };
__fiscalDrizzleDb?: unknown;
__financialIngestionSchemaStatus?: unknown;
};
globalState.__fiscalSqliteClient?.close?.();
globalState.__fiscalSqliteClient = undefined;
globalState.__fiscalDrizzleDb = undefined;
globalState.__financialIngestionSchemaStatus = undefined;
}
function setFinancialIngestionSchemaStatus(input: {
ok: boolean;
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
missingIndexes?: string[];
duplicateGroups?: number;
}) {
const globalState = globalThis as typeof globalThis & {
__financialIngestionSchemaStatus?: {
ok: boolean;
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
requestedMode: 'auto' | 'check-only' | 'off';
missingIndexes: string[];
duplicateGroups: number;
lastCheckedAt: string;
repair: null;
error: string | null;
};
};
globalState.__financialIngestionSchemaStatus = {
ok: input.ok,
mode: input.mode,
requestedMode: 'auto',
missingIndexes: input.missingIndexes ?? [],
duplicateGroups: input.duplicateGroups ?? 0,
lastCheckedAt: new Date().toISOString(),
repair: null,
error: input.ok ? null : 'schema drift injected by test'
};
}
function applySqlMigrations(client: { exec: (query: string) => void }) {
@@ -250,6 +283,10 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
runStatuses.clear();
runCounter = 0;
workflowBackendHealthy = true;
setFinancialIngestionSchemaStatus({
ok: true,
mode: 'healthy'
});
});
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
@@ -808,9 +845,100 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
const healthy = await jsonRequest('GET', '/api/health');
expect(healthy.response.status).toBe(200);
expect((healthy.json as { status: string; workflow: { ok: boolean } }).status).toBe('ok');
expect((healthy.json as {
status: string;
workflow: { ok: boolean };
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).status).toBe('ok');
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
expect((healthy.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.ok).toBe(true);
expect((healthy.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
};
};
}).database.ingestionSchema.mode).toBe('healthy');
setFinancialIngestionSchemaStatus({
ok: false,
mode: 'drifted',
missingIndexes: ['company_financial_bundle_uidx'],
duplicateGroups: 1
});
const schemaDrifted = await jsonRequest('GET', '/api/health');
expect(schemaDrifted.response.status).toBe(503);
expect((schemaDrifted.json as {
status: string;
workflow: { ok: boolean };
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).status).toBe('degraded');
expect((schemaDrifted.json as {
workflow: { ok: boolean };
}).workflow.ok).toBe(true);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.ok).toBe(false);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.mode).toBe('drifted');
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
missingIndexes: string[];
};
};
}).database.ingestionSchema.missingIndexes).toEqual(['company_financial_bundle_uidx']);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
duplicateGroups: number;
};
};
}).database.ingestionSchema.duplicateGroups).toBe(1);
setFinancialIngestionSchemaStatus({
ok: true,
mode: 'healthy'
});
workflowBackendHealthy = false;
const degraded = await jsonRequest('GET', '/api/health');
expect(degraded.response.status).toBe(503);

View File

@@ -3,6 +3,10 @@ import { dirname, join } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { load as loadSqliteVec } from 'sqlite-vec';
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from './financial-ingestion-schema';
import { schema } from './schema';
type AppDrizzleDb = ReturnType<typeof createDb>;
@@ -564,6 +568,9 @@ export function getSqliteClient() {
client.exec('PRAGMA busy_timeout = 5000;');
loadSqliteExtensions(client);
ensureLocalSqliteSchema(client);
ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
});
ensureSearchVirtualTables(client);
globalThis.__fiscalSqliteClient = client;

View File

@@ -1586,6 +1586,99 @@ describe('financial taxonomy internals', () => {
expect(merged[0]?.provenanceType).toBe('taxonomy');
});
it('builds faithful rows when persisted statement rows are missing sourceFactIds', () => {
const malformedSnapshot = {
...createSnapshot({
filingId: 19,
filingType: '10-K',
filingDate: '2026-02-20',
statement: 'income',
periods: [
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
]
}),
statement_rows: {
income: [{
...createRow({
key: 'revenue',
label: 'Revenue',
statement: 'income',
values: { '2025-fy': 123_000_000 }
}),
sourceFactIds: undefined
} as unknown as TaxonomyStatementRow],
balance: [],
cash_flow: [],
equity: [],
comprehensive_income: []
}
} satisfies FilingTaxonomySnapshotRecord;
const rows = __financialTaxonomyInternals.buildRows(
[malformedSnapshot],
'income',
new Set(['2025-fy'])
);
expect(rows).toHaveLength(1);
expect(rows[0]?.key).toBe('revenue');
expect(rows[0]?.sourceFactIds).toEqual([]);
});
it('aggregates persisted surface rows when legacy snapshots are missing source arrays', () => {
const snapshot = {
...createSnapshot({
filingId: 20,
filingType: '10-K',
filingDate: '2026-02-21',
statement: 'income',
periods: [
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
]
}),
surface_rows: {
income: [{
key: 'revenue',
label: 'Revenue',
category: 'revenue',
templateSection: 'statement',
order: 10,
unit: 'currency',
values: { '2025-fy': 123_000_000 },
sourceConcepts: undefined,
sourceRowKeys: undefined,
sourceFactIds: undefined,
formulaKey: null,
hasDimensions: false,
resolvedSourceRowKeys: {},
statement: 'income',
detailCount: 0,
resolutionMethod: 'direct',
confidence: 'high',
warningCodes: []
} as unknown as FilingTaxonomySnapshotRecord['surface_rows']['income'][number]],
balance: [],
cash_flow: [],
equity: [],
comprehensive_income: []
}
} satisfies FilingTaxonomySnapshotRecord;
const rows = __financialTaxonomyInternals.aggregateSurfaceRows({
snapshots: [snapshot],
statement: 'income',
selectedPeriodIds: new Set(['2025-fy'])
});
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({
key: 'revenue',
sourceConcepts: [],
sourceRowKeys: [],
sourceFactIds: []
});
});
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
const snapshot = {
...createSnapshot({
@@ -1610,11 +1703,81 @@ describe('financial taxonomy internals', () => {
} satisfies FilingTaxonomySnapshotRecord;
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: 'bank_lender',
parserVersion: '0.1.0',
surfaceRowCount: 5,
detailRowCount: 3,
kpiRowCount: 2,
unmappedRowCount: 4,
materialUnmappedRowCount: 1
materialUnmappedRowCount: 1,
warnings: []
});
});
it('aggregates normalization counts and warning codes across snapshots while using the latest parser identity', () => {
const olderSnapshot = {
...createSnapshot({
filingId: 17,
filingType: '10-K',
filingDate: '2025-02-13',
statement: 'income',
periods: [
{ id: '2024-fy', periodStart: '2024-01-01', periodEnd: '2024-12-31', periodLabel: '2024 FY' }
]
}),
parser_engine: 'fiscal-xbrl',
parser_version: '0.9.0',
fiscal_pack: 'core',
normalization_summary: {
surfaceRowCount: 4,
detailRowCount: 2,
kpiRowCount: 1,
unmappedRowCount: 3,
materialUnmappedRowCount: 1,
warnings: ['balance_residual_detected', 'income_sparse_mapping']
}
} satisfies FilingTaxonomySnapshotRecord;
const latestSnapshot = {
...createSnapshot({
filingId: 18,
filingType: '10-Q',
filingDate: '2026-02-13',
statement: 'income',
periods: [
{ id: '2025-q4', periodStart: '2025-10-01', periodEnd: '2025-12-31', periodLabel: '2025 Q4' }
]
}),
parser_engine: 'fiscal-xbrl',
parser_version: '1.1.0',
fiscal_pack: 'bank_lender',
normalization_summary: {
surfaceRowCount: 6,
detailRowCount: 5,
kpiRowCount: 4,
unmappedRowCount: 2,
materialUnmappedRowCount: 0,
warnings: ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
}
} satisfies FilingTaxonomySnapshotRecord;
expect(__financialTaxonomyInternals.buildNormalizationMetadata([olderSnapshot, latestSnapshot])).toEqual({
parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: 'bank_lender',
parserVersion: '1.1.0',
surfaceRowCount: 10,
detailRowCount: 7,
kpiRowCount: 5,
unmappedRowCount: 5,
materialUnmappedRowCount: 1,
warnings: [
'balance_residual_detected',
'income_sparse_mapping',
'unmapped_cash_flow_bridge'
]
});
});

View File

@@ -177,7 +177,7 @@ function buildKpiDimensionBreakdown(input: {
continue;
}
const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id));
const matchedFacts = input.facts.filter((fact) => (row.sourceFactIds ?? []).includes(fact.id));
if (matchedFacts.length === 0) {
continue;
}
@@ -213,9 +213,9 @@ function latestPeriodDate(period: FinancialStatementPeriod) {
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
return {
...row,
values: { ...row.values },
sourceConcepts: [...row.sourceConcepts],
sourceFactIds: [...row.sourceFactIds]
values: { ...(row.values ?? {}) },
sourceConcepts: [...(row.sourceConcepts ?? [])],
sourceFactIds: [...(row.sourceFactIds ?? [])]
};
}
@@ -230,7 +230,7 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
continue;
}
for (const [periodId, value] of Object.entries(row.values)) {
for (const [periodId, value] of Object.entries(row.values ?? {})) {
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
&& existing.values[periodId] !== null;
if (!hasExistingValue) {
@@ -238,9 +238,9 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
}
}
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])]
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...(row.sourceConcepts ?? [])])]
.sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])]
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...(row.sourceFactIds ?? [])])]
.sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.segment ??= row.segment;
@@ -260,11 +260,16 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
function emptyNormalizationMetadata(): NormalizationMetadata {
return {
parserEngine: 'unknown',
regime: 'unknown',
fiscalPack: null,
parserVersion: '0.0.0',
surfaceRowCount: 0,
detailRowCount: 0,
kpiRowCount: 0,
unmappedRowCount: 0,
materialUnmappedRowCount: 0
materialUnmappedRowCount: 0,
warnings: []
};
}
@@ -277,9 +282,22 @@ function buildNormalizationMetadata(
}
return {
parserEngine: latestSnapshot.parser_engine,
regime: latestSnapshot.taxonomy_regime,
fiscalPack: latestSnapshot.fiscal_pack,
parserVersion: latestSnapshot.parser_version,
surfaceRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.surfaceRowCount ?? 0),
0
),
detailRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.detailRowCount ?? 0),
0
),
kpiRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.kpiRowCount ?? 0),
0
),
unmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
0
@@ -287,7 +305,9 @@ function buildNormalizationMetadata(
materialUnmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
0
)
),
warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))]
.sort((left, right) => left.localeCompare(right))
};
}
@@ -329,8 +349,12 @@ function aggregateSurfaceRows(input: {
for (const snapshot of input.snapshots) {
const rows = snapshot.surface_rows?.[input.statement] ?? [];
for (const row of rows) {
const sourceConcepts = row.sourceConcepts ?? [];
const sourceRowKeys = row.sourceRowKeys ?? [];
const sourceFactIds = row.sourceFactIds ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
const filteredResolvedSourceRowKeys = Object.fromEntries(
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
@@ -345,9 +369,9 @@ function aggregateSurfaceRows(input: {
...row,
values: filteredValues,
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
sourceConcepts: [...row.sourceConcepts],
sourceRowKeys: [...row.sourceRowKeys],
sourceFactIds: [...row.sourceFactIds],
sourceConcepts: [...sourceConcepts],
sourceRowKeys: [...sourceRowKeys],
sourceFactIds: [...sourceFactIds],
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
});
continue;
@@ -365,9 +389,9 @@ function aggregateSurfaceRows(input: {
}
}
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...row.sourceRowKeys])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...sourceRowKeys])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.order = Math.min(existing.order, row.order);
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
@@ -406,8 +430,11 @@ function aggregateDetailRows(input: {
}
for (const row of rows) {
const sourceFactIds = row.sourceFactIds ?? [];
const dimensionsSummary = row.dimensionsSummary ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -418,8 +445,8 @@ function aggregateDetailRows(input: {
bucket.set(row.key, {
...row,
values: filteredValues,
sourceFactIds: [...row.sourceFactIds],
dimensionsSummary: [...row.dimensionsSummary]
sourceFactIds: [...sourceFactIds],
dimensionsSummary: [...dimensionsSummary]
});
continue;
}
@@ -430,8 +457,8 @@ function aggregateDetailRows(input: {
}
}
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...row.dimensionsSummary])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...dimensionsSummary])].sort((left, right) => left.localeCompare(right));
existing.isExtension = existing.isExtension || row.isExtension;
existing.residualFlag = existing.residualFlag || row.residualFlag;
}
@@ -521,8 +548,11 @@ function aggregatePersistedKpiRows(input: {
for (const snapshot of input.snapshots) {
for (const row of snapshot.kpi_rows ?? []) {
const sourceConcepts = row.sourceConcepts ?? [];
const sourceFactIds = row.sourceFactIds ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -533,8 +563,8 @@ function aggregatePersistedKpiRows(input: {
rowMap.set(row.key, {
...row,
values: filteredValues,
sourceConcepts: [...row.sourceConcepts],
sourceFactIds: [...row.sourceFactIds]
sourceConcepts: [...sourceConcepts],
sourceFactIds: [...sourceFactIds]
});
continue;
}
@@ -543,8 +573,8 @@ function aggregatePersistedKpiRows(input: {
...existing.values,
...filteredValues
};
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
}
}

View File

@@ -282,17 +282,20 @@ export function buildRows(
const rows = snapshot.statement_rows?.[statement] ?? [];
for (const row of rows) {
const rowValues = row.values ?? {};
const rowUnits = row.units ?? {};
const sourceFactIds = row.sourceFactIds ?? [];
const existing = rowMap.get(row.key);
if (!existing) {
rowMap.set(row.key, {
...row,
values: Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
units: Object.fromEntries(
Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
Object.entries(rowUnits).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
sourceFactIds: [...row.sourceFactIds]
sourceFactIds: [...sourceFactIds]
});
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
@@ -308,19 +311,19 @@ export function buildRows(
existing.parentKey = row.parentKey;
}
for (const [periodId, value] of Object.entries(row.values)) {
for (const [periodId, value] of Object.entries(rowValues)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
existing.values[periodId] = value;
}
}
for (const [periodId, unit] of Object.entries(row.units)) {
for (const [periodId, unit] of Object.entries(rowUnits)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
existing.units[periodId] = unit;
}
}
for (const factId of row.sourceFactIds) {
for (const factId of sourceFactIds) {
if (!existing.sourceFactIds.includes(factId)) {
existing.sourceFactIds.push(factId);
}

View File

@@ -1,5 +1,9 @@
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
function buildYahooChartUrl(ticker: string, params: string) {
return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
}
function fallbackQuote(ticker: string) {
const normalized = ticker.trim().toUpperCase();
let hash = 0;
@@ -15,7 +19,7 @@ export async function getQuote(ticker: string): Promise<number> {
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -47,7 +51,7 @@ export async function getQuoteOrNull(ticker: string): Promise<number | null> {
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -87,7 +91,7 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
}
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=20y'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -143,7 +147,7 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1wk&range=1y`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1wk&range=20y'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -195,11 +199,13 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
const now = Date.now();
const base = fallbackQuote(normalizedTicker);
return Array.from({ length: 26 }, (_, index) => {
const step = 25 - index;
const date = new Date(now - step * 14 * 24 * 60 * 60 * 1000).toISOString();
const wave = Math.sin(index / 3.5) * 0.05;
const trend = (index - 13) * 0.006;
const totalWeeks = 20 * 52;
return Array.from({ length: totalWeeks }, (_, index) => {
const step = (totalWeeks - 1) - index;
const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
const wave = Math.sin(index / 8) * 0.06;
const trend = (index - totalWeeks / 2) * 0.0009;
const close = Math.max(base * (1 + wave + trend), 1);
return {

View File

@@ -3,7 +3,8 @@ import type {
FinancialCadence,
FinancialSurfaceKind
} from '@/lib/types';
import { db } from '@/lib/server/db';
import { db, getSqliteClient } from '@/lib/server/db';
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import { companyFinancialBundle } from '@/lib/server/db/schema';
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
@@ -64,34 +65,38 @@ export async function upsertCompanyFinancialBundle(input: {
}) {
const now = new Date().toISOString();
const [saved] = await db
.insert(companyFinancialBundle)
.values({
ticker: input.ticker.trim().toUpperCase(),
surface_kind: input.surfaceKind,
cadence: input.cadence,
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
created_at: now,
updated_at: now
})
.onConflictDoUpdate({
target: [
companyFinancialBundle.ticker,
companyFinancialBundle.surface_kind,
companyFinancialBundle.cadence
],
set: {
const [saved] = await withFinancialIngestionSchemaRetry({
client: getSqliteClient(),
context: 'company-financial-bundle-upsert',
operation: async () => await db
.insert(companyFinancialBundle)
.values({
ticker: input.ticker.trim().toUpperCase(),
surface_kind: input.surfaceKind,
cadence: input.cadence,
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
created_at: now,
updated_at: now
}
})
.returning();
})
.onConflictDoUpdate({
target: [
companyFinancialBundle.ticker,
companyFinancialBundle.surface_kind,
companyFinancialBundle.cadence
],
set: {
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
updated_at: now
}
})
.returning()
});
return toBundleRecord(saved);
}

View File

@@ -11,7 +11,8 @@ import type {
TaxonomyFactRow,
TaxonomyStatementRow
} from '@/lib/types';
import { db } from '@/lib/server/db';
import { db, getSqliteClient } from '@/lib/server/db';
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import {
filingTaxonomyAsset,
filingTaxonomyConcept,
@@ -552,38 +553,13 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
const now = new Date().toISOString();
const [saved] = await db
.insert(filingTaxonomySnapshot)
.values({
filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
parser_engine: input.parser_engine,
parser_version: input.parser_version,
taxonomy_regime: input.taxonomy_regime,
fiscal_pack: input.fiscal_pack,
periods: input.periods,
faithful_rows: input.faithful_rows,
statement_rows: input.statement_rows,
surface_rows: input.surface_rows,
detail_rows: input.detail_rows,
kpi_rows: input.kpi_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
normalization_summary: input.normalization_summary,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
created_at: now,
updated_at: now
})
.onConflictDoUpdate({
target: filingTaxonomySnapshot.filing_id,
set: {
const [saved] = await withFinancialIngestionSchemaRetry({
client: getSqliteClient(),
context: 'filing-taxonomy-snapshot-upsert',
operation: async () => await db
.insert(filingTaxonomySnapshot)
.values({
filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
@@ -606,10 +582,39 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
created_at: now,
updated_at: now
}
})
.returning();
})
.onConflictDoUpdate({
target: filingTaxonomySnapshot.filing_id,
set: {
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
parser_engine: input.parser_engine,
parser_version: input.parser_version,
taxonomy_regime: input.taxonomy_regime,
fiscal_pack: input.fiscal_pack,
periods: input.periods,
faithful_rows: input.faithful_rows,
statement_rows: input.statement_rows,
surface_rows: input.surface_rows,
detail_rows: input.detail_rows,
kpi_rows: input.kpi_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
normalization_summary: input.normalization_summary,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
updated_at: now
}
})
.returning()
});
const snapshotId = saved.id;

View File

@@ -498,11 +498,16 @@ export type NormalizationSummary = {
};
export type NormalizationMetadata = {
parserEngine: string;
regime: 'us-gaap' | 'ifrs-full' | 'unknown';
fiscalPack: string | null;
parserVersion: string;
surfaceRowCount: number;
detailRowCount: number;
kpiRowCount: number;
unmappedRowCount: number;
materialUnmappedRowCount: number;
warnings: string[];
};
export type RatioRow = DerivedFinancialRow & {
@@ -741,6 +746,7 @@ export type CompanyAnalysis = {
quote: number;
position: Holding | null;
priceHistory: Array<{ date: string; close: number }>;
benchmarkHistory: Array<{ date: string; close: number }>;
financials: CompanyFinancialPoint[];
filings: Filing[];
aiReports: CompanyAiReport[];
@@ -788,3 +794,88 @@ export type ActiveContext = {
pathname: string;
activeTicker: string | null;
};
// Chart Types
export type ChartType = 'line' | 'combination';
export type TimeRange = '1W' | '1M' | '3M' | '1Y' | '3Y' | '5Y' | '10Y' | '20Y';
// Chart Data Formats
export type PriceDataPoint = {
date: string;
price: number;
};
export type OHLCVDataPoint = {
date: string;
open: number;
high: number;
low: number;
close: number;
volume: number;
};
export type ChartDataPoint = PriceDataPoint | OHLCVDataPoint;
export type DataSeries<T extends ChartDataPoint = ChartDataPoint> = {
id: string;
label: string;
data: T[];
color?: string;
type?: 'line' | 'area' | 'bar';
visible?: boolean;
};
// Chart Configuration
export type ChartZoomState = {
startIndex: number;
endIndex: number;
isZoomed: boolean;
};
export type ChartColorPalette = {
primary: string;
secondary: string;
positive: string;
negative: string;
grid: string;
text: string;
muted: string;
tooltipBg: string;
tooltipBorder: string;
volume: string;
};
export type InteractivePriceChartProps = {
// Data
data: ChartDataPoint[];
dataSeries?: DataSeries[];
// Configuration
defaultChartType?: ChartType;
defaultTimeRange?: TimeRange;
showVolume?: boolean;
showToolbar?: boolean;
height?: number;
// Customization
colors?: Partial<ChartColorPalette>;
formatters?: {
price?: (value: number) => string;
date?: (value: string) => string;
volume?: (value: number) => string;
};
// Event handlers
onChartTypeChange?: (type: ChartType) => void;
onTimeRangeChange?: (range: TimeRange) => void;
onDataPointHover?: (point: ChartDataPoint | null) => void;
onZoomChange?: (state: ChartZoomState) => void;
// State
loading?: boolean;
error?: string | null;
// Accessibility
ariaLabel?: string;
description?: string;
};

2
next-env.d.ts vendored
View File

@@ -1,6 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/types/routes.d.ts";
import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

View File

@@ -21,6 +21,7 @@
"backfill:search-index": "bun run scripts/backfill-search-index.ts",
"backfill:taxonomy-snapshots": "bun run scripts/backfill-taxonomy-snapshots.ts",
"compare:fiscal-ai": "bun run scripts/compare-fiscal-ai-statements.ts",
"repair:financial-ingestion-schema": "bun run scripts/repair-financial-ingestion-schema.ts",
"report:taxonomy-health": "bun run scripts/report-taxonomy-health.ts",
"db:generate": "bun x drizzle-kit generate",
"db:migrate": "bun x drizzle-kit migrate",
@@ -43,6 +44,7 @@
"date-fns": "^4.1.0",
"drizzle-orm": "^0.41.0",
"elysia": "latest",
"html-to-image": "^1.11.13",
"lucide-react": "^0.575.0",
"next": "^16.1.6",
"react": "^19.2.4",

View File

@@ -4,6 +4,10 @@ import { dirname } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from '../lib/server/db/financial-ingestion-schema';
import { resolveSqlitePath } from './dev-env';
function trim(value: string | undefined) {
@@ -74,6 +78,13 @@ function runDatabaseMigrations() {
try {
client.exec('PRAGMA foreign_keys = ON;');
migrate(drizzle(client), { migrationsFolder: './drizzle' });
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
});
if (!repairResult.ok) {
throw new Error(repairResult.error ?? `financial ingestion schema is ${repairResult.mode}`);
}
} finally {
client.close();
}

View File

@@ -3,6 +3,10 @@ import { mkdirSync, readFileSync } from 'node:fs';
import { Database } from 'bun:sqlite';
import { createServer } from 'node:net';
import { dirname, join } from 'node:path';
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from '../lib/server/db/financial-ingestion-schema';
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
type DrizzleJournal = {
@@ -126,6 +130,33 @@ mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? '.workflow-data', { recursive: true });
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
const client = new Database(databasePath, { create: true });
try {
client.exec('PRAGMA foreign_keys = ON;');
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
});
if (repairResult.mode === 'repaired') {
console.info(
`[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(', ') || 'none'}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? 'yes' : 'no'})`
);
} else if (repairResult.mode === 'drifted') {
console.warn(
`[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(', ') || 'none'}; duplicate groups: ${repairResult.duplicateGroups})`
);
} else if (repairResult.mode === 'failed') {
console.warn(
`[dev] financial ingestion schema repair failed: ${repairResult.error ?? 'unknown error'}`
);
}
} finally {
client.close();
}
}
console.info(`[dev] local origin ${config.publicOrigin}`);
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);