WIP main worktree changes before merge
This commit is contained in:
@@ -137,7 +137,9 @@ function AnalysisPageContent() {
|
||||
<PriceHistoryCard
|
||||
loading={loading}
|
||||
priceHistory={analysis.priceHistory}
|
||||
benchmarkHistory={analysis.benchmarkHistory}
|
||||
quote={analysis.quote}
|
||||
position={analysis.position}
|
||||
/>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -700,6 +700,10 @@ function FinancialsPageContent() {
|
||||
return null;
|
||||
}, [displayMode, financials?.statementRows, surfaceKind]);
|
||||
|
||||
const hasUnmappedResidualRows = useMemo(() => {
|
||||
return (financials?.statementDetails?.unmapped?.length ?? 0) > 0;
|
||||
}, [financials?.statementDetails]);
|
||||
|
||||
const trendSeries = financials?.trendSeries ?? [];
|
||||
const chartData = useMemo(() => {
|
||||
return periods.map((period) => ({
|
||||
@@ -1042,9 +1046,16 @@ function FinancialsPageContent() {
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
{isStatementSurfaceKind(surfaceKind) ? (
|
||||
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
|
||||
USD · {valueScaleLabel}
|
||||
</p>
|
||||
<div className="space-y-2">
|
||||
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
|
||||
USD · {valueScaleLabel}
|
||||
</p>
|
||||
{isTreeStatementMode && hasUnmappedResidualRows ? (
|
||||
<p className="text-sm text-[color:var(--terminal-muted)]">
|
||||
Parser residual rows are available under the <span className="text-[color:var(--terminal-bright)]">Unmapped / Residual</span> section.
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{isTreeStatementMode && treeModel ? (
|
||||
<StatementMatrix
|
||||
|
||||
3
bun.lock
3
bun.lock
@@ -17,6 +17,7 @@
|
||||
"date-fns": "^4.1.0",
|
||||
"drizzle-orm": "^0.41.0",
|
||||
"elysia": "latest",
|
||||
"html-to-image": "^1.11.13",
|
||||
"lucide-react": "^0.575.0",
|
||||
"next": "^16.1.6",
|
||||
"react": "^19.2.4",
|
||||
@@ -1068,6 +1069,8 @@
|
||||
|
||||
"hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="],
|
||||
|
||||
"html-to-image": ["html-to-image@1.11.13", "", {}, "sha512-cuOPoI7WApyhBElTTb9oqsawRvZ0rHhaHwghRLlTuffoD1B2aDemlCruLeZrUIIdvG7gs9xeELEPm6PhuASqrg=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
|
||||
|
||||
@@ -1,120 +1,127 @@
|
||||
'use client';
|
||||
|
||||
import { format } from 'date-fns';
|
||||
import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts';
|
||||
import { Panel } from '@/components/ui/panel';
|
||||
import { formatCurrency } from '@/lib/format';
|
||||
import { InteractivePriceChart } from '@/components/charts/interactive-price-chart';
|
||||
import type { DataSeries, Holding } from '@/lib/types';
|
||||
|
||||
type PriceHistoryCardProps = {
|
||||
loading: boolean;
|
||||
priceHistory: Array<{ date: string; close: number }>;
|
||||
benchmarkHistory: Array<{ date: string; close: number }>;
|
||||
quote: number;
|
||||
position: Holding | null;
|
||||
};
|
||||
|
||||
const CHART_TEXT = '#f3f5f7';
|
||||
const CHART_MUTED = '#a1a9b3';
|
||||
const CHART_GRID = 'rgba(196, 202, 211, 0.18)';
|
||||
const CHART_TOOLTIP_BG = 'rgba(31, 34, 39, 0.96)';
|
||||
const CHART_TOOLTIP_BORDER = 'rgba(220, 226, 234, 0.24)';
|
||||
|
||||
function formatShortDate(value: string) {
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM yy');
|
||||
}
|
||||
|
||||
function formatLongDate(value: string) {
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM dd, yyyy');
|
||||
}
|
||||
|
||||
function asFiniteNumber(value: string | number | null | undefined) {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = typeof value === 'number' ? value : Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
export function PriceHistoryCard(props: PriceHistoryCardProps) {
|
||||
const series = props.priceHistory.map((point) => ({
|
||||
...point,
|
||||
label: formatShortDate(point.date)
|
||||
}));
|
||||
const firstPoint = props.priceHistory[0] ?? null;
|
||||
const lastPoint = props.priceHistory[props.priceHistory.length - 1] ?? null;
|
||||
const change = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
|
||||
const changePct = firstPoint && lastPoint && firstPoint.close !== 0
|
||||
? ((lastPoint.close - firstPoint.close) / firstPoint.close) * 100
|
||||
const firstPoint = props.priceHistory[0];
|
||||
const lastPoint = props.priceHistory[props.priceHistory.length - 1];
|
||||
const inPortfolio = props.position !== null;
|
||||
|
||||
const defaultChange = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
|
||||
const defaultChangePct = firstPoint && firstPoint.close > 0 && defaultChange !== null
|
||||
? (defaultChange / firstPoint.close) * 100
|
||||
: null;
|
||||
const holdingChange = asFiniteNumber(props.position?.gain_loss);
|
||||
const holdingChangePct = asFiniteNumber(props.position?.gain_loss_pct);
|
||||
|
||||
const change = inPortfolio ? holdingChange : defaultChange;
|
||||
const changePct = inPortfolio ? holdingChangePct : defaultChangePct;
|
||||
const rangeLabel = inPortfolio ? 'Holding return' : '1Y return';
|
||||
const changeLabel = inPortfolio ? 'Holding P/L' : '1Y change';
|
||||
const dateRange = inPortfolio
|
||||
? props.position?.created_at && (props.position.last_price_at ?? lastPoint?.date)
|
||||
? `${formatLongDate(props.position.created_at)} to ${formatLongDate(props.position.last_price_at ?? lastPoint?.date ?? '')}`
|
||||
: 'No holding period available'
|
||||
: firstPoint && lastPoint
|
||||
? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}`
|
||||
: 'No comparison range';
|
||||
const statusToneClass = inPortfolio ? 'text-[#96f5bf]' : 'text-[color:var(--terminal-bright)]';
|
||||
const performanceToneClass = change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]';
|
||||
|
||||
const chartData = props.priceHistory.map(point => ({
|
||||
date: point.date,
|
||||
price: point.close
|
||||
}));
|
||||
const comparisonSeries: DataSeries[] = [
|
||||
{
|
||||
id: 'stock',
|
||||
label: 'Stock',
|
||||
data: chartData,
|
||||
color: '#96f5bf',
|
||||
type: 'line'
|
||||
},
|
||||
{
|
||||
id: 'sp500',
|
||||
label: 'S&P 500',
|
||||
data: props.benchmarkHistory.map((point) => ({
|
||||
date: point.date,
|
||||
price: point.close
|
||||
})),
|
||||
color: '#8ba0b8',
|
||||
type: 'line'
|
||||
}
|
||||
];
|
||||
|
||||
const helperText = Number.isFinite(props.quote)
|
||||
? `Spot price ${formatCurrency(props.quote)}`
|
||||
: 'Spot price unavailable';
|
||||
|
||||
return (
|
||||
<Panel
|
||||
title="Price chart"
|
||||
subtitle="One-year weekly close with current spot price and trailing move."
|
||||
className="h-full pt-2"
|
||||
>
|
||||
<div className="grid gap-4 xl:grid-cols-[minmax(0,1fr)_220px]">
|
||||
<div>
|
||||
{props.loading ? (
|
||||
<p className="text-sm text-[color:var(--terminal-muted)]">Loading price history...</p>
|
||||
) : series.length === 0 ? (
|
||||
<p className="text-sm text-[color:var(--terminal-muted)]">No price history available.</p>
|
||||
) : (
|
||||
<div className="h-[320px]">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<LineChart data={series}>
|
||||
<CartesianGrid strokeDasharray="2 2" stroke={CHART_GRID} />
|
||||
<XAxis
|
||||
dataKey="label"
|
||||
minTickGap={32}
|
||||
stroke={CHART_MUTED}
|
||||
fontSize={12}
|
||||
axisLine={{ stroke: CHART_MUTED }}
|
||||
tickLine={{ stroke: CHART_MUTED }}
|
||||
tick={{ fill: CHART_MUTED }}
|
||||
/>
|
||||
<YAxis
|
||||
stroke={CHART_MUTED}
|
||||
fontSize={12}
|
||||
axisLine={{ stroke: CHART_MUTED }}
|
||||
tickLine={{ stroke: CHART_MUTED }}
|
||||
tick={{ fill: CHART_MUTED }}
|
||||
tickFormatter={(value: number) => `$${value.toFixed(0)}`}
|
||||
domain={[(dataMin) => dataMin * 0.05, (dataMax) => dataMax * 1.05]}
|
||||
allowDataOverflow
|
||||
/>
|
||||
<Tooltip
|
||||
formatter={(value) => formatCurrency(Array.isArray(value) ? value[0] : value)}
|
||||
labelFormatter={(value) => String(value)}
|
||||
contentStyle={{
|
||||
backgroundColor: CHART_TOOLTIP_BG,
|
||||
border: `1px solid ${CHART_TOOLTIP_BORDER}`,
|
||||
borderRadius: '0.75rem'
|
||||
}}
|
||||
labelStyle={{ color: CHART_TEXT }}
|
||||
itemStyle={{ color: CHART_TEXT }}
|
||||
cursor={{ stroke: 'rgba(220, 226, 234, 0.28)', strokeWidth: 1 }}
|
||||
/>
|
||||
<Line type="monotone" dataKey="close" stroke="#d9dee5" strokeWidth={2.5} dot={false} />
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid gap-0 border-t border-[color:var(--line-weak)] sm:grid-cols-3 xl:grid-cols-1 xl:border-l">
|
||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b xl:border-l-0">
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Current price</p>
|
||||
<p className="mt-2 text-3xl font-semibold text-[color:var(--terminal-bright)]">{formatCurrency(props.quote)}</p>
|
||||
<Panel title="Price chart" subtitle="Interactive chart with historical data">
|
||||
<div className="space-y-4">
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
|
||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Portfolio status</p>
|
||||
<p className={`mt-2 text-2xl font-semibold ${statusToneClass}`}>{inPortfolio ? 'In portfolio' : 'Not in portfolio'}</p>
|
||||
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">{helperText}</p>
|
||||
</div>
|
||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b">
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y change</p>
|
||||
<p className={`mt-2 text-2xl font-semibold ${change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
|
||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{changeLabel}</p>
|
||||
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
|
||||
{change === null ? 'n/a' : formatCurrency(change)}
|
||||
</p>
|
||||
</div>
|
||||
<div className="px-4 py-4">
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y return</p>
|
||||
<p className={`mt-2 text-2xl font-semibold ${changePct !== null && changePct < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
|
||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{rangeLabel}</p>
|
||||
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
|
||||
{changePct === null ? 'n/a' : `${changePct.toFixed(2)}%`}
|
||||
</p>
|
||||
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">
|
||||
{firstPoint && lastPoint ? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}` : 'No comparison range'}
|
||||
{dateRange}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<InteractivePriceChart
|
||||
data={chartData}
|
||||
dataSeries={comparisonSeries}
|
||||
defaultChartType="line"
|
||||
defaultTimeRange="1Y"
|
||||
showVolume={false}
|
||||
showToolbar={true}
|
||||
height={320}
|
||||
loading={props.loading}
|
||||
formatters={{
|
||||
price: formatCurrency,
|
||||
date: (date: string) => format(new Date(date), 'MMM dd, yyyy')
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</Panel>
|
||||
);
|
||||
|
||||
@@ -29,6 +29,7 @@ function SummaryCard(props: {
|
||||
|
||||
export function NormalizationSummary({ normalization }: NormalizationSummaryProps) {
|
||||
const hasMaterialUnmapped = normalization.materialUnmappedRowCount > 0;
|
||||
const hasWarnings = normalization.warnings.length > 0;
|
||||
|
||||
return (
|
||||
<Panel
|
||||
@@ -36,10 +37,13 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
|
||||
subtitle="Pack, parser, and residual mapping health for the compact statement surface."
|
||||
variant="surface"
|
||||
>
|
||||
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-5">
|
||||
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-8">
|
||||
<SummaryCard label="Pack" value={normalization.fiscalPack ?? 'unknown'} />
|
||||
<SummaryCard label="Regime" value={normalization.regime} />
|
||||
<SummaryCard label="Parser" value={`fiscal-xbrl ${normalization.parserVersion}`} />
|
||||
<SummaryCard label="Parser" value={`${normalization.parserEngine} ${normalization.parserVersion}`} />
|
||||
<SummaryCard label="Surface Rows" value={String(normalization.surfaceRowCount)} />
|
||||
<SummaryCard label="Detail Rows" value={String(normalization.detailRowCount)} />
|
||||
<SummaryCard label="KPI Rows" value={String(normalization.kpiRowCount)} />
|
||||
<SummaryCard label="Unmapped Rows" value={String(normalization.unmappedRowCount)} />
|
||||
<SummaryCard
|
||||
label="Material Unmapped"
|
||||
@@ -47,6 +51,23 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
|
||||
tone={hasMaterialUnmapped ? 'warning' : 'default'}
|
||||
/>
|
||||
</div>
|
||||
{hasWarnings ? (
|
||||
<div className="mt-3 rounded-xl border border-[color:var(--line-weak)] bg-[color:var(--panel-soft)] px-3 py-3">
|
||||
<p className="panel-heading text-[10px] uppercase tracking-[0.16em] text-[color:var(--terminal-muted)]">
|
||||
Parser Warnings
|
||||
</p>
|
||||
<div className="mt-2 flex flex-wrap gap-2">
|
||||
{normalization.warnings.map((warning) => (
|
||||
<span
|
||||
key={warning}
|
||||
className="rounded-full border border-[color:var(--line-weak)] bg-[rgba(88,102,122,0.16)] px-3 py-1 text-xs text-[color:var(--terminal-bright)]"
|
||||
>
|
||||
{warning}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{hasMaterialUnmapped ? (
|
||||
<div className="mt-3 flex items-start gap-2 rounded-xl border border-[#7f6250] bg-[rgba(91,66,46,0.18)] px-3 py-3 text-sm text-[#f5d5c0]">
|
||||
<AlertTriangle className="mt-0.5 size-4 shrink-0" />
|
||||
|
||||
@@ -37,6 +37,9 @@ function renderList(values: string[]) {
|
||||
|
||||
export function StatementRowInspector(props: StatementRowInspectorProps) {
|
||||
const selection = props.selection;
|
||||
const parentSurfaceLabel = selection?.kind === 'detail'
|
||||
? selection.parentSurfaceRow?.label ?? (selection.row.parentSurfaceKey === 'unmapped' ? 'Unmapped / Residual' : selection.row.parentSurfaceKey)
|
||||
: null;
|
||||
|
||||
return (
|
||||
<Panel
|
||||
@@ -122,7 +125,7 @@ export function StatementRowInspector(props: StatementRowInspectorProps) {
|
||||
<div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-4">
|
||||
<InspectorCard label="Label" value={selection.row.label} />
|
||||
<InspectorCard label="Key" value={selection.row.key} />
|
||||
<InspectorCard label="Parent Surface" value={selection.parentSurfaceRow?.label ?? selection.row.parentSurfaceKey} />
|
||||
<InspectorCard label="Parent Surface" value={parentSurfaceLabel ?? selection.row.parentSurfaceKey} />
|
||||
<InspectorCard label="Residual" value={selection.row.residualFlag ? 'Yes' : 'No'} />
|
||||
</div>
|
||||
|
||||
|
||||
@@ -242,6 +242,23 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
|
||||
dimensionsSummary: [],
|
||||
residualFlag: false
|
||||
}
|
||||
],
|
||||
unmapped: [
|
||||
{
|
||||
key: 'other_income_unmapped',
|
||||
parentSurfaceKey: 'unmapped',
|
||||
label: 'Other Income Residual',
|
||||
conceptKey: 'other_income_unmapped',
|
||||
qname: 'us-gaap:OtherNonoperatingIncomeExpense',
|
||||
namespaceUri: 'http://fasb.org/us-gaap/2024',
|
||||
localName: 'OtherNonoperatingIncomeExpense',
|
||||
unit: 'USD',
|
||||
values: { [`${prefix}-fy24`]: 1_200, [`${prefix}-fy25`]: 1_450 },
|
||||
sourceFactIds: [107],
|
||||
isExtension: false,
|
||||
dimensionsSummary: [],
|
||||
residualFlag: true
|
||||
}
|
||||
]
|
||||
},
|
||||
ratioRows: [],
|
||||
@@ -285,11 +302,16 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
|
||||
validation: null
|
||||
},
|
||||
normalization: {
|
||||
parserEngine: 'fiscal-xbrl',
|
||||
regime: 'us-gaap',
|
||||
fiscalPack: isBank ? 'bank_lender' : 'core',
|
||||
parserVersion: '0.1.0',
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0
|
||||
surfaceRowCount: 8,
|
||||
detailRowCount: isBank ? 0 : 4,
|
||||
kpiRowCount: 0,
|
||||
unmappedRowCount: isBank ? 0 : 1,
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: isBank ? [] : ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
|
||||
},
|
||||
dimensionBreakdown: null
|
||||
}
|
||||
@@ -316,6 +338,11 @@ test('renders the standardized operating expense tree and inspector details', as
|
||||
await page.goto('/financials?ticker=MSFT');
|
||||
|
||||
await expect(page.getByText('Normalization Summary')).toBeVisible();
|
||||
await expect(page.getByText('fiscal-xbrl 0.1.0')).toBeVisible();
|
||||
await expect(page.getByText('Parser Warnings')).toBeVisible();
|
||||
await expect(page.getByText('income_sparse_mapping')).toBeVisible();
|
||||
await expect(page.getByText('unmapped_cash_flow_bridge')).toBeVisible();
|
||||
await expect(page.getByText('Parser residual rows are available under the Unmapped / Residual section.')).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Expand Operating Expenses details' })).toBeVisible();
|
||||
|
||||
await page.getByRole('button', { name: 'Expand Operating Expenses details' }).click();
|
||||
@@ -327,6 +354,11 @@ test('renders the standardized operating expense tree and inspector details', as
|
||||
await expect(page.getByText('Row Details')).toBeVisible();
|
||||
await expect(page.getByText('selling_general_and_administrative', { exact: true }).first()).toBeVisible();
|
||||
await expect(page.getByText('Corporate SG&A')).toBeVisible();
|
||||
await expect(page.getByText('Unmapped / Residual')).toBeVisible();
|
||||
|
||||
await page.getByRole('button', { name: /^Other Income Residual/ }).click();
|
||||
await expect(page.getByText('other_income_unmapped', { exact: true })).toBeVisible();
|
||||
await expect(page.getByText('Unmapped / Residual', { exact: true }).last()).toBeVisible();
|
||||
});
|
||||
|
||||
test('shows not meaningful expense breakdown rows for bank pack filings', async ({ page }, testInfo) => {
|
||||
|
||||
@@ -230,11 +230,16 @@ function createFinancialsPayload(input: {
|
||||
validation: null
|
||||
},
|
||||
normalization: {
|
||||
parserEngine: 'fiscal-xbrl',
|
||||
regime: 'unknown',
|
||||
fiscalPack,
|
||||
parserVersion: '0.0.0',
|
||||
surfaceRowCount: 0,
|
||||
detailRowCount: 0,
|
||||
kpiRowCount: 0,
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: []
|
||||
},
|
||||
dimensionBreakdown: null
|
||||
}
|
||||
|
||||
@@ -210,4 +210,86 @@ describe('statement view model', () => {
|
||||
row: { key: 'gp_unmapped', residualFlag: true }
|
||||
});
|
||||
});
|
||||
|
||||
it('renders unmapped detail rows in a dedicated residual section and counts them', () => {
|
||||
const model = buildStatementTree({
|
||||
surfaceKind: 'income_statement',
|
||||
rows: [
|
||||
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
|
||||
],
|
||||
statementDetails: {
|
||||
unmapped: [
|
||||
createDetailRow({
|
||||
key: 'unmapped_other_income',
|
||||
label: 'Other income residual',
|
||||
parentSurfaceKey: 'unmapped',
|
||||
values: { p1: 5 },
|
||||
residualFlag: true
|
||||
})
|
||||
]
|
||||
},
|
||||
categories: [],
|
||||
searchQuery: '',
|
||||
expandedRowKeys: new Set()
|
||||
});
|
||||
|
||||
expect(model.sections).toHaveLength(2);
|
||||
expect(model.sections[1]).toMatchObject({
|
||||
key: 'unmapped_residual',
|
||||
label: 'Unmapped / Residual'
|
||||
});
|
||||
expect(model.sections[1]?.nodes[0]).toMatchObject({
|
||||
kind: 'detail',
|
||||
row: { key: 'unmapped_other_income', parentSurfaceKey: 'unmapped' }
|
||||
});
|
||||
expect(model.visibleNodeCount).toBe(2);
|
||||
expect(model.totalNodeCount).toBe(2);
|
||||
});
|
||||
|
||||
it('matches search and resolves selection for unmapped detail rows without a real parent surface', () => {
|
||||
const rows = [
|
||||
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
|
||||
];
|
||||
const statementDetails = {
|
||||
unmapped: [
|
||||
createDetailRow({
|
||||
key: 'unmapped_fx_gain',
|
||||
label: 'FX gain residual',
|
||||
parentSurfaceKey: 'unmapped',
|
||||
values: { p1: 2 },
|
||||
residualFlag: true
|
||||
})
|
||||
]
|
||||
};
|
||||
|
||||
const model = buildStatementTree({
|
||||
surfaceKind: 'income_statement',
|
||||
rows,
|
||||
statementDetails,
|
||||
categories: [],
|
||||
searchQuery: 'fx gain',
|
||||
expandedRowKeys: new Set()
|
||||
});
|
||||
|
||||
expect(model.sections).toHaveLength(1);
|
||||
expect(model.sections[0]).toMatchObject({
|
||||
key: 'unmapped_residual',
|
||||
label: 'Unmapped / Residual'
|
||||
});
|
||||
expect(model.visibleNodeCount).toBe(1);
|
||||
expect(model.totalNodeCount).toBe(2);
|
||||
|
||||
const selection = resolveStatementSelection({
|
||||
surfaceKind: 'income_statement',
|
||||
rows,
|
||||
statementDetails,
|
||||
selection: { kind: 'detail', key: 'unmapped_fx_gain', parentKey: 'unmapped' }
|
||||
});
|
||||
|
||||
expect(selection).toMatchObject({
|
||||
kind: 'detail',
|
||||
row: { key: 'unmapped_fx_gain', parentSurfaceKey: 'unmapped' },
|
||||
parentSurfaceRow: null
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -79,6 +79,10 @@ type Categories = Array<{
|
||||
count: number;
|
||||
}>;
|
||||
|
||||
const UNMAPPED_DETAIL_GROUP_KEY = 'unmapped';
|
||||
const UNMAPPED_SECTION_KEY = 'unmapped_residual';
|
||||
const UNMAPPED_SECTION_LABEL = 'Unmapped / Residual';
|
||||
|
||||
function surfaceConfigForKind(surfaceKind: Extract<FinancialSurfaceKind, 'income_statement' | 'balance_sheet' | 'cash_flow_statement'>) {
|
||||
return SURFACE_CHILDREN[surfaceKind] ?? {};
|
||||
}
|
||||
@@ -129,6 +133,25 @@ function sortDetailRows(left: DetailFinancialRow, right: DetailFinancialRow) {
|
||||
return left.label.localeCompare(right.label);
|
||||
}
|
||||
|
||||
function buildUnmappedDetailNodes(input: {
|
||||
statementDetails: SurfaceDetailMap | null;
|
||||
searchQuery: string;
|
||||
}) {
|
||||
const normalizedSearch = normalize(input.searchQuery);
|
||||
|
||||
return [...(input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? [])]
|
||||
.sort(sortDetailRows)
|
||||
.filter((detail) => normalizedSearch.length === 0 || searchTextForDetail(detail).includes(normalizedSearch))
|
||||
.map((detail) => ({
|
||||
kind: 'detail',
|
||||
id: detailNodeId(UNMAPPED_DETAIL_GROUP_KEY, detail),
|
||||
level: 0,
|
||||
row: detail,
|
||||
parentSurfaceKey: UNMAPPED_DETAIL_GROUP_KEY,
|
||||
matchesSearch: normalizedSearch.length > 0 && searchTextForDetail(detail).includes(normalizedSearch)
|
||||
}) satisfies StatementTreeDetailNode);
|
||||
}
|
||||
|
||||
function countNodes(nodes: StatementTreeNode[]) {
|
||||
let count = 0;
|
||||
|
||||
@@ -223,10 +246,26 @@ export function buildStatementTree(input: {
|
||||
.filter((node): node is StatementTreeSurfaceNode => Boolean(node));
|
||||
|
||||
if (input.categories.length === 0) {
|
||||
const sections: StatementTreeSection[] = rootNodes.length > 0
|
||||
? [{ key: 'ungrouped', label: null, nodes: rootNodes }]
|
||||
: [];
|
||||
const unmappedNodes = buildUnmappedDetailNodes({
|
||||
statementDetails: input.statementDetails,
|
||||
searchQuery: input.searchQuery
|
||||
});
|
||||
|
||||
if (unmappedNodes.length > 0) {
|
||||
sections.push({
|
||||
key: UNMAPPED_SECTION_KEY,
|
||||
label: UNMAPPED_SECTION_LABEL,
|
||||
nodes: unmappedNodes
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
sections: [{ key: 'ungrouped', label: null, nodes: rootNodes }],
|
||||
sections,
|
||||
autoExpandedKeys,
|
||||
visibleNodeCount: countNodes(rootNodes),
|
||||
visibleNodeCount: sections.reduce((sum, section) => sum + countNodes(section.nodes), 0),
|
||||
totalNodeCount: input.rows.length + Object.values(input.statementDetails ?? {}).reduce((sum, rows) => sum + rows.length, 0)
|
||||
};
|
||||
}
|
||||
@@ -254,6 +293,18 @@ export function buildStatementTree(input: {
|
||||
});
|
||||
}
|
||||
|
||||
const unmappedNodes = buildUnmappedDetailNodes({
|
||||
statementDetails: input.statementDetails,
|
||||
searchQuery: input.searchQuery
|
||||
});
|
||||
if (unmappedNodes.length > 0) {
|
||||
sections.push({
|
||||
key: UNMAPPED_SECTION_KEY,
|
||||
label: UNMAPPED_SECTION_LABEL,
|
||||
nodes: unmappedNodes
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
sections,
|
||||
autoExpandedKeys,
|
||||
@@ -297,9 +348,11 @@ export function resolveStatementSelection(input: {
|
||||
}
|
||||
|
||||
const parentSurfaceKey = selection.parentKey ?? null;
|
||||
const detailRows = parentSurfaceKey
|
||||
? input.statementDetails?.[parentSurfaceKey] ?? []
|
||||
: Object.values(input.statementDetails ?? {}).flat();
|
||||
const detailRows = parentSurfaceKey === UNMAPPED_DETAIL_GROUP_KEY
|
||||
? input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? []
|
||||
: parentSurfaceKey
|
||||
? input.statementDetails?.[parentSurfaceKey] ?? []
|
||||
: Object.values(input.statementDetails ?? {}).flat();
|
||||
const row = detailRows.find((candidate) => candidate.key === selection.key) ?? null;
|
||||
|
||||
if (!row) {
|
||||
|
||||
@@ -104,11 +104,16 @@ function createFinancials(input: {
|
||||
validation: null
|
||||
},
|
||||
normalization: {
|
||||
parserEngine: 'fiscal-xbrl',
|
||||
regime: 'unknown',
|
||||
fiscalPack: input.fiscalPack ?? null,
|
||||
parserVersion: '0.0.0',
|
||||
surfaceRowCount: 0,
|
||||
detailRowCount: 0,
|
||||
kpiRowCount: 0,
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: []
|
||||
},
|
||||
dimensionBreakdown: null
|
||||
} satisfies CompanyFinancialStatementsResponse;
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
} from '@/lib/types';
|
||||
import { auth } from '@/lib/auth';
|
||||
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
|
||||
import { getLatestFinancialIngestionSchemaStatus } from '@/lib/server/db/financial-ingestion-schema';
|
||||
import { asErrorMessage, jsonError } from '@/lib/server/http';
|
||||
import { buildPortfolioSummary } from '@/lib/server/portfolio';
|
||||
import {
|
||||
@@ -391,16 +392,36 @@ export const app = new Elysia({ prefix: '/api' })
|
||||
getTaskQueueSnapshot(),
|
||||
checkWorkflowBackend()
|
||||
]);
|
||||
const ingestionSchema = getLatestFinancialIngestionSchemaStatus();
|
||||
const ingestionSchemaPayload = ingestionSchema
|
||||
? {
|
||||
ok: ingestionSchema.ok,
|
||||
mode: ingestionSchema.mode,
|
||||
missingIndexes: ingestionSchema.missingIndexes,
|
||||
duplicateGroups: ingestionSchema.duplicateGroups,
|
||||
lastCheckedAt: ingestionSchema.lastCheckedAt
|
||||
}
|
||||
: {
|
||||
ok: false,
|
||||
mode: 'failed' as const,
|
||||
missingIndexes: [],
|
||||
duplicateGroups: 0,
|
||||
lastCheckedAt: new Date().toISOString()
|
||||
};
|
||||
const schemaHealthy = ingestionSchema?.ok ?? false;
|
||||
|
||||
if (!workflowBackend.ok) {
|
||||
if (!workflowBackend.ok || !schemaHealthy) {
|
||||
return Response.json({
|
||||
status: 'degraded',
|
||||
version: '4.0.0',
|
||||
timestamp: new Date().toISOString(),
|
||||
queue,
|
||||
database: {
|
||||
ingestionSchema: ingestionSchemaPayload
|
||||
},
|
||||
workflow: {
|
||||
ok: false,
|
||||
reason: workflowBackend.reason
|
||||
ok: workflowBackend.ok,
|
||||
...(workflowBackend.ok ? {} : { reason: workflowBackend.reason })
|
||||
}
|
||||
}, { status: 503 });
|
||||
}
|
||||
@@ -410,6 +431,9 @@ export const app = new Elysia({ prefix: '/api' })
|
||||
version: '4.0.0',
|
||||
timestamp: new Date().toISOString(),
|
||||
queue,
|
||||
database: {
|
||||
ingestionSchema: ingestionSchemaPayload
|
||||
},
|
||||
workflow: {
|
||||
ok: true
|
||||
}
|
||||
@@ -1366,12 +1390,13 @@ export const app = new Elysia({ prefix: '/api' })
|
||||
return jsonError('ticker is required');
|
||||
}
|
||||
|
||||
const [filings, holding, watchlistItem, liveQuote, priceHistory, journalPreview, memo, secProfile] = await Promise.all([
|
||||
const [filings, holding, watchlistItem, liveQuote, priceHistory, benchmarkHistory, journalPreview, memo, secProfile] = await Promise.all([
|
||||
listFilingsRecords({ ticker, limit: 40 }),
|
||||
getHoldingByTicker(session.user.id, ticker),
|
||||
getWatchlistItemByTicker(session.user.id, ticker),
|
||||
getQuote(ticker),
|
||||
getPriceHistory(ticker),
|
||||
getPriceHistory('^GSPC'),
|
||||
listResearchJournalEntries(session.user.id, ticker, 6),
|
||||
getResearchMemoByTicker(session.user.id, ticker),
|
||||
getSecCompanyProfile(ticker)
|
||||
@@ -1478,6 +1503,7 @@ export const app = new Elysia({ prefix: '/api' })
|
||||
quote: liveQuote,
|
||||
position: holding,
|
||||
priceHistory,
|
||||
benchmarkHistory,
|
||||
financials,
|
||||
filings: redactedFilings.slice(0, 20),
|
||||
aiReports,
|
||||
|
||||
@@ -74,11 +74,44 @@ function resetDbSingletons() {
|
||||
const globalState = globalThis as typeof globalThis & {
|
||||
__fiscalSqliteClient?: { close?: () => void };
|
||||
__fiscalDrizzleDb?: unknown;
|
||||
__financialIngestionSchemaStatus?: unknown;
|
||||
};
|
||||
|
||||
globalState.__fiscalSqliteClient?.close?.();
|
||||
globalState.__fiscalSqliteClient = undefined;
|
||||
globalState.__fiscalDrizzleDb = undefined;
|
||||
globalState.__financialIngestionSchemaStatus = undefined;
|
||||
}
|
||||
|
||||
function setFinancialIngestionSchemaStatus(input: {
|
||||
ok: boolean;
|
||||
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
|
||||
missingIndexes?: string[];
|
||||
duplicateGroups?: number;
|
||||
}) {
|
||||
const globalState = globalThis as typeof globalThis & {
|
||||
__financialIngestionSchemaStatus?: {
|
||||
ok: boolean;
|
||||
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
|
||||
requestedMode: 'auto' | 'check-only' | 'off';
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
lastCheckedAt: string;
|
||||
repair: null;
|
||||
error: string | null;
|
||||
};
|
||||
};
|
||||
|
||||
globalState.__financialIngestionSchemaStatus = {
|
||||
ok: input.ok,
|
||||
mode: input.mode,
|
||||
requestedMode: 'auto',
|
||||
missingIndexes: input.missingIndexes ?? [],
|
||||
duplicateGroups: input.duplicateGroups ?? 0,
|
||||
lastCheckedAt: new Date().toISOString(),
|
||||
repair: null,
|
||||
error: input.ok ? null : 'schema drift injected by test'
|
||||
};
|
||||
}
|
||||
|
||||
function applySqlMigrations(client: { exec: (query: string) => void }) {
|
||||
@@ -250,6 +283,10 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
|
||||
runStatuses.clear();
|
||||
runCounter = 0;
|
||||
workflowBackendHealthy = true;
|
||||
setFinancialIngestionSchemaStatus({
|
||||
ok: true,
|
||||
mode: 'healthy'
|
||||
});
|
||||
});
|
||||
|
||||
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
|
||||
@@ -808,9 +845,100 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
|
||||
|
||||
const healthy = await jsonRequest('GET', '/api/health');
|
||||
expect(healthy.response.status).toBe(200);
|
||||
expect((healthy.json as { status: string; workflow: { ok: boolean } }).status).toBe('ok');
|
||||
expect((healthy.json as {
|
||||
status: string;
|
||||
workflow: { ok: boolean };
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).status).toBe('ok');
|
||||
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
|
||||
expect((healthy.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.ok).toBe(true);
|
||||
expect((healthy.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.mode).toBe('healthy');
|
||||
|
||||
setFinancialIngestionSchemaStatus({
|
||||
ok: false,
|
||||
mode: 'drifted',
|
||||
missingIndexes: ['company_financial_bundle_uidx'],
|
||||
duplicateGroups: 1
|
||||
});
|
||||
const schemaDrifted = await jsonRequest('GET', '/api/health');
|
||||
expect(schemaDrifted.response.status).toBe(503);
|
||||
expect((schemaDrifted.json as {
|
||||
status: string;
|
||||
workflow: { ok: boolean };
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).status).toBe('degraded');
|
||||
expect((schemaDrifted.json as {
|
||||
workflow: { ok: boolean };
|
||||
}).workflow.ok).toBe(true);
|
||||
expect((schemaDrifted.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.ok).toBe(false);
|
||||
expect((schemaDrifted.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
ok: boolean;
|
||||
mode: string;
|
||||
missingIndexes: string[];
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.mode).toBe('drifted');
|
||||
expect((schemaDrifted.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
missingIndexes: string[];
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.missingIndexes).toEqual(['company_financial_bundle_uidx']);
|
||||
expect((schemaDrifted.json as {
|
||||
database: {
|
||||
ingestionSchema: {
|
||||
duplicateGroups: number;
|
||||
};
|
||||
};
|
||||
}).database.ingestionSchema.duplicateGroups).toBe(1);
|
||||
|
||||
setFinancialIngestionSchemaStatus({
|
||||
ok: true,
|
||||
mode: 'healthy'
|
||||
});
|
||||
workflowBackendHealthy = false;
|
||||
const degraded = await jsonRequest('GET', '/api/health');
|
||||
expect(degraded.response.status).toBe(503);
|
||||
|
||||
@@ -3,6 +3,10 @@ import { dirname, join } from 'node:path';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
||||
import { load as loadSqliteVec } from 'sqlite-vec';
|
||||
import {
|
||||
ensureFinancialIngestionSchemaHealthy,
|
||||
resolveFinancialSchemaRepairMode
|
||||
} from './financial-ingestion-schema';
|
||||
import { schema } from './schema';
|
||||
|
||||
type AppDrizzleDb = ReturnType<typeof createDb>;
|
||||
@@ -564,6 +568,9 @@ export function getSqliteClient() {
|
||||
client.exec('PRAGMA busy_timeout = 5000;');
|
||||
loadSqliteExtensions(client);
|
||||
ensureLocalSqliteSchema(client);
|
||||
ensureFinancialIngestionSchemaHealthy(client, {
|
||||
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||
});
|
||||
ensureSearchVirtualTables(client);
|
||||
|
||||
globalThis.__fiscalSqliteClient = client;
|
||||
|
||||
@@ -1586,6 +1586,99 @@ describe('financial taxonomy internals', () => {
|
||||
expect(merged[0]?.provenanceType).toBe('taxonomy');
|
||||
});
|
||||
|
||||
it('builds faithful rows when persisted statement rows are missing sourceFactIds', () => {
|
||||
const malformedSnapshot = {
|
||||
...createSnapshot({
|
||||
filingId: 19,
|
||||
filingType: '10-K',
|
||||
filingDate: '2026-02-20',
|
||||
statement: 'income',
|
||||
periods: [
|
||||
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
|
||||
]
|
||||
}),
|
||||
statement_rows: {
|
||||
income: [{
|
||||
...createRow({
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
statement: 'income',
|
||||
values: { '2025-fy': 123_000_000 }
|
||||
}),
|
||||
sourceFactIds: undefined
|
||||
} as unknown as TaxonomyStatementRow],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
}
|
||||
} satisfies FilingTaxonomySnapshotRecord;
|
||||
|
||||
const rows = __financialTaxonomyInternals.buildRows(
|
||||
[malformedSnapshot],
|
||||
'income',
|
||||
new Set(['2025-fy'])
|
||||
);
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]?.key).toBe('revenue');
|
||||
expect(rows[0]?.sourceFactIds).toEqual([]);
|
||||
});
|
||||
|
||||
it('aggregates persisted surface rows when legacy snapshots are missing source arrays', () => {
|
||||
const snapshot = {
|
||||
...createSnapshot({
|
||||
filingId: 20,
|
||||
filingType: '10-K',
|
||||
filingDate: '2026-02-21',
|
||||
statement: 'income',
|
||||
periods: [
|
||||
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
|
||||
]
|
||||
}),
|
||||
surface_rows: {
|
||||
income: [{
|
||||
key: 'revenue',
|
||||
label: 'Revenue',
|
||||
category: 'revenue',
|
||||
templateSection: 'statement',
|
||||
order: 10,
|
||||
unit: 'currency',
|
||||
values: { '2025-fy': 123_000_000 },
|
||||
sourceConcepts: undefined,
|
||||
sourceRowKeys: undefined,
|
||||
sourceFactIds: undefined,
|
||||
formulaKey: null,
|
||||
hasDimensions: false,
|
||||
resolvedSourceRowKeys: {},
|
||||
statement: 'income',
|
||||
detailCount: 0,
|
||||
resolutionMethod: 'direct',
|
||||
confidence: 'high',
|
||||
warningCodes: []
|
||||
} as unknown as FilingTaxonomySnapshotRecord['surface_rows']['income'][number]],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
}
|
||||
} satisfies FilingTaxonomySnapshotRecord;
|
||||
|
||||
const rows = __financialTaxonomyInternals.aggregateSurfaceRows({
|
||||
snapshots: [snapshot],
|
||||
statement: 'income',
|
||||
selectedPeriodIds: new Set(['2025-fy'])
|
||||
});
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({
|
||||
key: 'revenue',
|
||||
sourceConcepts: [],
|
||||
sourceRowKeys: [],
|
||||
sourceFactIds: []
|
||||
});
|
||||
});
|
||||
|
||||
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
|
||||
const snapshot = {
|
||||
...createSnapshot({
|
||||
@@ -1610,11 +1703,81 @@ describe('financial taxonomy internals', () => {
|
||||
} satisfies FilingTaxonomySnapshotRecord;
|
||||
|
||||
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
|
||||
parserEngine: 'fiscal-xbrl',
|
||||
regime: 'us-gaap',
|
||||
fiscalPack: 'bank_lender',
|
||||
parserVersion: '0.1.0',
|
||||
surfaceRowCount: 5,
|
||||
detailRowCount: 3,
|
||||
kpiRowCount: 2,
|
||||
unmappedRowCount: 4,
|
||||
materialUnmappedRowCount: 1
|
||||
materialUnmappedRowCount: 1,
|
||||
warnings: []
|
||||
});
|
||||
});
|
||||
|
||||
it('aggregates normalization counts and warning codes across snapshots while using the latest parser identity', () => {
|
||||
const olderSnapshot = {
|
||||
...createSnapshot({
|
||||
filingId: 17,
|
||||
filingType: '10-K',
|
||||
filingDate: '2025-02-13',
|
||||
statement: 'income',
|
||||
periods: [
|
||||
{ id: '2024-fy', periodStart: '2024-01-01', periodEnd: '2024-12-31', periodLabel: '2024 FY' }
|
||||
]
|
||||
}),
|
||||
parser_engine: 'fiscal-xbrl',
|
||||
parser_version: '0.9.0',
|
||||
fiscal_pack: 'core',
|
||||
normalization_summary: {
|
||||
surfaceRowCount: 4,
|
||||
detailRowCount: 2,
|
||||
kpiRowCount: 1,
|
||||
unmappedRowCount: 3,
|
||||
materialUnmappedRowCount: 1,
|
||||
warnings: ['balance_residual_detected', 'income_sparse_mapping']
|
||||
}
|
||||
} satisfies FilingTaxonomySnapshotRecord;
|
||||
|
||||
const latestSnapshot = {
|
||||
...createSnapshot({
|
||||
filingId: 18,
|
||||
filingType: '10-Q',
|
||||
filingDate: '2026-02-13',
|
||||
statement: 'income',
|
||||
periods: [
|
||||
{ id: '2025-q4', periodStart: '2025-10-01', periodEnd: '2025-12-31', periodLabel: '2025 Q4' }
|
||||
]
|
||||
}),
|
||||
parser_engine: 'fiscal-xbrl',
|
||||
parser_version: '1.1.0',
|
||||
fiscal_pack: 'bank_lender',
|
||||
normalization_summary: {
|
||||
surfaceRowCount: 6,
|
||||
detailRowCount: 5,
|
||||
kpiRowCount: 4,
|
||||
unmappedRowCount: 2,
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
|
||||
}
|
||||
} satisfies FilingTaxonomySnapshotRecord;
|
||||
|
||||
expect(__financialTaxonomyInternals.buildNormalizationMetadata([olderSnapshot, latestSnapshot])).toEqual({
|
||||
parserEngine: 'fiscal-xbrl',
|
||||
regime: 'us-gaap',
|
||||
fiscalPack: 'bank_lender',
|
||||
parserVersion: '1.1.0',
|
||||
surfaceRowCount: 10,
|
||||
detailRowCount: 7,
|
||||
kpiRowCount: 5,
|
||||
unmappedRowCount: 5,
|
||||
materialUnmappedRowCount: 1,
|
||||
warnings: [
|
||||
'balance_residual_detected',
|
||||
'income_sparse_mapping',
|
||||
'unmapped_cash_flow_bridge'
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ function buildKpiDimensionBreakdown(input: {
|
||||
continue;
|
||||
}
|
||||
|
||||
const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id));
|
||||
const matchedFacts = input.facts.filter((fact) => (row.sourceFactIds ?? []).includes(fact.id));
|
||||
if (matchedFacts.length === 0) {
|
||||
continue;
|
||||
}
|
||||
@@ -213,9 +213,9 @@ function latestPeriodDate(period: FinancialStatementPeriod) {
|
||||
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
|
||||
return {
|
||||
...row,
|
||||
values: { ...row.values },
|
||||
sourceConcepts: [...row.sourceConcepts],
|
||||
sourceFactIds: [...row.sourceFactIds]
|
||||
values: { ...(row.values ?? {}) },
|
||||
sourceConcepts: [...(row.sourceConcepts ?? [])],
|
||||
sourceFactIds: [...(row.sourceFactIds ?? [])]
|
||||
};
|
||||
}
|
||||
|
||||
@@ -230,7 +230,7 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const [periodId, value] of Object.entries(row.values)) {
|
||||
for (const [periodId, value] of Object.entries(row.values ?? {})) {
|
||||
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
|
||||
&& existing.values[periodId] !== null;
|
||||
if (!hasExistingValue) {
|
||||
@@ -238,9 +238,9 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
||||
}
|
||||
}
|
||||
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])]
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...(row.sourceConcepts ?? [])])]
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])]
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...(row.sourceFactIds ?? [])])]
|
||||
.sort((left, right) => left - right);
|
||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||
existing.segment ??= row.segment;
|
||||
@@ -260,11 +260,16 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
||||
|
||||
function emptyNormalizationMetadata(): NormalizationMetadata {
|
||||
return {
|
||||
parserEngine: 'unknown',
|
||||
regime: 'unknown',
|
||||
fiscalPack: null,
|
||||
parserVersion: '0.0.0',
|
||||
surfaceRowCount: 0,
|
||||
detailRowCount: 0,
|
||||
kpiRowCount: 0,
|
||||
unmappedRowCount: 0,
|
||||
materialUnmappedRowCount: 0
|
||||
materialUnmappedRowCount: 0,
|
||||
warnings: []
|
||||
};
|
||||
}
|
||||
|
||||
@@ -277,9 +282,22 @@ function buildNormalizationMetadata(
|
||||
}
|
||||
|
||||
return {
|
||||
parserEngine: latestSnapshot.parser_engine,
|
||||
regime: latestSnapshot.taxonomy_regime,
|
||||
fiscalPack: latestSnapshot.fiscal_pack,
|
||||
parserVersion: latestSnapshot.parser_version,
|
||||
surfaceRowCount: snapshots.reduce(
|
||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.surfaceRowCount ?? 0),
|
||||
0
|
||||
),
|
||||
detailRowCount: snapshots.reduce(
|
||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.detailRowCount ?? 0),
|
||||
0
|
||||
),
|
||||
kpiRowCount: snapshots.reduce(
|
||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.kpiRowCount ?? 0),
|
||||
0
|
||||
),
|
||||
unmappedRowCount: snapshots.reduce(
|
||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
|
||||
0
|
||||
@@ -287,7 +305,9 @@ function buildNormalizationMetadata(
|
||||
materialUnmappedRowCount: snapshots.reduce(
|
||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
|
||||
0
|
||||
)
|
||||
),
|
||||
warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))]
|
||||
.sort((left, right) => left.localeCompare(right))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -329,8 +349,12 @@ function aggregateSurfaceRows(input: {
|
||||
for (const snapshot of input.snapshots) {
|
||||
const rows = snapshot.surface_rows?.[input.statement] ?? [];
|
||||
for (const row of rows) {
|
||||
const sourceConcepts = row.sourceConcepts ?? [];
|
||||
const sourceRowKeys = row.sourceRowKeys ?? [];
|
||||
const sourceFactIds = row.sourceFactIds ?? [];
|
||||
const rowValues = row.values ?? {};
|
||||
const filteredValues = Object.fromEntries(
|
||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
);
|
||||
const filteredResolvedSourceRowKeys = Object.fromEntries(
|
||||
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
@@ -345,9 +369,9 @@ function aggregateSurfaceRows(input: {
|
||||
...row,
|
||||
values: filteredValues,
|
||||
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
|
||||
sourceConcepts: [...row.sourceConcepts],
|
||||
sourceRowKeys: [...row.sourceRowKeys],
|
||||
sourceFactIds: [...row.sourceFactIds],
|
||||
sourceConcepts: [...sourceConcepts],
|
||||
sourceRowKeys: [...sourceRowKeys],
|
||||
sourceFactIds: [...sourceFactIds],
|
||||
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
|
||||
});
|
||||
continue;
|
||||
@@ -365,9 +389,9 @@ function aggregateSurfaceRows(input: {
|
||||
}
|
||||
}
|
||||
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...row.sourceRowKeys])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...sourceRowKeys])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||
existing.order = Math.min(existing.order, row.order);
|
||||
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
|
||||
@@ -406,8 +430,11 @@ function aggregateDetailRows(input: {
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const sourceFactIds = row.sourceFactIds ?? [];
|
||||
const dimensionsSummary = row.dimensionsSummary ?? [];
|
||||
const rowValues = row.values ?? {};
|
||||
const filteredValues = Object.fromEntries(
|
||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
);
|
||||
if (!rowHasValues(filteredValues)) {
|
||||
continue;
|
||||
@@ -418,8 +445,8 @@ function aggregateDetailRows(input: {
|
||||
bucket.set(row.key, {
|
||||
...row,
|
||||
values: filteredValues,
|
||||
sourceFactIds: [...row.sourceFactIds],
|
||||
dimensionsSummary: [...row.dimensionsSummary]
|
||||
sourceFactIds: [...sourceFactIds],
|
||||
dimensionsSummary: [...dimensionsSummary]
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@@ -430,8 +457,8 @@ function aggregateDetailRows(input: {
|
||||
}
|
||||
}
|
||||
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...row.dimensionsSummary])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...dimensionsSummary])].sort((left, right) => left.localeCompare(right));
|
||||
existing.isExtension = existing.isExtension || row.isExtension;
|
||||
existing.residualFlag = existing.residualFlag || row.residualFlag;
|
||||
}
|
||||
@@ -521,8 +548,11 @@ function aggregatePersistedKpiRows(input: {
|
||||
|
||||
for (const snapshot of input.snapshots) {
|
||||
for (const row of snapshot.kpi_rows ?? []) {
|
||||
const sourceConcepts = row.sourceConcepts ?? [];
|
||||
const sourceFactIds = row.sourceFactIds ?? [];
|
||||
const rowValues = row.values ?? {};
|
||||
const filteredValues = Object.fromEntries(
|
||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||
);
|
||||
if (!rowHasValues(filteredValues)) {
|
||||
continue;
|
||||
@@ -533,8 +563,8 @@ function aggregatePersistedKpiRows(input: {
|
||||
rowMap.set(row.key, {
|
||||
...row,
|
||||
values: filteredValues,
|
||||
sourceConcepts: [...row.sourceConcepts],
|
||||
sourceFactIds: [...row.sourceFactIds]
|
||||
sourceConcepts: [...sourceConcepts],
|
||||
sourceFactIds: [...sourceFactIds]
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@@ -543,8 +573,8 @@ function aggregatePersistedKpiRows(input: {
|
||||
...existing.values,
|
||||
...filteredValues
|
||||
};
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,17 +282,20 @@ export function buildRows(
|
||||
const rows = snapshot.statement_rows?.[statement] ?? [];
|
||||
|
||||
for (const row of rows) {
|
||||
const rowValues = row.values ?? {};
|
||||
const rowUnits = row.units ?? {};
|
||||
const sourceFactIds = row.sourceFactIds ?? [];
|
||||
const existing = rowMap.get(row.key);
|
||||
if (!existing) {
|
||||
rowMap.set(row.key, {
|
||||
...row,
|
||||
values: Object.fromEntries(
|
||||
Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
Object.entries(rowValues).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
),
|
||||
units: Object.fromEntries(
|
||||
Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
Object.entries(rowUnits).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
),
|
||||
sourceFactIds: [...row.sourceFactIds]
|
||||
sourceFactIds: [...sourceFactIds]
|
||||
});
|
||||
|
||||
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
|
||||
@@ -308,19 +311,19 @@ export function buildRows(
|
||||
existing.parentKey = row.parentKey;
|
||||
}
|
||||
|
||||
for (const [periodId, value] of Object.entries(row.values)) {
|
||||
for (const [periodId, value] of Object.entries(rowValues)) {
|
||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
|
||||
existing.values[periodId] = value;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [periodId, unit] of Object.entries(row.units)) {
|
||||
for (const [periodId, unit] of Object.entries(rowUnits)) {
|
||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
|
||||
existing.units[periodId] = unit;
|
||||
}
|
||||
}
|
||||
|
||||
for (const factId of row.sourceFactIds) {
|
||||
for (const factId of sourceFactIds) {
|
||||
if (!existing.sourceFactIds.includes(factId)) {
|
||||
existing.sourceFactIds.push(factId);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
|
||||
|
||||
function buildYahooChartUrl(ticker: string, params: string) {
|
||||
return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
|
||||
}
|
||||
|
||||
function fallbackQuote(ticker: string) {
|
||||
const normalized = ticker.trim().toUpperCase();
|
||||
let hash = 0;
|
||||
@@ -15,7 +19,7 @@ export async function getQuote(ticker: string): Promise<number> {
|
||||
const normalizedTicker = ticker.trim().toUpperCase();
|
||||
|
||||
try {
|
||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
|
||||
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||
},
|
||||
@@ -47,7 +51,7 @@ export async function getQuoteOrNull(ticker: string): Promise<number | null> {
|
||||
const normalizedTicker = ticker.trim().toUpperCase();
|
||||
|
||||
try {
|
||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
|
||||
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||
},
|
||||
@@ -87,7 +91,7 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, {
|
||||
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=20y'), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||
},
|
||||
@@ -143,7 +147,7 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
|
||||
const normalizedTicker = ticker.trim().toUpperCase();
|
||||
|
||||
try {
|
||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1wk&range=1y`, {
|
||||
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1wk&range=20y'), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||
},
|
||||
@@ -195,11 +199,13 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
|
||||
const now = Date.now();
|
||||
const base = fallbackQuote(normalizedTicker);
|
||||
|
||||
return Array.from({ length: 26 }, (_, index) => {
|
||||
const step = 25 - index;
|
||||
const date = new Date(now - step * 14 * 24 * 60 * 60 * 1000).toISOString();
|
||||
const wave = Math.sin(index / 3.5) * 0.05;
|
||||
const trend = (index - 13) * 0.006;
|
||||
const totalWeeks = 20 * 52;
|
||||
|
||||
return Array.from({ length: totalWeeks }, (_, index) => {
|
||||
const step = (totalWeeks - 1) - index;
|
||||
const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
|
||||
const wave = Math.sin(index / 8) * 0.06;
|
||||
const trend = (index - totalWeeks / 2) * 0.0009;
|
||||
const close = Math.max(base * (1 + wave + trend), 1);
|
||||
|
||||
return {
|
||||
|
||||
@@ -3,7 +3,8 @@ import type {
|
||||
FinancialCadence,
|
||||
FinancialSurfaceKind
|
||||
} from '@/lib/types';
|
||||
import { db } from '@/lib/server/db';
|
||||
import { db, getSqliteClient } from '@/lib/server/db';
|
||||
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
|
||||
import { companyFinancialBundle } from '@/lib/server/db/schema';
|
||||
|
||||
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
|
||||
@@ -64,34 +65,38 @@ export async function upsertCompanyFinancialBundle(input: {
|
||||
}) {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const [saved] = await db
|
||||
.insert(companyFinancialBundle)
|
||||
.values({
|
||||
ticker: input.ticker.trim().toUpperCase(),
|
||||
surface_kind: input.surfaceKind,
|
||||
cadence: input.cadence,
|
||||
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
||||
source_snapshot_ids: input.sourceSnapshotIds,
|
||||
source_signature: input.sourceSignature,
|
||||
payload: input.payload,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
companyFinancialBundle.ticker,
|
||||
companyFinancialBundle.surface_kind,
|
||||
companyFinancialBundle.cadence
|
||||
],
|
||||
set: {
|
||||
const [saved] = await withFinancialIngestionSchemaRetry({
|
||||
client: getSqliteClient(),
|
||||
context: 'company-financial-bundle-upsert',
|
||||
operation: async () => await db
|
||||
.insert(companyFinancialBundle)
|
||||
.values({
|
||||
ticker: input.ticker.trim().toUpperCase(),
|
||||
surface_kind: input.surfaceKind,
|
||||
cadence: input.cadence,
|
||||
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
||||
source_snapshot_ids: input.sourceSnapshotIds,
|
||||
source_signature: input.sourceSignature,
|
||||
payload: input.payload,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
}
|
||||
})
|
||||
.returning();
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
companyFinancialBundle.ticker,
|
||||
companyFinancialBundle.surface_kind,
|
||||
companyFinancialBundle.cadence
|
||||
],
|
||||
set: {
|
||||
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
||||
source_snapshot_ids: input.sourceSnapshotIds,
|
||||
source_signature: input.sourceSignature,
|
||||
payload: input.payload,
|
||||
updated_at: now
|
||||
}
|
||||
})
|
||||
.returning()
|
||||
});
|
||||
|
||||
return toBundleRecord(saved);
|
||||
}
|
||||
|
||||
@@ -11,7 +11,8 @@ import type {
|
||||
TaxonomyFactRow,
|
||||
TaxonomyStatementRow
|
||||
} from '@/lib/types';
|
||||
import { db } from '@/lib/server/db';
|
||||
import { db, getSqliteClient } from '@/lib/server/db';
|
||||
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
|
||||
import {
|
||||
filingTaxonomyAsset,
|
||||
filingTaxonomyConcept,
|
||||
@@ -552,38 +553,13 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
|
||||
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const [saved] = await db
|
||||
.insert(filingTaxonomySnapshot)
|
||||
.values({
|
||||
filing_id: input.filing_id,
|
||||
ticker: input.ticker,
|
||||
filing_date: input.filing_date,
|
||||
filing_type: input.filing_type,
|
||||
parse_status: input.parse_status,
|
||||
parse_error: input.parse_error,
|
||||
source: input.source,
|
||||
parser_engine: input.parser_engine,
|
||||
parser_version: input.parser_version,
|
||||
taxonomy_regime: input.taxonomy_regime,
|
||||
fiscal_pack: input.fiscal_pack,
|
||||
periods: input.periods,
|
||||
faithful_rows: input.faithful_rows,
|
||||
statement_rows: input.statement_rows,
|
||||
surface_rows: input.surface_rows,
|
||||
detail_rows: input.detail_rows,
|
||||
kpi_rows: input.kpi_rows,
|
||||
derived_metrics: input.derived_metrics,
|
||||
validation_result: input.validation_result,
|
||||
normalization_summary: input.normalization_summary,
|
||||
facts_count: input.facts_count,
|
||||
concepts_count: input.concepts_count,
|
||||
dimensions_count: input.dimensions_count,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: filingTaxonomySnapshot.filing_id,
|
||||
set: {
|
||||
const [saved] = await withFinancialIngestionSchemaRetry({
|
||||
client: getSqliteClient(),
|
||||
context: 'filing-taxonomy-snapshot-upsert',
|
||||
operation: async () => await db
|
||||
.insert(filingTaxonomySnapshot)
|
||||
.values({
|
||||
filing_id: input.filing_id,
|
||||
ticker: input.ticker,
|
||||
filing_date: input.filing_date,
|
||||
filing_type: input.filing_type,
|
||||
@@ -606,10 +582,39 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
|
||||
facts_count: input.facts_count,
|
||||
concepts_count: input.concepts_count,
|
||||
dimensions_count: input.dimensions_count,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
}
|
||||
})
|
||||
.returning();
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: filingTaxonomySnapshot.filing_id,
|
||||
set: {
|
||||
ticker: input.ticker,
|
||||
filing_date: input.filing_date,
|
||||
filing_type: input.filing_type,
|
||||
parse_status: input.parse_status,
|
||||
parse_error: input.parse_error,
|
||||
source: input.source,
|
||||
parser_engine: input.parser_engine,
|
||||
parser_version: input.parser_version,
|
||||
taxonomy_regime: input.taxonomy_regime,
|
||||
fiscal_pack: input.fiscal_pack,
|
||||
periods: input.periods,
|
||||
faithful_rows: input.faithful_rows,
|
||||
statement_rows: input.statement_rows,
|
||||
surface_rows: input.surface_rows,
|
||||
detail_rows: input.detail_rows,
|
||||
kpi_rows: input.kpi_rows,
|
||||
derived_metrics: input.derived_metrics,
|
||||
validation_result: input.validation_result,
|
||||
normalization_summary: input.normalization_summary,
|
||||
facts_count: input.facts_count,
|
||||
concepts_count: input.concepts_count,
|
||||
dimensions_count: input.dimensions_count,
|
||||
updated_at: now
|
||||
}
|
||||
})
|
||||
.returning()
|
||||
});
|
||||
|
||||
const snapshotId = saved.id;
|
||||
|
||||
|
||||
91
lib/types.ts
91
lib/types.ts
@@ -498,11 +498,16 @@ export type NormalizationSummary = {
|
||||
};
|
||||
|
||||
export type NormalizationMetadata = {
|
||||
parserEngine: string;
|
||||
regime: 'us-gaap' | 'ifrs-full' | 'unknown';
|
||||
fiscalPack: string | null;
|
||||
parserVersion: string;
|
||||
surfaceRowCount: number;
|
||||
detailRowCount: number;
|
||||
kpiRowCount: number;
|
||||
unmappedRowCount: number;
|
||||
materialUnmappedRowCount: number;
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export type RatioRow = DerivedFinancialRow & {
|
||||
@@ -741,6 +746,7 @@ export type CompanyAnalysis = {
|
||||
quote: number;
|
||||
position: Holding | null;
|
||||
priceHistory: Array<{ date: string; close: number }>;
|
||||
benchmarkHistory: Array<{ date: string; close: number }>;
|
||||
financials: CompanyFinancialPoint[];
|
||||
filings: Filing[];
|
||||
aiReports: CompanyAiReport[];
|
||||
@@ -788,3 +794,88 @@ export type ActiveContext = {
|
||||
pathname: string;
|
||||
activeTicker: string | null;
|
||||
};
|
||||
|
||||
// Chart Types
|
||||
export type ChartType = 'line' | 'combination';
|
||||
export type TimeRange = '1W' | '1M' | '3M' | '1Y' | '3Y' | '5Y' | '10Y' | '20Y';
|
||||
|
||||
// Chart Data Formats
|
||||
export type PriceDataPoint = {
|
||||
date: string;
|
||||
price: number;
|
||||
};
|
||||
|
||||
export type OHLCVDataPoint = {
|
||||
date: string;
|
||||
open: number;
|
||||
high: number;
|
||||
low: number;
|
||||
close: number;
|
||||
volume: number;
|
||||
};
|
||||
|
||||
export type ChartDataPoint = PriceDataPoint | OHLCVDataPoint;
|
||||
|
||||
export type DataSeries<T extends ChartDataPoint = ChartDataPoint> = {
|
||||
id: string;
|
||||
label: string;
|
||||
data: T[];
|
||||
color?: string;
|
||||
type?: 'line' | 'area' | 'bar';
|
||||
visible?: boolean;
|
||||
};
|
||||
|
||||
// Chart Configuration
|
||||
export type ChartZoomState = {
|
||||
startIndex: number;
|
||||
endIndex: number;
|
||||
isZoomed: boolean;
|
||||
};
|
||||
|
||||
export type ChartColorPalette = {
|
||||
primary: string;
|
||||
secondary: string;
|
||||
positive: string;
|
||||
negative: string;
|
||||
grid: string;
|
||||
text: string;
|
||||
muted: string;
|
||||
tooltipBg: string;
|
||||
tooltipBorder: string;
|
||||
volume: string;
|
||||
};
|
||||
|
||||
export type InteractivePriceChartProps = {
|
||||
// Data
|
||||
data: ChartDataPoint[];
|
||||
dataSeries?: DataSeries[];
|
||||
|
||||
// Configuration
|
||||
defaultChartType?: ChartType;
|
||||
defaultTimeRange?: TimeRange;
|
||||
showVolume?: boolean;
|
||||
showToolbar?: boolean;
|
||||
height?: number;
|
||||
|
||||
// Customization
|
||||
colors?: Partial<ChartColorPalette>;
|
||||
formatters?: {
|
||||
price?: (value: number) => string;
|
||||
date?: (value: string) => string;
|
||||
volume?: (value: number) => string;
|
||||
};
|
||||
|
||||
// Event handlers
|
||||
onChartTypeChange?: (type: ChartType) => void;
|
||||
onTimeRangeChange?: (range: TimeRange) => void;
|
||||
onDataPointHover?: (point: ChartDataPoint | null) => void;
|
||||
onZoomChange?: (state: ChartZoomState) => void;
|
||||
|
||||
// State
|
||||
loading?: boolean;
|
||||
error?: string | null;
|
||||
|
||||
// Accessibility
|
||||
ariaLabel?: string;
|
||||
description?: string;
|
||||
};
|
||||
|
||||
2
next-env.d.ts
vendored
2
next-env.d.ts
vendored
@@ -1,6 +1,6 @@
|
||||
/// <reference types="next" />
|
||||
/// <reference types="next/image-types/global" />
|
||||
import "./.next/types/routes.d.ts";
|
||||
import "./.next/dev/types/routes.d.ts";
|
||||
|
||||
// NOTE: This file should not be edited
|
||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"backfill:search-index": "bun run scripts/backfill-search-index.ts",
|
||||
"backfill:taxonomy-snapshots": "bun run scripts/backfill-taxonomy-snapshots.ts",
|
||||
"compare:fiscal-ai": "bun run scripts/compare-fiscal-ai-statements.ts",
|
||||
"repair:financial-ingestion-schema": "bun run scripts/repair-financial-ingestion-schema.ts",
|
||||
"report:taxonomy-health": "bun run scripts/report-taxonomy-health.ts",
|
||||
"db:generate": "bun x drizzle-kit generate",
|
||||
"db:migrate": "bun x drizzle-kit migrate",
|
||||
@@ -43,6 +44,7 @@
|
||||
"date-fns": "^4.1.0",
|
||||
"drizzle-orm": "^0.41.0",
|
||||
"elysia": "latest",
|
||||
"html-to-image": "^1.11.13",
|
||||
"lucide-react": "^0.575.0",
|
||||
"next": "^16.1.6",
|
||||
"react": "^19.2.4",
|
||||
|
||||
@@ -4,6 +4,10 @@ import { dirname } from 'node:path';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
||||
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
||||
import {
|
||||
ensureFinancialIngestionSchemaHealthy,
|
||||
resolveFinancialSchemaRepairMode
|
||||
} from '../lib/server/db/financial-ingestion-schema';
|
||||
import { resolveSqlitePath } from './dev-env';
|
||||
|
||||
function trim(value: string | undefined) {
|
||||
@@ -74,6 +78,13 @@ function runDatabaseMigrations() {
|
||||
try {
|
||||
client.exec('PRAGMA foreign_keys = ON;');
|
||||
migrate(drizzle(client), { migrationsFolder: './drizzle' });
|
||||
|
||||
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
|
||||
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||
});
|
||||
if (!repairResult.ok) {
|
||||
throw new Error(repairResult.error ?? `financial ingestion schema is ${repairResult.mode}`);
|
||||
}
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@ import { mkdirSync, readFileSync } from 'node:fs';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { createServer } from 'node:net';
|
||||
import { dirname, join } from 'node:path';
|
||||
import {
|
||||
ensureFinancialIngestionSchemaHealthy,
|
||||
resolveFinancialSchemaRepairMode
|
||||
} from '../lib/server/db/financial-ingestion-schema';
|
||||
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
|
||||
|
||||
type DrizzleJournal = {
|
||||
@@ -126,6 +130,33 @@ mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? '.workflow-data', { recursive: true });
|
||||
|
||||
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
|
||||
|
||||
if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
|
||||
const client = new Database(databasePath, { create: true });
|
||||
|
||||
try {
|
||||
client.exec('PRAGMA foreign_keys = ON;');
|
||||
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
|
||||
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||
});
|
||||
|
||||
if (repairResult.mode === 'repaired') {
|
||||
console.info(
|
||||
`[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(', ') || 'none'}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? 'yes' : 'no'})`
|
||||
);
|
||||
} else if (repairResult.mode === 'drifted') {
|
||||
console.warn(
|
||||
`[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(', ') || 'none'}; duplicate groups: ${repairResult.duplicateGroups})`
|
||||
);
|
||||
} else if (repairResult.mode === 'failed') {
|
||||
console.warn(
|
||||
`[dev] financial ingestion schema repair failed: ${repairResult.error ?? 'unknown error'}`
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
console.info(`[dev] local origin ${config.publicOrigin}`);
|
||||
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
|
||||
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);
|
||||
|
||||
Reference in New Issue
Block a user