WIP main worktree changes before merge
This commit is contained in:
@@ -137,7 +137,9 @@ function AnalysisPageContent() {
|
|||||||
<PriceHistoryCard
|
<PriceHistoryCard
|
||||||
loading={loading}
|
loading={loading}
|
||||||
priceHistory={analysis.priceHistory}
|
priceHistory={analysis.priceHistory}
|
||||||
|
benchmarkHistory={analysis.benchmarkHistory}
|
||||||
quote={analysis.quote}
|
quote={analysis.quote}
|
||||||
|
position={analysis.position}
|
||||||
/>
|
/>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
|||||||
@@ -700,6 +700,10 @@ function FinancialsPageContent() {
|
|||||||
return null;
|
return null;
|
||||||
}, [displayMode, financials?.statementRows, surfaceKind]);
|
}, [displayMode, financials?.statementRows, surfaceKind]);
|
||||||
|
|
||||||
|
const hasUnmappedResidualRows = useMemo(() => {
|
||||||
|
return (financials?.statementDetails?.unmapped?.length ?? 0) > 0;
|
||||||
|
}, [financials?.statementDetails]);
|
||||||
|
|
||||||
const trendSeries = financials?.trendSeries ?? [];
|
const trendSeries = financials?.trendSeries ?? [];
|
||||||
const chartData = useMemo(() => {
|
const chartData = useMemo(() => {
|
||||||
return periods.map((period) => ({
|
return periods.map((period) => ({
|
||||||
@@ -1042,9 +1046,16 @@ function FinancialsPageContent() {
|
|||||||
) : (
|
) : (
|
||||||
<div className="space-y-3">
|
<div className="space-y-3">
|
||||||
{isStatementSurfaceKind(surfaceKind) ? (
|
{isStatementSurfaceKind(surfaceKind) ? (
|
||||||
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
|
<div className="space-y-2">
|
||||||
USD · {valueScaleLabel}
|
<p className="text-xs uppercase tracking-[0.24em] text-[color:var(--terminal-muted)]">
|
||||||
</p>
|
USD · {valueScaleLabel}
|
||||||
|
</p>
|
||||||
|
{isTreeStatementMode && hasUnmappedResidualRows ? (
|
||||||
|
<p className="text-sm text-[color:var(--terminal-muted)]">
|
||||||
|
Parser residual rows are available under the <span className="text-[color:var(--terminal-bright)]">Unmapped / Residual</span> section.
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
{isTreeStatementMode && treeModel ? (
|
{isTreeStatementMode && treeModel ? (
|
||||||
<StatementMatrix
|
<StatementMatrix
|
||||||
|
|||||||
3
bun.lock
3
bun.lock
@@ -17,6 +17,7 @@
|
|||||||
"date-fns": "^4.1.0",
|
"date-fns": "^4.1.0",
|
||||||
"drizzle-orm": "^0.41.0",
|
"drizzle-orm": "^0.41.0",
|
||||||
"elysia": "latest",
|
"elysia": "latest",
|
||||||
|
"html-to-image": "^1.11.13",
|
||||||
"lucide-react": "^0.575.0",
|
"lucide-react": "^0.575.0",
|
||||||
"next": "^16.1.6",
|
"next": "^16.1.6",
|
||||||
"react": "^19.2.4",
|
"react": "^19.2.4",
|
||||||
@@ -1068,6 +1069,8 @@
|
|||||||
|
|
||||||
"hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="],
|
"hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="],
|
||||||
|
|
||||||
|
"html-to-image": ["html-to-image@1.11.13", "", {}, "sha512-cuOPoI7WApyhBElTTb9oqsawRvZ0rHhaHwghRLlTuffoD1B2aDemlCruLeZrUIIdvG7gs9xeELEPm6PhuASqrg=="],
|
||||||
|
|
||||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||||
|
|
||||||
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
|
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
|
||||||
|
|||||||
@@ -1,120 +1,127 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { format } from 'date-fns';
|
import { format } from 'date-fns';
|
||||||
import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts';
|
|
||||||
import { Panel } from '@/components/ui/panel';
|
import { Panel } from '@/components/ui/panel';
|
||||||
import { formatCurrency } from '@/lib/format';
|
import { formatCurrency } from '@/lib/format';
|
||||||
|
import { InteractivePriceChart } from '@/components/charts/interactive-price-chart';
|
||||||
|
import type { DataSeries, Holding } from '@/lib/types';
|
||||||
|
|
||||||
type PriceHistoryCardProps = {
|
type PriceHistoryCardProps = {
|
||||||
loading: boolean;
|
loading: boolean;
|
||||||
priceHistory: Array<{ date: string; close: number }>;
|
priceHistory: Array<{ date: string; close: number }>;
|
||||||
|
benchmarkHistory: Array<{ date: string; close: number }>;
|
||||||
quote: number;
|
quote: number;
|
||||||
|
position: Holding | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
const CHART_TEXT = '#f3f5f7';
|
|
||||||
const CHART_MUTED = '#a1a9b3';
|
|
||||||
const CHART_GRID = 'rgba(196, 202, 211, 0.18)';
|
|
||||||
const CHART_TOOLTIP_BG = 'rgba(31, 34, 39, 0.96)';
|
|
||||||
const CHART_TOOLTIP_BORDER = 'rgba(220, 226, 234, 0.24)';
|
|
||||||
|
|
||||||
function formatShortDate(value: string) {
|
|
||||||
const parsed = new Date(value);
|
|
||||||
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM yy');
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatLongDate(value: string) {
|
function formatLongDate(value: string) {
|
||||||
const parsed = new Date(value);
|
const parsed = new Date(value);
|
||||||
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM dd, yyyy');
|
return Number.isNaN(parsed.getTime()) ? value : format(parsed, 'MMM dd, yyyy');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function asFiniteNumber(value: string | number | null | undefined) {
|
||||||
|
if (value === null || value === undefined) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = typeof value === 'number' ? value : Number(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
export function PriceHistoryCard(props: PriceHistoryCardProps) {
|
export function PriceHistoryCard(props: PriceHistoryCardProps) {
|
||||||
const series = props.priceHistory.map((point) => ({
|
const firstPoint = props.priceHistory[0];
|
||||||
...point,
|
const lastPoint = props.priceHistory[props.priceHistory.length - 1];
|
||||||
label: formatShortDate(point.date)
|
const inPortfolio = props.position !== null;
|
||||||
}));
|
|
||||||
const firstPoint = props.priceHistory[0] ?? null;
|
const defaultChange = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
|
||||||
const lastPoint = props.priceHistory[props.priceHistory.length - 1] ?? null;
|
const defaultChangePct = firstPoint && firstPoint.close > 0 && defaultChange !== null
|
||||||
const change = firstPoint && lastPoint ? lastPoint.close - firstPoint.close : null;
|
? (defaultChange / firstPoint.close) * 100
|
||||||
const changePct = firstPoint && lastPoint && firstPoint.close !== 0
|
|
||||||
? ((lastPoint.close - firstPoint.close) / firstPoint.close) * 100
|
|
||||||
: null;
|
: null;
|
||||||
|
const holdingChange = asFiniteNumber(props.position?.gain_loss);
|
||||||
|
const holdingChangePct = asFiniteNumber(props.position?.gain_loss_pct);
|
||||||
|
|
||||||
|
const change = inPortfolio ? holdingChange : defaultChange;
|
||||||
|
const changePct = inPortfolio ? holdingChangePct : defaultChangePct;
|
||||||
|
const rangeLabel = inPortfolio ? 'Holding return' : '1Y return';
|
||||||
|
const changeLabel = inPortfolio ? 'Holding P/L' : '1Y change';
|
||||||
|
const dateRange = inPortfolio
|
||||||
|
? props.position?.created_at && (props.position.last_price_at ?? lastPoint?.date)
|
||||||
|
? `${formatLongDate(props.position.created_at)} to ${formatLongDate(props.position.last_price_at ?? lastPoint?.date ?? '')}`
|
||||||
|
: 'No holding period available'
|
||||||
|
: firstPoint && lastPoint
|
||||||
|
? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}`
|
||||||
|
: 'No comparison range';
|
||||||
|
const statusToneClass = inPortfolio ? 'text-[#96f5bf]' : 'text-[color:var(--terminal-bright)]';
|
||||||
|
const performanceToneClass = change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]';
|
||||||
|
|
||||||
|
const chartData = props.priceHistory.map(point => ({
|
||||||
|
date: point.date,
|
||||||
|
price: point.close
|
||||||
|
}));
|
||||||
|
const comparisonSeries: DataSeries[] = [
|
||||||
|
{
|
||||||
|
id: 'stock',
|
||||||
|
label: 'Stock',
|
||||||
|
data: chartData,
|
||||||
|
color: '#96f5bf',
|
||||||
|
type: 'line'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'sp500',
|
||||||
|
label: 'S&P 500',
|
||||||
|
data: props.benchmarkHistory.map((point) => ({
|
||||||
|
date: point.date,
|
||||||
|
price: point.close
|
||||||
|
})),
|
||||||
|
color: '#8ba0b8',
|
||||||
|
type: 'line'
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
const helperText = Number.isFinite(props.quote)
|
||||||
|
? `Spot price ${formatCurrency(props.quote)}`
|
||||||
|
: 'Spot price unavailable';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Panel
|
<Panel title="Price chart" subtitle="Interactive chart with historical data">
|
||||||
title="Price chart"
|
<div className="space-y-4">
|
||||||
subtitle="One-year weekly close with current spot price and trailing move."
|
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
|
||||||
className="h-full pt-2"
|
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
|
||||||
>
|
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Portfolio status</p>
|
||||||
<div className="grid gap-4 xl:grid-cols-[minmax(0,1fr)_220px]">
|
<p className={`mt-2 text-2xl font-semibold ${statusToneClass}`}>{inPortfolio ? 'In portfolio' : 'Not in portfolio'}</p>
|
||||||
<div>
|
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">{helperText}</p>
|
||||||
{props.loading ? (
|
|
||||||
<p className="text-sm text-[color:var(--terminal-muted)]">Loading price history...</p>
|
|
||||||
) : series.length === 0 ? (
|
|
||||||
<p className="text-sm text-[color:var(--terminal-muted)]">No price history available.</p>
|
|
||||||
) : (
|
|
||||||
<div className="h-[320px]">
|
|
||||||
<ResponsiveContainer width="100%" height="100%">
|
|
||||||
<LineChart data={series}>
|
|
||||||
<CartesianGrid strokeDasharray="2 2" stroke={CHART_GRID} />
|
|
||||||
<XAxis
|
|
||||||
dataKey="label"
|
|
||||||
minTickGap={32}
|
|
||||||
stroke={CHART_MUTED}
|
|
||||||
fontSize={12}
|
|
||||||
axisLine={{ stroke: CHART_MUTED }}
|
|
||||||
tickLine={{ stroke: CHART_MUTED }}
|
|
||||||
tick={{ fill: CHART_MUTED }}
|
|
||||||
/>
|
|
||||||
<YAxis
|
|
||||||
stroke={CHART_MUTED}
|
|
||||||
fontSize={12}
|
|
||||||
axisLine={{ stroke: CHART_MUTED }}
|
|
||||||
tickLine={{ stroke: CHART_MUTED }}
|
|
||||||
tick={{ fill: CHART_MUTED }}
|
|
||||||
tickFormatter={(value: number) => `$${value.toFixed(0)}`}
|
|
||||||
domain={[(dataMin) => dataMin * 0.05, (dataMax) => dataMax * 1.05]}
|
|
||||||
allowDataOverflow
|
|
||||||
/>
|
|
||||||
<Tooltip
|
|
||||||
formatter={(value) => formatCurrency(Array.isArray(value) ? value[0] : value)}
|
|
||||||
labelFormatter={(value) => String(value)}
|
|
||||||
contentStyle={{
|
|
||||||
backgroundColor: CHART_TOOLTIP_BG,
|
|
||||||
border: `1px solid ${CHART_TOOLTIP_BORDER}`,
|
|
||||||
borderRadius: '0.75rem'
|
|
||||||
}}
|
|
||||||
labelStyle={{ color: CHART_TEXT }}
|
|
||||||
itemStyle={{ color: CHART_TEXT }}
|
|
||||||
cursor={{ stroke: 'rgba(220, 226, 234, 0.28)', strokeWidth: 1 }}
|
|
||||||
/>
|
|
||||||
<Line type="monotone" dataKey="close" stroke="#d9dee5" strokeWidth={2.5} dot={false} />
|
|
||||||
</LineChart>
|
|
||||||
</ResponsiveContainer>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="grid gap-0 border-t border-[color:var(--line-weak)] sm:grid-cols-3 xl:grid-cols-1 xl:border-l">
|
|
||||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b xl:border-l-0">
|
|
||||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">Current price</p>
|
|
||||||
<p className="mt-2 text-3xl font-semibold text-[color:var(--terminal-bright)]">{formatCurrency(props.quote)}</p>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 xl:border-b">
|
<div className="border-b border-[color:var(--line-weak)] px-4 py-4 sm:border-b-0 sm:border-r">
|
||||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y change</p>
|
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{changeLabel}</p>
|
||||||
<p className={`mt-2 text-2xl font-semibold ${change !== null && change < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
|
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
|
||||||
{change === null ? 'n/a' : formatCurrency(change)}
|
{change === null ? 'n/a' : formatCurrency(change)}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<div className="px-4 py-4">
|
<div className="px-4 py-4">
|
||||||
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">1Y return</p>
|
<p className="text-xs uppercase tracking-[0.14em] text-[color:var(--terminal-muted)]">{rangeLabel}</p>
|
||||||
<p className={`mt-2 text-2xl font-semibold ${changePct !== null && changePct < 0 ? 'text-[#ff9f9f]' : 'text-[#96f5bf]'}`}>
|
<p className={`mt-2 text-2xl font-semibold ${performanceToneClass}`}>
|
||||||
{changePct === null ? 'n/a' : `${changePct.toFixed(2)}%`}
|
{changePct === null ? 'n/a' : `${changePct.toFixed(2)}%`}
|
||||||
</p>
|
</p>
|
||||||
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">
|
<p className="mt-2 text-xs text-[color:var(--terminal-muted)]">
|
||||||
{firstPoint && lastPoint ? `${formatLongDate(firstPoint.date)} to ${formatLongDate(lastPoint.date)}` : 'No comparison range'}
|
{dateRange}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<InteractivePriceChart
|
||||||
|
data={chartData}
|
||||||
|
dataSeries={comparisonSeries}
|
||||||
|
defaultChartType="line"
|
||||||
|
defaultTimeRange="1Y"
|
||||||
|
showVolume={false}
|
||||||
|
showToolbar={true}
|
||||||
|
height={320}
|
||||||
|
loading={props.loading}
|
||||||
|
formatters={{
|
||||||
|
price: formatCurrency,
|
||||||
|
date: (date: string) => format(new Date(date), 'MMM dd, yyyy')
|
||||||
|
}}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Panel>
|
</Panel>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ function SummaryCard(props: {
|
|||||||
|
|
||||||
export function NormalizationSummary({ normalization }: NormalizationSummaryProps) {
|
export function NormalizationSummary({ normalization }: NormalizationSummaryProps) {
|
||||||
const hasMaterialUnmapped = normalization.materialUnmappedRowCount > 0;
|
const hasMaterialUnmapped = normalization.materialUnmappedRowCount > 0;
|
||||||
|
const hasWarnings = normalization.warnings.length > 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Panel
|
<Panel
|
||||||
@@ -36,10 +37,13 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
|
|||||||
subtitle="Pack, parser, and residual mapping health for the compact statement surface."
|
subtitle="Pack, parser, and residual mapping health for the compact statement surface."
|
||||||
variant="surface"
|
variant="surface"
|
||||||
>
|
>
|
||||||
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-5">
|
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-8">
|
||||||
<SummaryCard label="Pack" value={normalization.fiscalPack ?? 'unknown'} />
|
<SummaryCard label="Pack" value={normalization.fiscalPack ?? 'unknown'} />
|
||||||
<SummaryCard label="Regime" value={normalization.regime} />
|
<SummaryCard label="Regime" value={normalization.regime} />
|
||||||
<SummaryCard label="Parser" value={`fiscal-xbrl ${normalization.parserVersion}`} />
|
<SummaryCard label="Parser" value={`${normalization.parserEngine} ${normalization.parserVersion}`} />
|
||||||
|
<SummaryCard label="Surface Rows" value={String(normalization.surfaceRowCount)} />
|
||||||
|
<SummaryCard label="Detail Rows" value={String(normalization.detailRowCount)} />
|
||||||
|
<SummaryCard label="KPI Rows" value={String(normalization.kpiRowCount)} />
|
||||||
<SummaryCard label="Unmapped Rows" value={String(normalization.unmappedRowCount)} />
|
<SummaryCard label="Unmapped Rows" value={String(normalization.unmappedRowCount)} />
|
||||||
<SummaryCard
|
<SummaryCard
|
||||||
label="Material Unmapped"
|
label="Material Unmapped"
|
||||||
@@ -47,6 +51,23 @@ export function NormalizationSummary({ normalization }: NormalizationSummaryProp
|
|||||||
tone={hasMaterialUnmapped ? 'warning' : 'default'}
|
tone={hasMaterialUnmapped ? 'warning' : 'default'}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
{hasWarnings ? (
|
||||||
|
<div className="mt-3 rounded-xl border border-[color:var(--line-weak)] bg-[color:var(--panel-soft)] px-3 py-3">
|
||||||
|
<p className="panel-heading text-[10px] uppercase tracking-[0.16em] text-[color:var(--terminal-muted)]">
|
||||||
|
Parser Warnings
|
||||||
|
</p>
|
||||||
|
<div className="mt-2 flex flex-wrap gap-2">
|
||||||
|
{normalization.warnings.map((warning) => (
|
||||||
|
<span
|
||||||
|
key={warning}
|
||||||
|
className="rounded-full border border-[color:var(--line-weak)] bg-[rgba(88,102,122,0.16)] px-3 py-1 text-xs text-[color:var(--terminal-bright)]"
|
||||||
|
>
|
||||||
|
{warning}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
{hasMaterialUnmapped ? (
|
{hasMaterialUnmapped ? (
|
||||||
<div className="mt-3 flex items-start gap-2 rounded-xl border border-[#7f6250] bg-[rgba(91,66,46,0.18)] px-3 py-3 text-sm text-[#f5d5c0]">
|
<div className="mt-3 flex items-start gap-2 rounded-xl border border-[#7f6250] bg-[rgba(91,66,46,0.18)] px-3 py-3 text-sm text-[#f5d5c0]">
|
||||||
<AlertTriangle className="mt-0.5 size-4 shrink-0" />
|
<AlertTriangle className="mt-0.5 size-4 shrink-0" />
|
||||||
|
|||||||
@@ -37,6 +37,9 @@ function renderList(values: string[]) {
|
|||||||
|
|
||||||
export function StatementRowInspector(props: StatementRowInspectorProps) {
|
export function StatementRowInspector(props: StatementRowInspectorProps) {
|
||||||
const selection = props.selection;
|
const selection = props.selection;
|
||||||
|
const parentSurfaceLabel = selection?.kind === 'detail'
|
||||||
|
? selection.parentSurfaceRow?.label ?? (selection.row.parentSurfaceKey === 'unmapped' ? 'Unmapped / Residual' : selection.row.parentSurfaceKey)
|
||||||
|
: null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Panel
|
<Panel
|
||||||
@@ -122,7 +125,7 @@ export function StatementRowInspector(props: StatementRowInspectorProps) {
|
|||||||
<div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-4">
|
<div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-4">
|
||||||
<InspectorCard label="Label" value={selection.row.label} />
|
<InspectorCard label="Label" value={selection.row.label} />
|
||||||
<InspectorCard label="Key" value={selection.row.key} />
|
<InspectorCard label="Key" value={selection.row.key} />
|
||||||
<InspectorCard label="Parent Surface" value={selection.parentSurfaceRow?.label ?? selection.row.parentSurfaceKey} />
|
<InspectorCard label="Parent Surface" value={parentSurfaceLabel ?? selection.row.parentSurfaceKey} />
|
||||||
<InspectorCard label="Residual" value={selection.row.residualFlag ? 'Yes' : 'No'} />
|
<InspectorCard label="Residual" value={selection.row.residualFlag ? 'Yes' : 'No'} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -242,6 +242,23 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
|
|||||||
dimensionsSummary: [],
|
dimensionsSummary: [],
|
||||||
residualFlag: false
|
residualFlag: false
|
||||||
}
|
}
|
||||||
|
],
|
||||||
|
unmapped: [
|
||||||
|
{
|
||||||
|
key: 'other_income_unmapped',
|
||||||
|
parentSurfaceKey: 'unmapped',
|
||||||
|
label: 'Other Income Residual',
|
||||||
|
conceptKey: 'other_income_unmapped',
|
||||||
|
qname: 'us-gaap:OtherNonoperatingIncomeExpense',
|
||||||
|
namespaceUri: 'http://fasb.org/us-gaap/2024',
|
||||||
|
localName: 'OtherNonoperatingIncomeExpense',
|
||||||
|
unit: 'USD',
|
||||||
|
values: { [`${prefix}-fy24`]: 1_200, [`${prefix}-fy25`]: 1_450 },
|
||||||
|
sourceFactIds: [107],
|
||||||
|
isExtension: false,
|
||||||
|
dimensionsSummary: [],
|
||||||
|
residualFlag: true
|
||||||
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
ratioRows: [],
|
ratioRows: [],
|
||||||
@@ -285,11 +302,16 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
|
|||||||
validation: null
|
validation: null
|
||||||
},
|
},
|
||||||
normalization: {
|
normalization: {
|
||||||
|
parserEngine: 'fiscal-xbrl',
|
||||||
regime: 'us-gaap',
|
regime: 'us-gaap',
|
||||||
fiscalPack: isBank ? 'bank_lender' : 'core',
|
fiscalPack: isBank ? 'bank_lender' : 'core',
|
||||||
parserVersion: '0.1.0',
|
parserVersion: '0.1.0',
|
||||||
unmappedRowCount: 0,
|
surfaceRowCount: 8,
|
||||||
materialUnmappedRowCount: 0
|
detailRowCount: isBank ? 0 : 4,
|
||||||
|
kpiRowCount: 0,
|
||||||
|
unmappedRowCount: isBank ? 0 : 1,
|
||||||
|
materialUnmappedRowCount: 0,
|
||||||
|
warnings: isBank ? [] : ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
|
||||||
},
|
},
|
||||||
dimensionBreakdown: null
|
dimensionBreakdown: null
|
||||||
}
|
}
|
||||||
@@ -316,6 +338,11 @@ test('renders the standardized operating expense tree and inspector details', as
|
|||||||
await page.goto('/financials?ticker=MSFT');
|
await page.goto('/financials?ticker=MSFT');
|
||||||
|
|
||||||
await expect(page.getByText('Normalization Summary')).toBeVisible();
|
await expect(page.getByText('Normalization Summary')).toBeVisible();
|
||||||
|
await expect(page.getByText('fiscal-xbrl 0.1.0')).toBeVisible();
|
||||||
|
await expect(page.getByText('Parser Warnings')).toBeVisible();
|
||||||
|
await expect(page.getByText('income_sparse_mapping')).toBeVisible();
|
||||||
|
await expect(page.getByText('unmapped_cash_flow_bridge')).toBeVisible();
|
||||||
|
await expect(page.getByText('Parser residual rows are available under the Unmapped / Residual section.')).toBeVisible();
|
||||||
await expect(page.getByRole('button', { name: 'Expand Operating Expenses details' })).toBeVisible();
|
await expect(page.getByRole('button', { name: 'Expand Operating Expenses details' })).toBeVisible();
|
||||||
|
|
||||||
await page.getByRole('button', { name: 'Expand Operating Expenses details' }).click();
|
await page.getByRole('button', { name: 'Expand Operating Expenses details' }).click();
|
||||||
@@ -327,6 +354,11 @@ test('renders the standardized operating expense tree and inspector details', as
|
|||||||
await expect(page.getByText('Row Details')).toBeVisible();
|
await expect(page.getByText('Row Details')).toBeVisible();
|
||||||
await expect(page.getByText('selling_general_and_administrative', { exact: true }).first()).toBeVisible();
|
await expect(page.getByText('selling_general_and_administrative', { exact: true }).first()).toBeVisible();
|
||||||
await expect(page.getByText('Corporate SG&A')).toBeVisible();
|
await expect(page.getByText('Corporate SG&A')).toBeVisible();
|
||||||
|
await expect(page.getByText('Unmapped / Residual')).toBeVisible();
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: /^Other Income Residual/ }).click();
|
||||||
|
await expect(page.getByText('other_income_unmapped', { exact: true })).toBeVisible();
|
||||||
|
await expect(page.getByText('Unmapped / Residual', { exact: true }).last()).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('shows not meaningful expense breakdown rows for bank pack filings', async ({ page }, testInfo) => {
|
test('shows not meaningful expense breakdown rows for bank pack filings', async ({ page }, testInfo) => {
|
||||||
|
|||||||
@@ -230,11 +230,16 @@ function createFinancialsPayload(input: {
|
|||||||
validation: null
|
validation: null
|
||||||
},
|
},
|
||||||
normalization: {
|
normalization: {
|
||||||
|
parserEngine: 'fiscal-xbrl',
|
||||||
regime: 'unknown',
|
regime: 'unknown',
|
||||||
fiscalPack,
|
fiscalPack,
|
||||||
parserVersion: '0.0.0',
|
parserVersion: '0.0.0',
|
||||||
|
surfaceRowCount: 0,
|
||||||
|
detailRowCount: 0,
|
||||||
|
kpiRowCount: 0,
|
||||||
unmappedRowCount: 0,
|
unmappedRowCount: 0,
|
||||||
materialUnmappedRowCount: 0
|
materialUnmappedRowCount: 0,
|
||||||
|
warnings: []
|
||||||
},
|
},
|
||||||
dimensionBreakdown: null
|
dimensionBreakdown: null
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -210,4 +210,86 @@ describe('statement view model', () => {
|
|||||||
row: { key: 'gp_unmapped', residualFlag: true }
|
row: { key: 'gp_unmapped', residualFlag: true }
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('renders unmapped detail rows in a dedicated residual section and counts them', () => {
|
||||||
|
const model = buildStatementTree({
|
||||||
|
surfaceKind: 'income_statement',
|
||||||
|
rows: [
|
||||||
|
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
|
||||||
|
],
|
||||||
|
statementDetails: {
|
||||||
|
unmapped: [
|
||||||
|
createDetailRow({
|
||||||
|
key: 'unmapped_other_income',
|
||||||
|
label: 'Other income residual',
|
||||||
|
parentSurfaceKey: 'unmapped',
|
||||||
|
values: { p1: 5 },
|
||||||
|
residualFlag: true
|
||||||
|
})
|
||||||
|
]
|
||||||
|
},
|
||||||
|
categories: [],
|
||||||
|
searchQuery: '',
|
||||||
|
expandedRowKeys: new Set()
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(model.sections).toHaveLength(2);
|
||||||
|
expect(model.sections[1]).toMatchObject({
|
||||||
|
key: 'unmapped_residual',
|
||||||
|
label: 'Unmapped / Residual'
|
||||||
|
});
|
||||||
|
expect(model.sections[1]?.nodes[0]).toMatchObject({
|
||||||
|
kind: 'detail',
|
||||||
|
row: { key: 'unmapped_other_income', parentSurfaceKey: 'unmapped' }
|
||||||
|
});
|
||||||
|
expect(model.visibleNodeCount).toBe(2);
|
||||||
|
expect(model.totalNodeCount).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('matches search and resolves selection for unmapped detail rows without a real parent surface', () => {
|
||||||
|
const rows = [
|
||||||
|
createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
|
||||||
|
];
|
||||||
|
const statementDetails = {
|
||||||
|
unmapped: [
|
||||||
|
createDetailRow({
|
||||||
|
key: 'unmapped_fx_gain',
|
||||||
|
label: 'FX gain residual',
|
||||||
|
parentSurfaceKey: 'unmapped',
|
||||||
|
values: { p1: 2 },
|
||||||
|
residualFlag: true
|
||||||
|
})
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
const model = buildStatementTree({
|
||||||
|
surfaceKind: 'income_statement',
|
||||||
|
rows,
|
||||||
|
statementDetails,
|
||||||
|
categories: [],
|
||||||
|
searchQuery: 'fx gain',
|
||||||
|
expandedRowKeys: new Set()
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(model.sections).toHaveLength(1);
|
||||||
|
expect(model.sections[0]).toMatchObject({
|
||||||
|
key: 'unmapped_residual',
|
||||||
|
label: 'Unmapped / Residual'
|
||||||
|
});
|
||||||
|
expect(model.visibleNodeCount).toBe(1);
|
||||||
|
expect(model.totalNodeCount).toBe(2);
|
||||||
|
|
||||||
|
const selection = resolveStatementSelection({
|
||||||
|
surfaceKind: 'income_statement',
|
||||||
|
rows,
|
||||||
|
statementDetails,
|
||||||
|
selection: { kind: 'detail', key: 'unmapped_fx_gain', parentKey: 'unmapped' }
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(selection).toMatchObject({
|
||||||
|
kind: 'detail',
|
||||||
|
row: { key: 'unmapped_fx_gain', parentSurfaceKey: 'unmapped' },
|
||||||
|
parentSurfaceRow: null
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ type Categories = Array<{
|
|||||||
count: number;
|
count: number;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
const UNMAPPED_DETAIL_GROUP_KEY = 'unmapped';
|
||||||
|
const UNMAPPED_SECTION_KEY = 'unmapped_residual';
|
||||||
|
const UNMAPPED_SECTION_LABEL = 'Unmapped / Residual';
|
||||||
|
|
||||||
function surfaceConfigForKind(surfaceKind: Extract<FinancialSurfaceKind, 'income_statement' | 'balance_sheet' | 'cash_flow_statement'>) {
|
function surfaceConfigForKind(surfaceKind: Extract<FinancialSurfaceKind, 'income_statement' | 'balance_sheet' | 'cash_flow_statement'>) {
|
||||||
return SURFACE_CHILDREN[surfaceKind] ?? {};
|
return SURFACE_CHILDREN[surfaceKind] ?? {};
|
||||||
}
|
}
|
||||||
@@ -129,6 +133,25 @@ function sortDetailRows(left: DetailFinancialRow, right: DetailFinancialRow) {
|
|||||||
return left.label.localeCompare(right.label);
|
return left.label.localeCompare(right.label);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildUnmappedDetailNodes(input: {
|
||||||
|
statementDetails: SurfaceDetailMap | null;
|
||||||
|
searchQuery: string;
|
||||||
|
}) {
|
||||||
|
const normalizedSearch = normalize(input.searchQuery);
|
||||||
|
|
||||||
|
return [...(input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? [])]
|
||||||
|
.sort(sortDetailRows)
|
||||||
|
.filter((detail) => normalizedSearch.length === 0 || searchTextForDetail(detail).includes(normalizedSearch))
|
||||||
|
.map((detail) => ({
|
||||||
|
kind: 'detail',
|
||||||
|
id: detailNodeId(UNMAPPED_DETAIL_GROUP_KEY, detail),
|
||||||
|
level: 0,
|
||||||
|
row: detail,
|
||||||
|
parentSurfaceKey: UNMAPPED_DETAIL_GROUP_KEY,
|
||||||
|
matchesSearch: normalizedSearch.length > 0 && searchTextForDetail(detail).includes(normalizedSearch)
|
||||||
|
}) satisfies StatementTreeDetailNode);
|
||||||
|
}
|
||||||
|
|
||||||
function countNodes(nodes: StatementTreeNode[]) {
|
function countNodes(nodes: StatementTreeNode[]) {
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
|
||||||
@@ -223,10 +246,26 @@ export function buildStatementTree(input: {
|
|||||||
.filter((node): node is StatementTreeSurfaceNode => Boolean(node));
|
.filter((node): node is StatementTreeSurfaceNode => Boolean(node));
|
||||||
|
|
||||||
if (input.categories.length === 0) {
|
if (input.categories.length === 0) {
|
||||||
|
const sections: StatementTreeSection[] = rootNodes.length > 0
|
||||||
|
? [{ key: 'ungrouped', label: null, nodes: rootNodes }]
|
||||||
|
: [];
|
||||||
|
const unmappedNodes = buildUnmappedDetailNodes({
|
||||||
|
statementDetails: input.statementDetails,
|
||||||
|
searchQuery: input.searchQuery
|
||||||
|
});
|
||||||
|
|
||||||
|
if (unmappedNodes.length > 0) {
|
||||||
|
sections.push({
|
||||||
|
key: UNMAPPED_SECTION_KEY,
|
||||||
|
label: UNMAPPED_SECTION_LABEL,
|
||||||
|
nodes: unmappedNodes
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
sections: [{ key: 'ungrouped', label: null, nodes: rootNodes }],
|
sections,
|
||||||
autoExpandedKeys,
|
autoExpandedKeys,
|
||||||
visibleNodeCount: countNodes(rootNodes),
|
visibleNodeCount: sections.reduce((sum, section) => sum + countNodes(section.nodes), 0),
|
||||||
totalNodeCount: input.rows.length + Object.values(input.statementDetails ?? {}).reduce((sum, rows) => sum + rows.length, 0)
|
totalNodeCount: input.rows.length + Object.values(input.statementDetails ?? {}).reduce((sum, rows) => sum + rows.length, 0)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -254,6 +293,18 @@ export function buildStatementTree(input: {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const unmappedNodes = buildUnmappedDetailNodes({
|
||||||
|
statementDetails: input.statementDetails,
|
||||||
|
searchQuery: input.searchQuery
|
||||||
|
});
|
||||||
|
if (unmappedNodes.length > 0) {
|
||||||
|
sections.push({
|
||||||
|
key: UNMAPPED_SECTION_KEY,
|
||||||
|
label: UNMAPPED_SECTION_LABEL,
|
||||||
|
nodes: unmappedNodes
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
sections,
|
sections,
|
||||||
autoExpandedKeys,
|
autoExpandedKeys,
|
||||||
@@ -297,9 +348,11 @@ export function resolveStatementSelection(input: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const parentSurfaceKey = selection.parentKey ?? null;
|
const parentSurfaceKey = selection.parentKey ?? null;
|
||||||
const detailRows = parentSurfaceKey
|
const detailRows = parentSurfaceKey === UNMAPPED_DETAIL_GROUP_KEY
|
||||||
? input.statementDetails?.[parentSurfaceKey] ?? []
|
? input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? []
|
||||||
: Object.values(input.statementDetails ?? {}).flat();
|
: parentSurfaceKey
|
||||||
|
? input.statementDetails?.[parentSurfaceKey] ?? []
|
||||||
|
: Object.values(input.statementDetails ?? {}).flat();
|
||||||
const row = detailRows.find((candidate) => candidate.key === selection.key) ?? null;
|
const row = detailRows.find((candidate) => candidate.key === selection.key) ?? null;
|
||||||
|
|
||||||
if (!row) {
|
if (!row) {
|
||||||
|
|||||||
@@ -104,11 +104,16 @@ function createFinancials(input: {
|
|||||||
validation: null
|
validation: null
|
||||||
},
|
},
|
||||||
normalization: {
|
normalization: {
|
||||||
|
parserEngine: 'fiscal-xbrl',
|
||||||
regime: 'unknown',
|
regime: 'unknown',
|
||||||
fiscalPack: input.fiscalPack ?? null,
|
fiscalPack: input.fiscalPack ?? null,
|
||||||
parserVersion: '0.0.0',
|
parserVersion: '0.0.0',
|
||||||
|
surfaceRowCount: 0,
|
||||||
|
detailRowCount: 0,
|
||||||
|
kpiRowCount: 0,
|
||||||
unmappedRowCount: 0,
|
unmappedRowCount: 0,
|
||||||
materialUnmappedRowCount: 0
|
materialUnmappedRowCount: 0,
|
||||||
|
warnings: []
|
||||||
},
|
},
|
||||||
dimensionBreakdown: null
|
dimensionBreakdown: null
|
||||||
} satisfies CompanyFinancialStatementsResponse;
|
} satisfies CompanyFinancialStatementsResponse;
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import type {
|
|||||||
} from '@/lib/types';
|
} from '@/lib/types';
|
||||||
import { auth } from '@/lib/auth';
|
import { auth } from '@/lib/auth';
|
||||||
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
|
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
|
||||||
|
import { getLatestFinancialIngestionSchemaStatus } from '@/lib/server/db/financial-ingestion-schema';
|
||||||
import { asErrorMessage, jsonError } from '@/lib/server/http';
|
import { asErrorMessage, jsonError } from '@/lib/server/http';
|
||||||
import { buildPortfolioSummary } from '@/lib/server/portfolio';
|
import { buildPortfolioSummary } from '@/lib/server/portfolio';
|
||||||
import {
|
import {
|
||||||
@@ -391,16 +392,36 @@ export const app = new Elysia({ prefix: '/api' })
|
|||||||
getTaskQueueSnapshot(),
|
getTaskQueueSnapshot(),
|
||||||
checkWorkflowBackend()
|
checkWorkflowBackend()
|
||||||
]);
|
]);
|
||||||
|
const ingestionSchema = getLatestFinancialIngestionSchemaStatus();
|
||||||
|
const ingestionSchemaPayload = ingestionSchema
|
||||||
|
? {
|
||||||
|
ok: ingestionSchema.ok,
|
||||||
|
mode: ingestionSchema.mode,
|
||||||
|
missingIndexes: ingestionSchema.missingIndexes,
|
||||||
|
duplicateGroups: ingestionSchema.duplicateGroups,
|
||||||
|
lastCheckedAt: ingestionSchema.lastCheckedAt
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
ok: false,
|
||||||
|
mode: 'failed' as const,
|
||||||
|
missingIndexes: [],
|
||||||
|
duplicateGroups: 0,
|
||||||
|
lastCheckedAt: new Date().toISOString()
|
||||||
|
};
|
||||||
|
const schemaHealthy = ingestionSchema?.ok ?? false;
|
||||||
|
|
||||||
if (!workflowBackend.ok) {
|
if (!workflowBackend.ok || !schemaHealthy) {
|
||||||
return Response.json({
|
return Response.json({
|
||||||
status: 'degraded',
|
status: 'degraded',
|
||||||
version: '4.0.0',
|
version: '4.0.0',
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
queue,
|
queue,
|
||||||
|
database: {
|
||||||
|
ingestionSchema: ingestionSchemaPayload
|
||||||
|
},
|
||||||
workflow: {
|
workflow: {
|
||||||
ok: false,
|
ok: workflowBackend.ok,
|
||||||
reason: workflowBackend.reason
|
...(workflowBackend.ok ? {} : { reason: workflowBackend.reason })
|
||||||
}
|
}
|
||||||
}, { status: 503 });
|
}, { status: 503 });
|
||||||
}
|
}
|
||||||
@@ -410,6 +431,9 @@ export const app = new Elysia({ prefix: '/api' })
|
|||||||
version: '4.0.0',
|
version: '4.0.0',
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
queue,
|
queue,
|
||||||
|
database: {
|
||||||
|
ingestionSchema: ingestionSchemaPayload
|
||||||
|
},
|
||||||
workflow: {
|
workflow: {
|
||||||
ok: true
|
ok: true
|
||||||
}
|
}
|
||||||
@@ -1366,12 +1390,13 @@ export const app = new Elysia({ prefix: '/api' })
|
|||||||
return jsonError('ticker is required');
|
return jsonError('ticker is required');
|
||||||
}
|
}
|
||||||
|
|
||||||
const [filings, holding, watchlistItem, liveQuote, priceHistory, journalPreview, memo, secProfile] = await Promise.all([
|
const [filings, holding, watchlistItem, liveQuote, priceHistory, benchmarkHistory, journalPreview, memo, secProfile] = await Promise.all([
|
||||||
listFilingsRecords({ ticker, limit: 40 }),
|
listFilingsRecords({ ticker, limit: 40 }),
|
||||||
getHoldingByTicker(session.user.id, ticker),
|
getHoldingByTicker(session.user.id, ticker),
|
||||||
getWatchlistItemByTicker(session.user.id, ticker),
|
getWatchlistItemByTicker(session.user.id, ticker),
|
||||||
getQuote(ticker),
|
getQuote(ticker),
|
||||||
getPriceHistory(ticker),
|
getPriceHistory(ticker),
|
||||||
|
getPriceHistory('^GSPC'),
|
||||||
listResearchJournalEntries(session.user.id, ticker, 6),
|
listResearchJournalEntries(session.user.id, ticker, 6),
|
||||||
getResearchMemoByTicker(session.user.id, ticker),
|
getResearchMemoByTicker(session.user.id, ticker),
|
||||||
getSecCompanyProfile(ticker)
|
getSecCompanyProfile(ticker)
|
||||||
@@ -1478,6 +1503,7 @@ export const app = new Elysia({ prefix: '/api' })
|
|||||||
quote: liveQuote,
|
quote: liveQuote,
|
||||||
position: holding,
|
position: holding,
|
||||||
priceHistory,
|
priceHistory,
|
||||||
|
benchmarkHistory,
|
||||||
financials,
|
financials,
|
||||||
filings: redactedFilings.slice(0, 20),
|
filings: redactedFilings.slice(0, 20),
|
||||||
aiReports,
|
aiReports,
|
||||||
|
|||||||
@@ -74,11 +74,44 @@ function resetDbSingletons() {
|
|||||||
const globalState = globalThis as typeof globalThis & {
|
const globalState = globalThis as typeof globalThis & {
|
||||||
__fiscalSqliteClient?: { close?: () => void };
|
__fiscalSqliteClient?: { close?: () => void };
|
||||||
__fiscalDrizzleDb?: unknown;
|
__fiscalDrizzleDb?: unknown;
|
||||||
|
__financialIngestionSchemaStatus?: unknown;
|
||||||
};
|
};
|
||||||
|
|
||||||
globalState.__fiscalSqliteClient?.close?.();
|
globalState.__fiscalSqliteClient?.close?.();
|
||||||
globalState.__fiscalSqliteClient = undefined;
|
globalState.__fiscalSqliteClient = undefined;
|
||||||
globalState.__fiscalDrizzleDb = undefined;
|
globalState.__fiscalDrizzleDb = undefined;
|
||||||
|
globalState.__financialIngestionSchemaStatus = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setFinancialIngestionSchemaStatus(input: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
|
||||||
|
missingIndexes?: string[];
|
||||||
|
duplicateGroups?: number;
|
||||||
|
}) {
|
||||||
|
const globalState = globalThis as typeof globalThis & {
|
||||||
|
__financialIngestionSchemaStatus?: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
|
||||||
|
requestedMode: 'auto' | 'check-only' | 'off';
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
lastCheckedAt: string;
|
||||||
|
repair: null;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
globalState.__financialIngestionSchemaStatus = {
|
||||||
|
ok: input.ok,
|
||||||
|
mode: input.mode,
|
||||||
|
requestedMode: 'auto',
|
||||||
|
missingIndexes: input.missingIndexes ?? [],
|
||||||
|
duplicateGroups: input.duplicateGroups ?? 0,
|
||||||
|
lastCheckedAt: new Date().toISOString(),
|
||||||
|
repair: null,
|
||||||
|
error: input.ok ? null : 'schema drift injected by test'
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function applySqlMigrations(client: { exec: (query: string) => void }) {
|
function applySqlMigrations(client: { exec: (query: string) => void }) {
|
||||||
@@ -250,6 +283,10 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
|
|||||||
runStatuses.clear();
|
runStatuses.clear();
|
||||||
runCounter = 0;
|
runCounter = 0;
|
||||||
workflowBackendHealthy = true;
|
workflowBackendHealthy = true;
|
||||||
|
setFinancialIngestionSchemaStatus({
|
||||||
|
ok: true,
|
||||||
|
mode: 'healthy'
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
|
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
|
||||||
@@ -808,9 +845,100 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
|
|||||||
|
|
||||||
const healthy = await jsonRequest('GET', '/api/health');
|
const healthy = await jsonRequest('GET', '/api/health');
|
||||||
expect(healthy.response.status).toBe(200);
|
expect(healthy.response.status).toBe(200);
|
||||||
expect((healthy.json as { status: string; workflow: { ok: boolean } }).status).toBe('ok');
|
expect((healthy.json as {
|
||||||
|
status: string;
|
||||||
|
workflow: { ok: boolean };
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).status).toBe('ok');
|
||||||
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
|
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
|
||||||
|
expect((healthy.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.ok).toBe(true);
|
||||||
|
expect((healthy.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.mode).toBe('healthy');
|
||||||
|
|
||||||
|
setFinancialIngestionSchemaStatus({
|
||||||
|
ok: false,
|
||||||
|
mode: 'drifted',
|
||||||
|
missingIndexes: ['company_financial_bundle_uidx'],
|
||||||
|
duplicateGroups: 1
|
||||||
|
});
|
||||||
|
const schemaDrifted = await jsonRequest('GET', '/api/health');
|
||||||
|
expect(schemaDrifted.response.status).toBe(503);
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
status: string;
|
||||||
|
workflow: { ok: boolean };
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).status).toBe('degraded');
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
workflow: { ok: boolean };
|
||||||
|
}).workflow.ok).toBe(true);
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.ok).toBe(false);
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
ok: boolean;
|
||||||
|
mode: string;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.mode).toBe('drifted');
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
missingIndexes: string[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.missingIndexes).toEqual(['company_financial_bundle_uidx']);
|
||||||
|
expect((schemaDrifted.json as {
|
||||||
|
database: {
|
||||||
|
ingestionSchema: {
|
||||||
|
duplicateGroups: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}).database.ingestionSchema.duplicateGroups).toBe(1);
|
||||||
|
|
||||||
|
setFinancialIngestionSchemaStatus({
|
||||||
|
ok: true,
|
||||||
|
mode: 'healthy'
|
||||||
|
});
|
||||||
workflowBackendHealthy = false;
|
workflowBackendHealthy = false;
|
||||||
const degraded = await jsonRequest('GET', '/api/health');
|
const degraded = await jsonRequest('GET', '/api/health');
|
||||||
expect(degraded.response.status).toBe(503);
|
expect(degraded.response.status).toBe(503);
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ import { dirname, join } from 'node:path';
|
|||||||
import { Database } from 'bun:sqlite';
|
import { Database } from 'bun:sqlite';
|
||||||
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
||||||
import { load as loadSqliteVec } from 'sqlite-vec';
|
import { load as loadSqliteVec } from 'sqlite-vec';
|
||||||
|
import {
|
||||||
|
ensureFinancialIngestionSchemaHealthy,
|
||||||
|
resolveFinancialSchemaRepairMode
|
||||||
|
} from './financial-ingestion-schema';
|
||||||
import { schema } from './schema';
|
import { schema } from './schema';
|
||||||
|
|
||||||
type AppDrizzleDb = ReturnType<typeof createDb>;
|
type AppDrizzleDb = ReturnType<typeof createDb>;
|
||||||
@@ -564,6 +568,9 @@ export function getSqliteClient() {
|
|||||||
client.exec('PRAGMA busy_timeout = 5000;');
|
client.exec('PRAGMA busy_timeout = 5000;');
|
||||||
loadSqliteExtensions(client);
|
loadSqliteExtensions(client);
|
||||||
ensureLocalSqliteSchema(client);
|
ensureLocalSqliteSchema(client);
|
||||||
|
ensureFinancialIngestionSchemaHealthy(client, {
|
||||||
|
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||||
|
});
|
||||||
ensureSearchVirtualTables(client);
|
ensureSearchVirtualTables(client);
|
||||||
|
|
||||||
globalThis.__fiscalSqliteClient = client;
|
globalThis.__fiscalSqliteClient = client;
|
||||||
|
|||||||
@@ -1586,6 +1586,99 @@ describe('financial taxonomy internals', () => {
|
|||||||
expect(merged[0]?.provenanceType).toBe('taxonomy');
|
expect(merged[0]?.provenanceType).toBe('taxonomy');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('builds faithful rows when persisted statement rows are missing sourceFactIds', () => {
|
||||||
|
const malformedSnapshot = {
|
||||||
|
...createSnapshot({
|
||||||
|
filingId: 19,
|
||||||
|
filingType: '10-K',
|
||||||
|
filingDate: '2026-02-20',
|
||||||
|
statement: 'income',
|
||||||
|
periods: [
|
||||||
|
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
statement_rows: {
|
||||||
|
income: [{
|
||||||
|
...createRow({
|
||||||
|
key: 'revenue',
|
||||||
|
label: 'Revenue',
|
||||||
|
statement: 'income',
|
||||||
|
values: { '2025-fy': 123_000_000 }
|
||||||
|
}),
|
||||||
|
sourceFactIds: undefined
|
||||||
|
} as unknown as TaxonomyStatementRow],
|
||||||
|
balance: [],
|
||||||
|
cash_flow: [],
|
||||||
|
equity: [],
|
||||||
|
comprehensive_income: []
|
||||||
|
}
|
||||||
|
} satisfies FilingTaxonomySnapshotRecord;
|
||||||
|
|
||||||
|
const rows = __financialTaxonomyInternals.buildRows(
|
||||||
|
[malformedSnapshot],
|
||||||
|
'income',
|
||||||
|
new Set(['2025-fy'])
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0]?.key).toBe('revenue');
|
||||||
|
expect(rows[0]?.sourceFactIds).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('aggregates persisted surface rows when legacy snapshots are missing source arrays', () => {
|
||||||
|
const snapshot = {
|
||||||
|
...createSnapshot({
|
||||||
|
filingId: 20,
|
||||||
|
filingType: '10-K',
|
||||||
|
filingDate: '2026-02-21',
|
||||||
|
statement: 'income',
|
||||||
|
periods: [
|
||||||
|
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
surface_rows: {
|
||||||
|
income: [{
|
||||||
|
key: 'revenue',
|
||||||
|
label: 'Revenue',
|
||||||
|
category: 'revenue',
|
||||||
|
templateSection: 'statement',
|
||||||
|
order: 10,
|
||||||
|
unit: 'currency',
|
||||||
|
values: { '2025-fy': 123_000_000 },
|
||||||
|
sourceConcepts: undefined,
|
||||||
|
sourceRowKeys: undefined,
|
||||||
|
sourceFactIds: undefined,
|
||||||
|
formulaKey: null,
|
||||||
|
hasDimensions: false,
|
||||||
|
resolvedSourceRowKeys: {},
|
||||||
|
statement: 'income',
|
||||||
|
detailCount: 0,
|
||||||
|
resolutionMethod: 'direct',
|
||||||
|
confidence: 'high',
|
||||||
|
warningCodes: []
|
||||||
|
} as unknown as FilingTaxonomySnapshotRecord['surface_rows']['income'][number]],
|
||||||
|
balance: [],
|
||||||
|
cash_flow: [],
|
||||||
|
equity: [],
|
||||||
|
comprehensive_income: []
|
||||||
|
}
|
||||||
|
} satisfies FilingTaxonomySnapshotRecord;
|
||||||
|
|
||||||
|
const rows = __financialTaxonomyInternals.aggregateSurfaceRows({
|
||||||
|
snapshots: [snapshot],
|
||||||
|
statement: 'income',
|
||||||
|
selectedPeriodIds: new Set(['2025-fy'])
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0]).toMatchObject({
|
||||||
|
key: 'revenue',
|
||||||
|
sourceConcepts: [],
|
||||||
|
sourceRowKeys: [],
|
||||||
|
sourceFactIds: []
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
|
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
|
||||||
const snapshot = {
|
const snapshot = {
|
||||||
...createSnapshot({
|
...createSnapshot({
|
||||||
@@ -1610,11 +1703,81 @@ describe('financial taxonomy internals', () => {
|
|||||||
} satisfies FilingTaxonomySnapshotRecord;
|
} satisfies FilingTaxonomySnapshotRecord;
|
||||||
|
|
||||||
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
|
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
|
||||||
|
parserEngine: 'fiscal-xbrl',
|
||||||
regime: 'us-gaap',
|
regime: 'us-gaap',
|
||||||
fiscalPack: 'bank_lender',
|
fiscalPack: 'bank_lender',
|
||||||
parserVersion: '0.1.0',
|
parserVersion: '0.1.0',
|
||||||
|
surfaceRowCount: 5,
|
||||||
|
detailRowCount: 3,
|
||||||
|
kpiRowCount: 2,
|
||||||
unmappedRowCount: 4,
|
unmappedRowCount: 4,
|
||||||
materialUnmappedRowCount: 1
|
materialUnmappedRowCount: 1,
|
||||||
|
warnings: []
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('aggregates normalization counts and warning codes across snapshots while using the latest parser identity', () => {
|
||||||
|
const olderSnapshot = {
|
||||||
|
...createSnapshot({
|
||||||
|
filingId: 17,
|
||||||
|
filingType: '10-K',
|
||||||
|
filingDate: '2025-02-13',
|
||||||
|
statement: 'income',
|
||||||
|
periods: [
|
||||||
|
{ id: '2024-fy', periodStart: '2024-01-01', periodEnd: '2024-12-31', periodLabel: '2024 FY' }
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
parser_engine: 'fiscal-xbrl',
|
||||||
|
parser_version: '0.9.0',
|
||||||
|
fiscal_pack: 'core',
|
||||||
|
normalization_summary: {
|
||||||
|
surfaceRowCount: 4,
|
||||||
|
detailRowCount: 2,
|
||||||
|
kpiRowCount: 1,
|
||||||
|
unmappedRowCount: 3,
|
||||||
|
materialUnmappedRowCount: 1,
|
||||||
|
warnings: ['balance_residual_detected', 'income_sparse_mapping']
|
||||||
|
}
|
||||||
|
} satisfies FilingTaxonomySnapshotRecord;
|
||||||
|
|
||||||
|
const latestSnapshot = {
|
||||||
|
...createSnapshot({
|
||||||
|
filingId: 18,
|
||||||
|
filingType: '10-Q',
|
||||||
|
filingDate: '2026-02-13',
|
||||||
|
statement: 'income',
|
||||||
|
periods: [
|
||||||
|
{ id: '2025-q4', periodStart: '2025-10-01', periodEnd: '2025-12-31', periodLabel: '2025 Q4' }
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
parser_engine: 'fiscal-xbrl',
|
||||||
|
parser_version: '1.1.0',
|
||||||
|
fiscal_pack: 'bank_lender',
|
||||||
|
normalization_summary: {
|
||||||
|
surfaceRowCount: 6,
|
||||||
|
detailRowCount: 5,
|
||||||
|
kpiRowCount: 4,
|
||||||
|
unmappedRowCount: 2,
|
||||||
|
materialUnmappedRowCount: 0,
|
||||||
|
warnings: ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
|
||||||
|
}
|
||||||
|
} satisfies FilingTaxonomySnapshotRecord;
|
||||||
|
|
||||||
|
expect(__financialTaxonomyInternals.buildNormalizationMetadata([olderSnapshot, latestSnapshot])).toEqual({
|
||||||
|
parserEngine: 'fiscal-xbrl',
|
||||||
|
regime: 'us-gaap',
|
||||||
|
fiscalPack: 'bank_lender',
|
||||||
|
parserVersion: '1.1.0',
|
||||||
|
surfaceRowCount: 10,
|
||||||
|
detailRowCount: 7,
|
||||||
|
kpiRowCount: 5,
|
||||||
|
unmappedRowCount: 5,
|
||||||
|
materialUnmappedRowCount: 1,
|
||||||
|
warnings: [
|
||||||
|
'balance_residual_detected',
|
||||||
|
'income_sparse_mapping',
|
||||||
|
'unmapped_cash_flow_bridge'
|
||||||
|
]
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -177,7 +177,7 @@ function buildKpiDimensionBreakdown(input: {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id));
|
const matchedFacts = input.facts.filter((fact) => (row.sourceFactIds ?? []).includes(fact.id));
|
||||||
if (matchedFacts.length === 0) {
|
if (matchedFacts.length === 0) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -213,9 +213,9 @@ function latestPeriodDate(period: FinancialStatementPeriod) {
|
|||||||
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
|
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
|
||||||
return {
|
return {
|
||||||
...row,
|
...row,
|
||||||
values: { ...row.values },
|
values: { ...(row.values ?? {}) },
|
||||||
sourceConcepts: [...row.sourceConcepts],
|
sourceConcepts: [...(row.sourceConcepts ?? [])],
|
||||||
sourceFactIds: [...row.sourceFactIds]
|
sourceFactIds: [...(row.sourceFactIds ?? [])]
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,7 +230,7 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [periodId, value] of Object.entries(row.values)) {
|
for (const [periodId, value] of Object.entries(row.values ?? {})) {
|
||||||
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
|
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
|
||||||
&& existing.values[periodId] !== null;
|
&& existing.values[periodId] !== null;
|
||||||
if (!hasExistingValue) {
|
if (!hasExistingValue) {
|
||||||
@@ -238,9 +238,9 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])]
|
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...(row.sourceConcepts ?? [])])]
|
||||||
.sort((left, right) => left.localeCompare(right));
|
.sort((left, right) => left.localeCompare(right));
|
||||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])]
|
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...(row.sourceFactIds ?? [])])]
|
||||||
.sort((left, right) => left - right);
|
.sort((left, right) => left - right);
|
||||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||||
existing.segment ??= row.segment;
|
existing.segment ??= row.segment;
|
||||||
@@ -260,11 +260,16 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
|
|||||||
|
|
||||||
function emptyNormalizationMetadata(): NormalizationMetadata {
|
function emptyNormalizationMetadata(): NormalizationMetadata {
|
||||||
return {
|
return {
|
||||||
|
parserEngine: 'unknown',
|
||||||
regime: 'unknown',
|
regime: 'unknown',
|
||||||
fiscalPack: null,
|
fiscalPack: null,
|
||||||
parserVersion: '0.0.0',
|
parserVersion: '0.0.0',
|
||||||
|
surfaceRowCount: 0,
|
||||||
|
detailRowCount: 0,
|
||||||
|
kpiRowCount: 0,
|
||||||
unmappedRowCount: 0,
|
unmappedRowCount: 0,
|
||||||
materialUnmappedRowCount: 0
|
materialUnmappedRowCount: 0,
|
||||||
|
warnings: []
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -277,9 +282,22 @@ function buildNormalizationMetadata(
|
|||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
parserEngine: latestSnapshot.parser_engine,
|
||||||
regime: latestSnapshot.taxonomy_regime,
|
regime: latestSnapshot.taxonomy_regime,
|
||||||
fiscalPack: latestSnapshot.fiscal_pack,
|
fiscalPack: latestSnapshot.fiscal_pack,
|
||||||
parserVersion: latestSnapshot.parser_version,
|
parserVersion: latestSnapshot.parser_version,
|
||||||
|
surfaceRowCount: snapshots.reduce(
|
||||||
|
(sum, snapshot) => sum + (snapshot.normalization_summary?.surfaceRowCount ?? 0),
|
||||||
|
0
|
||||||
|
),
|
||||||
|
detailRowCount: snapshots.reduce(
|
||||||
|
(sum, snapshot) => sum + (snapshot.normalization_summary?.detailRowCount ?? 0),
|
||||||
|
0
|
||||||
|
),
|
||||||
|
kpiRowCount: snapshots.reduce(
|
||||||
|
(sum, snapshot) => sum + (snapshot.normalization_summary?.kpiRowCount ?? 0),
|
||||||
|
0
|
||||||
|
),
|
||||||
unmappedRowCount: snapshots.reduce(
|
unmappedRowCount: snapshots.reduce(
|
||||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
|
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
|
||||||
0
|
0
|
||||||
@@ -287,7 +305,9 @@ function buildNormalizationMetadata(
|
|||||||
materialUnmappedRowCount: snapshots.reduce(
|
materialUnmappedRowCount: snapshots.reduce(
|
||||||
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
|
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
|
||||||
0
|
0
|
||||||
)
|
),
|
||||||
|
warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))]
|
||||||
|
.sort((left, right) => left.localeCompare(right))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,8 +349,12 @@ function aggregateSurfaceRows(input: {
|
|||||||
for (const snapshot of input.snapshots) {
|
for (const snapshot of input.snapshots) {
|
||||||
const rows = snapshot.surface_rows?.[input.statement] ?? [];
|
const rows = snapshot.surface_rows?.[input.statement] ?? [];
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
|
const sourceConcepts = row.sourceConcepts ?? [];
|
||||||
|
const sourceRowKeys = row.sourceRowKeys ?? [];
|
||||||
|
const sourceFactIds = row.sourceFactIds ?? [];
|
||||||
|
const rowValues = row.values ?? {};
|
||||||
const filteredValues = Object.fromEntries(
|
const filteredValues = Object.fromEntries(
|
||||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||||
);
|
);
|
||||||
const filteredResolvedSourceRowKeys = Object.fromEntries(
|
const filteredResolvedSourceRowKeys = Object.fromEntries(
|
||||||
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||||
@@ -345,9 +369,9 @@ function aggregateSurfaceRows(input: {
|
|||||||
...row,
|
...row,
|
||||||
values: filteredValues,
|
values: filteredValues,
|
||||||
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
|
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
|
||||||
sourceConcepts: [...row.sourceConcepts],
|
sourceConcepts: [...sourceConcepts],
|
||||||
sourceRowKeys: [...row.sourceRowKeys],
|
sourceRowKeys: [...sourceRowKeys],
|
||||||
sourceFactIds: [...row.sourceFactIds],
|
sourceFactIds: [...sourceFactIds],
|
||||||
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
|
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
|
||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
@@ -365,9 +389,9 @@ function aggregateSurfaceRows(input: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||||
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...row.sourceRowKeys])].sort((left, right) => left.localeCompare(right));
|
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...sourceRowKeys])].sort((left, right) => left.localeCompare(right));
|
||||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||||
existing.order = Math.min(existing.order, row.order);
|
existing.order = Math.min(existing.order, row.order);
|
||||||
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
|
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
|
||||||
@@ -406,8 +430,11 @@ function aggregateDetailRows(input: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
|
const sourceFactIds = row.sourceFactIds ?? [];
|
||||||
|
const dimensionsSummary = row.dimensionsSummary ?? [];
|
||||||
|
const rowValues = row.values ?? {};
|
||||||
const filteredValues = Object.fromEntries(
|
const filteredValues = Object.fromEntries(
|
||||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||||
);
|
);
|
||||||
if (!rowHasValues(filteredValues)) {
|
if (!rowHasValues(filteredValues)) {
|
||||||
continue;
|
continue;
|
||||||
@@ -418,8 +445,8 @@ function aggregateDetailRows(input: {
|
|||||||
bucket.set(row.key, {
|
bucket.set(row.key, {
|
||||||
...row,
|
...row,
|
||||||
values: filteredValues,
|
values: filteredValues,
|
||||||
sourceFactIds: [...row.sourceFactIds],
|
sourceFactIds: [...sourceFactIds],
|
||||||
dimensionsSummary: [...row.dimensionsSummary]
|
dimensionsSummary: [...dimensionsSummary]
|
||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -430,8 +457,8 @@ function aggregateDetailRows(input: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||||
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...row.dimensionsSummary])].sort((left, right) => left.localeCompare(right));
|
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...dimensionsSummary])].sort((left, right) => left.localeCompare(right));
|
||||||
existing.isExtension = existing.isExtension || row.isExtension;
|
existing.isExtension = existing.isExtension || row.isExtension;
|
||||||
existing.residualFlag = existing.residualFlag || row.residualFlag;
|
existing.residualFlag = existing.residualFlag || row.residualFlag;
|
||||||
}
|
}
|
||||||
@@ -521,8 +548,11 @@ function aggregatePersistedKpiRows(input: {
|
|||||||
|
|
||||||
for (const snapshot of input.snapshots) {
|
for (const snapshot of input.snapshots) {
|
||||||
for (const row of snapshot.kpi_rows ?? []) {
|
for (const row of snapshot.kpi_rows ?? []) {
|
||||||
|
const sourceConcepts = row.sourceConcepts ?? [];
|
||||||
|
const sourceFactIds = row.sourceFactIds ?? [];
|
||||||
|
const rowValues = row.values ?? {};
|
||||||
const filteredValues = Object.fromEntries(
|
const filteredValues = Object.fromEntries(
|
||||||
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
|
||||||
);
|
);
|
||||||
if (!rowHasValues(filteredValues)) {
|
if (!rowHasValues(filteredValues)) {
|
||||||
continue;
|
continue;
|
||||||
@@ -533,8 +563,8 @@ function aggregatePersistedKpiRows(input: {
|
|||||||
rowMap.set(row.key, {
|
rowMap.set(row.key, {
|
||||||
...row,
|
...row,
|
||||||
values: filteredValues,
|
values: filteredValues,
|
||||||
sourceConcepts: [...row.sourceConcepts],
|
sourceConcepts: [...sourceConcepts],
|
||||||
sourceFactIds: [...row.sourceFactIds]
|
sourceFactIds: [...sourceFactIds]
|
||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -543,8 +573,8 @@ function aggregatePersistedKpiRows(input: {
|
|||||||
...existing.values,
|
...existing.values,
|
||||||
...filteredValues
|
...filteredValues
|
||||||
};
|
};
|
||||||
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
|
||||||
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
|
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
|
||||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -282,17 +282,20 @@ export function buildRows(
|
|||||||
const rows = snapshot.statement_rows?.[statement] ?? [];
|
const rows = snapshot.statement_rows?.[statement] ?? [];
|
||||||
|
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
|
const rowValues = row.values ?? {};
|
||||||
|
const rowUnits = row.units ?? {};
|
||||||
|
const sourceFactIds = row.sourceFactIds ?? [];
|
||||||
const existing = rowMap.get(row.key);
|
const existing = rowMap.get(row.key);
|
||||||
if (!existing) {
|
if (!existing) {
|
||||||
rowMap.set(row.key, {
|
rowMap.set(row.key, {
|
||||||
...row,
|
...row,
|
||||||
values: Object.fromEntries(
|
values: Object.fromEntries(
|
||||||
Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
Object.entries(rowValues).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||||
),
|
),
|
||||||
units: Object.fromEntries(
|
units: Object.fromEntries(
|
||||||
Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
Object.entries(rowUnits).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||||
),
|
),
|
||||||
sourceFactIds: [...row.sourceFactIds]
|
sourceFactIds: [...sourceFactIds]
|
||||||
});
|
});
|
||||||
|
|
||||||
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
|
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
|
||||||
@@ -308,19 +311,19 @@ export function buildRows(
|
|||||||
existing.parentKey = row.parentKey;
|
existing.parentKey = row.parentKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [periodId, value] of Object.entries(row.values)) {
|
for (const [periodId, value] of Object.entries(rowValues)) {
|
||||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
|
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
|
||||||
existing.values[periodId] = value;
|
existing.values[periodId] = value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [periodId, unit] of Object.entries(row.units)) {
|
for (const [periodId, unit] of Object.entries(rowUnits)) {
|
||||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
|
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
|
||||||
existing.units[periodId] = unit;
|
existing.units[periodId] = unit;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const factId of row.sourceFactIds) {
|
for (const factId of sourceFactIds) {
|
||||||
if (!existing.sourceFactIds.includes(factId)) {
|
if (!existing.sourceFactIds.includes(factId)) {
|
||||||
existing.sourceFactIds.push(factId);
|
existing.sourceFactIds.push(factId);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
|
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
|
||||||
|
|
||||||
|
function buildYahooChartUrl(ticker: string, params: string) {
|
||||||
|
return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
|
||||||
|
}
|
||||||
|
|
||||||
function fallbackQuote(ticker: string) {
|
function fallbackQuote(ticker: string) {
|
||||||
const normalized = ticker.trim().toUpperCase();
|
const normalized = ticker.trim().toUpperCase();
|
||||||
let hash = 0;
|
let hash = 0;
|
||||||
@@ -15,7 +19,7 @@ export async function getQuote(ticker: string): Promise<number> {
|
|||||||
const normalizedTicker = ticker.trim().toUpperCase();
|
const normalizedTicker = ticker.trim().toUpperCase();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
|
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||||
},
|
},
|
||||||
@@ -47,7 +51,7 @@ export async function getQuoteOrNull(ticker: string): Promise<number | null> {
|
|||||||
const normalizedTicker = ticker.trim().toUpperCase();
|
const normalizedTicker = ticker.trim().toUpperCase();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
|
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||||
},
|
},
|
||||||
@@ -87,7 +91,7 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, {
|
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=20y'), {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||||
},
|
},
|
||||||
@@ -143,7 +147,7 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
|
|||||||
const normalizedTicker = ticker.trim().toUpperCase();
|
const normalizedTicker = ticker.trim().toUpperCase();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1wk&range=1y`, {
|
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1wk&range=20y'), {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
|
||||||
},
|
},
|
||||||
@@ -195,11 +199,13 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
|
|||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const base = fallbackQuote(normalizedTicker);
|
const base = fallbackQuote(normalizedTicker);
|
||||||
|
|
||||||
return Array.from({ length: 26 }, (_, index) => {
|
const totalWeeks = 20 * 52;
|
||||||
const step = 25 - index;
|
|
||||||
const date = new Date(now - step * 14 * 24 * 60 * 60 * 1000).toISOString();
|
return Array.from({ length: totalWeeks }, (_, index) => {
|
||||||
const wave = Math.sin(index / 3.5) * 0.05;
|
const step = (totalWeeks - 1) - index;
|
||||||
const trend = (index - 13) * 0.006;
|
const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
|
||||||
|
const wave = Math.sin(index / 8) * 0.06;
|
||||||
|
const trend = (index - totalWeeks / 2) * 0.0009;
|
||||||
const close = Math.max(base * (1 + wave + trend), 1);
|
const close = Math.max(base * (1 + wave + trend), 1);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -3,7 +3,8 @@ import type {
|
|||||||
FinancialCadence,
|
FinancialCadence,
|
||||||
FinancialSurfaceKind
|
FinancialSurfaceKind
|
||||||
} from '@/lib/types';
|
} from '@/lib/types';
|
||||||
import { db } from '@/lib/server/db';
|
import { db, getSqliteClient } from '@/lib/server/db';
|
||||||
|
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
|
||||||
import { companyFinancialBundle } from '@/lib/server/db/schema';
|
import { companyFinancialBundle } from '@/lib/server/db/schema';
|
||||||
|
|
||||||
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
|
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
|
||||||
@@ -64,34 +65,38 @@ export async function upsertCompanyFinancialBundle(input: {
|
|||||||
}) {
|
}) {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
|
||||||
const [saved] = await db
|
const [saved] = await withFinancialIngestionSchemaRetry({
|
||||||
.insert(companyFinancialBundle)
|
client: getSqliteClient(),
|
||||||
.values({
|
context: 'company-financial-bundle-upsert',
|
||||||
ticker: input.ticker.trim().toUpperCase(),
|
operation: async () => await db
|
||||||
surface_kind: input.surfaceKind,
|
.insert(companyFinancialBundle)
|
||||||
cadence: input.cadence,
|
.values({
|
||||||
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
ticker: input.ticker.trim().toUpperCase(),
|
||||||
source_snapshot_ids: input.sourceSnapshotIds,
|
surface_kind: input.surfaceKind,
|
||||||
source_signature: input.sourceSignature,
|
cadence: input.cadence,
|
||||||
payload: input.payload,
|
|
||||||
created_at: now,
|
|
||||||
updated_at: now
|
|
||||||
})
|
|
||||||
.onConflictDoUpdate({
|
|
||||||
target: [
|
|
||||||
companyFinancialBundle.ticker,
|
|
||||||
companyFinancialBundle.surface_kind,
|
|
||||||
companyFinancialBundle.cadence
|
|
||||||
],
|
|
||||||
set: {
|
|
||||||
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
||||||
source_snapshot_ids: input.sourceSnapshotIds,
|
source_snapshot_ids: input.sourceSnapshotIds,
|
||||||
source_signature: input.sourceSignature,
|
source_signature: input.sourceSignature,
|
||||||
payload: input.payload,
|
payload: input.payload,
|
||||||
|
created_at: now,
|
||||||
updated_at: now
|
updated_at: now
|
||||||
}
|
})
|
||||||
})
|
.onConflictDoUpdate({
|
||||||
.returning();
|
target: [
|
||||||
|
companyFinancialBundle.ticker,
|
||||||
|
companyFinancialBundle.surface_kind,
|
||||||
|
companyFinancialBundle.cadence
|
||||||
|
],
|
||||||
|
set: {
|
||||||
|
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
|
||||||
|
source_snapshot_ids: input.sourceSnapshotIds,
|
||||||
|
source_signature: input.sourceSignature,
|
||||||
|
payload: input.payload,
|
||||||
|
updated_at: now
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.returning()
|
||||||
|
});
|
||||||
|
|
||||||
return toBundleRecord(saved);
|
return toBundleRecord(saved);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ import type {
|
|||||||
TaxonomyFactRow,
|
TaxonomyFactRow,
|
||||||
TaxonomyStatementRow
|
TaxonomyStatementRow
|
||||||
} from '@/lib/types';
|
} from '@/lib/types';
|
||||||
import { db } from '@/lib/server/db';
|
import { db, getSqliteClient } from '@/lib/server/db';
|
||||||
|
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
|
||||||
import {
|
import {
|
||||||
filingTaxonomyAsset,
|
filingTaxonomyAsset,
|
||||||
filingTaxonomyConcept,
|
filingTaxonomyConcept,
|
||||||
@@ -552,38 +553,13 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
|
|||||||
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
|
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
|
||||||
const [saved] = await db
|
const [saved] = await withFinancialIngestionSchemaRetry({
|
||||||
.insert(filingTaxonomySnapshot)
|
client: getSqliteClient(),
|
||||||
.values({
|
context: 'filing-taxonomy-snapshot-upsert',
|
||||||
filing_id: input.filing_id,
|
operation: async () => await db
|
||||||
ticker: input.ticker,
|
.insert(filingTaxonomySnapshot)
|
||||||
filing_date: input.filing_date,
|
.values({
|
||||||
filing_type: input.filing_type,
|
filing_id: input.filing_id,
|
||||||
parse_status: input.parse_status,
|
|
||||||
parse_error: input.parse_error,
|
|
||||||
source: input.source,
|
|
||||||
parser_engine: input.parser_engine,
|
|
||||||
parser_version: input.parser_version,
|
|
||||||
taxonomy_regime: input.taxonomy_regime,
|
|
||||||
fiscal_pack: input.fiscal_pack,
|
|
||||||
periods: input.periods,
|
|
||||||
faithful_rows: input.faithful_rows,
|
|
||||||
statement_rows: input.statement_rows,
|
|
||||||
surface_rows: input.surface_rows,
|
|
||||||
detail_rows: input.detail_rows,
|
|
||||||
kpi_rows: input.kpi_rows,
|
|
||||||
derived_metrics: input.derived_metrics,
|
|
||||||
validation_result: input.validation_result,
|
|
||||||
normalization_summary: input.normalization_summary,
|
|
||||||
facts_count: input.facts_count,
|
|
||||||
concepts_count: input.concepts_count,
|
|
||||||
dimensions_count: input.dimensions_count,
|
|
||||||
created_at: now,
|
|
||||||
updated_at: now
|
|
||||||
})
|
|
||||||
.onConflictDoUpdate({
|
|
||||||
target: filingTaxonomySnapshot.filing_id,
|
|
||||||
set: {
|
|
||||||
ticker: input.ticker,
|
ticker: input.ticker,
|
||||||
filing_date: input.filing_date,
|
filing_date: input.filing_date,
|
||||||
filing_type: input.filing_type,
|
filing_type: input.filing_type,
|
||||||
@@ -606,10 +582,39 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
|
|||||||
facts_count: input.facts_count,
|
facts_count: input.facts_count,
|
||||||
concepts_count: input.concepts_count,
|
concepts_count: input.concepts_count,
|
||||||
dimensions_count: input.dimensions_count,
|
dimensions_count: input.dimensions_count,
|
||||||
|
created_at: now,
|
||||||
updated_at: now
|
updated_at: now
|
||||||
}
|
})
|
||||||
})
|
.onConflictDoUpdate({
|
||||||
.returning();
|
target: filingTaxonomySnapshot.filing_id,
|
||||||
|
set: {
|
||||||
|
ticker: input.ticker,
|
||||||
|
filing_date: input.filing_date,
|
||||||
|
filing_type: input.filing_type,
|
||||||
|
parse_status: input.parse_status,
|
||||||
|
parse_error: input.parse_error,
|
||||||
|
source: input.source,
|
||||||
|
parser_engine: input.parser_engine,
|
||||||
|
parser_version: input.parser_version,
|
||||||
|
taxonomy_regime: input.taxonomy_regime,
|
||||||
|
fiscal_pack: input.fiscal_pack,
|
||||||
|
periods: input.periods,
|
||||||
|
faithful_rows: input.faithful_rows,
|
||||||
|
statement_rows: input.statement_rows,
|
||||||
|
surface_rows: input.surface_rows,
|
||||||
|
detail_rows: input.detail_rows,
|
||||||
|
kpi_rows: input.kpi_rows,
|
||||||
|
derived_metrics: input.derived_metrics,
|
||||||
|
validation_result: input.validation_result,
|
||||||
|
normalization_summary: input.normalization_summary,
|
||||||
|
facts_count: input.facts_count,
|
||||||
|
concepts_count: input.concepts_count,
|
||||||
|
dimensions_count: input.dimensions_count,
|
||||||
|
updated_at: now
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.returning()
|
||||||
|
});
|
||||||
|
|
||||||
const snapshotId = saved.id;
|
const snapshotId = saved.id;
|
||||||
|
|
||||||
|
|||||||
91
lib/types.ts
91
lib/types.ts
@@ -498,11 +498,16 @@ export type NormalizationSummary = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export type NormalizationMetadata = {
|
export type NormalizationMetadata = {
|
||||||
|
parserEngine: string;
|
||||||
regime: 'us-gaap' | 'ifrs-full' | 'unknown';
|
regime: 'us-gaap' | 'ifrs-full' | 'unknown';
|
||||||
fiscalPack: string | null;
|
fiscalPack: string | null;
|
||||||
parserVersion: string;
|
parserVersion: string;
|
||||||
|
surfaceRowCount: number;
|
||||||
|
detailRowCount: number;
|
||||||
|
kpiRowCount: number;
|
||||||
unmappedRowCount: number;
|
unmappedRowCount: number;
|
||||||
materialUnmappedRowCount: number;
|
materialUnmappedRowCount: number;
|
||||||
|
warnings: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
export type RatioRow = DerivedFinancialRow & {
|
export type RatioRow = DerivedFinancialRow & {
|
||||||
@@ -741,6 +746,7 @@ export type CompanyAnalysis = {
|
|||||||
quote: number;
|
quote: number;
|
||||||
position: Holding | null;
|
position: Holding | null;
|
||||||
priceHistory: Array<{ date: string; close: number }>;
|
priceHistory: Array<{ date: string; close: number }>;
|
||||||
|
benchmarkHistory: Array<{ date: string; close: number }>;
|
||||||
financials: CompanyFinancialPoint[];
|
financials: CompanyFinancialPoint[];
|
||||||
filings: Filing[];
|
filings: Filing[];
|
||||||
aiReports: CompanyAiReport[];
|
aiReports: CompanyAiReport[];
|
||||||
@@ -788,3 +794,88 @@ export type ActiveContext = {
|
|||||||
pathname: string;
|
pathname: string;
|
||||||
activeTicker: string | null;
|
activeTicker: string | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Chart Types
|
||||||
|
export type ChartType = 'line' | 'combination';
|
||||||
|
export type TimeRange = '1W' | '1M' | '3M' | '1Y' | '3Y' | '5Y' | '10Y' | '20Y';
|
||||||
|
|
||||||
|
// Chart Data Formats
|
||||||
|
export type PriceDataPoint = {
|
||||||
|
date: string;
|
||||||
|
price: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type OHLCVDataPoint = {
|
||||||
|
date: string;
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ChartDataPoint = PriceDataPoint | OHLCVDataPoint;
|
||||||
|
|
||||||
|
export type DataSeries<T extends ChartDataPoint = ChartDataPoint> = {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
data: T[];
|
||||||
|
color?: string;
|
||||||
|
type?: 'line' | 'area' | 'bar';
|
||||||
|
visible?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Chart Configuration
|
||||||
|
export type ChartZoomState = {
|
||||||
|
startIndex: number;
|
||||||
|
endIndex: number;
|
||||||
|
isZoomed: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ChartColorPalette = {
|
||||||
|
primary: string;
|
||||||
|
secondary: string;
|
||||||
|
positive: string;
|
||||||
|
negative: string;
|
||||||
|
grid: string;
|
||||||
|
text: string;
|
||||||
|
muted: string;
|
||||||
|
tooltipBg: string;
|
||||||
|
tooltipBorder: string;
|
||||||
|
volume: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InteractivePriceChartProps = {
|
||||||
|
// Data
|
||||||
|
data: ChartDataPoint[];
|
||||||
|
dataSeries?: DataSeries[];
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
defaultChartType?: ChartType;
|
||||||
|
defaultTimeRange?: TimeRange;
|
||||||
|
showVolume?: boolean;
|
||||||
|
showToolbar?: boolean;
|
||||||
|
height?: number;
|
||||||
|
|
||||||
|
// Customization
|
||||||
|
colors?: Partial<ChartColorPalette>;
|
||||||
|
formatters?: {
|
||||||
|
price?: (value: number) => string;
|
||||||
|
date?: (value: string) => string;
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Event handlers
|
||||||
|
onChartTypeChange?: (type: ChartType) => void;
|
||||||
|
onTimeRangeChange?: (range: TimeRange) => void;
|
||||||
|
onDataPointHover?: (point: ChartDataPoint | null) => void;
|
||||||
|
onZoomChange?: (state: ChartZoomState) => void;
|
||||||
|
|
||||||
|
// State
|
||||||
|
loading?: boolean;
|
||||||
|
error?: string | null;
|
||||||
|
|
||||||
|
// Accessibility
|
||||||
|
ariaLabel?: string;
|
||||||
|
description?: string;
|
||||||
|
};
|
||||||
|
|||||||
2
next-env.d.ts
vendored
2
next-env.d.ts
vendored
@@ -1,6 +1,6 @@
|
|||||||
/// <reference types="next" />
|
/// <reference types="next" />
|
||||||
/// <reference types="next/image-types/global" />
|
/// <reference types="next/image-types/global" />
|
||||||
import "./.next/types/routes.d.ts";
|
import "./.next/dev/types/routes.d.ts";
|
||||||
|
|
||||||
// NOTE: This file should not be edited
|
// NOTE: This file should not be edited
|
||||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||||
|
|||||||
@@ -21,6 +21,7 @@
|
|||||||
"backfill:search-index": "bun run scripts/backfill-search-index.ts",
|
"backfill:search-index": "bun run scripts/backfill-search-index.ts",
|
||||||
"backfill:taxonomy-snapshots": "bun run scripts/backfill-taxonomy-snapshots.ts",
|
"backfill:taxonomy-snapshots": "bun run scripts/backfill-taxonomy-snapshots.ts",
|
||||||
"compare:fiscal-ai": "bun run scripts/compare-fiscal-ai-statements.ts",
|
"compare:fiscal-ai": "bun run scripts/compare-fiscal-ai-statements.ts",
|
||||||
|
"repair:financial-ingestion-schema": "bun run scripts/repair-financial-ingestion-schema.ts",
|
||||||
"report:taxonomy-health": "bun run scripts/report-taxonomy-health.ts",
|
"report:taxonomy-health": "bun run scripts/report-taxonomy-health.ts",
|
||||||
"db:generate": "bun x drizzle-kit generate",
|
"db:generate": "bun x drizzle-kit generate",
|
||||||
"db:migrate": "bun x drizzle-kit migrate",
|
"db:migrate": "bun x drizzle-kit migrate",
|
||||||
@@ -43,6 +44,7 @@
|
|||||||
"date-fns": "^4.1.0",
|
"date-fns": "^4.1.0",
|
||||||
"drizzle-orm": "^0.41.0",
|
"drizzle-orm": "^0.41.0",
|
||||||
"elysia": "latest",
|
"elysia": "latest",
|
||||||
|
"html-to-image": "^1.11.13",
|
||||||
"lucide-react": "^0.575.0",
|
"lucide-react": "^0.575.0",
|
||||||
"next": "^16.1.6",
|
"next": "^16.1.6",
|
||||||
"react": "^19.2.4",
|
"react": "^19.2.4",
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import { dirname } from 'node:path';
|
|||||||
import { Database } from 'bun:sqlite';
|
import { Database } from 'bun:sqlite';
|
||||||
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
||||||
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
||||||
|
import {
|
||||||
|
ensureFinancialIngestionSchemaHealthy,
|
||||||
|
resolveFinancialSchemaRepairMode
|
||||||
|
} from '../lib/server/db/financial-ingestion-schema';
|
||||||
import { resolveSqlitePath } from './dev-env';
|
import { resolveSqlitePath } from './dev-env';
|
||||||
|
|
||||||
function trim(value: string | undefined) {
|
function trim(value: string | undefined) {
|
||||||
@@ -74,6 +78,13 @@ function runDatabaseMigrations() {
|
|||||||
try {
|
try {
|
||||||
client.exec('PRAGMA foreign_keys = ON;');
|
client.exec('PRAGMA foreign_keys = ON;');
|
||||||
migrate(drizzle(client), { migrationsFolder: './drizzle' });
|
migrate(drizzle(client), { migrationsFolder: './drizzle' });
|
||||||
|
|
||||||
|
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
|
||||||
|
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||||
|
});
|
||||||
|
if (!repairResult.ok) {
|
||||||
|
throw new Error(repairResult.error ?? `financial ingestion schema is ${repairResult.mode}`);
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
client.close();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ import { mkdirSync, readFileSync } from 'node:fs';
|
|||||||
import { Database } from 'bun:sqlite';
|
import { Database } from 'bun:sqlite';
|
||||||
import { createServer } from 'node:net';
|
import { createServer } from 'node:net';
|
||||||
import { dirname, join } from 'node:path';
|
import { dirname, join } from 'node:path';
|
||||||
|
import {
|
||||||
|
ensureFinancialIngestionSchemaHealthy,
|
||||||
|
resolveFinancialSchemaRepairMode
|
||||||
|
} from '../lib/server/db/financial-ingestion-schema';
|
||||||
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
|
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
|
||||||
|
|
||||||
type DrizzleJournal = {
|
type DrizzleJournal = {
|
||||||
@@ -126,6 +130,33 @@ mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? '.workflow-data', { recursive: true });
|
|||||||
|
|
||||||
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
|
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
|
||||||
|
|
||||||
|
if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
|
||||||
|
const client = new Database(databasePath, { create: true });
|
||||||
|
|
||||||
|
try {
|
||||||
|
client.exec('PRAGMA foreign_keys = ON;');
|
||||||
|
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
|
||||||
|
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (repairResult.mode === 'repaired') {
|
||||||
|
console.info(
|
||||||
|
`[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(', ') || 'none'}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? 'yes' : 'no'})`
|
||||||
|
);
|
||||||
|
} else if (repairResult.mode === 'drifted') {
|
||||||
|
console.warn(
|
||||||
|
`[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(', ') || 'none'}; duplicate groups: ${repairResult.duplicateGroups})`
|
||||||
|
);
|
||||||
|
} else if (repairResult.mode === 'failed') {
|
||||||
|
console.warn(
|
||||||
|
`[dev] financial ingestion schema repair failed: ${repairResult.error ?? 'unknown error'}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
console.info(`[dev] local origin ${config.publicOrigin}`);
|
console.info(`[dev] local origin ${config.publicOrigin}`);
|
||||||
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
|
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
|
||||||
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);
|
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);
|
||||||
|
|||||||
Reference in New Issue
Block a user