Add untracked chart and schema files
This commit is contained in:
38
components/charts/hooks/use-chart-export.ts
Normal file
38
components/charts/hooks/use-chart-export.ts
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
import { useCallback, useRef } from 'react';
|
||||||
|
import { toPng } from 'html-to-image';
|
||||||
|
import { getChartColors, getComputedColors } from '../utils/chart-colors';
|
||||||
|
|
||||||
|
export function useChartExport() {
|
||||||
|
const chartRef = useRef<HTMLDivElement>(null);
|
||||||
|
|
||||||
|
const exportChart = useCallback(async (filename: string = 'chart.png') => {
|
||||||
|
if (!chartRef.current) {
|
||||||
|
console.error('Chart ref not available');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get background color from CSS variable
|
||||||
|
const colors = getChartColors();
|
||||||
|
const computedColors = getComputedColors(colors);
|
||||||
|
const backgroundColor = computedColors.tooltipBg || colors.tooltipBg;
|
||||||
|
|
||||||
|
const dataUrl = await toPng(chartRef.current, {
|
||||||
|
quality: 1.0,
|
||||||
|
pixelRatio: 2, // High DPI export
|
||||||
|
backgroundColor: backgroundColor
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create download link
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.download = filename;
|
||||||
|
link.href = dataUrl;
|
||||||
|
link.click();
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to export chart:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return { chartRef, exportChart };
|
||||||
|
}
|
||||||
44
components/charts/hooks/use-chart-zoom.ts
Normal file
44
components/charts/hooks/use-chart-zoom.ts
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
import { useState, useCallback } from 'react';
|
||||||
|
import type { ChartZoomState } from '@/lib/types';
|
||||||
|
|
||||||
|
export function useChartZoom(dataLength: number) {
|
||||||
|
const [zoomState, setZoomState] = useState<ChartZoomState>({
|
||||||
|
startIndex: 0,
|
||||||
|
endIndex: Math.max(0, dataLength - 1),
|
||||||
|
isZoomed: false
|
||||||
|
});
|
||||||
|
|
||||||
|
const handleZoomChange = useCallback(
|
||||||
|
(brushData: { startIndex?: number; endIndex?: number }) => {
|
||||||
|
if (
|
||||||
|
brushData.startIndex === undefined ||
|
||||||
|
brushData.endIndex === undefined
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setZoomState({
|
||||||
|
startIndex: brushData.startIndex,
|
||||||
|
endIndex: brushData.endIndex,
|
||||||
|
isZoomed:
|
||||||
|
brushData.startIndex !== 0 ||
|
||||||
|
brushData.endIndex !== dataLength - 1
|
||||||
|
});
|
||||||
|
},
|
||||||
|
[dataLength]
|
||||||
|
);
|
||||||
|
|
||||||
|
const resetZoom = useCallback(() => {
|
||||||
|
setZoomState({
|
||||||
|
startIndex: 0,
|
||||||
|
endIndex: Math.max(0, dataLength - 1),
|
||||||
|
isZoomed: false
|
||||||
|
});
|
||||||
|
}, [dataLength]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
zoomState,
|
||||||
|
handleZoomChange,
|
||||||
|
resetZoom
|
||||||
|
};
|
||||||
|
}
|
||||||
92
components/charts/interactive-price-chart.tsx
Normal file
92
components/charts/interactive-price-chart.tsx
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState, useMemo } from 'react';
|
||||||
|
import type { InteractivePriceChartProps, ChartType, TimeRange } from '@/lib/types';
|
||||||
|
import { filterByTimeRange, isOHLCVData } from './utils/chart-data-transformers';
|
||||||
|
import { ChartContainer } from './primitives/chart-container';
|
||||||
|
import { ChartToolbar } from './primitives/chart-toolbar';
|
||||||
|
import { LineChartView } from './renderers/line-chart-view';
|
||||||
|
import { CombinationChartView } from './renderers/combination-chart-view';
|
||||||
|
import { VolumeIndicator } from './primitives/volume-indicator';
|
||||||
|
import { useChartExport } from './hooks/use-chart-export';
|
||||||
|
|
||||||
|
export function InteractivePriceChart({
|
||||||
|
data,
|
||||||
|
dataSeries,
|
||||||
|
defaultChartType = 'line',
|
||||||
|
defaultTimeRange = '1Y',
|
||||||
|
showVolume = false,
|
||||||
|
showToolbar = true,
|
||||||
|
height = 400,
|
||||||
|
loading = false,
|
||||||
|
error = null,
|
||||||
|
formatters,
|
||||||
|
onChartTypeChange,
|
||||||
|
onTimeRangeChange
|
||||||
|
}: InteractivePriceChartProps) {
|
||||||
|
const [chartType, setChartType] = useState<ChartType>(defaultChartType);
|
||||||
|
const [timeRange, setTimeRange] = useState<TimeRange>(defaultTimeRange);
|
||||||
|
const filteredData = useMemo(() => filterByTimeRange(data, timeRange), [data, timeRange]);
|
||||||
|
const filteredDataSeries = useMemo(
|
||||||
|
() => dataSeries?.map((series) => ({
|
||||||
|
...series,
|
||||||
|
data: filterByTimeRange(series.data, timeRange)
|
||||||
|
})),
|
||||||
|
[dataSeries, timeRange]
|
||||||
|
);
|
||||||
|
const { chartRef, exportChart } = useChartExport();
|
||||||
|
|
||||||
|
const handleChartTypeChange = (type: ChartType) => {
|
||||||
|
setChartType(type);
|
||||||
|
onChartTypeChange?.(type);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleTimeRangeChange = (range: TimeRange) => {
|
||||||
|
setTimeRange(range);
|
||||||
|
onTimeRangeChange?.(range);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleExport = () => {
|
||||||
|
exportChart(`chart-${Date.now()}.png`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const shouldShowVolume = showVolume && filteredData.some(isOHLCVData);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div ref={chartRef} className="w-full">
|
||||||
|
{showToolbar && (
|
||||||
|
<ChartToolbar
|
||||||
|
chartType={chartType}
|
||||||
|
timeRange={timeRange}
|
||||||
|
onChartTypeChange={handleChartTypeChange}
|
||||||
|
onTimeRangeChange={handleTimeRangeChange}
|
||||||
|
onExport={handleExport}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<ChartContainer height={height} loading={loading} error={error}>
|
||||||
|
{chartType === 'line' && (
|
||||||
|
<LineChartView
|
||||||
|
data={filteredData}
|
||||||
|
formatters={formatters}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{chartType === 'combination' && filteredDataSeries && (
|
||||||
|
<CombinationChartView
|
||||||
|
dataSeries={filteredDataSeries}
|
||||||
|
formatters={formatters}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</ChartContainer>
|
||||||
|
|
||||||
|
{shouldShowVolume && (
|
||||||
|
<VolumeIndicator
|
||||||
|
data={filteredData.filter(isOHLCVData)}
|
||||||
|
height={80}
|
||||||
|
formatters={formatters}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
45
components/charts/primitives/chart-container.tsx
Normal file
45
components/charts/primitives/chart-container.tsx
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
import { cn } from '@/lib/utils';
|
||||||
|
|
||||||
|
type ChartContainerProps = {
|
||||||
|
children: React.ReactNode;
|
||||||
|
height?: number;
|
||||||
|
loading?: boolean;
|
||||||
|
error?: string | null;
|
||||||
|
className?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function ChartContainer({
|
||||||
|
children,
|
||||||
|
height = 400,
|
||||||
|
loading = false,
|
||||||
|
error = null,
|
||||||
|
className
|
||||||
|
}: ChartContainerProps) {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={cn(
|
||||||
|
'relative w-full rounded-xl border border-[color:var(--line-weak)] bg-[color:var(--panel-soft)]',
|
||||||
|
className
|
||||||
|
)}
|
||||||
|
style={{ height: `${height}px` }}
|
||||||
|
>
|
||||||
|
{loading && (
|
||||||
|
<div className="absolute inset-0 flex items-center justify-center">
|
||||||
|
<div className="text-sm text-[color:var(--terminal-muted)]">Loading chart...</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="absolute inset-0 flex items-center justify-center">
|
||||||
|
<div className="text-sm text-[color:var(--danger)]">{error}</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!loading && !error && (
|
||||||
|
<div className="h-full w-full p-4">
|
||||||
|
{children}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
76
components/charts/primitives/chart-toolbar.tsx
Normal file
76
components/charts/primitives/chart-toolbar.tsx
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
import { Download } from 'lucide-react';
|
||||||
|
import { cn } from '@/lib/utils';
|
||||||
|
import type { ChartType, TimeRange } from '@/lib/types';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
|
||||||
|
type ChartToolbarProps = {
|
||||||
|
chartType: ChartType;
|
||||||
|
timeRange: TimeRange;
|
||||||
|
onChartTypeChange: (type: ChartType) => void;
|
||||||
|
onTimeRangeChange: (range: TimeRange) => void;
|
||||||
|
onExport: () => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
const TIME_RANGES: TimeRange[] = ['1W', '1M', '3M', '1Y', '3Y', '5Y', '10Y', '20Y'];
|
||||||
|
const CHART_TYPES: { value: ChartType; label: string }[] = [
|
||||||
|
{ value: 'line', label: 'Line' },
|
||||||
|
{ value: 'combination', label: 'Compare' }
|
||||||
|
];
|
||||||
|
|
||||||
|
export function ChartToolbar({
|
||||||
|
chartType,
|
||||||
|
timeRange,
|
||||||
|
onChartTypeChange,
|
||||||
|
onTimeRangeChange,
|
||||||
|
onExport
|
||||||
|
}: ChartToolbarProps) {
|
||||||
|
return (
|
||||||
|
<div className="mb-4 flex flex-col gap-3 lg:flex-row lg:items-center lg:justify-between">
|
||||||
|
<div className="flex flex-wrap gap-1.5">
|
||||||
|
{TIME_RANGES.map((range) => (
|
||||||
|
<button
|
||||||
|
key={range}
|
||||||
|
type="button"
|
||||||
|
onClick={() => onTimeRangeChange(range)}
|
||||||
|
className={cn(
|
||||||
|
'min-h-9 rounded-lg px-3 py-1.5 text-xs font-medium transition-colors',
|
||||||
|
timeRange === range
|
||||||
|
? 'bg-[color:var(--accent)] text-[#16181c]'
|
||||||
|
: 'bg-[color:var(--panel-soft)] text-[color:var(--terminal-muted)] hover:bg-[color:var(--panel-bright)] hover:text-[color:var(--terminal-bright)]'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{range}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<div className="flex gap-1">
|
||||||
|
{CHART_TYPES.map((type) => (
|
||||||
|
<button
|
||||||
|
key={type.value}
|
||||||
|
type="button"
|
||||||
|
onClick={() => onChartTypeChange(type.value)}
|
||||||
|
className={cn(
|
||||||
|
'rounded-lg px-3 py-1.5 text-xs font-medium transition-colors',
|
||||||
|
chartType === type.value
|
||||||
|
? 'bg-[color:var(--accent)] text-[#16181c]'
|
||||||
|
: 'bg-[color:var(--panel-soft)] text-[color:var(--terminal-muted)] hover:bg-[color:var(--panel-bright)] hover:text-[color:var(--terminal-bright)]'
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{type.label}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
onClick={onExport}
|
||||||
|
className="min-h-9 gap-1.5 px-2.5 py-1.5 text-xs"
|
||||||
|
>
|
||||||
|
<Download className="h-3.5 w-3.5" />
|
||||||
|
Export
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
79
components/charts/primitives/chart-tooltip.tsx
Normal file
79
components/charts/primitives/chart-tooltip.tsx
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
import type { TooltipContentProps, TooltipPayloadEntry } from 'recharts';
|
||||||
|
import { formatCurrency, formatCompactCurrency } from '@/lib/format';
|
||||||
|
import { getChartColors } from '../utils/chart-colors';
|
||||||
|
import { isOHLCVData } from '../utils/chart-data-transformers';
|
||||||
|
|
||||||
|
type ChartTooltipProps = TooltipContentProps & {
|
||||||
|
formatters?: {
|
||||||
|
price?: (value: number) => string;
|
||||||
|
date?: (value: string) => string;
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function ChartTooltip(props: ChartTooltipProps) {
|
||||||
|
const { active, payload, label, formatters } = props;
|
||||||
|
|
||||||
|
if (!active || !payload || payload.length === 0) return null;
|
||||||
|
|
||||||
|
const colors = getChartColors();
|
||||||
|
const formatDate = formatters?.date || ((date: string) => new Date(date).toLocaleDateString());
|
||||||
|
const formatPrice = formatters?.price || formatCurrency;
|
||||||
|
const formatVolume = formatters?.volume || formatCompactCurrency;
|
||||||
|
|
||||||
|
const data = payload[0].payload;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="min-w-[180px] rounded-xl border p-3 backdrop-blur-sm"
|
||||||
|
style={{
|
||||||
|
backgroundColor: colors.tooltipBg,
|
||||||
|
borderColor: colors.tooltipBorder
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div className="mb-2 text-xs uppercase tracking-wider" style={{ color: colors.muted }}>
|
||||||
|
{formatDate(label as string)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{isOHLCVData(data) ? (
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
<TooltipRow label="Open" value={formatPrice(data.open)} color={colors.text} />
|
||||||
|
<TooltipRow label="High" value={formatPrice(data.high)} color={colors.positive} />
|
||||||
|
<TooltipRow label="Low" value={formatPrice(data.low)} color={colors.negative} />
|
||||||
|
<TooltipRow label="Close" value={formatPrice(data.close)} color={colors.text} />
|
||||||
|
<div className="mt-2 border-t pt-2" style={{ borderColor: colors.tooltipBorder }}>
|
||||||
|
<TooltipRow label="Volume" value={formatVolume(data.volume)} color={colors.muted} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
{payload.map((entry: TooltipPayloadEntry, index: number) => (
|
||||||
|
<TooltipRow
|
||||||
|
key={index}
|
||||||
|
label={String(entry.name || 'Value')}
|
||||||
|
value={formatPrice(entry.value as number)}
|
||||||
|
color={entry.color || colors.text}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
type TooltipRowProps = {
|
||||||
|
label: string;
|
||||||
|
value: string;
|
||||||
|
color: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function TooltipRow({ label, value, color }: TooltipRowProps) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center justify-between gap-4 text-xs">
|
||||||
|
<span style={{ color: color }}>{label}:</span>
|
||||||
|
<span className="font-mono font-medium" style={{ color: color }}>
|
||||||
|
{value}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
58
components/charts/primitives/volume-indicator.tsx
Normal file
58
components/charts/primitives/volume-indicator.tsx
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
import { BarChart, Bar, YAxis, Tooltip, ResponsiveContainer } from 'recharts';
|
||||||
|
import type { OHLCVDataPoint } from '@/lib/types';
|
||||||
|
import { getChartColors } from '../utils/chart-colors';
|
||||||
|
import { formatCompactCurrency } from '@/lib/format';
|
||||||
|
|
||||||
|
type VolumeIndicatorProps = {
|
||||||
|
data: OHLCVDataPoint[];
|
||||||
|
height?: number;
|
||||||
|
formatters?: {
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function VolumeIndicator({
|
||||||
|
data,
|
||||||
|
height = 80,
|
||||||
|
formatters
|
||||||
|
}: VolumeIndicatorProps) {
|
||||||
|
const colors = getChartColors();
|
||||||
|
if (data.length === 0) return null;
|
||||||
|
|
||||||
|
const formatVolume = formatters?.volume || formatCompactCurrency;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ height: `${height}px`, width: '100%' }} className="mt-2">
|
||||||
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
<BarChart data={data}>
|
||||||
|
<YAxis
|
||||||
|
orientation="right"
|
||||||
|
tickFormatter={formatVolume}
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={10}
|
||||||
|
width={60}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
formatter={(value) => [
|
||||||
|
formatVolume(typeof value === 'number' ? value : Number(value ?? 0)),
|
||||||
|
'Volume'
|
||||||
|
]}
|
||||||
|
labelFormatter={(label) => `Date: ${label}`}
|
||||||
|
contentStyle={{
|
||||||
|
backgroundColor: colors.tooltipBg,
|
||||||
|
border: `1px solid ${colors.tooltipBorder}`,
|
||||||
|
borderRadius: '0.75rem',
|
||||||
|
fontSize: '0.75rem'
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<Bar
|
||||||
|
dataKey="volume"
|
||||||
|
fill={colors.volume}
|
||||||
|
opacity={0.3}
|
||||||
|
isAnimationActive={data.length <= 500}
|
||||||
|
/>
|
||||||
|
</BarChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
71
components/charts/renderers/candlestick-chart-view.tsx
Normal file
71
components/charts/renderers/candlestick-chart-view.tsx
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
import {
|
||||||
|
ComposedChart,
|
||||||
|
XAxis,
|
||||||
|
YAxis,
|
||||||
|
CartesianGrid,
|
||||||
|
Tooltip,
|
||||||
|
ResponsiveContainer,
|
||||||
|
Scatter
|
||||||
|
} from 'recharts';
|
||||||
|
import type { ChartDataPoint } from '@/lib/types';
|
||||||
|
import { getChartColors } from '../utils/chart-colors';
|
||||||
|
import { ChartTooltip } from '../primitives/chart-tooltip';
|
||||||
|
import { CandlestickShape } from '../utils/candlestick-shapes';
|
||||||
|
import { isOHLCVData } from '../utils/chart-data-transformers';
|
||||||
|
|
||||||
|
type CandlestickChartViewProps = {
|
||||||
|
data: ChartDataPoint[];
|
||||||
|
formatters?: {
|
||||||
|
price?: (value: number) => string;
|
||||||
|
date?: (value: string) => string;
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function CandlestickChartView({
|
||||||
|
data,
|
||||||
|
formatters
|
||||||
|
}: CandlestickChartViewProps) {
|
||||||
|
const colors = getChartColors();
|
||||||
|
|
||||||
|
const ohlcvData = data.filter(isOHLCVData);
|
||||||
|
|
||||||
|
if (ohlcvData.length === 0) {
|
||||||
|
return (
|
||||||
|
<div className="flex h-full items-center justify-center text-sm text-[color:var(--terminal-muted)]">
|
||||||
|
Candlestick chart requires OHLCV data
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
<ComposedChart data={ohlcvData}>
|
||||||
|
<CartesianGrid strokeDasharray="2 2" stroke={colors.grid} />
|
||||||
|
<XAxis
|
||||||
|
dataKey="date"
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={formatters?.date}
|
||||||
|
minTickGap={32}
|
||||||
|
/>
|
||||||
|
<YAxis
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={formatters?.price}
|
||||||
|
width={60}
|
||||||
|
domain={['auto', 'auto']}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
content={(tooltipProps) => <ChartTooltip {...tooltipProps} formatters={formatters} />}
|
||||||
|
cursor={{ stroke: colors.muted, strokeDasharray: '3 3' }}
|
||||||
|
/>
|
||||||
|
<Scatter
|
||||||
|
dataKey="close"
|
||||||
|
shape={<CandlestickShape />}
|
||||||
|
isAnimationActive={false}
|
||||||
|
/>
|
||||||
|
</ComposedChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
);
|
||||||
|
}
|
||||||
112
components/charts/renderers/combination-chart-view.tsx
Normal file
112
components/charts/renderers/combination-chart-view.tsx
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
import {
|
||||||
|
ComposedChart,
|
||||||
|
Line,
|
||||||
|
XAxis,
|
||||||
|
YAxis,
|
||||||
|
CartesianGrid,
|
||||||
|
Tooltip,
|
||||||
|
ResponsiveContainer,
|
||||||
|
Legend
|
||||||
|
} from 'recharts';
|
||||||
|
import type { DataSeries } from '@/lib/types';
|
||||||
|
import { getChartColors } from '../utils/chart-colors';
|
||||||
|
import { ChartTooltip } from '../primitives/chart-tooltip';
|
||||||
|
import { mergeDataSeries } from '../utils/chart-data-transformers';
|
||||||
|
|
||||||
|
type CombinationChartViewProps = {
|
||||||
|
dataSeries: DataSeries[];
|
||||||
|
formatters?: {
|
||||||
|
price?: (value: number) => string;
|
||||||
|
date?: (value: string) => string;
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function CombinationChartView({
|
||||||
|
dataSeries,
|
||||||
|
formatters
|
||||||
|
}: CombinationChartViewProps) {
|
||||||
|
const colors = getChartColors();
|
||||||
|
|
||||||
|
if (!dataSeries || dataSeries.length === 0) {
|
||||||
|
return (
|
||||||
|
<div className="flex h-full items-center justify-center text-sm text-[color:var(--terminal-muted)]">
|
||||||
|
No data series provided
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergedData = mergeDataSeries(dataSeries);
|
||||||
|
const visibleSeries = dataSeries.filter(series => series.visible !== false);
|
||||||
|
const baseValues = Object.fromEntries(visibleSeries.map((series) => {
|
||||||
|
const initialPoint = mergedData.find((entry) => typeof entry[series.id] === 'number');
|
||||||
|
const baseValue = typeof initialPoint?.[series.id] === 'number' ? Number(initialPoint[series.id]) : null;
|
||||||
|
return [series.id, baseValue];
|
||||||
|
}));
|
||||||
|
const normalizedData = mergedData.map((point) => {
|
||||||
|
const normalizedPoint: Record<string, string | number | null> = { date: point.date };
|
||||||
|
|
||||||
|
visibleSeries.forEach((series) => {
|
||||||
|
const baseValue = baseValues[series.id];
|
||||||
|
const currentValue = typeof point[series.id] === 'number' ? Number(point[series.id]) : null;
|
||||||
|
|
||||||
|
normalizedPoint[series.id] = baseValue && currentValue
|
||||||
|
? ((currentValue / baseValue) - 1) * 100
|
||||||
|
: null;
|
||||||
|
});
|
||||||
|
|
||||||
|
return normalizedPoint;
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
<ComposedChart data={normalizedData}>
|
||||||
|
<CartesianGrid strokeDasharray="2 2" stroke={colors.grid} />
|
||||||
|
<XAxis
|
||||||
|
dataKey="date"
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={formatters?.date}
|
||||||
|
minTickGap={32}
|
||||||
|
/>
|
||||||
|
<YAxis
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={(value) => `${Number(value).toFixed(0)}%`}
|
||||||
|
width={60}
|
||||||
|
domain={['auto', 'auto']}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
content={(tooltipProps) => (
|
||||||
|
<ChartTooltip
|
||||||
|
{...tooltipProps}
|
||||||
|
formatters={{
|
||||||
|
...formatters,
|
||||||
|
price: (value: number) => `${value.toFixed(2)}%`
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
cursor={{ stroke: colors.muted, strokeDasharray: '3 3' }}
|
||||||
|
/>
|
||||||
|
<Legend />
|
||||||
|
|
||||||
|
{visibleSeries.map(series => {
|
||||||
|
const seriesColor = series.color || colors.primary;
|
||||||
|
return (
|
||||||
|
<Line
|
||||||
|
key={series.id}
|
||||||
|
type="monotone"
|
||||||
|
dataKey={series.id}
|
||||||
|
name={series.label}
|
||||||
|
stroke={seriesColor}
|
||||||
|
strokeWidth={2}
|
||||||
|
dot={false}
|
||||||
|
connectNulls={false}
|
||||||
|
isAnimationActive={normalizedData.length <= 500}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</ComposedChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
);
|
||||||
|
}
|
||||||
100
components/charts/renderers/line-chart-view.tsx
Normal file
100
components/charts/renderers/line-chart-view.tsx
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
import {
|
||||||
|
LineChart,
|
||||||
|
Line,
|
||||||
|
XAxis,
|
||||||
|
YAxis,
|
||||||
|
CartesianGrid,
|
||||||
|
Tooltip,
|
||||||
|
ResponsiveContainer
|
||||||
|
} from 'recharts';
|
||||||
|
import type { ChartDataPoint } from '@/lib/types';
|
||||||
|
import { getChartColors } from '../utils/chart-colors';
|
||||||
|
import { ChartTooltip } from '../primitives/chart-tooltip';
|
||||||
|
import { isPriceData, isOHLCVData } from '../utils/chart-data-transformers';
|
||||||
|
|
||||||
|
type LineChartViewProps = {
|
||||||
|
data: ChartDataPoint[];
|
||||||
|
formatters?: {
|
||||||
|
price?: (value: number) => string;
|
||||||
|
date?: (value: string) => string;
|
||||||
|
volume?: (value: number) => string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export function LineChartView({
|
||||||
|
data,
|
||||||
|
formatters
|
||||||
|
}: LineChartViewProps) {
|
||||||
|
const colors = getChartColors();
|
||||||
|
|
||||||
|
const chartData = data.map(point => {
|
||||||
|
if (isOHLCVData(point)) {
|
||||||
|
return {
|
||||||
|
date: point.date,
|
||||||
|
price: point.close,
|
||||||
|
open: point.open,
|
||||||
|
high: point.high,
|
||||||
|
low: point.low,
|
||||||
|
close: point.close,
|
||||||
|
volume: point.volume
|
||||||
|
};
|
||||||
|
} else if (isPriceData(point)) {
|
||||||
|
return {
|
||||||
|
date: point.date,
|
||||||
|
price: point.price
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return point;
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
<LineChart data={chartData} margin={{ top: 5, right: 5, left: 5, bottom: 5 }}>
|
||||||
|
<CartesianGrid
|
||||||
|
strokeDasharray="3 3"
|
||||||
|
stroke={colors.grid}
|
||||||
|
vertical={false}
|
||||||
|
/>
|
||||||
|
<XAxis
|
||||||
|
dataKey="date"
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={formatters?.date}
|
||||||
|
minTickGap={50}
|
||||||
|
axisLine={{ stroke: colors.grid }}
|
||||||
|
tickLine={false}
|
||||||
|
/>
|
||||||
|
<YAxis
|
||||||
|
stroke={colors.muted}
|
||||||
|
fontSize={11}
|
||||||
|
tickFormatter={formatters?.price}
|
||||||
|
width={65}
|
||||||
|
domain={['auto', 'auto']}
|
||||||
|
axisLine={false}
|
||||||
|
tickLine={false}
|
||||||
|
orientation="right"
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
content={(tooltipProps) => <ChartTooltip {...tooltipProps} formatters={formatters} />}
|
||||||
|
cursor={{ stroke: colors.muted, strokeWidth: 1, strokeDasharray: '5 5' }}
|
||||||
|
isAnimationActive={false}
|
||||||
|
/>
|
||||||
|
<Line
|
||||||
|
type="linear"
|
||||||
|
dataKey="price"
|
||||||
|
stroke={colors.primary}
|
||||||
|
strokeWidth={2}
|
||||||
|
dot={false}
|
||||||
|
activeDot={{
|
||||||
|
r: 4,
|
||||||
|
stroke: colors.primary,
|
||||||
|
strokeWidth: 2,
|
||||||
|
fill: colors.tooltipBg
|
||||||
|
}}
|
||||||
|
isAnimationActive={false}
|
||||||
|
connectNulls={true}
|
||||||
|
/>
|
||||||
|
</LineChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
);
|
||||||
|
}
|
||||||
68
components/charts/utils/candlestick-shapes.tsx
Normal file
68
components/charts/utils/candlestick-shapes.tsx
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
import { getPriceChangeColor } from './chart-colors';
|
||||||
|
|
||||||
|
type CandlestickShapeProps = {
|
||||||
|
cx?: number;
|
||||||
|
payload?: {
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Custom candlestick shape component for Recharts
|
||||||
|
* Renders candlestick with wick and body
|
||||||
|
*/
|
||||||
|
export function CandlestickShape(props: CandlestickShapeProps) {
|
||||||
|
const { cx, payload } = props;
|
||||||
|
|
||||||
|
if (!payload || !cx) return null;
|
||||||
|
|
||||||
|
const { open, high, low, close } = payload;
|
||||||
|
const isPositive = close >= open;
|
||||||
|
const color = getPriceChangeColor(close - open);
|
||||||
|
|
||||||
|
// Calculate positions
|
||||||
|
const bodyTop = Math.min(open, close);
|
||||||
|
const bodyBottom = Math.max(open, close);
|
||||||
|
const bodyHeight = Math.max(bodyBottom - bodyTop, 1);
|
||||||
|
|
||||||
|
// Candlestick width
|
||||||
|
const width = 8;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<g>
|
||||||
|
{/* Upper wick */}
|
||||||
|
<line
|
||||||
|
x1={cx}
|
||||||
|
y1={high}
|
||||||
|
x2={cx}
|
||||||
|
y2={bodyTop}
|
||||||
|
stroke={color}
|
||||||
|
strokeWidth={1}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Body */}
|
||||||
|
<rect
|
||||||
|
x={cx - width / 2}
|
||||||
|
y={bodyTop}
|
||||||
|
width={width}
|
||||||
|
height={bodyHeight}
|
||||||
|
fill={isPositive ? 'transparent' : color}
|
||||||
|
stroke={color}
|
||||||
|
strokeWidth={1}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Lower wick */}
|
||||||
|
<line
|
||||||
|
x1={cx}
|
||||||
|
y1={bodyBottom}
|
||||||
|
x2={cx}
|
||||||
|
y2={low}
|
||||||
|
stroke={color}
|
||||||
|
strokeWidth={1}
|
||||||
|
/>
|
||||||
|
</g>
|
||||||
|
);
|
||||||
|
}
|
||||||
65
components/charts/utils/chart-colors.ts
Normal file
65
components/charts/utils/chart-colors.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
import type { ChartColorPalette } from '@/lib/types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get chart color palette using CSS variables for theming
|
||||||
|
* These colors match the existing dark theme in the codebase
|
||||||
|
*/
|
||||||
|
export function getChartColors(): ChartColorPalette {
|
||||||
|
return {
|
||||||
|
primary: 'var(--accent)',
|
||||||
|
secondary: 'var(--terminal-muted)',
|
||||||
|
positive: '#96f5bf', // Green - matches existing price-history-card
|
||||||
|
negative: '#ff9f9f', // Red - matches existing price-history-card
|
||||||
|
grid: 'var(--line-weak)',
|
||||||
|
text: 'var(--terminal-bright)',
|
||||||
|
muted: 'var(--terminal-muted)',
|
||||||
|
tooltipBg: 'rgba(31, 34, 39, 0.96)',
|
||||||
|
tooltipBorder: 'var(--line-strong)',
|
||||||
|
volume: 'var(--terminal-muted)'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get color for price change (positive/negative)
|
||||||
|
*/
|
||||||
|
export function getPriceChangeColor(change: number): string {
|
||||||
|
const colors = getChartColors();
|
||||||
|
return change >= 0 ? colors.positive : colors.negative;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert CSS variable to computed color value
|
||||||
|
* Used for chart export since html-to-image can't render CSS variables
|
||||||
|
*/
|
||||||
|
export function cssVarToColor(cssVar: string): string {
|
||||||
|
if (typeof window === 'undefined') return cssVar;
|
||||||
|
|
||||||
|
// If it's already a color value, return as-is
|
||||||
|
if (!cssVar.startsWith('var(')) return cssVar;
|
||||||
|
|
||||||
|
// Extract variable name from var(--name)
|
||||||
|
const varName = cssVar.match(/var\((--[^)]+)\)/)?.[1];
|
||||||
|
if (!varName) return cssVar;
|
||||||
|
|
||||||
|
// Get computed color from CSS variable
|
||||||
|
const computedColor = getComputedStyle(document.documentElement)
|
||||||
|
.getPropertyValue(varName)
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
return computedColor || cssVar;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert entire color palette to computed colors for export
|
||||||
|
*/
|
||||||
|
export function getComputedColors(palette: Partial<ChartColorPalette>): Partial<ChartColorPalette> {
|
||||||
|
const computed: Partial<ChartColorPalette> = {};
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(palette)) {
|
||||||
|
if (value) {
|
||||||
|
computed[key as keyof ChartColorPalette] = cssVarToColor(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return computed;
|
||||||
|
}
|
||||||
157
components/charts/utils/chart-data-transformers.ts
Normal file
157
components/charts/utils/chart-data-transformers.ts
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
import { subWeeks, subMonths, subYears } from 'date-fns';
|
||||||
|
import type { TimeRange, ChartDataPoint } from '@/lib/types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Filter chart data by time range
|
||||||
|
*/
|
||||||
|
export function filterByTimeRange<T extends ChartDataPoint>(
|
||||||
|
data: T[],
|
||||||
|
range: TimeRange
|
||||||
|
): T[] {
|
||||||
|
if (data.length === 0) return data;
|
||||||
|
|
||||||
|
const now = new Date();
|
||||||
|
const cutoffDate = {
|
||||||
|
'1W': subWeeks(now, 1),
|
||||||
|
'1M': subMonths(now, 1),
|
||||||
|
'3M': subMonths(now, 3),
|
||||||
|
'1Y': subYears(now, 1),
|
||||||
|
'3Y': subYears(now, 3),
|
||||||
|
'5Y': subYears(now, 5),
|
||||||
|
'10Y': subYears(now, 10),
|
||||||
|
'20Y': subYears(now, 20)
|
||||||
|
}[range];
|
||||||
|
|
||||||
|
return data.filter(point => new Date(point.date) >= cutoffDate);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if data point has OHLCV fields
|
||||||
|
*/
|
||||||
|
export function isOHLCVData(data: ChartDataPoint): data is {
|
||||||
|
date: string;
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
} {
|
||||||
|
return 'open' in data && 'high' in data && 'low' in data && 'close' in data && 'volume' in data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if data point has simple price field
|
||||||
|
*/
|
||||||
|
export function isPriceData(data: ChartDataPoint): data is { date: string; price: number } {
|
||||||
|
return 'price' in data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize data to ensure consistent structure
|
||||||
|
* Converts price data to OHLCV-like structure for candlestick charts
|
||||||
|
*/
|
||||||
|
export function normalizeChartData<T extends ChartDataPoint>(data: T[]): T[] {
|
||||||
|
if (!data || data.length === 0) return [];
|
||||||
|
|
||||||
|
// Sort by date ascending
|
||||||
|
return [...data].sort((a, b) =>
|
||||||
|
new Date(a.date).getTime() - new Date(b.date).getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample data for performance with large datasets
|
||||||
|
* Keeps every Nth point when dataset is too large
|
||||||
|
*/
|
||||||
|
export function sampleData<T extends ChartDataPoint>(
|
||||||
|
data: T[],
|
||||||
|
maxPoints: number = 1000
|
||||||
|
): T[] {
|
||||||
|
if (data.length <= maxPoints) return data;
|
||||||
|
|
||||||
|
const samplingRate = Math.ceil(data.length / maxPoints);
|
||||||
|
const sampled: T[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < data.length; i += samplingRate) {
|
||||||
|
sampled.push(data[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Always include the last point
|
||||||
|
if (sampled[sampled.length - 1] !== data[data.length - 1]) {
|
||||||
|
sampled.push(data[data.length - 1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sampled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate min/max values for Y-axis domain
|
||||||
|
*/
|
||||||
|
export function calculateYAxisDomain<T extends ChartDataPoint>(
|
||||||
|
data: T[],
|
||||||
|
padding: number = 0.1
|
||||||
|
): [number, number] {
|
||||||
|
if (data.length === 0) return [0, 100];
|
||||||
|
|
||||||
|
let min = Infinity;
|
||||||
|
let max = -Infinity;
|
||||||
|
|
||||||
|
data.forEach(point => {
|
||||||
|
if (isOHLCVData(point)) {
|
||||||
|
min = Math.min(min, point.low);
|
||||||
|
max = Math.max(max, point.high);
|
||||||
|
} else if (isPriceData(point)) {
|
||||||
|
min = Math.min(min, point.price);
|
||||||
|
max = Math.max(max, point.price);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add padding
|
||||||
|
const range = max - min;
|
||||||
|
const paddedMin = min - (range * padding);
|
||||||
|
const paddedMax = max + (range * padding);
|
||||||
|
|
||||||
|
return [paddedMin, paddedMax];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate volume max for volume indicator Y-axis
|
||||||
|
*/
|
||||||
|
export function calculateVolumeMax<T extends ChartDataPoint>(data: T[]): number {
|
||||||
|
if (data.length === 0 || !isOHLCVData(data[0])) return 0;
|
||||||
|
|
||||||
|
return Math.max(...data.map(d => (isOHLCVData(d) ? d.volume : 0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge multiple data series by date for combination charts
|
||||||
|
*/
|
||||||
|
export function mergeDataSeries<T extends ChartDataPoint>(
|
||||||
|
seriesArray: Array<{ id: string; data: T[] }>
|
||||||
|
): Array<T & Record<string, number>> {
|
||||||
|
if (!seriesArray || seriesArray.length === 0) return [];
|
||||||
|
|
||||||
|
// Create map indexed by date
|
||||||
|
const dateMap = new Map<string, T & Record<string, number>>();
|
||||||
|
|
||||||
|
seriesArray.forEach(series => {
|
||||||
|
series.data.forEach(point => {
|
||||||
|
const date = point.date;
|
||||||
|
const existing = dateMap.get(date) || { date } as T & Record<string, number>;
|
||||||
|
|
||||||
|
// Add value for this series
|
||||||
|
if (isOHLCVData(point)) {
|
||||||
|
existing[series.id] = point.close;
|
||||||
|
} else if (isPriceData(point)) {
|
||||||
|
existing[series.id] = point.price;
|
||||||
|
}
|
||||||
|
|
||||||
|
dateMap.set(date, existing);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Convert to array and sort by date
|
||||||
|
return Array.from(dateMap.values()).sort((a, b) =>
|
||||||
|
new Date(a.date).getTime() - new Date(b.date).getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
180
lib/server/db/financial-ingestion-schema.test.ts
Normal file
180
lib/server/db/financial-ingestion-schema.test.ts
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
import { describe, expect, it } from 'bun:test';
|
||||||
|
import { Database } from 'bun:sqlite';
|
||||||
|
import {
|
||||||
|
ensureFinancialIngestionSchemaHealthy,
|
||||||
|
inspectFinancialIngestionSchema,
|
||||||
|
withFinancialIngestionSchemaRetry
|
||||||
|
} from './financial-ingestion-schema';
|
||||||
|
|
||||||
|
function createBundleTable(client: Database) {
|
||||||
|
client.exec(`
|
||||||
|
CREATE TABLE \`company_financial_bundle\` (
|
||||||
|
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
\`ticker\` text NOT NULL,
|
||||||
|
\`surface_kind\` text NOT NULL,
|
||||||
|
\`cadence\` text NOT NULL,
|
||||||
|
\`bundle_version\` integer NOT NULL,
|
||||||
|
\`source_snapshot_ids\` text NOT NULL,
|
||||||
|
\`source_signature\` text NOT NULL,
|
||||||
|
\`payload\` text NOT NULL,
|
||||||
|
\`created_at\` text NOT NULL,
|
||||||
|
\`updated_at\` text NOT NULL
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createSnapshotTable(client: Database) {
|
||||||
|
client.exec(`
|
||||||
|
CREATE TABLE \`filing_taxonomy_snapshot\` (
|
||||||
|
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
\`filing_id\` integer NOT NULL,
|
||||||
|
\`ticker\` text NOT NULL,
|
||||||
|
\`filing_date\` text NOT NULL,
|
||||||
|
\`filing_type\` text NOT NULL,
|
||||||
|
\`parse_status\` text NOT NULL,
|
||||||
|
\`updated_at\` text NOT NULL
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMetricValidationTable(client: Database) {
|
||||||
|
client.exec(`
|
||||||
|
CREATE TABLE \`filing_taxonomy_metric_validation\` (
|
||||||
|
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
\`snapshot_id\` integer NOT NULL,
|
||||||
|
\`metric_key\` text NOT NULL,
|
||||||
|
\`status\` text NOT NULL,
|
||||||
|
\`updated_at\` text NOT NULL
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHealthyIndexes(client: Database) {
|
||||||
|
client.exec('CREATE UNIQUE INDEX `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`);');
|
||||||
|
client.exec('CREATE INDEX `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`);');
|
||||||
|
client.exec('CREATE UNIQUE INDEX `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);');
|
||||||
|
client.exec('CREATE INDEX `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);');
|
||||||
|
client.exec('CREATE INDEX `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);');
|
||||||
|
client.exec('CREATE UNIQUE INDEX `filing_taxonomy_metric_validation_uidx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`metric_key`);');
|
||||||
|
client.exec('CREATE INDEX `filing_taxonomy_metric_validation_snapshot_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`);');
|
||||||
|
client.exec('CREATE INDEX `filing_taxonomy_metric_validation_status_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`status`);');
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('financial ingestion schema repair', () => {
|
||||||
|
it('reports a healthy schema when all critical indexes exist', () => {
|
||||||
|
const client = new Database(':memory:');
|
||||||
|
try {
|
||||||
|
createBundleTable(client);
|
||||||
|
createSnapshotTable(client);
|
||||||
|
createMetricValidationTable(client);
|
||||||
|
createHealthyIndexes(client);
|
||||||
|
|
||||||
|
const report = inspectFinancialIngestionSchema(client);
|
||||||
|
expect(report.ok).toBe(true);
|
||||||
|
expect(report.missingIndexes).toEqual([]);
|
||||||
|
expect(report.duplicateGroups).toBe(0);
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('repairs missing company financial bundle indexes and dedupes rows', () => {
|
||||||
|
const client = new Database(':memory:');
|
||||||
|
try {
|
||||||
|
createBundleTable(client);
|
||||||
|
createSnapshotTable(client);
|
||||||
|
createMetricValidationTable(client);
|
||||||
|
createHealthyIndexes(client);
|
||||||
|
client.exec('DROP INDEX `company_financial_bundle_uidx`;');
|
||||||
|
client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;');
|
||||||
|
client.exec(`
|
||||||
|
INSERT INTO \`company_financial_bundle\` (
|
||||||
|
\`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\`
|
||||||
|
) VALUES
|
||||||
|
('MSFT', 'income_statement', 'annual', 14, '[]', 'old', '{}', '2026-03-12T10:00:00.000Z', '2026-03-12T10:00:00.000Z'),
|
||||||
|
('MSFT', 'income_statement', 'annual', 14, '[]', 'new', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z');
|
||||||
|
`);
|
||||||
|
|
||||||
|
const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
|
||||||
|
const rows = client.query('SELECT `source_signature` FROM `company_financial_bundle`').all() as Array<{ source_signature: string }>;
|
||||||
|
const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>;
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(result.mode).toBe('repaired');
|
||||||
|
expect(rows).toEqual([{ source_signature: 'new' }]);
|
||||||
|
expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true);
|
||||||
|
expect(indexes.some((row) => row.name === 'company_financial_bundle_ticker_idx')).toBe(true);
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('dedupes filing taxonomy snapshots and clears bundle cache before recreating indexes', () => {
|
||||||
|
const client = new Database(':memory:');
|
||||||
|
try {
|
||||||
|
createBundleTable(client);
|
||||||
|
createSnapshotTable(client);
|
||||||
|
createMetricValidationTable(client);
|
||||||
|
createHealthyIndexes(client);
|
||||||
|
client.exec('DROP INDEX `filing_taxonomy_snapshot_filing_uidx`;');
|
||||||
|
client.exec(`
|
||||||
|
INSERT INTO \`filing_taxonomy_snapshot\` (
|
||||||
|
\`filing_id\`, \`ticker\`, \`filing_date\`, \`filing_type\`, \`parse_status\`, \`updated_at\`
|
||||||
|
) VALUES
|
||||||
|
(10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T10:00:00.000Z'),
|
||||||
|
(10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T11:00:00.000Z');
|
||||||
|
`);
|
||||||
|
client.exec(`
|
||||||
|
INSERT INTO \`company_financial_bundle\` (
|
||||||
|
\`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\`
|
||||||
|
) VALUES
|
||||||
|
('MSFT', 'income_statement', 'annual', 14, '[1,2]', 'cached', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z');
|
||||||
|
`);
|
||||||
|
|
||||||
|
const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
|
||||||
|
const snapshotCount = client.query('SELECT COUNT(*) AS count FROM `filing_taxonomy_snapshot`').get() as { count: number };
|
||||||
|
const bundleCount = client.query('SELECT COUNT(*) AS count FROM `company_financial_bundle`').get() as { count: number };
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(result.repair?.snapshotDuplicateRowsDeleted).toBe(1);
|
||||||
|
expect(result.repair?.bundleCacheCleared).toBe(true);
|
||||||
|
expect(snapshotCount.count).toBe(1);
|
||||||
|
expect(bundleCount.count).toBe(0);
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retries once after repairing the schema for ON CONFLICT drift errors', async () => {
|
||||||
|
const client = new Database(':memory:');
|
||||||
|
try {
|
||||||
|
createBundleTable(client);
|
||||||
|
createSnapshotTable(client);
|
||||||
|
createMetricValidationTable(client);
|
||||||
|
createHealthyIndexes(client);
|
||||||
|
client.exec('DROP INDEX `company_financial_bundle_uidx`;');
|
||||||
|
client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;');
|
||||||
|
|
||||||
|
let calls = 0;
|
||||||
|
const result = await withFinancialIngestionSchemaRetry({
|
||||||
|
client,
|
||||||
|
context: 'test-retry',
|
||||||
|
operation: async () => {
|
||||||
|
calls += 1;
|
||||||
|
if (calls === 1) {
|
||||||
|
throw new Error('ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint');
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'ok';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('ok');
|
||||||
|
expect(calls).toBe(2);
|
||||||
|
const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>;
|
||||||
|
expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true);
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
541
lib/server/db/financial-ingestion-schema.ts
Normal file
541
lib/server/db/financial-ingestion-schema.ts
Normal file
@@ -0,0 +1,541 @@
|
|||||||
|
import type { Database } from 'bun:sqlite';
|
||||||
|
|
||||||
|
export type FinancialSchemaRepairMode = 'auto' | 'check-only' | 'off';
|
||||||
|
export type FinancialIngestionHealthMode = 'healthy' | 'repaired' | 'drifted' | 'failed';
|
||||||
|
|
||||||
|
type CriticalIndexDefinition = {
|
||||||
|
name: string;
|
||||||
|
table: 'company_financial_bundle' | 'filing_taxonomy_snapshot' | 'filing_taxonomy_metric_validation';
|
||||||
|
unique: boolean;
|
||||||
|
columns: string[];
|
||||||
|
createSql: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type DuplicateRule = {
|
||||||
|
table: 'company_financial_bundle' | 'filing_taxonomy_snapshot' | 'filing_taxonomy_metric_validation';
|
||||||
|
partitionColumns: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FinancialIngestionIndexStatus = {
|
||||||
|
name: string;
|
||||||
|
table: string;
|
||||||
|
expectedColumns: string[];
|
||||||
|
actualColumns: string[];
|
||||||
|
unique: boolean;
|
||||||
|
present: boolean;
|
||||||
|
healthy: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FinancialIngestionDuplicateStatus = {
|
||||||
|
table: string;
|
||||||
|
duplicateGroups: number;
|
||||||
|
duplicateRows: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FinancialIngestionSchemaReport = {
|
||||||
|
ok: boolean;
|
||||||
|
checkedAt: string;
|
||||||
|
indexes: FinancialIngestionIndexStatus[];
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
duplicateRows: number;
|
||||||
|
duplicates: FinancialIngestionDuplicateStatus[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FinancialIngestionSchemaRepairResult = {
|
||||||
|
attempted: boolean;
|
||||||
|
requestedMode: FinancialSchemaRepairMode;
|
||||||
|
missingIndexesBefore: string[];
|
||||||
|
duplicateGroupsBefore: number;
|
||||||
|
indexesCreated: string[];
|
||||||
|
indexesRecreated: string[];
|
||||||
|
duplicateRowsDeleted: number;
|
||||||
|
duplicateGroupsResolved: number;
|
||||||
|
bundleCacheCleared: boolean;
|
||||||
|
snapshotDuplicateRowsDeleted: number;
|
||||||
|
reportAfter: FinancialIngestionSchemaReport;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FinancialIngestionSchemaEnsureResult = {
|
||||||
|
ok: boolean;
|
||||||
|
mode: FinancialIngestionHealthMode;
|
||||||
|
requestedMode: FinancialSchemaRepairMode;
|
||||||
|
missingIndexes: string[];
|
||||||
|
duplicateGroups: number;
|
||||||
|
lastCheckedAt: string;
|
||||||
|
repair: FinancialIngestionSchemaRepairResult | null;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
declare global {
|
||||||
|
// eslint-disable-next-line no-var
|
||||||
|
var __financialIngestionSchemaStatus: FinancialIngestionSchemaEnsureResult | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CRITICAL_INDEX_DEFINITIONS: CriticalIndexDefinition[] = [
|
||||||
|
{
|
||||||
|
table: 'company_financial_bundle',
|
||||||
|
name: 'company_financial_bundle_uidx',
|
||||||
|
unique: true,
|
||||||
|
columns: ['ticker', 'surface_kind', 'cadence'],
|
||||||
|
createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'company_financial_bundle',
|
||||||
|
name: 'company_financial_bundle_ticker_idx',
|
||||||
|
unique: false,
|
||||||
|
columns: ['ticker', 'updated_at'],
|
||||||
|
createSql: 'CREATE INDEX IF NOT EXISTS `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_snapshot',
|
||||||
|
name: 'filing_taxonomy_snapshot_filing_uidx',
|
||||||
|
unique: true,
|
||||||
|
columns: ['filing_id'],
|
||||||
|
createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_snapshot',
|
||||||
|
name: 'filing_taxonomy_snapshot_ticker_date_idx',
|
||||||
|
unique: false,
|
||||||
|
columns: ['ticker', 'filing_date'],
|
||||||
|
createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_snapshot',
|
||||||
|
name: 'filing_taxonomy_snapshot_status_idx',
|
||||||
|
unique: false,
|
||||||
|
columns: ['parse_status'],
|
||||||
|
createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_metric_validation',
|
||||||
|
name: 'filing_taxonomy_metric_validation_uidx',
|
||||||
|
unique: true,
|
||||||
|
columns: ['snapshot_id', 'metric_key'],
|
||||||
|
createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_uidx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`metric_key`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_metric_validation',
|
||||||
|
name: 'filing_taxonomy_metric_validation_snapshot_idx',
|
||||||
|
unique: false,
|
||||||
|
columns: ['snapshot_id'],
|
||||||
|
createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_snapshot_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`);'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_metric_validation',
|
||||||
|
name: 'filing_taxonomy_metric_validation_status_idx',
|
||||||
|
unique: false,
|
||||||
|
columns: ['snapshot_id', 'status'],
|
||||||
|
createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_status_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`status`);'
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
const UNIQUE_DUPLICATE_RULES: DuplicateRule[] = [
|
||||||
|
{
|
||||||
|
table: 'company_financial_bundle',
|
||||||
|
partitionColumns: ['ticker', 'surface_kind', 'cadence']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_snapshot',
|
||||||
|
partitionColumns: ['filing_id']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
table: 'filing_taxonomy_metric_validation',
|
||||||
|
partitionColumns: ['snapshot_id', 'metric_key']
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
function hasTable(client: Database, tableName: string) {
|
||||||
|
const row = client
|
||||||
|
.query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
|
||||||
|
.get('table', tableName) as { name: string } | null;
|
||||||
|
|
||||||
|
return row !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nowIso() {
|
||||||
|
return new Date().toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveFinancialSchemaRepairMode(value: string | undefined): FinancialSchemaRepairMode {
|
||||||
|
const normalized = value?.trim().toLowerCase();
|
||||||
|
if (normalized === 'check-only') {
|
||||||
|
return 'check-only';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized === 'off') {
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'auto';
|
||||||
|
}
|
||||||
|
|
||||||
|
function arrayEqual(left: string[], right: string[]) {
|
||||||
|
return left.length === right.length && left.every((value, index) => value === right[index]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function queryIndexColumns(client: Database, indexName: string) {
|
||||||
|
return (client.query(`PRAGMA index_info(\`${indexName}\`)`).all() as Array<{ seqno: number; name: string }>)
|
||||||
|
.sort((left, right) => left.seqno - right.seqno)
|
||||||
|
.map((row) => row.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inspectIndex(client: Database, definition: CriticalIndexDefinition): FinancialIngestionIndexStatus {
|
||||||
|
if (!hasTable(client, definition.table)) {
|
||||||
|
return {
|
||||||
|
name: definition.name,
|
||||||
|
table: definition.table,
|
||||||
|
expectedColumns: [...definition.columns],
|
||||||
|
actualColumns: [],
|
||||||
|
unique: definition.unique,
|
||||||
|
present: false,
|
||||||
|
healthy: false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const indexList = client.query(`PRAGMA index_list(\`${definition.table}\`)`).all() as Array<{
|
||||||
|
name: string;
|
||||||
|
unique: number;
|
||||||
|
}>;
|
||||||
|
const existing = indexList.find((row) => row.name === definition.name) ?? null;
|
||||||
|
const actualColumns = existing ? queryIndexColumns(client, definition.name) : [];
|
||||||
|
const isUnique = existing ? existing.unique === 1 : false;
|
||||||
|
const healthy = Boolean(existing) && isUnique === definition.unique && arrayEqual(actualColumns, definition.columns);
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: definition.name,
|
||||||
|
table: definition.table,
|
||||||
|
expectedColumns: [...definition.columns],
|
||||||
|
actualColumns,
|
||||||
|
unique: definition.unique,
|
||||||
|
present: existing !== null,
|
||||||
|
healthy
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function inspectDuplicates(client: Database, rule: DuplicateRule): FinancialIngestionDuplicateStatus {
|
||||||
|
if (!hasTable(client, rule.table)) {
|
||||||
|
return {
|
||||||
|
table: rule.table,
|
||||||
|
duplicateGroups: 0,
|
||||||
|
duplicateRows: 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const groupBy = rule.partitionColumns.map((column) => `\`${column}\``).join(', ');
|
||||||
|
const row = client.query(`
|
||||||
|
SELECT
|
||||||
|
COUNT(*) AS duplicate_groups,
|
||||||
|
COALESCE(SUM(cnt - 1), 0) AS duplicate_rows
|
||||||
|
FROM (
|
||||||
|
SELECT COUNT(*) AS cnt
|
||||||
|
FROM \`${rule.table}\`
|
||||||
|
GROUP BY ${groupBy}
|
||||||
|
HAVING COUNT(*) > 1
|
||||||
|
)
|
||||||
|
`).get() as { duplicate_groups: number | null; duplicate_rows: number | null } | null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
table: rule.table,
|
||||||
|
duplicateGroups: Number(row?.duplicate_groups ?? 0),
|
||||||
|
duplicateRows: Number(row?.duplicate_rows ?? 0)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inspectFinancialIngestionSchema(client: Database): FinancialIngestionSchemaReport {
|
||||||
|
const indexes = CRITICAL_INDEX_DEFINITIONS.map((definition) => inspectIndex(client, definition));
|
||||||
|
const duplicates = UNIQUE_DUPLICATE_RULES.map((rule) => inspectDuplicates(client, rule));
|
||||||
|
const missingIndexes = indexes.filter((index) => !index.healthy).map((index) => index.name);
|
||||||
|
|
||||||
|
return {
|
||||||
|
ok: missingIndexes.length === 0 && duplicates.every((entry) => entry.duplicateGroups === 0),
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
indexes,
|
||||||
|
missingIndexes,
|
||||||
|
duplicateGroups: duplicates.reduce((sum, entry) => sum + entry.duplicateGroups, 0),
|
||||||
|
duplicateRows: duplicates.reduce((sum, entry) => sum + entry.duplicateRows, 0),
|
||||||
|
duplicates
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTransaction<T>(client: Database, work: () => T) {
|
||||||
|
client.exec('BEGIN IMMEDIATE;');
|
||||||
|
try {
|
||||||
|
const result = work();
|
||||||
|
client.exec('COMMIT;');
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
client.exec('ROLLBACK;');
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function deleteDuplicateRows(client: Database, rule: DuplicateRule) {
|
||||||
|
if (!hasTable(client, rule.table)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const partitionBy = rule.partitionColumns.map((column) => `\`${column}\``).join(', ');
|
||||||
|
client.exec(`
|
||||||
|
DELETE FROM \`${rule.table}\`
|
||||||
|
WHERE \`id\` IN (
|
||||||
|
WITH ranked AS (
|
||||||
|
SELECT
|
||||||
|
\`id\`,
|
||||||
|
ROW_NUMBER() OVER (
|
||||||
|
PARTITION BY ${partitionBy}
|
||||||
|
ORDER BY \`updated_at\` DESC, \`id\` DESC
|
||||||
|
) AS rn
|
||||||
|
FROM \`${rule.table}\`
|
||||||
|
)
|
||||||
|
SELECT \`id\`
|
||||||
|
FROM ranked
|
||||||
|
WHERE rn > 1
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
const row = client.query('SELECT changes() AS count').get() as { count: number } | null;
|
||||||
|
return Number(row?.count ?? 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearBundleCache(client: Database) {
|
||||||
|
if (!hasTable(client, 'company_financial_bundle')) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
client.exec('DELETE FROM `company_financial_bundle`;');
|
||||||
|
const row = client.query('SELECT changes() AS count').get() as { count: number } | null;
|
||||||
|
return Number(row?.count ?? 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createOrRecreateIndex(client: Database, definition: CriticalIndexDefinition, status: FinancialIngestionIndexStatus) {
|
||||||
|
if (status.present && !status.healthy) {
|
||||||
|
client.exec(`DROP INDEX IF EXISTS \`${definition.name}\`;`);
|
||||||
|
}
|
||||||
|
|
||||||
|
client.exec(definition.createSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function repairFinancialIngestionSchema(
|
||||||
|
client: Database,
|
||||||
|
options: {
|
||||||
|
mode?: FinancialSchemaRepairMode;
|
||||||
|
} = {}
|
||||||
|
): FinancialIngestionSchemaRepairResult {
|
||||||
|
const requestedMode = options.mode ?? 'auto';
|
||||||
|
const before = inspectFinancialIngestionSchema(client);
|
||||||
|
|
||||||
|
if (requestedMode !== 'auto' || before.ok) {
|
||||||
|
return {
|
||||||
|
attempted: false,
|
||||||
|
requestedMode,
|
||||||
|
missingIndexesBefore: before.missingIndexes,
|
||||||
|
duplicateGroupsBefore: before.duplicateGroups,
|
||||||
|
indexesCreated: [],
|
||||||
|
indexesRecreated: [],
|
||||||
|
duplicateRowsDeleted: 0,
|
||||||
|
duplicateGroupsResolved: 0,
|
||||||
|
bundleCacheCleared: false,
|
||||||
|
snapshotDuplicateRowsDeleted: 0,
|
||||||
|
reportAfter: before
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let duplicateRowsDeleted = 0;
|
||||||
|
let duplicateGroupsResolved = 0;
|
||||||
|
let snapshotDuplicateRowsDeleted = 0;
|
||||||
|
let bundleCacheCleared = false;
|
||||||
|
const indexesCreated: string[] = [];
|
||||||
|
const indexesRecreated: string[] = [];
|
||||||
|
|
||||||
|
const snapshotStatusBefore = before.duplicates.find((entry) => entry.table === 'filing_taxonomy_snapshot');
|
||||||
|
const companyBundleStatusBefore = before.duplicates.find((entry) => entry.table === 'company_financial_bundle');
|
||||||
|
const metricValidationStatusBefore = before.duplicates.find((entry) => entry.table === 'filing_taxonomy_metric_validation');
|
||||||
|
|
||||||
|
if ((snapshotStatusBefore?.duplicateGroups ?? 0) > 0) {
|
||||||
|
snapshotDuplicateRowsDeleted = runTransaction(client, () => {
|
||||||
|
return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'filing_taxonomy_snapshot')!);
|
||||||
|
});
|
||||||
|
duplicateRowsDeleted += snapshotDuplicateRowsDeleted;
|
||||||
|
duplicateGroupsResolved += snapshotStatusBefore?.duplicateGroups ?? 0;
|
||||||
|
|
||||||
|
if (snapshotDuplicateRowsDeleted > 0) {
|
||||||
|
runTransaction(client, () => {
|
||||||
|
clearBundleCache(client);
|
||||||
|
});
|
||||||
|
bundleCacheCleared = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!bundleCacheCleared && (companyBundleStatusBefore?.duplicateGroups ?? 0) > 0) {
|
||||||
|
const deleted = runTransaction(client, () => {
|
||||||
|
return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'company_financial_bundle')!);
|
||||||
|
});
|
||||||
|
duplicateRowsDeleted += deleted;
|
||||||
|
duplicateGroupsResolved += companyBundleStatusBefore?.duplicateGroups ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((metricValidationStatusBefore?.duplicateGroups ?? 0) > 0) {
|
||||||
|
const deleted = runTransaction(client, () => {
|
||||||
|
return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'filing_taxonomy_metric_validation')!);
|
||||||
|
});
|
||||||
|
duplicateRowsDeleted += deleted;
|
||||||
|
duplicateGroupsResolved += metricValidationStatusBefore?.duplicateGroups ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const definition of CRITICAL_INDEX_DEFINITIONS) {
|
||||||
|
const status = before.indexes.find((entry) => entry.name === definition.name);
|
||||||
|
if (!status || status.healthy) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
runTransaction(client, () => {
|
||||||
|
createOrRecreateIndex(client, definition, status);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (status.present) {
|
||||||
|
indexesRecreated.push(definition.name);
|
||||||
|
} else {
|
||||||
|
indexesCreated.push(definition.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const reportAfter = inspectFinancialIngestionSchema(client);
|
||||||
|
|
||||||
|
return {
|
||||||
|
attempted: true,
|
||||||
|
requestedMode,
|
||||||
|
missingIndexesBefore: before.missingIndexes,
|
||||||
|
duplicateGroupsBefore: before.duplicateGroups,
|
||||||
|
indexesCreated,
|
||||||
|
indexesRecreated,
|
||||||
|
duplicateRowsDeleted,
|
||||||
|
duplicateGroupsResolved,
|
||||||
|
bundleCacheCleared,
|
||||||
|
snapshotDuplicateRowsDeleted,
|
||||||
|
reportAfter
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function cacheEnsureResult(result: FinancialIngestionSchemaEnsureResult) {
|
||||||
|
globalThis.__financialIngestionSchemaStatus = result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getLatestFinancialIngestionSchemaStatus() {
|
||||||
|
return globalThis.__financialIngestionSchemaStatus ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ensureFinancialIngestionSchemaHealthy(
|
||||||
|
client: Database,
|
||||||
|
options: {
|
||||||
|
mode?: FinancialSchemaRepairMode;
|
||||||
|
} = {}
|
||||||
|
): FinancialIngestionSchemaEnsureResult {
|
||||||
|
const requestedMode = options.mode ?? 'auto';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const before = inspectFinancialIngestionSchema(client);
|
||||||
|
if (before.ok) {
|
||||||
|
const result: FinancialIngestionSchemaEnsureResult = {
|
||||||
|
ok: true,
|
||||||
|
mode: 'healthy',
|
||||||
|
requestedMode,
|
||||||
|
missingIndexes: [],
|
||||||
|
duplicateGroups: 0,
|
||||||
|
lastCheckedAt: before.checkedAt,
|
||||||
|
repair: null,
|
||||||
|
error: null
|
||||||
|
};
|
||||||
|
cacheEnsureResult(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (requestedMode !== 'auto') {
|
||||||
|
const result: FinancialIngestionSchemaEnsureResult = {
|
||||||
|
ok: false,
|
||||||
|
mode: 'drifted',
|
||||||
|
requestedMode,
|
||||||
|
missingIndexes: before.missingIndexes,
|
||||||
|
duplicateGroups: before.duplicateGroups,
|
||||||
|
lastCheckedAt: before.checkedAt,
|
||||||
|
repair: null,
|
||||||
|
error: null
|
||||||
|
};
|
||||||
|
cacheEnsureResult(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const repair = repairFinancialIngestionSchema(client, { mode: requestedMode });
|
||||||
|
const finalReport = repair.reportAfter;
|
||||||
|
const result: FinancialIngestionSchemaEnsureResult = {
|
||||||
|
ok: finalReport.ok,
|
||||||
|
mode: finalReport.ok ? 'repaired' : 'failed',
|
||||||
|
requestedMode,
|
||||||
|
missingIndexes: finalReport.missingIndexes,
|
||||||
|
duplicateGroups: finalReport.duplicateGroups,
|
||||||
|
lastCheckedAt: finalReport.checkedAt,
|
||||||
|
repair,
|
||||||
|
error: finalReport.ok ? null : 'Financial ingestion schema remains drifted after automatic repair.'
|
||||||
|
};
|
||||||
|
cacheEnsureResult(result);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
const result: FinancialIngestionSchemaEnsureResult = {
|
||||||
|
ok: false,
|
||||||
|
mode: 'failed',
|
||||||
|
requestedMode,
|
||||||
|
missingIndexes: [],
|
||||||
|
duplicateGroups: 0,
|
||||||
|
lastCheckedAt: nowIso(),
|
||||||
|
repair: null,
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
};
|
||||||
|
cacheEnsureResult(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isMissingOnConflictConstraintError(error: unknown) {
|
||||||
|
return error instanceof Error
|
||||||
|
&& error.message.toLowerCase().includes('on conflict clause does not match any primary key or unique constraint');
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function withFinancialIngestionSchemaRetry<T>(input: {
|
||||||
|
client: Database;
|
||||||
|
operation: () => Promise<T>;
|
||||||
|
context: string;
|
||||||
|
}) {
|
||||||
|
try {
|
||||||
|
return await input.operation();
|
||||||
|
} catch (error) {
|
||||||
|
if (!isMissingOnConflictConstraintError(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ensureResult = ensureFinancialIngestionSchemaHealthy(input.client, {
|
||||||
|
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await input.operation();
|
||||||
|
} catch (retryError) {
|
||||||
|
const suffix = ensureResult.error
|
||||||
|
? ` repair_error=${ensureResult.error}`
|
||||||
|
: ` missing_indexes=${ensureResult.missingIndexes.join(',') || 'none'} duplicate_groups=${ensureResult.duplicateGroups}`;
|
||||||
|
const reason = retryError instanceof Error ? retryError.message : String(retryError);
|
||||||
|
throw new Error(`[${input.context}] ingestion schema retry failed:${suffix}; cause=${reason}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const __financialIngestionSchemaInternals = {
|
||||||
|
CRITICAL_INDEX_DEFINITIONS,
|
||||||
|
UNIQUE_DUPLICATE_RULES,
|
||||||
|
clearBundleCache,
|
||||||
|
deleteDuplicateRows,
|
||||||
|
hasTable,
|
||||||
|
inspectDuplicates,
|
||||||
|
inspectIndex,
|
||||||
|
runTransaction
|
||||||
|
};
|
||||||
68
scripts/repair-financial-ingestion-schema.ts
Normal file
68
scripts/repair-financial-ingestion-schema.ts
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
import { Database } from 'bun:sqlite';
|
||||||
|
import {
|
||||||
|
ensureFinancialIngestionSchemaHealthy,
|
||||||
|
inspectFinancialIngestionSchema,
|
||||||
|
resolveFinancialSchemaRepairMode
|
||||||
|
} from '../lib/server/db/financial-ingestion-schema';
|
||||||
|
import { resolveSqlitePath } from './dev-env';
|
||||||
|
|
||||||
|
function parseArgs(argv: string[]) {
|
||||||
|
const flags = new Set(argv);
|
||||||
|
|
||||||
|
return {
|
||||||
|
dryRun: flags.has('--dry-run'),
|
||||||
|
verbose: flags.has('--verbose'),
|
||||||
|
failOnDrift: flags.has('--fail-on-drift')
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDatabasePath() {
|
||||||
|
const raw = process.env.DATABASE_URL?.trim() || 'file:data/fiscal.sqlite';
|
||||||
|
const databasePath = resolveSqlitePath(raw);
|
||||||
|
|
||||||
|
if (!databasePath || databasePath.includes('://')) {
|
||||||
|
throw new Error(`DATABASE_URL must resolve to a SQLite file path. Received: ${raw}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return databasePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
const options = parseArgs(process.argv.slice(2));
|
||||||
|
const databasePath = resolveDatabasePath();
|
||||||
|
const client = new Database(databasePath, { create: true });
|
||||||
|
|
||||||
|
try {
|
||||||
|
client.exec('PRAGMA foreign_keys = ON;');
|
||||||
|
|
||||||
|
const mode = options.dryRun
|
||||||
|
? 'check-only'
|
||||||
|
: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE);
|
||||||
|
const before = inspectFinancialIngestionSchema(client);
|
||||||
|
const result = ensureFinancialIngestionSchemaHealthy(client, { mode });
|
||||||
|
|
||||||
|
console.info(`[repair-financial-ingestion-schema] db=${databasePath}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] mode=${mode}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] status=${result.mode}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] missing_indexes_before=${before.missingIndexes.join(',') || 'none'}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] duplicate_groups_before=${before.duplicateGroups}`);
|
||||||
|
|
||||||
|
if (result.repair) {
|
||||||
|
console.info(`[repair-financial-ingestion-schema] indexes_created=${result.repair.indexesCreated.join(',') || 'none'}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] indexes_recreated=${result.repair.indexesRecreated.join(',') || 'none'}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] duplicate_rows_deleted=${result.repair.duplicateRowsDeleted}`);
|
||||||
|
console.info(`[repair-financial-ingestion-schema] bundle_cache_cleared=${result.repair.bundleCacheCleared ? 'yes' : 'no'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.verbose) {
|
||||||
|
console.info(JSON.stringify({
|
||||||
|
before,
|
||||||
|
after: result
|
||||||
|
}, null, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.failOnDrift && !result.ok) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user