From 01199d489aaee48c88364b24371a4e83c0cdbeaf Mon Sep 17 00:00:00 2001 From: francy51 Date: Fri, 13 Mar 2026 00:11:59 -0400 Subject: [PATCH] Add untracked chart and schema files --- components/charts/hooks/use-chart-export.ts | 38 ++ components/charts/hooks/use-chart-zoom.ts | 44 ++ components/charts/interactive-price-chart.tsx | 92 +++ .../charts/primitives/chart-container.tsx | 45 ++ .../charts/primitives/chart-toolbar.tsx | 76 +++ .../charts/primitives/chart-tooltip.tsx | 79 +++ .../charts/primitives/volume-indicator.tsx | 58 ++ .../renderers/candlestick-chart-view.tsx | 71 +++ .../renderers/combination-chart-view.tsx | 112 ++++ .../charts/renderers/line-chart-view.tsx | 100 ++++ .../charts/utils/candlestick-shapes.tsx | 68 +++ components/charts/utils/chart-colors.ts | 65 +++ .../charts/utils/chart-data-transformers.ts | 157 +++++ .../db/financial-ingestion-schema.test.ts | 180 ++++++ lib/server/db/financial-ingestion-schema.ts | 541 ++++++++++++++++++ scripts/repair-financial-ingestion-schema.ts | 68 +++ 16 files changed, 1794 insertions(+) create mode 100644 components/charts/hooks/use-chart-export.ts create mode 100644 components/charts/hooks/use-chart-zoom.ts create mode 100644 components/charts/interactive-price-chart.tsx create mode 100644 components/charts/primitives/chart-container.tsx create mode 100644 components/charts/primitives/chart-toolbar.tsx create mode 100644 components/charts/primitives/chart-tooltip.tsx create mode 100644 components/charts/primitives/volume-indicator.tsx create mode 100644 components/charts/renderers/candlestick-chart-view.tsx create mode 100644 components/charts/renderers/combination-chart-view.tsx create mode 100644 components/charts/renderers/line-chart-view.tsx create mode 100644 components/charts/utils/candlestick-shapes.tsx create mode 100644 components/charts/utils/chart-colors.ts create mode 100644 components/charts/utils/chart-data-transformers.ts create mode 100644 lib/server/db/financial-ingestion-schema.test.ts create mode 100644 lib/server/db/financial-ingestion-schema.ts create mode 100644 scripts/repair-financial-ingestion-schema.ts diff --git a/components/charts/hooks/use-chart-export.ts b/components/charts/hooks/use-chart-export.ts new file mode 100644 index 0000000..b0ec500 --- /dev/null +++ b/components/charts/hooks/use-chart-export.ts @@ -0,0 +1,38 @@ +import { useCallback, useRef } from 'react'; +import { toPng } from 'html-to-image'; +import { getChartColors, getComputedColors } from '../utils/chart-colors'; + +export function useChartExport() { + const chartRef = useRef(null); + + const exportChart = useCallback(async (filename: string = 'chart.png') => { + if (!chartRef.current) { + console.error('Chart ref not available'); + return; + } + + try { + // Get background color from CSS variable + const colors = getChartColors(); + const computedColors = getComputedColors(colors); + const backgroundColor = computedColors.tooltipBg || colors.tooltipBg; + + const dataUrl = await toPng(chartRef.current, { + quality: 1.0, + pixelRatio: 2, // High DPI export + backgroundColor: backgroundColor + }); + + // Create download link + const link = document.createElement('a'); + link.download = filename; + link.href = dataUrl; + link.click(); + } catch (error) { + console.error('Failed to export chart:', error); + throw error; + } + }, []); + + return { chartRef, exportChart }; +} diff --git a/components/charts/hooks/use-chart-zoom.ts b/components/charts/hooks/use-chart-zoom.ts new file mode 100644 index 0000000..adb2930 --- /dev/null +++ b/components/charts/hooks/use-chart-zoom.ts @@ -0,0 +1,44 @@ +import { useState, useCallback } from 'react'; +import type { ChartZoomState } from '@/lib/types'; + +export function useChartZoom(dataLength: number) { + const [zoomState, setZoomState] = useState({ + startIndex: 0, + endIndex: Math.max(0, dataLength - 1), + isZoomed: false + }); + + const handleZoomChange = useCallback( + (brushData: { startIndex?: number; endIndex?: number }) => { + if ( + brushData.startIndex === undefined || + brushData.endIndex === undefined + ) { + return; + } + + setZoomState({ + startIndex: brushData.startIndex, + endIndex: brushData.endIndex, + isZoomed: + brushData.startIndex !== 0 || + brushData.endIndex !== dataLength - 1 + }); + }, + [dataLength] + ); + + const resetZoom = useCallback(() => { + setZoomState({ + startIndex: 0, + endIndex: Math.max(0, dataLength - 1), + isZoomed: false + }); + }, [dataLength]); + + return { + zoomState, + handleZoomChange, + resetZoom + }; +} diff --git a/components/charts/interactive-price-chart.tsx b/components/charts/interactive-price-chart.tsx new file mode 100644 index 0000000..5f351e0 --- /dev/null +++ b/components/charts/interactive-price-chart.tsx @@ -0,0 +1,92 @@ +'use client'; + +import { useState, useMemo } from 'react'; +import type { InteractivePriceChartProps, ChartType, TimeRange } from '@/lib/types'; +import { filterByTimeRange, isOHLCVData } from './utils/chart-data-transformers'; +import { ChartContainer } from './primitives/chart-container'; +import { ChartToolbar } from './primitives/chart-toolbar'; +import { LineChartView } from './renderers/line-chart-view'; +import { CombinationChartView } from './renderers/combination-chart-view'; +import { VolumeIndicator } from './primitives/volume-indicator'; +import { useChartExport } from './hooks/use-chart-export'; + +export function InteractivePriceChart({ + data, + dataSeries, + defaultChartType = 'line', + defaultTimeRange = '1Y', + showVolume = false, + showToolbar = true, + height = 400, + loading = false, + error = null, + formatters, + onChartTypeChange, + onTimeRangeChange +}: InteractivePriceChartProps) { + const [chartType, setChartType] = useState(defaultChartType); + const [timeRange, setTimeRange] = useState(defaultTimeRange); + const filteredData = useMemo(() => filterByTimeRange(data, timeRange), [data, timeRange]); + const filteredDataSeries = useMemo( + () => dataSeries?.map((series) => ({ + ...series, + data: filterByTimeRange(series.data, timeRange) + })), + [dataSeries, timeRange] + ); + const { chartRef, exportChart } = useChartExport(); + + const handleChartTypeChange = (type: ChartType) => { + setChartType(type); + onChartTypeChange?.(type); + }; + + const handleTimeRangeChange = (range: TimeRange) => { + setTimeRange(range); + onTimeRangeChange?.(range); + }; + + const handleExport = () => { + exportChart(`chart-${Date.now()}.png`); + }; + + const shouldShowVolume = showVolume && filteredData.some(isOHLCVData); + + return ( +
+ {showToolbar && ( + + )} + + + {chartType === 'line' && ( + + )} + + {chartType === 'combination' && filteredDataSeries && ( + + )} + + + {shouldShowVolume && ( + + )} +
+ ); +} diff --git a/components/charts/primitives/chart-container.tsx b/components/charts/primitives/chart-container.tsx new file mode 100644 index 0000000..6051d88 --- /dev/null +++ b/components/charts/primitives/chart-container.tsx @@ -0,0 +1,45 @@ +import { cn } from '@/lib/utils'; + +type ChartContainerProps = { + children: React.ReactNode; + height?: number; + loading?: boolean; + error?: string | null; + className?: string; +}; + +export function ChartContainer({ + children, + height = 400, + loading = false, + error = null, + className +}: ChartContainerProps) { + return ( +
+ {loading && ( +
+
Loading chart...
+
+ )} + + {error && ( +
+
{error}
+
+ )} + + {!loading && !error && ( +
+ {children} +
+ )} +
+ ); +} diff --git a/components/charts/primitives/chart-toolbar.tsx b/components/charts/primitives/chart-toolbar.tsx new file mode 100644 index 0000000..849be30 --- /dev/null +++ b/components/charts/primitives/chart-toolbar.tsx @@ -0,0 +1,76 @@ +import { Download } from 'lucide-react'; +import { cn } from '@/lib/utils'; +import type { ChartType, TimeRange } from '@/lib/types'; +import { Button } from '@/components/ui/button'; + +type ChartToolbarProps = { + chartType: ChartType; + timeRange: TimeRange; + onChartTypeChange: (type: ChartType) => void; + onTimeRangeChange: (range: TimeRange) => void; + onExport: () => void; +}; + +const TIME_RANGES: TimeRange[] = ['1W', '1M', '3M', '1Y', '3Y', '5Y', '10Y', '20Y']; +const CHART_TYPES: { value: ChartType; label: string }[] = [ + { value: 'line', label: 'Line' }, + { value: 'combination', label: 'Compare' } +]; + +export function ChartToolbar({ + chartType, + timeRange, + onChartTypeChange, + onTimeRangeChange, + onExport +}: ChartToolbarProps) { + return ( +
+
+ {TIME_RANGES.map((range) => ( + + ))} +
+ +
+
+ {CHART_TYPES.map((type) => ( + + ))} +
+ +
+
+ ); +} diff --git a/components/charts/primitives/chart-tooltip.tsx b/components/charts/primitives/chart-tooltip.tsx new file mode 100644 index 0000000..54e23c9 --- /dev/null +++ b/components/charts/primitives/chart-tooltip.tsx @@ -0,0 +1,79 @@ +import type { TooltipContentProps, TooltipPayloadEntry } from 'recharts'; +import { formatCurrency, formatCompactCurrency } from '@/lib/format'; +import { getChartColors } from '../utils/chart-colors'; +import { isOHLCVData } from '../utils/chart-data-transformers'; + +type ChartTooltipProps = TooltipContentProps & { + formatters?: { + price?: (value: number) => string; + date?: (value: string) => string; + volume?: (value: number) => string; + }; +}; + +export function ChartTooltip(props: ChartTooltipProps) { + const { active, payload, label, formatters } = props; + + if (!active || !payload || payload.length === 0) return null; + + const colors = getChartColors(); + const formatDate = formatters?.date || ((date: string) => new Date(date).toLocaleDateString()); + const formatPrice = formatters?.price || formatCurrency; + const formatVolume = formatters?.volume || formatCompactCurrency; + + const data = payload[0].payload; + + return ( +
+
+ {formatDate(label as string)} +
+ + {isOHLCVData(data) ? ( +
+ + + + +
+ +
+
+ ) : ( +
+ {payload.map((entry: TooltipPayloadEntry, index: number) => ( + + ))} +
+ )} +
+ ); +} + +type TooltipRowProps = { + label: string; + value: string; + color: string; +}; + +function TooltipRow({ label, value, color }: TooltipRowProps) { + return ( +
+ {label}: + + {value} + +
+ ); +} diff --git a/components/charts/primitives/volume-indicator.tsx b/components/charts/primitives/volume-indicator.tsx new file mode 100644 index 0000000..e937d7d --- /dev/null +++ b/components/charts/primitives/volume-indicator.tsx @@ -0,0 +1,58 @@ +import { BarChart, Bar, YAxis, Tooltip, ResponsiveContainer } from 'recharts'; +import type { OHLCVDataPoint } from '@/lib/types'; +import { getChartColors } from '../utils/chart-colors'; +import { formatCompactCurrency } from '@/lib/format'; + +type VolumeIndicatorProps = { + data: OHLCVDataPoint[]; + height?: number; + formatters?: { + volume?: (value: number) => string; + }; +}; + +export function VolumeIndicator({ + data, + height = 80, + formatters +}: VolumeIndicatorProps) { + const colors = getChartColors(); + if (data.length === 0) return null; + + const formatVolume = formatters?.volume || formatCompactCurrency; + + return ( +
+ + + + [ + formatVolume(typeof value === 'number' ? value : Number(value ?? 0)), + 'Volume' + ]} + labelFormatter={(label) => `Date: ${label}`} + contentStyle={{ + backgroundColor: colors.tooltipBg, + border: `1px solid ${colors.tooltipBorder}`, + borderRadius: '0.75rem', + fontSize: '0.75rem' + }} + /> + + + +
+ ); +} diff --git a/components/charts/renderers/candlestick-chart-view.tsx b/components/charts/renderers/candlestick-chart-view.tsx new file mode 100644 index 0000000..90fb781 --- /dev/null +++ b/components/charts/renderers/candlestick-chart-view.tsx @@ -0,0 +1,71 @@ +import { + ComposedChart, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer, + Scatter +} from 'recharts'; +import type { ChartDataPoint } from '@/lib/types'; +import { getChartColors } from '../utils/chart-colors'; +import { ChartTooltip } from '../primitives/chart-tooltip'; +import { CandlestickShape } from '../utils/candlestick-shapes'; +import { isOHLCVData } from '../utils/chart-data-transformers'; + +type CandlestickChartViewProps = { + data: ChartDataPoint[]; + formatters?: { + price?: (value: number) => string; + date?: (value: string) => string; + volume?: (value: number) => string; + }; +}; + +export function CandlestickChartView({ + data, + formatters +}: CandlestickChartViewProps) { + const colors = getChartColors(); + + const ohlcvData = data.filter(isOHLCVData); + + if (ohlcvData.length === 0) { + return ( +
+ Candlestick chart requires OHLCV data +
+ ); + } + + return ( + + + + + + } + cursor={{ stroke: colors.muted, strokeDasharray: '3 3' }} + /> + } + isAnimationActive={false} + /> + + + ); +} diff --git a/components/charts/renderers/combination-chart-view.tsx b/components/charts/renderers/combination-chart-view.tsx new file mode 100644 index 0000000..3d98d20 --- /dev/null +++ b/components/charts/renderers/combination-chart-view.tsx @@ -0,0 +1,112 @@ +import { + ComposedChart, + Line, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer, + Legend +} from 'recharts'; +import type { DataSeries } from '@/lib/types'; +import { getChartColors } from '../utils/chart-colors'; +import { ChartTooltip } from '../primitives/chart-tooltip'; +import { mergeDataSeries } from '../utils/chart-data-transformers'; + +type CombinationChartViewProps = { + dataSeries: DataSeries[]; + formatters?: { + price?: (value: number) => string; + date?: (value: string) => string; + volume?: (value: number) => string; + }; +}; + +export function CombinationChartView({ + dataSeries, + formatters +}: CombinationChartViewProps) { + const colors = getChartColors(); + + if (!dataSeries || dataSeries.length === 0) { + return ( +
+ No data series provided +
+ ); + } + + const mergedData = mergeDataSeries(dataSeries); + const visibleSeries = dataSeries.filter(series => series.visible !== false); + const baseValues = Object.fromEntries(visibleSeries.map((series) => { + const initialPoint = mergedData.find((entry) => typeof entry[series.id] === 'number'); + const baseValue = typeof initialPoint?.[series.id] === 'number' ? Number(initialPoint[series.id]) : null; + return [series.id, baseValue]; + })); + const normalizedData = mergedData.map((point) => { + const normalizedPoint: Record = { date: point.date }; + + visibleSeries.forEach((series) => { + const baseValue = baseValues[series.id]; + const currentValue = typeof point[series.id] === 'number' ? Number(point[series.id]) : null; + + normalizedPoint[series.id] = baseValue && currentValue + ? ((currentValue / baseValue) - 1) * 100 + : null; + }); + + return normalizedPoint; + }); + + return ( + + + + + `${Number(value).toFixed(0)}%`} + width={60} + domain={['auto', 'auto']} + /> + ( + `${value.toFixed(2)}%` + }} + /> + )} + cursor={{ stroke: colors.muted, strokeDasharray: '3 3' }} + /> + + + {visibleSeries.map(series => { + const seriesColor = series.color || colors.primary; + return ( + + ); + })} + + + ); +} diff --git a/components/charts/renderers/line-chart-view.tsx b/components/charts/renderers/line-chart-view.tsx new file mode 100644 index 0000000..3883c37 --- /dev/null +++ b/components/charts/renderers/line-chart-view.tsx @@ -0,0 +1,100 @@ +import { + LineChart, + Line, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer +} from 'recharts'; +import type { ChartDataPoint } from '@/lib/types'; +import { getChartColors } from '../utils/chart-colors'; +import { ChartTooltip } from '../primitives/chart-tooltip'; +import { isPriceData, isOHLCVData } from '../utils/chart-data-transformers'; + +type LineChartViewProps = { + data: ChartDataPoint[]; + formatters?: { + price?: (value: number) => string; + date?: (value: string) => string; + volume?: (value: number) => string; + }; +}; + +export function LineChartView({ + data, + formatters +}: LineChartViewProps) { + const colors = getChartColors(); + + const chartData = data.map(point => { + if (isOHLCVData(point)) { + return { + date: point.date, + price: point.close, + open: point.open, + high: point.high, + low: point.low, + close: point.close, + volume: point.volume + }; + } else if (isPriceData(point)) { + return { + date: point.date, + price: point.price + }; + } + return point; + }); + + return ( + + + + + + } + cursor={{ stroke: colors.muted, strokeWidth: 1, strokeDasharray: '5 5' }} + isAnimationActive={false} + /> + + + + ); +} diff --git a/components/charts/utils/candlestick-shapes.tsx b/components/charts/utils/candlestick-shapes.tsx new file mode 100644 index 0000000..d7f2601 --- /dev/null +++ b/components/charts/utils/candlestick-shapes.tsx @@ -0,0 +1,68 @@ +import { getPriceChangeColor } from './chart-colors'; + +type CandlestickShapeProps = { + cx?: number; + payload?: { + open: number; + high: number; + low: number; + close: number; + }; +}; + +/** + * Custom candlestick shape component for Recharts + * Renders candlestick with wick and body + */ +export function CandlestickShape(props: CandlestickShapeProps) { + const { cx, payload } = props; + + if (!payload || !cx) return null; + + const { open, high, low, close } = payload; + const isPositive = close >= open; + const color = getPriceChangeColor(close - open); + + // Calculate positions + const bodyTop = Math.min(open, close); + const bodyBottom = Math.max(open, close); + const bodyHeight = Math.max(bodyBottom - bodyTop, 1); + + // Candlestick width + const width = 8; + + return ( + + {/* Upper wick */} + + + {/* Body */} + + + {/* Lower wick */} + + + ); +} diff --git a/components/charts/utils/chart-colors.ts b/components/charts/utils/chart-colors.ts new file mode 100644 index 0000000..405c369 --- /dev/null +++ b/components/charts/utils/chart-colors.ts @@ -0,0 +1,65 @@ +import type { ChartColorPalette } from '@/lib/types'; + +/** + * Get chart color palette using CSS variables for theming + * These colors match the existing dark theme in the codebase + */ +export function getChartColors(): ChartColorPalette { + return { + primary: 'var(--accent)', + secondary: 'var(--terminal-muted)', + positive: '#96f5bf', // Green - matches existing price-history-card + negative: '#ff9f9f', // Red - matches existing price-history-card + grid: 'var(--line-weak)', + text: 'var(--terminal-bright)', + muted: 'var(--terminal-muted)', + tooltipBg: 'rgba(31, 34, 39, 0.96)', + tooltipBorder: 'var(--line-strong)', + volume: 'var(--terminal-muted)' + }; +} + +/** + * Get color for price change (positive/negative) + */ +export function getPriceChangeColor(change: number): string { + const colors = getChartColors(); + return change >= 0 ? colors.positive : colors.negative; +} + +/** + * Convert CSS variable to computed color value + * Used for chart export since html-to-image can't render CSS variables + */ +export function cssVarToColor(cssVar: string): string { + if (typeof window === 'undefined') return cssVar; + + // If it's already a color value, return as-is + if (!cssVar.startsWith('var(')) return cssVar; + + // Extract variable name from var(--name) + const varName = cssVar.match(/var\((--[^)]+)\)/)?.[1]; + if (!varName) return cssVar; + + // Get computed color from CSS variable + const computedColor = getComputedStyle(document.documentElement) + .getPropertyValue(varName) + .trim(); + + return computedColor || cssVar; +} + +/** + * Convert entire color palette to computed colors for export + */ +export function getComputedColors(palette: Partial): Partial { + const computed: Partial = {}; + + for (const [key, value] of Object.entries(palette)) { + if (value) { + computed[key as keyof ChartColorPalette] = cssVarToColor(value); + } + } + + return computed; +} diff --git a/components/charts/utils/chart-data-transformers.ts b/components/charts/utils/chart-data-transformers.ts new file mode 100644 index 0000000..3c060af --- /dev/null +++ b/components/charts/utils/chart-data-transformers.ts @@ -0,0 +1,157 @@ +import { subWeeks, subMonths, subYears } from 'date-fns'; +import type { TimeRange, ChartDataPoint } from '@/lib/types'; + +/** + * Filter chart data by time range + */ +export function filterByTimeRange( + data: T[], + range: TimeRange +): T[] { + if (data.length === 0) return data; + + const now = new Date(); + const cutoffDate = { + '1W': subWeeks(now, 1), + '1M': subMonths(now, 1), + '3M': subMonths(now, 3), + '1Y': subYears(now, 1), + '3Y': subYears(now, 3), + '5Y': subYears(now, 5), + '10Y': subYears(now, 10), + '20Y': subYears(now, 20) + }[range]; + + return data.filter(point => new Date(point.date) >= cutoffDate); +} + +/** + * Check if data point has OHLCV fields + */ +export function isOHLCVData(data: ChartDataPoint): data is { + date: string; + open: number; + high: number; + low: number; + close: number; + volume: number; +} { + return 'open' in data && 'high' in data && 'low' in data && 'close' in data && 'volume' in data; +} + +/** + * Check if data point has simple price field + */ +export function isPriceData(data: ChartDataPoint): data is { date: string; price: number } { + return 'price' in data; +} + +/** + * Normalize data to ensure consistent structure + * Converts price data to OHLCV-like structure for candlestick charts + */ +export function normalizeChartData(data: T[]): T[] { + if (!data || data.length === 0) return []; + + // Sort by date ascending + return [...data].sort((a, b) => + new Date(a.date).getTime() - new Date(b.date).getTime() + ); +} + +/** + * Sample data for performance with large datasets + * Keeps every Nth point when dataset is too large + */ +export function sampleData( + data: T[], + maxPoints: number = 1000 +): T[] { + if (data.length <= maxPoints) return data; + + const samplingRate = Math.ceil(data.length / maxPoints); + const sampled: T[] = []; + + for (let i = 0; i < data.length; i += samplingRate) { + sampled.push(data[i]); + } + + // Always include the last point + if (sampled[sampled.length - 1] !== data[data.length - 1]) { + sampled.push(data[data.length - 1]); + } + + return sampled; +} + +/** + * Calculate min/max values for Y-axis domain + */ +export function calculateYAxisDomain( + data: T[], + padding: number = 0.1 +): [number, number] { + if (data.length === 0) return [0, 100]; + + let min = Infinity; + let max = -Infinity; + + data.forEach(point => { + if (isOHLCVData(point)) { + min = Math.min(min, point.low); + max = Math.max(max, point.high); + } else if (isPriceData(point)) { + min = Math.min(min, point.price); + max = Math.max(max, point.price); + } + }); + + // Add padding + const range = max - min; + const paddedMin = min - (range * padding); + const paddedMax = max + (range * padding); + + return [paddedMin, paddedMax]; +} + +/** + * Calculate volume max for volume indicator Y-axis + */ +export function calculateVolumeMax(data: T[]): number { + if (data.length === 0 || !isOHLCVData(data[0])) return 0; + + return Math.max(...data.map(d => (isOHLCVData(d) ? d.volume : 0))); +} + +/** + * Merge multiple data series by date for combination charts + */ +export function mergeDataSeries( + seriesArray: Array<{ id: string; data: T[] }> +): Array> { + if (!seriesArray || seriesArray.length === 0) return []; + + // Create map indexed by date + const dateMap = new Map>(); + + seriesArray.forEach(series => { + series.data.forEach(point => { + const date = point.date; + const existing = dateMap.get(date) || { date } as T & Record; + + // Add value for this series + if (isOHLCVData(point)) { + existing[series.id] = point.close; + } else if (isPriceData(point)) { + existing[series.id] = point.price; + } + + dateMap.set(date, existing); + }); + }); + + // Convert to array and sort by date + return Array.from(dateMap.values()).sort((a, b) => + new Date(a.date).getTime() - new Date(b.date).getTime() + ); +} diff --git a/lib/server/db/financial-ingestion-schema.test.ts b/lib/server/db/financial-ingestion-schema.test.ts new file mode 100644 index 0000000..377f67e --- /dev/null +++ b/lib/server/db/financial-ingestion-schema.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { + ensureFinancialIngestionSchemaHealthy, + inspectFinancialIngestionSchema, + withFinancialIngestionSchemaRetry +} from './financial-ingestion-schema'; + +function createBundleTable(client: Database) { + client.exec(` + CREATE TABLE \`company_financial_bundle\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`ticker\` text NOT NULL, + \`surface_kind\` text NOT NULL, + \`cadence\` text NOT NULL, + \`bundle_version\` integer NOT NULL, + \`source_snapshot_ids\` text NOT NULL, + \`source_signature\` text NOT NULL, + \`payload\` text NOT NULL, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL + ); + `); +} + +function createSnapshotTable(client: Database) { + client.exec(` + CREATE TABLE \`filing_taxonomy_snapshot\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`filing_id\` integer NOT NULL, + \`ticker\` text NOT NULL, + \`filing_date\` text NOT NULL, + \`filing_type\` text NOT NULL, + \`parse_status\` text NOT NULL, + \`updated_at\` text NOT NULL + ); + `); +} + +function createMetricValidationTable(client: Database) { + client.exec(` + CREATE TABLE \`filing_taxonomy_metric_validation\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`snapshot_id\` integer NOT NULL, + \`metric_key\` text NOT NULL, + \`status\` text NOT NULL, + \`updated_at\` text NOT NULL + ); + `); +} + +function createHealthyIndexes(client: Database) { + client.exec('CREATE UNIQUE INDEX `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`);'); + client.exec('CREATE INDEX `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`);'); + client.exec('CREATE UNIQUE INDEX `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);'); + client.exec('CREATE INDEX `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);'); + client.exec('CREATE INDEX `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);'); + client.exec('CREATE UNIQUE INDEX `filing_taxonomy_metric_validation_uidx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`metric_key`);'); + client.exec('CREATE INDEX `filing_taxonomy_metric_validation_snapshot_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`);'); + client.exec('CREATE INDEX `filing_taxonomy_metric_validation_status_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`status`);'); +} + +describe('financial ingestion schema repair', () => { + it('reports a healthy schema when all critical indexes exist', () => { + const client = new Database(':memory:'); + try { + createBundleTable(client); + createSnapshotTable(client); + createMetricValidationTable(client); + createHealthyIndexes(client); + + const report = inspectFinancialIngestionSchema(client); + expect(report.ok).toBe(true); + expect(report.missingIndexes).toEqual([]); + expect(report.duplicateGroups).toBe(0); + } finally { + client.close(); + } + }); + + it('repairs missing company financial bundle indexes and dedupes rows', () => { + const client = new Database(':memory:'); + try { + createBundleTable(client); + createSnapshotTable(client); + createMetricValidationTable(client); + createHealthyIndexes(client); + client.exec('DROP INDEX `company_financial_bundle_uidx`;'); + client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;'); + client.exec(` + INSERT INTO \`company_financial_bundle\` ( + \`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\` + ) VALUES + ('MSFT', 'income_statement', 'annual', 14, '[]', 'old', '{}', '2026-03-12T10:00:00.000Z', '2026-03-12T10:00:00.000Z'), + ('MSFT', 'income_statement', 'annual', 14, '[]', 'new', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z'); + `); + + const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' }); + const rows = client.query('SELECT `source_signature` FROM `company_financial_bundle`').all() as Array<{ source_signature: string }>; + const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>; + + expect(result.ok).toBe(true); + expect(result.mode).toBe('repaired'); + expect(rows).toEqual([{ source_signature: 'new' }]); + expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true); + expect(indexes.some((row) => row.name === 'company_financial_bundle_ticker_idx')).toBe(true); + } finally { + client.close(); + } + }); + + it('dedupes filing taxonomy snapshots and clears bundle cache before recreating indexes', () => { + const client = new Database(':memory:'); + try { + createBundleTable(client); + createSnapshotTable(client); + createMetricValidationTable(client); + createHealthyIndexes(client); + client.exec('DROP INDEX `filing_taxonomy_snapshot_filing_uidx`;'); + client.exec(` + INSERT INTO \`filing_taxonomy_snapshot\` ( + \`filing_id\`, \`ticker\`, \`filing_date\`, \`filing_type\`, \`parse_status\`, \`updated_at\` + ) VALUES + (10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T10:00:00.000Z'), + (10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T11:00:00.000Z'); + `); + client.exec(` + INSERT INTO \`company_financial_bundle\` ( + \`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\` + ) VALUES + ('MSFT', 'income_statement', 'annual', 14, '[1,2]', 'cached', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z'); + `); + + const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' }); + const snapshotCount = client.query('SELECT COUNT(*) AS count FROM `filing_taxonomy_snapshot`').get() as { count: number }; + const bundleCount = client.query('SELECT COUNT(*) AS count FROM `company_financial_bundle`').get() as { count: number }; + + expect(result.ok).toBe(true); + expect(result.repair?.snapshotDuplicateRowsDeleted).toBe(1); + expect(result.repair?.bundleCacheCleared).toBe(true); + expect(snapshotCount.count).toBe(1); + expect(bundleCount.count).toBe(0); + } finally { + client.close(); + } + }); + + it('retries once after repairing the schema for ON CONFLICT drift errors', async () => { + const client = new Database(':memory:'); + try { + createBundleTable(client); + createSnapshotTable(client); + createMetricValidationTable(client); + createHealthyIndexes(client); + client.exec('DROP INDEX `company_financial_bundle_uidx`;'); + client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;'); + + let calls = 0; + const result = await withFinancialIngestionSchemaRetry({ + client, + context: 'test-retry', + operation: async () => { + calls += 1; + if (calls === 1) { + throw new Error('ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint'); + } + + return 'ok'; + } + }); + + expect(result).toBe('ok'); + expect(calls).toBe(2); + const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>; + expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true); + } finally { + client.close(); + } + }); +}); diff --git a/lib/server/db/financial-ingestion-schema.ts b/lib/server/db/financial-ingestion-schema.ts new file mode 100644 index 0000000..0964c4b --- /dev/null +++ b/lib/server/db/financial-ingestion-schema.ts @@ -0,0 +1,541 @@ +import type { Database } from 'bun:sqlite'; + +export type FinancialSchemaRepairMode = 'auto' | 'check-only' | 'off'; +export type FinancialIngestionHealthMode = 'healthy' | 'repaired' | 'drifted' | 'failed'; + +type CriticalIndexDefinition = { + name: string; + table: 'company_financial_bundle' | 'filing_taxonomy_snapshot' | 'filing_taxonomy_metric_validation'; + unique: boolean; + columns: string[]; + createSql: string; +}; + +type DuplicateRule = { + table: 'company_financial_bundle' | 'filing_taxonomy_snapshot' | 'filing_taxonomy_metric_validation'; + partitionColumns: string[]; +}; + +export type FinancialIngestionIndexStatus = { + name: string; + table: string; + expectedColumns: string[]; + actualColumns: string[]; + unique: boolean; + present: boolean; + healthy: boolean; +}; + +export type FinancialIngestionDuplicateStatus = { + table: string; + duplicateGroups: number; + duplicateRows: number; +}; + +export type FinancialIngestionSchemaReport = { + ok: boolean; + checkedAt: string; + indexes: FinancialIngestionIndexStatus[]; + missingIndexes: string[]; + duplicateGroups: number; + duplicateRows: number; + duplicates: FinancialIngestionDuplicateStatus[]; +}; + +export type FinancialIngestionSchemaRepairResult = { + attempted: boolean; + requestedMode: FinancialSchemaRepairMode; + missingIndexesBefore: string[]; + duplicateGroupsBefore: number; + indexesCreated: string[]; + indexesRecreated: string[]; + duplicateRowsDeleted: number; + duplicateGroupsResolved: number; + bundleCacheCleared: boolean; + snapshotDuplicateRowsDeleted: number; + reportAfter: FinancialIngestionSchemaReport; +}; + +export type FinancialIngestionSchemaEnsureResult = { + ok: boolean; + mode: FinancialIngestionHealthMode; + requestedMode: FinancialSchemaRepairMode; + missingIndexes: string[]; + duplicateGroups: number; + lastCheckedAt: string; + repair: FinancialIngestionSchemaRepairResult | null; + error: string | null; +}; + +declare global { + // eslint-disable-next-line no-var + var __financialIngestionSchemaStatus: FinancialIngestionSchemaEnsureResult | undefined; +} + +const CRITICAL_INDEX_DEFINITIONS: CriticalIndexDefinition[] = [ + { + table: 'company_financial_bundle', + name: 'company_financial_bundle_uidx', + unique: true, + columns: ['ticker', 'surface_kind', 'cadence'], + createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`);' + }, + { + table: 'company_financial_bundle', + name: 'company_financial_bundle_ticker_idx', + unique: false, + columns: ['ticker', 'updated_at'], + createSql: 'CREATE INDEX IF NOT EXISTS `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`);' + }, + { + table: 'filing_taxonomy_snapshot', + name: 'filing_taxonomy_snapshot_filing_uidx', + unique: true, + columns: ['filing_id'], + createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);' + }, + { + table: 'filing_taxonomy_snapshot', + name: 'filing_taxonomy_snapshot_ticker_date_idx', + unique: false, + columns: ['ticker', 'filing_date'], + createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);' + }, + { + table: 'filing_taxonomy_snapshot', + name: 'filing_taxonomy_snapshot_status_idx', + unique: false, + columns: ['parse_status'], + createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);' + }, + { + table: 'filing_taxonomy_metric_validation', + name: 'filing_taxonomy_metric_validation_uidx', + unique: true, + columns: ['snapshot_id', 'metric_key'], + createSql: 'CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_uidx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`metric_key`);' + }, + { + table: 'filing_taxonomy_metric_validation', + name: 'filing_taxonomy_metric_validation_snapshot_idx', + unique: false, + columns: ['snapshot_id'], + createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_snapshot_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`);' + }, + { + table: 'filing_taxonomy_metric_validation', + name: 'filing_taxonomy_metric_validation_status_idx', + unique: false, + columns: ['snapshot_id', 'status'], + createSql: 'CREATE INDEX IF NOT EXISTS `filing_taxonomy_metric_validation_status_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`status`);' + } +]; + +const UNIQUE_DUPLICATE_RULES: DuplicateRule[] = [ + { + table: 'company_financial_bundle', + partitionColumns: ['ticker', 'surface_kind', 'cadence'] + }, + { + table: 'filing_taxonomy_snapshot', + partitionColumns: ['filing_id'] + }, + { + table: 'filing_taxonomy_metric_validation', + partitionColumns: ['snapshot_id', 'metric_key'] + } +]; + +function hasTable(client: Database, tableName: string) { + const row = client + .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') + .get('table', tableName) as { name: string } | null; + + return row !== null; +} + +function nowIso() { + return new Date().toISOString(); +} + +export function resolveFinancialSchemaRepairMode(value: string | undefined): FinancialSchemaRepairMode { + const normalized = value?.trim().toLowerCase(); + if (normalized === 'check-only') { + return 'check-only'; + } + + if (normalized === 'off') { + return 'off'; + } + + return 'auto'; +} + +function arrayEqual(left: string[], right: string[]) { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function queryIndexColumns(client: Database, indexName: string) { + return (client.query(`PRAGMA index_info(\`${indexName}\`)`).all() as Array<{ seqno: number; name: string }>) + .sort((left, right) => left.seqno - right.seqno) + .map((row) => row.name); +} + +function inspectIndex(client: Database, definition: CriticalIndexDefinition): FinancialIngestionIndexStatus { + if (!hasTable(client, definition.table)) { + return { + name: definition.name, + table: definition.table, + expectedColumns: [...definition.columns], + actualColumns: [], + unique: definition.unique, + present: false, + healthy: false + }; + } + + const indexList = client.query(`PRAGMA index_list(\`${definition.table}\`)`).all() as Array<{ + name: string; + unique: number; + }>; + const existing = indexList.find((row) => row.name === definition.name) ?? null; + const actualColumns = existing ? queryIndexColumns(client, definition.name) : []; + const isUnique = existing ? existing.unique === 1 : false; + const healthy = Boolean(existing) && isUnique === definition.unique && arrayEqual(actualColumns, definition.columns); + + return { + name: definition.name, + table: definition.table, + expectedColumns: [...definition.columns], + actualColumns, + unique: definition.unique, + present: existing !== null, + healthy + }; +} + +function inspectDuplicates(client: Database, rule: DuplicateRule): FinancialIngestionDuplicateStatus { + if (!hasTable(client, rule.table)) { + return { + table: rule.table, + duplicateGroups: 0, + duplicateRows: 0 + }; + } + + const groupBy = rule.partitionColumns.map((column) => `\`${column}\``).join(', '); + const row = client.query(` + SELECT + COUNT(*) AS duplicate_groups, + COALESCE(SUM(cnt - 1), 0) AS duplicate_rows + FROM ( + SELECT COUNT(*) AS cnt + FROM \`${rule.table}\` + GROUP BY ${groupBy} + HAVING COUNT(*) > 1 + ) + `).get() as { duplicate_groups: number | null; duplicate_rows: number | null } | null; + + return { + table: rule.table, + duplicateGroups: Number(row?.duplicate_groups ?? 0), + duplicateRows: Number(row?.duplicate_rows ?? 0) + }; +} + +export function inspectFinancialIngestionSchema(client: Database): FinancialIngestionSchemaReport { + const indexes = CRITICAL_INDEX_DEFINITIONS.map((definition) => inspectIndex(client, definition)); + const duplicates = UNIQUE_DUPLICATE_RULES.map((rule) => inspectDuplicates(client, rule)); + const missingIndexes = indexes.filter((index) => !index.healthy).map((index) => index.name); + + return { + ok: missingIndexes.length === 0 && duplicates.every((entry) => entry.duplicateGroups === 0), + checkedAt: nowIso(), + indexes, + missingIndexes, + duplicateGroups: duplicates.reduce((sum, entry) => sum + entry.duplicateGroups, 0), + duplicateRows: duplicates.reduce((sum, entry) => sum + entry.duplicateRows, 0), + duplicates + }; +} + +function runTransaction(client: Database, work: () => T) { + client.exec('BEGIN IMMEDIATE;'); + try { + const result = work(); + client.exec('COMMIT;'); + return result; + } catch (error) { + client.exec('ROLLBACK;'); + throw error; + } +} + +function deleteDuplicateRows(client: Database, rule: DuplicateRule) { + if (!hasTable(client, rule.table)) { + return 0; + } + + const partitionBy = rule.partitionColumns.map((column) => `\`${column}\``).join(', '); + client.exec(` + DELETE FROM \`${rule.table}\` + WHERE \`id\` IN ( + WITH ranked AS ( + SELECT + \`id\`, + ROW_NUMBER() OVER ( + PARTITION BY ${partitionBy} + ORDER BY \`updated_at\` DESC, \`id\` DESC + ) AS rn + FROM \`${rule.table}\` + ) + SELECT \`id\` + FROM ranked + WHERE rn > 1 + ); + `); + + const row = client.query('SELECT changes() AS count').get() as { count: number } | null; + return Number(row?.count ?? 0); +} + +function clearBundleCache(client: Database) { + if (!hasTable(client, 'company_financial_bundle')) { + return 0; + } + + client.exec('DELETE FROM `company_financial_bundle`;'); + const row = client.query('SELECT changes() AS count').get() as { count: number } | null; + return Number(row?.count ?? 0); +} + +function createOrRecreateIndex(client: Database, definition: CriticalIndexDefinition, status: FinancialIngestionIndexStatus) { + if (status.present && !status.healthy) { + client.exec(`DROP INDEX IF EXISTS \`${definition.name}\`;`); + } + + client.exec(definition.createSql); +} + +export function repairFinancialIngestionSchema( + client: Database, + options: { + mode?: FinancialSchemaRepairMode; + } = {} +): FinancialIngestionSchemaRepairResult { + const requestedMode = options.mode ?? 'auto'; + const before = inspectFinancialIngestionSchema(client); + + if (requestedMode !== 'auto' || before.ok) { + return { + attempted: false, + requestedMode, + missingIndexesBefore: before.missingIndexes, + duplicateGroupsBefore: before.duplicateGroups, + indexesCreated: [], + indexesRecreated: [], + duplicateRowsDeleted: 0, + duplicateGroupsResolved: 0, + bundleCacheCleared: false, + snapshotDuplicateRowsDeleted: 0, + reportAfter: before + }; + } + + let duplicateRowsDeleted = 0; + let duplicateGroupsResolved = 0; + let snapshotDuplicateRowsDeleted = 0; + let bundleCacheCleared = false; + const indexesCreated: string[] = []; + const indexesRecreated: string[] = []; + + const snapshotStatusBefore = before.duplicates.find((entry) => entry.table === 'filing_taxonomy_snapshot'); + const companyBundleStatusBefore = before.duplicates.find((entry) => entry.table === 'company_financial_bundle'); + const metricValidationStatusBefore = before.duplicates.find((entry) => entry.table === 'filing_taxonomy_metric_validation'); + + if ((snapshotStatusBefore?.duplicateGroups ?? 0) > 0) { + snapshotDuplicateRowsDeleted = runTransaction(client, () => { + return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'filing_taxonomy_snapshot')!); + }); + duplicateRowsDeleted += snapshotDuplicateRowsDeleted; + duplicateGroupsResolved += snapshotStatusBefore?.duplicateGroups ?? 0; + + if (snapshotDuplicateRowsDeleted > 0) { + runTransaction(client, () => { + clearBundleCache(client); + }); + bundleCacheCleared = true; + } + } + + if (!bundleCacheCleared && (companyBundleStatusBefore?.duplicateGroups ?? 0) > 0) { + const deleted = runTransaction(client, () => { + return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'company_financial_bundle')!); + }); + duplicateRowsDeleted += deleted; + duplicateGroupsResolved += companyBundleStatusBefore?.duplicateGroups ?? 0; + } + + if ((metricValidationStatusBefore?.duplicateGroups ?? 0) > 0) { + const deleted = runTransaction(client, () => { + return deleteDuplicateRows(client, UNIQUE_DUPLICATE_RULES.find((rule) => rule.table === 'filing_taxonomy_metric_validation')!); + }); + duplicateRowsDeleted += deleted; + duplicateGroupsResolved += metricValidationStatusBefore?.duplicateGroups ?? 0; + } + + for (const definition of CRITICAL_INDEX_DEFINITIONS) { + const status = before.indexes.find((entry) => entry.name === definition.name); + if (!status || status.healthy) { + continue; + } + + runTransaction(client, () => { + createOrRecreateIndex(client, definition, status); + }); + + if (status.present) { + indexesRecreated.push(definition.name); + } else { + indexesCreated.push(definition.name); + } + } + + const reportAfter = inspectFinancialIngestionSchema(client); + + return { + attempted: true, + requestedMode, + missingIndexesBefore: before.missingIndexes, + duplicateGroupsBefore: before.duplicateGroups, + indexesCreated, + indexesRecreated, + duplicateRowsDeleted, + duplicateGroupsResolved, + bundleCacheCleared, + snapshotDuplicateRowsDeleted, + reportAfter + }; +} + +function cacheEnsureResult(result: FinancialIngestionSchemaEnsureResult) { + globalThis.__financialIngestionSchemaStatus = result; +} + +export function getLatestFinancialIngestionSchemaStatus() { + return globalThis.__financialIngestionSchemaStatus ?? null; +} + +export function ensureFinancialIngestionSchemaHealthy( + client: Database, + options: { + mode?: FinancialSchemaRepairMode; + } = {} +): FinancialIngestionSchemaEnsureResult { + const requestedMode = options.mode ?? 'auto'; + + try { + const before = inspectFinancialIngestionSchema(client); + if (before.ok) { + const result: FinancialIngestionSchemaEnsureResult = { + ok: true, + mode: 'healthy', + requestedMode, + missingIndexes: [], + duplicateGroups: 0, + lastCheckedAt: before.checkedAt, + repair: null, + error: null + }; + cacheEnsureResult(result); + return result; + } + + if (requestedMode !== 'auto') { + const result: FinancialIngestionSchemaEnsureResult = { + ok: false, + mode: 'drifted', + requestedMode, + missingIndexes: before.missingIndexes, + duplicateGroups: before.duplicateGroups, + lastCheckedAt: before.checkedAt, + repair: null, + error: null + }; + cacheEnsureResult(result); + return result; + } + + const repair = repairFinancialIngestionSchema(client, { mode: requestedMode }); + const finalReport = repair.reportAfter; + const result: FinancialIngestionSchemaEnsureResult = { + ok: finalReport.ok, + mode: finalReport.ok ? 'repaired' : 'failed', + requestedMode, + missingIndexes: finalReport.missingIndexes, + duplicateGroups: finalReport.duplicateGroups, + lastCheckedAt: finalReport.checkedAt, + repair, + error: finalReport.ok ? null : 'Financial ingestion schema remains drifted after automatic repair.' + }; + cacheEnsureResult(result); + return result; + } catch (error) { + const result: FinancialIngestionSchemaEnsureResult = { + ok: false, + mode: 'failed', + requestedMode, + missingIndexes: [], + duplicateGroups: 0, + lastCheckedAt: nowIso(), + repair: null, + error: error instanceof Error ? error.message : String(error) + }; + cacheEnsureResult(result); + return result; + } +} + +export function isMissingOnConflictConstraintError(error: unknown) { + return error instanceof Error + && error.message.toLowerCase().includes('on conflict clause does not match any primary key or unique constraint'); +} + +export async function withFinancialIngestionSchemaRetry(input: { + client: Database; + operation: () => Promise; + context: string; +}) { + try { + return await input.operation(); + } catch (error) { + if (!isMissingOnConflictConstraintError(error)) { + throw error; + } + + const ensureResult = ensureFinancialIngestionSchemaHealthy(input.client, { + mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE) + }); + + try { + return await input.operation(); + } catch (retryError) { + const suffix = ensureResult.error + ? ` repair_error=${ensureResult.error}` + : ` missing_indexes=${ensureResult.missingIndexes.join(',') || 'none'} duplicate_groups=${ensureResult.duplicateGroups}`; + const reason = retryError instanceof Error ? retryError.message : String(retryError); + throw new Error(`[${input.context}] ingestion schema retry failed:${suffix}; cause=${reason}`); + } + } +} + +export const __financialIngestionSchemaInternals = { + CRITICAL_INDEX_DEFINITIONS, + UNIQUE_DUPLICATE_RULES, + clearBundleCache, + deleteDuplicateRows, + hasTable, + inspectDuplicates, + inspectIndex, + runTransaction +}; diff --git a/scripts/repair-financial-ingestion-schema.ts b/scripts/repair-financial-ingestion-schema.ts new file mode 100644 index 0000000..45d40a7 --- /dev/null +++ b/scripts/repair-financial-ingestion-schema.ts @@ -0,0 +1,68 @@ +import { Database } from 'bun:sqlite'; +import { + ensureFinancialIngestionSchemaHealthy, + inspectFinancialIngestionSchema, + resolveFinancialSchemaRepairMode +} from '../lib/server/db/financial-ingestion-schema'; +import { resolveSqlitePath } from './dev-env'; + +function parseArgs(argv: string[]) { + const flags = new Set(argv); + + return { + dryRun: flags.has('--dry-run'), + verbose: flags.has('--verbose'), + failOnDrift: flags.has('--fail-on-drift') + }; +} + +function resolveDatabasePath() { + const raw = process.env.DATABASE_URL?.trim() || 'file:data/fiscal.sqlite'; + const databasePath = resolveSqlitePath(raw); + + if (!databasePath || databasePath.includes('://')) { + throw new Error(`DATABASE_URL must resolve to a SQLite file path. Received: ${raw}`); + } + + return databasePath; +} + +const options = parseArgs(process.argv.slice(2)); +const databasePath = resolveDatabasePath(); +const client = new Database(databasePath, { create: true }); + +try { + client.exec('PRAGMA foreign_keys = ON;'); + + const mode = options.dryRun + ? 'check-only' + : resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE); + const before = inspectFinancialIngestionSchema(client); + const result = ensureFinancialIngestionSchemaHealthy(client, { mode }); + + console.info(`[repair-financial-ingestion-schema] db=${databasePath}`); + console.info(`[repair-financial-ingestion-schema] mode=${mode}`); + console.info(`[repair-financial-ingestion-schema] status=${result.mode}`); + console.info(`[repair-financial-ingestion-schema] missing_indexes_before=${before.missingIndexes.join(',') || 'none'}`); + console.info(`[repair-financial-ingestion-schema] duplicate_groups_before=${before.duplicateGroups}`); + + if (result.repair) { + console.info(`[repair-financial-ingestion-schema] indexes_created=${result.repair.indexesCreated.join(',') || 'none'}`); + console.info(`[repair-financial-ingestion-schema] indexes_recreated=${result.repair.indexesRecreated.join(',') || 'none'}`); + console.info(`[repair-financial-ingestion-schema] duplicate_rows_deleted=${result.repair.duplicateRowsDeleted}`); + console.info(`[repair-financial-ingestion-schema] bundle_cache_cleared=${result.repair.bundleCacheCleared ? 'yes' : 'no'}`); + } + + if (options.verbose) { + console.info(JSON.stringify({ + before, + after: result + }, null, 2)); + } + + if (options.failOnDrift && !result.ok) { + process.exit(1); + } +} finally { + client.close(); +}