+
-
+
+
+
+
+ {hasWarnings ? (
+
+
+ Parser Warnings
+
+
+ {normalization.warnings.map((warning) => (
+
+ {warning}
+
+ ))}
+
+
+ ) : null}
{hasMaterialUnmapped ? (
diff --git a/components/financials/statement-row-inspector.tsx b/components/financials/statement-row-inspector.tsx
index 82b4d6b..d3d0082 100644
--- a/components/financials/statement-row-inspector.tsx
+++ b/components/financials/statement-row-inspector.tsx
@@ -37,6 +37,9 @@ function renderList(values: string[]) {
export function StatementRowInspector(props: StatementRowInspectorProps) {
const selection = props.selection;
+ const parentSurfaceLabel = selection?.kind === 'detail'
+ ? selection.parentSurfaceRow?.label ?? (selection.row.parentSurfaceKey === 'unmapped' ? 'Unmapped / Residual' : selection.row.parentSurfaceKey)
+ : null;
return (
-
+
diff --git a/e2e/financials.spec.ts b/e2e/financials.spec.ts
index cff1321..86da3e7 100644
--- a/e2e/financials.spec.ts
+++ b/e2e/financials.spec.ts
@@ -242,6 +242,23 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
dimensionsSummary: [],
residualFlag: false
}
+ ],
+ unmapped: [
+ {
+ key: 'other_income_unmapped',
+ parentSurfaceKey: 'unmapped',
+ label: 'Other Income Residual',
+ conceptKey: 'other_income_unmapped',
+ qname: 'us-gaap:OtherNonoperatingIncomeExpense',
+ namespaceUri: 'http://fasb.org/us-gaap/2024',
+ localName: 'OtherNonoperatingIncomeExpense',
+ unit: 'USD',
+ values: { [`${prefix}-fy24`]: 1_200, [`${prefix}-fy25`]: 1_450 },
+ sourceFactIds: [107],
+ isExtension: false,
+ dimensionsSummary: [],
+ residualFlag: true
+ }
]
},
ratioRows: [],
@@ -285,11 +302,16 @@ function buildFinancialsPayload(ticker: 'MSFT' | 'JPM') {
validation: null
},
normalization: {
+ parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: isBank ? 'bank_lender' : 'core',
parserVersion: '0.1.0',
- unmappedRowCount: 0,
- materialUnmappedRowCount: 0
+ surfaceRowCount: 8,
+ detailRowCount: isBank ? 0 : 4,
+ kpiRowCount: 0,
+ unmappedRowCount: isBank ? 0 : 1,
+ materialUnmappedRowCount: 0,
+ warnings: isBank ? [] : ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
},
dimensionBreakdown: null
}
@@ -316,6 +338,11 @@ test('renders the standardized operating expense tree and inspector details', as
await page.goto('/financials?ticker=MSFT');
await expect(page.getByText('Normalization Summary')).toBeVisible();
+ await expect(page.getByText('fiscal-xbrl 0.1.0')).toBeVisible();
+ await expect(page.getByText('Parser Warnings')).toBeVisible();
+ await expect(page.getByText('income_sparse_mapping')).toBeVisible();
+ await expect(page.getByText('unmapped_cash_flow_bridge')).toBeVisible();
+ await expect(page.getByText('Parser residual rows are available under the Unmapped / Residual section.')).toBeVisible();
await expect(page.getByRole('button', { name: 'Expand Operating Expenses details' })).toBeVisible();
await page.getByRole('button', { name: 'Expand Operating Expenses details' }).click();
@@ -327,6 +354,11 @@ test('renders the standardized operating expense tree and inspector details', as
await expect(page.getByText('Row Details')).toBeVisible();
await expect(page.getByText('selling_general_and_administrative', { exact: true }).first()).toBeVisible();
await expect(page.getByText('Corporate SG&A')).toBeVisible();
+ await expect(page.getByText('Unmapped / Residual')).toBeVisible();
+
+ await page.getByRole('button', { name: /^Other Income Residual/ }).click();
+ await expect(page.getByText('other_income_unmapped', { exact: true })).toBeVisible();
+ await expect(page.getByText('Unmapped / Residual', { exact: true }).last()).toBeVisible();
});
test('shows not meaningful expense breakdown rows for bank pack filings', async ({ page }, testInfo) => {
diff --git a/e2e/graphing.spec.ts b/e2e/graphing.spec.ts
index 3890f94..7cdde6c 100644
--- a/e2e/graphing.spec.ts
+++ b/e2e/graphing.spec.ts
@@ -230,11 +230,16 @@ function createFinancialsPayload(input: {
validation: null
},
normalization: {
+ parserEngine: 'fiscal-xbrl',
regime: 'unknown',
fiscalPack,
parserVersion: '0.0.0',
+ surfaceRowCount: 0,
+ detailRowCount: 0,
+ kpiRowCount: 0,
unmappedRowCount: 0,
- materialUnmappedRowCount: 0
+ materialUnmappedRowCount: 0,
+ warnings: []
},
dimensionBreakdown: null
}
diff --git a/lib/financials/statement-view-model.test.ts b/lib/financials/statement-view-model.test.ts
index c09a2fd..2e0dd09 100644
--- a/lib/financials/statement-view-model.test.ts
+++ b/lib/financials/statement-view-model.test.ts
@@ -210,4 +210,86 @@ describe('statement view model', () => {
row: { key: 'gp_unmapped', residualFlag: true }
});
});
+
+ it('renders unmapped detail rows in a dedicated residual section and counts them', () => {
+ const model = buildStatementTree({
+ surfaceKind: 'income_statement',
+ rows: [
+ createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
+ ],
+ statementDetails: {
+ unmapped: [
+ createDetailRow({
+ key: 'unmapped_other_income',
+ label: 'Other income residual',
+ parentSurfaceKey: 'unmapped',
+ values: { p1: 5 },
+ residualFlag: true
+ })
+ ]
+ },
+ categories: [],
+ searchQuery: '',
+ expandedRowKeys: new Set()
+ });
+
+ expect(model.sections).toHaveLength(2);
+ expect(model.sections[1]).toMatchObject({
+ key: 'unmapped_residual',
+ label: 'Unmapped / Residual'
+ });
+ expect(model.sections[1]?.nodes[0]).toMatchObject({
+ kind: 'detail',
+ row: { key: 'unmapped_other_income', parentSurfaceKey: 'unmapped' }
+ });
+ expect(model.visibleNodeCount).toBe(2);
+ expect(model.totalNodeCount).toBe(2);
+ });
+
+ it('matches search and resolves selection for unmapped detail rows without a real parent surface', () => {
+ const rows = [
+ createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } })
+ ];
+ const statementDetails = {
+ unmapped: [
+ createDetailRow({
+ key: 'unmapped_fx_gain',
+ label: 'FX gain residual',
+ parentSurfaceKey: 'unmapped',
+ values: { p1: 2 },
+ residualFlag: true
+ })
+ ]
+ };
+
+ const model = buildStatementTree({
+ surfaceKind: 'income_statement',
+ rows,
+ statementDetails,
+ categories: [],
+ searchQuery: 'fx gain',
+ expandedRowKeys: new Set()
+ });
+
+ expect(model.sections).toHaveLength(1);
+ expect(model.sections[0]).toMatchObject({
+ key: 'unmapped_residual',
+ label: 'Unmapped / Residual'
+ });
+ expect(model.visibleNodeCount).toBe(1);
+ expect(model.totalNodeCount).toBe(2);
+
+ const selection = resolveStatementSelection({
+ surfaceKind: 'income_statement',
+ rows,
+ statementDetails,
+ selection: { kind: 'detail', key: 'unmapped_fx_gain', parentKey: 'unmapped' }
+ });
+
+ expect(selection).toMatchObject({
+ kind: 'detail',
+ row: { key: 'unmapped_fx_gain', parentSurfaceKey: 'unmapped' },
+ parentSurfaceRow: null
+ });
+ });
});
diff --git a/lib/financials/statement-view-model.ts b/lib/financials/statement-view-model.ts
index a6aa96b..b253f21 100644
--- a/lib/financials/statement-view-model.ts
+++ b/lib/financials/statement-view-model.ts
@@ -79,6 +79,10 @@ type Categories = Array<{
count: number;
}>;
+const UNMAPPED_DETAIL_GROUP_KEY = 'unmapped';
+const UNMAPPED_SECTION_KEY = 'unmapped_residual';
+const UNMAPPED_SECTION_LABEL = 'Unmapped / Residual';
+
function surfaceConfigForKind(surfaceKind: Extract
) {
return SURFACE_CHILDREN[surfaceKind] ?? {};
}
@@ -129,6 +133,25 @@ function sortDetailRows(left: DetailFinancialRow, right: DetailFinancialRow) {
return left.label.localeCompare(right.label);
}
+function buildUnmappedDetailNodes(input: {
+ statementDetails: SurfaceDetailMap | null;
+ searchQuery: string;
+}) {
+ const normalizedSearch = normalize(input.searchQuery);
+
+ return [...(input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? [])]
+ .sort(sortDetailRows)
+ .filter((detail) => normalizedSearch.length === 0 || searchTextForDetail(detail).includes(normalizedSearch))
+ .map((detail) => ({
+ kind: 'detail',
+ id: detailNodeId(UNMAPPED_DETAIL_GROUP_KEY, detail),
+ level: 0,
+ row: detail,
+ parentSurfaceKey: UNMAPPED_DETAIL_GROUP_KEY,
+ matchesSearch: normalizedSearch.length > 0 && searchTextForDetail(detail).includes(normalizedSearch)
+ }) satisfies StatementTreeDetailNode);
+}
+
function countNodes(nodes: StatementTreeNode[]) {
let count = 0;
@@ -223,10 +246,26 @@ export function buildStatementTree(input: {
.filter((node): node is StatementTreeSurfaceNode => Boolean(node));
if (input.categories.length === 0) {
+ const sections: StatementTreeSection[] = rootNodes.length > 0
+ ? [{ key: 'ungrouped', label: null, nodes: rootNodes }]
+ : [];
+ const unmappedNodes = buildUnmappedDetailNodes({
+ statementDetails: input.statementDetails,
+ searchQuery: input.searchQuery
+ });
+
+ if (unmappedNodes.length > 0) {
+ sections.push({
+ key: UNMAPPED_SECTION_KEY,
+ label: UNMAPPED_SECTION_LABEL,
+ nodes: unmappedNodes
+ });
+ }
+
return {
- sections: [{ key: 'ungrouped', label: null, nodes: rootNodes }],
+ sections,
autoExpandedKeys,
- visibleNodeCount: countNodes(rootNodes),
+ visibleNodeCount: sections.reduce((sum, section) => sum + countNodes(section.nodes), 0),
totalNodeCount: input.rows.length + Object.values(input.statementDetails ?? {}).reduce((sum, rows) => sum + rows.length, 0)
};
}
@@ -254,6 +293,18 @@ export function buildStatementTree(input: {
});
}
+ const unmappedNodes = buildUnmappedDetailNodes({
+ statementDetails: input.statementDetails,
+ searchQuery: input.searchQuery
+ });
+ if (unmappedNodes.length > 0) {
+ sections.push({
+ key: UNMAPPED_SECTION_KEY,
+ label: UNMAPPED_SECTION_LABEL,
+ nodes: unmappedNodes
+ });
+ }
+
return {
sections,
autoExpandedKeys,
@@ -297,9 +348,11 @@ export function resolveStatementSelection(input: {
}
const parentSurfaceKey = selection.parentKey ?? null;
- const detailRows = parentSurfaceKey
- ? input.statementDetails?.[parentSurfaceKey] ?? []
- : Object.values(input.statementDetails ?? {}).flat();
+ const detailRows = parentSurfaceKey === UNMAPPED_DETAIL_GROUP_KEY
+ ? input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY] ?? []
+ : parentSurfaceKey
+ ? input.statementDetails?.[parentSurfaceKey] ?? []
+ : Object.values(input.statementDetails ?? {}).flat();
const row = detailRows.find((candidate) => candidate.key === selection.key) ?? null;
if (!row) {
diff --git a/lib/graphing/series.test.ts b/lib/graphing/series.test.ts
index 435538a..1e24d09 100644
--- a/lib/graphing/series.test.ts
+++ b/lib/graphing/series.test.ts
@@ -104,11 +104,16 @@ function createFinancials(input: {
validation: null
},
normalization: {
+ parserEngine: 'fiscal-xbrl',
regime: 'unknown',
fiscalPack: input.fiscalPack ?? null,
parserVersion: '0.0.0',
+ surfaceRowCount: 0,
+ detailRowCount: 0,
+ kpiRowCount: 0,
unmappedRowCount: 0,
- materialUnmappedRowCount: 0
+ materialUnmappedRowCount: 0,
+ warnings: []
},
dimensionBreakdown: null
} satisfies CompanyFinancialStatementsResponse;
diff --git a/lib/server/api/app.ts b/lib/server/api/app.ts
index c78ee82..f333526 100644
--- a/lib/server/api/app.ts
+++ b/lib/server/api/app.ts
@@ -18,6 +18,7 @@ import type {
} from '@/lib/types';
import { auth } from '@/lib/auth';
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
+import { getLatestFinancialIngestionSchemaStatus } from '@/lib/server/db/financial-ingestion-schema';
import { asErrorMessage, jsonError } from '@/lib/server/http';
import { buildPortfolioSummary } from '@/lib/server/portfolio';
import {
@@ -391,16 +392,36 @@ export const app = new Elysia({ prefix: '/api' })
getTaskQueueSnapshot(),
checkWorkflowBackend()
]);
+ const ingestionSchema = getLatestFinancialIngestionSchemaStatus();
+ const ingestionSchemaPayload = ingestionSchema
+ ? {
+ ok: ingestionSchema.ok,
+ mode: ingestionSchema.mode,
+ missingIndexes: ingestionSchema.missingIndexes,
+ duplicateGroups: ingestionSchema.duplicateGroups,
+ lastCheckedAt: ingestionSchema.lastCheckedAt
+ }
+ : {
+ ok: false,
+ mode: 'failed' as const,
+ missingIndexes: [],
+ duplicateGroups: 0,
+ lastCheckedAt: new Date().toISOString()
+ };
+ const schemaHealthy = ingestionSchema?.ok ?? false;
- if (!workflowBackend.ok) {
+ if (!workflowBackend.ok || !schemaHealthy) {
return Response.json({
status: 'degraded',
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
+ database: {
+ ingestionSchema: ingestionSchemaPayload
+ },
workflow: {
- ok: false,
- reason: workflowBackend.reason
+ ok: workflowBackend.ok,
+ ...(workflowBackend.ok ? {} : { reason: workflowBackend.reason })
}
}, { status: 503 });
}
@@ -410,6 +431,9 @@ export const app = new Elysia({ prefix: '/api' })
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
+ database: {
+ ingestionSchema: ingestionSchemaPayload
+ },
workflow: {
ok: true
}
@@ -1366,12 +1390,13 @@ export const app = new Elysia({ prefix: '/api' })
return jsonError('ticker is required');
}
- const [filings, holding, watchlistItem, liveQuote, priceHistory, journalPreview, memo, secProfile] = await Promise.all([
+ const [filings, holding, watchlistItem, liveQuote, priceHistory, benchmarkHistory, journalPreview, memo, secProfile] = await Promise.all([
listFilingsRecords({ ticker, limit: 40 }),
getHoldingByTicker(session.user.id, ticker),
getWatchlistItemByTicker(session.user.id, ticker),
getQuote(ticker),
getPriceHistory(ticker),
+ getPriceHistory('^GSPC'),
listResearchJournalEntries(session.user.id, ticker, 6),
getResearchMemoByTicker(session.user.id, ticker),
getSecCompanyProfile(ticker)
@@ -1478,6 +1503,7 @@ export const app = new Elysia({ prefix: '/api' })
quote: liveQuote,
position: holding,
priceHistory,
+ benchmarkHistory,
financials,
filings: redactedFilings.slice(0, 20),
aiReports,
diff --git a/lib/server/api/task-workflow-hybrid.e2e.test.ts b/lib/server/api/task-workflow-hybrid.e2e.test.ts
index b655211..03fcc7d 100644
--- a/lib/server/api/task-workflow-hybrid.e2e.test.ts
+++ b/lib/server/api/task-workflow-hybrid.e2e.test.ts
@@ -74,11 +74,44 @@ function resetDbSingletons() {
const globalState = globalThis as typeof globalThis & {
__fiscalSqliteClient?: { close?: () => void };
__fiscalDrizzleDb?: unknown;
+ __financialIngestionSchemaStatus?: unknown;
};
globalState.__fiscalSqliteClient?.close?.();
globalState.__fiscalSqliteClient = undefined;
globalState.__fiscalDrizzleDb = undefined;
+ globalState.__financialIngestionSchemaStatus = undefined;
+}
+
+function setFinancialIngestionSchemaStatus(input: {
+ ok: boolean;
+ mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
+ missingIndexes?: string[];
+ duplicateGroups?: number;
+}) {
+ const globalState = globalThis as typeof globalThis & {
+ __financialIngestionSchemaStatus?: {
+ ok: boolean;
+ mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
+ requestedMode: 'auto' | 'check-only' | 'off';
+ missingIndexes: string[];
+ duplicateGroups: number;
+ lastCheckedAt: string;
+ repair: null;
+ error: string | null;
+ };
+ };
+
+ globalState.__financialIngestionSchemaStatus = {
+ ok: input.ok,
+ mode: input.mode,
+ requestedMode: 'auto',
+ missingIndexes: input.missingIndexes ?? [],
+ duplicateGroups: input.duplicateGroups ?? 0,
+ lastCheckedAt: new Date().toISOString(),
+ repair: null,
+ error: input.ok ? null : 'schema drift injected by test'
+ };
}
function applySqlMigrations(client: { exec: (query: string) => void }) {
@@ -250,6 +283,10 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
runStatuses.clear();
runCounter = 0;
workflowBackendHealthy = true;
+ setFinancialIngestionSchemaStatus({
+ ok: true,
+ mode: 'healthy'
+ });
});
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
@@ -808,9 +845,100 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
const healthy = await jsonRequest('GET', '/api/health');
expect(healthy.response.status).toBe(200);
- expect((healthy.json as { status: string; workflow: { ok: boolean } }).status).toBe('ok');
+ expect((healthy.json as {
+ status: string;
+ workflow: { ok: boolean };
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ missingIndexes: string[];
+ duplicateGroups: number;
+ };
+ };
+ }).status).toBe('ok');
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
+ expect((healthy.json as {
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ missingIndexes: string[];
+ duplicateGroups: number;
+ };
+ };
+ }).database.ingestionSchema.ok).toBe(true);
+ expect((healthy.json as {
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ };
+ };
+ }).database.ingestionSchema.mode).toBe('healthy');
+ setFinancialIngestionSchemaStatus({
+ ok: false,
+ mode: 'drifted',
+ missingIndexes: ['company_financial_bundle_uidx'],
+ duplicateGroups: 1
+ });
+ const schemaDrifted = await jsonRequest('GET', '/api/health');
+ expect(schemaDrifted.response.status).toBe(503);
+ expect((schemaDrifted.json as {
+ status: string;
+ workflow: { ok: boolean };
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ missingIndexes: string[];
+ duplicateGroups: number;
+ };
+ };
+ }).status).toBe('degraded');
+ expect((schemaDrifted.json as {
+ workflow: { ok: boolean };
+ }).workflow.ok).toBe(true);
+ expect((schemaDrifted.json as {
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ missingIndexes: string[];
+ duplicateGroups: number;
+ };
+ };
+ }).database.ingestionSchema.ok).toBe(false);
+ expect((schemaDrifted.json as {
+ database: {
+ ingestionSchema: {
+ ok: boolean;
+ mode: string;
+ missingIndexes: string[];
+ duplicateGroups: number;
+ };
+ };
+ }).database.ingestionSchema.mode).toBe('drifted');
+ expect((schemaDrifted.json as {
+ database: {
+ ingestionSchema: {
+ missingIndexes: string[];
+ };
+ };
+ }).database.ingestionSchema.missingIndexes).toEqual(['company_financial_bundle_uidx']);
+ expect((schemaDrifted.json as {
+ database: {
+ ingestionSchema: {
+ duplicateGroups: number;
+ };
+ };
+ }).database.ingestionSchema.duplicateGroups).toBe(1);
+
+ setFinancialIngestionSchemaStatus({
+ ok: true,
+ mode: 'healthy'
+ });
workflowBackendHealthy = false;
const degraded = await jsonRequest('GET', '/api/health');
expect(degraded.response.status).toBe(503);
diff --git a/lib/server/db/index.ts b/lib/server/db/index.ts
index 1019607..3a0d4c2 100644
--- a/lib/server/db/index.ts
+++ b/lib/server/db/index.ts
@@ -3,6 +3,10 @@ import { dirname, join } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { load as loadSqliteVec } from 'sqlite-vec';
+import {
+ ensureFinancialIngestionSchemaHealthy,
+ resolveFinancialSchemaRepairMode
+} from './financial-ingestion-schema';
import { schema } from './schema';
type AppDrizzleDb = ReturnType;
@@ -564,6 +568,9 @@ export function getSqliteClient() {
client.exec('PRAGMA busy_timeout = 5000;');
loadSqliteExtensions(client);
ensureLocalSqliteSchema(client);
+ ensureFinancialIngestionSchemaHealthy(client, {
+ mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
+ });
ensureSearchVirtualTables(client);
globalThis.__fiscalSqliteClient = client;
diff --git a/lib/server/financial-taxonomy.test.ts b/lib/server/financial-taxonomy.test.ts
index 67e52b8..36d135f 100644
--- a/lib/server/financial-taxonomy.test.ts
+++ b/lib/server/financial-taxonomy.test.ts
@@ -1586,6 +1586,99 @@ describe('financial taxonomy internals', () => {
expect(merged[0]?.provenanceType).toBe('taxonomy');
});
+ it('builds faithful rows when persisted statement rows are missing sourceFactIds', () => {
+ const malformedSnapshot = {
+ ...createSnapshot({
+ filingId: 19,
+ filingType: '10-K',
+ filingDate: '2026-02-20',
+ statement: 'income',
+ periods: [
+ { id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
+ ]
+ }),
+ statement_rows: {
+ income: [{
+ ...createRow({
+ key: 'revenue',
+ label: 'Revenue',
+ statement: 'income',
+ values: { '2025-fy': 123_000_000 }
+ }),
+ sourceFactIds: undefined
+ } as unknown as TaxonomyStatementRow],
+ balance: [],
+ cash_flow: [],
+ equity: [],
+ comprehensive_income: []
+ }
+ } satisfies FilingTaxonomySnapshotRecord;
+
+ const rows = __financialTaxonomyInternals.buildRows(
+ [malformedSnapshot],
+ 'income',
+ new Set(['2025-fy'])
+ );
+
+ expect(rows).toHaveLength(1);
+ expect(rows[0]?.key).toBe('revenue');
+ expect(rows[0]?.sourceFactIds).toEqual([]);
+ });
+
+ it('aggregates persisted surface rows when legacy snapshots are missing source arrays', () => {
+ const snapshot = {
+ ...createSnapshot({
+ filingId: 20,
+ filingType: '10-K',
+ filingDate: '2026-02-21',
+ statement: 'income',
+ periods: [
+ { id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
+ ]
+ }),
+ surface_rows: {
+ income: [{
+ key: 'revenue',
+ label: 'Revenue',
+ category: 'revenue',
+ templateSection: 'statement',
+ order: 10,
+ unit: 'currency',
+ values: { '2025-fy': 123_000_000 },
+ sourceConcepts: undefined,
+ sourceRowKeys: undefined,
+ sourceFactIds: undefined,
+ formulaKey: null,
+ hasDimensions: false,
+ resolvedSourceRowKeys: {},
+ statement: 'income',
+ detailCount: 0,
+ resolutionMethod: 'direct',
+ confidence: 'high',
+ warningCodes: []
+ } as unknown as FilingTaxonomySnapshotRecord['surface_rows']['income'][number]],
+ balance: [],
+ cash_flow: [],
+ equity: [],
+ comprehensive_income: []
+ }
+ } satisfies FilingTaxonomySnapshotRecord;
+
+ const rows = __financialTaxonomyInternals.aggregateSurfaceRows({
+ snapshots: [snapshot],
+ statement: 'income',
+ selectedPeriodIds: new Set(['2025-fy'])
+ });
+
+ expect(rows).toHaveLength(1);
+ expect(rows[0]).toMatchObject({
+ key: 'revenue',
+ sourceConcepts: [],
+ sourceRowKeys: [],
+ sourceFactIds: []
+ });
+ });
+
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
const snapshot = {
...createSnapshot({
@@ -1610,11 +1703,81 @@ describe('financial taxonomy internals', () => {
} satisfies FilingTaxonomySnapshotRecord;
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
+ parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: 'bank_lender',
parserVersion: '0.1.0',
+ surfaceRowCount: 5,
+ detailRowCount: 3,
+ kpiRowCount: 2,
unmappedRowCount: 4,
- materialUnmappedRowCount: 1
+ materialUnmappedRowCount: 1,
+ warnings: []
+ });
+ });
+
+ it('aggregates normalization counts and warning codes across snapshots while using the latest parser identity', () => {
+ const olderSnapshot = {
+ ...createSnapshot({
+ filingId: 17,
+ filingType: '10-K',
+ filingDate: '2025-02-13',
+ statement: 'income',
+ periods: [
+ { id: '2024-fy', periodStart: '2024-01-01', periodEnd: '2024-12-31', periodLabel: '2024 FY' }
+ ]
+ }),
+ parser_engine: 'fiscal-xbrl',
+ parser_version: '0.9.0',
+ fiscal_pack: 'core',
+ normalization_summary: {
+ surfaceRowCount: 4,
+ detailRowCount: 2,
+ kpiRowCount: 1,
+ unmappedRowCount: 3,
+ materialUnmappedRowCount: 1,
+ warnings: ['balance_residual_detected', 'income_sparse_mapping']
+ }
+ } satisfies FilingTaxonomySnapshotRecord;
+
+ const latestSnapshot = {
+ ...createSnapshot({
+ filingId: 18,
+ filingType: '10-Q',
+ filingDate: '2026-02-13',
+ statement: 'income',
+ periods: [
+ { id: '2025-q4', periodStart: '2025-10-01', periodEnd: '2025-12-31', periodLabel: '2025 Q4' }
+ ]
+ }),
+ parser_engine: 'fiscal-xbrl',
+ parser_version: '1.1.0',
+ fiscal_pack: 'bank_lender',
+ normalization_summary: {
+ surfaceRowCount: 6,
+ detailRowCount: 5,
+ kpiRowCount: 4,
+ unmappedRowCount: 2,
+ materialUnmappedRowCount: 0,
+ warnings: ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
+ }
+ } satisfies FilingTaxonomySnapshotRecord;
+
+ expect(__financialTaxonomyInternals.buildNormalizationMetadata([olderSnapshot, latestSnapshot])).toEqual({
+ parserEngine: 'fiscal-xbrl',
+ regime: 'us-gaap',
+ fiscalPack: 'bank_lender',
+ parserVersion: '1.1.0',
+ surfaceRowCount: 10,
+ detailRowCount: 7,
+ kpiRowCount: 5,
+ unmappedRowCount: 5,
+ materialUnmappedRowCount: 1,
+ warnings: [
+ 'balance_residual_detected',
+ 'income_sparse_mapping',
+ 'unmapped_cash_flow_bridge'
+ ]
});
});
diff --git a/lib/server/financial-taxonomy.ts b/lib/server/financial-taxonomy.ts
index 1840e25..d059cc0 100644
--- a/lib/server/financial-taxonomy.ts
+++ b/lib/server/financial-taxonomy.ts
@@ -177,7 +177,7 @@ function buildKpiDimensionBreakdown(input: {
continue;
}
- const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id));
+ const matchedFacts = input.facts.filter((fact) => (row.sourceFactIds ?? []).includes(fact.id));
if (matchedFacts.length === 0) {
continue;
}
@@ -213,9 +213,9 @@ function latestPeriodDate(period: FinancialStatementPeriod) {
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
return {
...row,
- values: { ...row.values },
- sourceConcepts: [...row.sourceConcepts],
- sourceFactIds: [...row.sourceFactIds]
+ values: { ...(row.values ?? {}) },
+ sourceConcepts: [...(row.sourceConcepts ?? [])],
+ sourceFactIds: [...(row.sourceFactIds ?? [])]
};
}
@@ -230,7 +230,7 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
continue;
}
- for (const [periodId, value] of Object.entries(row.values)) {
+ for (const [periodId, value] of Object.entries(row.values ?? {})) {
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
&& existing.values[periodId] !== null;
if (!hasExistingValue) {
@@ -238,9 +238,9 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
}
}
- existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])]
+ existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...(row.sourceConcepts ?? [])])]
.sort((left, right) => left.localeCompare(right));
- existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])]
+ existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...(row.sourceFactIds ?? [])])]
.sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.segment ??= row.segment;
@@ -260,11 +260,16 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
function emptyNormalizationMetadata(): NormalizationMetadata {
return {
+ parserEngine: 'unknown',
regime: 'unknown',
fiscalPack: null,
parserVersion: '0.0.0',
+ surfaceRowCount: 0,
+ detailRowCount: 0,
+ kpiRowCount: 0,
unmappedRowCount: 0,
- materialUnmappedRowCount: 0
+ materialUnmappedRowCount: 0,
+ warnings: []
};
}
@@ -277,9 +282,22 @@ function buildNormalizationMetadata(
}
return {
+ parserEngine: latestSnapshot.parser_engine,
regime: latestSnapshot.taxonomy_regime,
fiscalPack: latestSnapshot.fiscal_pack,
parserVersion: latestSnapshot.parser_version,
+ surfaceRowCount: snapshots.reduce(
+ (sum, snapshot) => sum + (snapshot.normalization_summary?.surfaceRowCount ?? 0),
+ 0
+ ),
+ detailRowCount: snapshots.reduce(
+ (sum, snapshot) => sum + (snapshot.normalization_summary?.detailRowCount ?? 0),
+ 0
+ ),
+ kpiRowCount: snapshots.reduce(
+ (sum, snapshot) => sum + (snapshot.normalization_summary?.kpiRowCount ?? 0),
+ 0
+ ),
unmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
0
@@ -287,7 +305,9 @@ function buildNormalizationMetadata(
materialUnmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
0
- )
+ ),
+ warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))]
+ .sort((left, right) => left.localeCompare(right))
};
}
@@ -329,8 +349,12 @@ function aggregateSurfaceRows(input: {
for (const snapshot of input.snapshots) {
const rows = snapshot.surface_rows?.[input.statement] ?? [];
for (const row of rows) {
+ const sourceConcepts = row.sourceConcepts ?? [];
+ const sourceRowKeys = row.sourceRowKeys ?? [];
+ const sourceFactIds = row.sourceFactIds ?? [];
+ const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
- Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
+ Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
const filteredResolvedSourceRowKeys = Object.fromEntries(
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
@@ -345,9 +369,9 @@ function aggregateSurfaceRows(input: {
...row,
values: filteredValues,
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
- sourceConcepts: [...row.sourceConcepts],
- sourceRowKeys: [...row.sourceRowKeys],
- sourceFactIds: [...row.sourceFactIds],
+ sourceConcepts: [...sourceConcepts],
+ sourceRowKeys: [...sourceRowKeys],
+ sourceFactIds: [...sourceFactIds],
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
});
continue;
@@ -365,9 +389,9 @@ function aggregateSurfaceRows(input: {
}
}
- existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
- existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...row.sourceRowKeys])].sort((left, right) => left.localeCompare(right));
- existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
+ existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
+ existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...sourceRowKeys])].sort((left, right) => left.localeCompare(right));
+ existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.order = Math.min(existing.order, row.order);
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
@@ -406,8 +430,11 @@ function aggregateDetailRows(input: {
}
for (const row of rows) {
+ const sourceFactIds = row.sourceFactIds ?? [];
+ const dimensionsSummary = row.dimensionsSummary ?? [];
+ const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
- Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
+ Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -418,8 +445,8 @@ function aggregateDetailRows(input: {
bucket.set(row.key, {
...row,
values: filteredValues,
- sourceFactIds: [...row.sourceFactIds],
- dimensionsSummary: [...row.dimensionsSummary]
+ sourceFactIds: [...sourceFactIds],
+ dimensionsSummary: [...dimensionsSummary]
});
continue;
}
@@ -430,8 +457,8 @@ function aggregateDetailRows(input: {
}
}
- existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
- existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...row.dimensionsSummary])].sort((left, right) => left.localeCompare(right));
+ existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
+ existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...dimensionsSummary])].sort((left, right) => left.localeCompare(right));
existing.isExtension = existing.isExtension || row.isExtension;
existing.residualFlag = existing.residualFlag || row.residualFlag;
}
@@ -521,8 +548,11 @@ function aggregatePersistedKpiRows(input: {
for (const snapshot of input.snapshots) {
for (const row of snapshot.kpi_rows ?? []) {
+ const sourceConcepts = row.sourceConcepts ?? [];
+ const sourceFactIds = row.sourceFactIds ?? [];
+ const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
- Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
+ Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -533,8 +563,8 @@ function aggregatePersistedKpiRows(input: {
rowMap.set(row.key, {
...row,
values: filteredValues,
- sourceConcepts: [...row.sourceConcepts],
- sourceFactIds: [...row.sourceFactIds]
+ sourceConcepts: [...sourceConcepts],
+ sourceFactIds: [...sourceFactIds]
});
continue;
}
@@ -543,8 +573,8 @@ function aggregatePersistedKpiRows(input: {
...existing.values,
...filteredValues
};
- existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
- existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
+ existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
+ existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
}
}
diff --git a/lib/server/financials/cadence.ts b/lib/server/financials/cadence.ts
index 1c73c1c..bbf9f08 100644
--- a/lib/server/financials/cadence.ts
+++ b/lib/server/financials/cadence.ts
@@ -282,17 +282,20 @@ export function buildRows(
const rows = snapshot.statement_rows?.[statement] ?? [];
for (const row of rows) {
+ const rowValues = row.values ?? {};
+ const rowUnits = row.units ?? {};
+ const sourceFactIds = row.sourceFactIds ?? [];
const existing = rowMap.get(row.key);
if (!existing) {
rowMap.set(row.key, {
...row,
values: Object.fromEntries(
- Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
+ Object.entries(rowValues).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
units: Object.fromEntries(
- Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
+ Object.entries(rowUnits).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
- sourceFactIds: [...row.sourceFactIds]
+ sourceFactIds: [...sourceFactIds]
});
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
@@ -308,19 +311,19 @@ export function buildRows(
existing.parentKey = row.parentKey;
}
- for (const [periodId, value] of Object.entries(row.values)) {
+ for (const [periodId, value] of Object.entries(rowValues)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
existing.values[periodId] = value;
}
}
- for (const [periodId, unit] of Object.entries(row.units)) {
+ for (const [periodId, unit] of Object.entries(rowUnits)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
existing.units[periodId] = unit;
}
}
- for (const factId of row.sourceFactIds) {
+ for (const factId of sourceFactIds) {
if (!existing.sourceFactIds.includes(factId)) {
existing.sourceFactIds.push(factId);
}
diff --git a/lib/server/prices.ts b/lib/server/prices.ts
index ac8ec69..6232f23 100644
--- a/lib/server/prices.ts
+++ b/lib/server/prices.ts
@@ -1,5 +1,9 @@
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
+function buildYahooChartUrl(ticker: string, params: string) {
+ return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
+}
+
function fallbackQuote(ticker: string) {
const normalized = ticker.trim().toUpperCase();
let hash = 0;
@@ -15,7 +19,7 @@ export async function getQuote(ticker: string): Promise {
const normalizedTicker = ticker.trim().toUpperCase();
try {
- const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
+ const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -47,7 +51,7 @@ export async function getQuoteOrNull(ticker: string): Promise {
const normalizedTicker = ticker.trim().toUpperCase();
try {
- const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
+ const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -87,7 +91,7 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
}
try {
- const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, {
+ const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=20y'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -143,7 +147,7 @@ export async function getPriceHistory(ticker: string): Promise {
- const step = 25 - index;
- const date = new Date(now - step * 14 * 24 * 60 * 60 * 1000).toISOString();
- const wave = Math.sin(index / 3.5) * 0.05;
- const trend = (index - 13) * 0.006;
+ const totalWeeks = 20 * 52;
+
+ return Array.from({ length: totalWeeks }, (_, index) => {
+ const step = (totalWeeks - 1) - index;
+ const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
+ const wave = Math.sin(index / 8) * 0.06;
+ const trend = (index - totalWeeks / 2) * 0.0009;
const close = Math.max(base * (1 + wave + trend), 1);
return {
diff --git a/lib/server/repos/company-financial-bundles.ts b/lib/server/repos/company-financial-bundles.ts
index d93f3b9..fa07a19 100644
--- a/lib/server/repos/company-financial-bundles.ts
+++ b/lib/server/repos/company-financial-bundles.ts
@@ -3,7 +3,8 @@ import type {
FinancialCadence,
FinancialSurfaceKind
} from '@/lib/types';
-import { db } from '@/lib/server/db';
+import { db, getSqliteClient } from '@/lib/server/db';
+import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import { companyFinancialBundle } from '@/lib/server/db/schema';
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
@@ -64,34 +65,38 @@ export async function upsertCompanyFinancialBundle(input: {
}) {
const now = new Date().toISOString();
- const [saved] = await db
- .insert(companyFinancialBundle)
- .values({
- ticker: input.ticker.trim().toUpperCase(),
- surface_kind: input.surfaceKind,
- cadence: input.cadence,
- bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
- source_snapshot_ids: input.sourceSnapshotIds,
- source_signature: input.sourceSignature,
- payload: input.payload,
- created_at: now,
- updated_at: now
- })
- .onConflictDoUpdate({
- target: [
- companyFinancialBundle.ticker,
- companyFinancialBundle.surface_kind,
- companyFinancialBundle.cadence
- ],
- set: {
+ const [saved] = await withFinancialIngestionSchemaRetry({
+ client: getSqliteClient(),
+ context: 'company-financial-bundle-upsert',
+ operation: async () => await db
+ .insert(companyFinancialBundle)
+ .values({
+ ticker: input.ticker.trim().toUpperCase(),
+ surface_kind: input.surfaceKind,
+ cadence: input.cadence,
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
+ created_at: now,
updated_at: now
- }
- })
- .returning();
+ })
+ .onConflictDoUpdate({
+ target: [
+ companyFinancialBundle.ticker,
+ companyFinancialBundle.surface_kind,
+ companyFinancialBundle.cadence
+ ],
+ set: {
+ bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
+ source_snapshot_ids: input.sourceSnapshotIds,
+ source_signature: input.sourceSignature,
+ payload: input.payload,
+ updated_at: now
+ }
+ })
+ .returning()
+ });
return toBundleRecord(saved);
}
diff --git a/lib/server/repos/filing-taxonomy.ts b/lib/server/repos/filing-taxonomy.ts
index c7e6d63..b52ab77 100644
--- a/lib/server/repos/filing-taxonomy.ts
+++ b/lib/server/repos/filing-taxonomy.ts
@@ -11,7 +11,8 @@ import type {
TaxonomyFactRow,
TaxonomyStatementRow
} from '@/lib/types';
-import { db } from '@/lib/server/db';
+import { db, getSqliteClient } from '@/lib/server/db';
+import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import {
filingTaxonomyAsset,
filingTaxonomyConcept,
@@ -552,38 +553,13 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
const now = new Date().toISOString();
- const [saved] = await db
- .insert(filingTaxonomySnapshot)
- .values({
- filing_id: input.filing_id,
- ticker: input.ticker,
- filing_date: input.filing_date,
- filing_type: input.filing_type,
- parse_status: input.parse_status,
- parse_error: input.parse_error,
- source: input.source,
- parser_engine: input.parser_engine,
- parser_version: input.parser_version,
- taxonomy_regime: input.taxonomy_regime,
- fiscal_pack: input.fiscal_pack,
- periods: input.periods,
- faithful_rows: input.faithful_rows,
- statement_rows: input.statement_rows,
- surface_rows: input.surface_rows,
- detail_rows: input.detail_rows,
- kpi_rows: input.kpi_rows,
- derived_metrics: input.derived_metrics,
- validation_result: input.validation_result,
- normalization_summary: input.normalization_summary,
- facts_count: input.facts_count,
- concepts_count: input.concepts_count,
- dimensions_count: input.dimensions_count,
- created_at: now,
- updated_at: now
- })
- .onConflictDoUpdate({
- target: filingTaxonomySnapshot.filing_id,
- set: {
+ const [saved] = await withFinancialIngestionSchemaRetry({
+ client: getSqliteClient(),
+ context: 'filing-taxonomy-snapshot-upsert',
+ operation: async () => await db
+ .insert(filingTaxonomySnapshot)
+ .values({
+ filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
@@ -606,10 +582,39 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
+ created_at: now,
updated_at: now
- }
- })
- .returning();
+ })
+ .onConflictDoUpdate({
+ target: filingTaxonomySnapshot.filing_id,
+ set: {
+ ticker: input.ticker,
+ filing_date: input.filing_date,
+ filing_type: input.filing_type,
+ parse_status: input.parse_status,
+ parse_error: input.parse_error,
+ source: input.source,
+ parser_engine: input.parser_engine,
+ parser_version: input.parser_version,
+ taxonomy_regime: input.taxonomy_regime,
+ fiscal_pack: input.fiscal_pack,
+ periods: input.periods,
+ faithful_rows: input.faithful_rows,
+ statement_rows: input.statement_rows,
+ surface_rows: input.surface_rows,
+ detail_rows: input.detail_rows,
+ kpi_rows: input.kpi_rows,
+ derived_metrics: input.derived_metrics,
+ validation_result: input.validation_result,
+ normalization_summary: input.normalization_summary,
+ facts_count: input.facts_count,
+ concepts_count: input.concepts_count,
+ dimensions_count: input.dimensions_count,
+ updated_at: now
+ }
+ })
+ .returning()
+ });
const snapshotId = saved.id;
diff --git a/lib/types.ts b/lib/types.ts
index d7428a2..717e8b1 100644
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -498,11 +498,16 @@ export type NormalizationSummary = {
};
export type NormalizationMetadata = {
+ parserEngine: string;
regime: 'us-gaap' | 'ifrs-full' | 'unknown';
fiscalPack: string | null;
parserVersion: string;
+ surfaceRowCount: number;
+ detailRowCount: number;
+ kpiRowCount: number;
unmappedRowCount: number;
materialUnmappedRowCount: number;
+ warnings: string[];
};
export type RatioRow = DerivedFinancialRow & {
@@ -741,6 +746,7 @@ export type CompanyAnalysis = {
quote: number;
position: Holding | null;
priceHistory: Array<{ date: string; close: number }>;
+ benchmarkHistory: Array<{ date: string; close: number }>;
financials: CompanyFinancialPoint[];
filings: Filing[];
aiReports: CompanyAiReport[];
@@ -788,3 +794,88 @@ export type ActiveContext = {
pathname: string;
activeTicker: string | null;
};
+
+// Chart Types
+export type ChartType = 'line' | 'combination';
+export type TimeRange = '1W' | '1M' | '3M' | '1Y' | '3Y' | '5Y' | '10Y' | '20Y';
+
+// Chart Data Formats
+export type PriceDataPoint = {
+ date: string;
+ price: number;
+};
+
+export type OHLCVDataPoint = {
+ date: string;
+ open: number;
+ high: number;
+ low: number;
+ close: number;
+ volume: number;
+};
+
+export type ChartDataPoint = PriceDataPoint | OHLCVDataPoint;
+
+export type DataSeries = {
+ id: string;
+ label: string;
+ data: T[];
+ color?: string;
+ type?: 'line' | 'area' | 'bar';
+ visible?: boolean;
+};
+
+// Chart Configuration
+export type ChartZoomState = {
+ startIndex: number;
+ endIndex: number;
+ isZoomed: boolean;
+};
+
+export type ChartColorPalette = {
+ primary: string;
+ secondary: string;
+ positive: string;
+ negative: string;
+ grid: string;
+ text: string;
+ muted: string;
+ tooltipBg: string;
+ tooltipBorder: string;
+ volume: string;
+};
+
+export type InteractivePriceChartProps = {
+ // Data
+ data: ChartDataPoint[];
+ dataSeries?: DataSeries[];
+
+ // Configuration
+ defaultChartType?: ChartType;
+ defaultTimeRange?: TimeRange;
+ showVolume?: boolean;
+ showToolbar?: boolean;
+ height?: number;
+
+ // Customization
+ colors?: Partial;
+ formatters?: {
+ price?: (value: number) => string;
+ date?: (value: string) => string;
+ volume?: (value: number) => string;
+ };
+
+ // Event handlers
+ onChartTypeChange?: (type: ChartType) => void;
+ onTimeRangeChange?: (range: TimeRange) => void;
+ onDataPointHover?: (point: ChartDataPoint | null) => void;
+ onZoomChange?: (state: ChartZoomState) => void;
+
+ // State
+ loading?: boolean;
+ error?: string | null;
+
+ // Accessibility
+ ariaLabel?: string;
+ description?: string;
+};
diff --git a/next-env.d.ts b/next-env.d.ts
index 9edff1c..c4b7818 100644
--- a/next-env.d.ts
+++ b/next-env.d.ts
@@ -1,6 +1,6 @@
///
///
-import "./.next/types/routes.d.ts";
+import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
diff --git a/package.json b/package.json
index a017507..4b4b224 100644
--- a/package.json
+++ b/package.json
@@ -21,6 +21,7 @@
"backfill:search-index": "bun run scripts/backfill-search-index.ts",
"backfill:taxonomy-snapshots": "bun run scripts/backfill-taxonomy-snapshots.ts",
"compare:fiscal-ai": "bun run scripts/compare-fiscal-ai-statements.ts",
+ "repair:financial-ingestion-schema": "bun run scripts/repair-financial-ingestion-schema.ts",
"report:taxonomy-health": "bun run scripts/report-taxonomy-health.ts",
"db:generate": "bun x drizzle-kit generate",
"db:migrate": "bun x drizzle-kit migrate",
@@ -43,6 +44,7 @@
"date-fns": "^4.1.0",
"drizzle-orm": "^0.41.0",
"elysia": "latest",
+ "html-to-image": "^1.11.13",
"lucide-react": "^0.575.0",
"next": "^16.1.6",
"react": "^19.2.4",
diff --git a/scripts/bootstrap-production.ts b/scripts/bootstrap-production.ts
index fdfb0ec..3a102ee 100644
--- a/scripts/bootstrap-production.ts
+++ b/scripts/bootstrap-production.ts
@@ -4,6 +4,10 @@ import { dirname } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
+import {
+ ensureFinancialIngestionSchemaHealthy,
+ resolveFinancialSchemaRepairMode
+} from '../lib/server/db/financial-ingestion-schema';
import { resolveSqlitePath } from './dev-env';
function trim(value: string | undefined) {
@@ -74,6 +78,13 @@ function runDatabaseMigrations() {
try {
client.exec('PRAGMA foreign_keys = ON;');
migrate(drizzle(client), { migrationsFolder: './drizzle' });
+
+ const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
+ mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
+ });
+ if (!repairResult.ok) {
+ throw new Error(repairResult.error ?? `financial ingestion schema is ${repairResult.mode}`);
+ }
} finally {
client.close();
}
diff --git a/scripts/dev.ts b/scripts/dev.ts
index a65a937..b87e1be 100644
--- a/scripts/dev.ts
+++ b/scripts/dev.ts
@@ -3,6 +3,10 @@ import { mkdirSync, readFileSync } from 'node:fs';
import { Database } from 'bun:sqlite';
import { createServer } from 'node:net';
import { dirname, join } from 'node:path';
+import {
+ ensureFinancialIngestionSchemaHealthy,
+ resolveFinancialSchemaRepairMode
+} from '../lib/server/db/financial-ingestion-schema';
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
type DrizzleJournal = {
@@ -126,6 +130,33 @@ mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? '.workflow-data', { recursive: true });
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
+if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
+ const client = new Database(databasePath, { create: true });
+
+ try {
+ client.exec('PRAGMA foreign_keys = ON;');
+ const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
+ mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
+ });
+
+ if (repairResult.mode === 'repaired') {
+ console.info(
+ `[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(', ') || 'none'}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? 'yes' : 'no'})`
+ );
+ } else if (repairResult.mode === 'drifted') {
+ console.warn(
+ `[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(', ') || 'none'}; duplicate groups: ${repairResult.duplicateGroups})`
+ );
+ } else if (repairResult.mode === 'failed') {
+ console.warn(
+ `[dev] financial ingestion schema repair failed: ${repairResult.error ?? 'unknown error'}`
+ );
+ }
+ } finally {
+ client.close();
+ }
+}
+
console.info(`[dev] local origin ${config.publicOrigin}`);
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);