Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

WIP main worktree changes before merge

Browse Source
This commit is contained in:
francy51
2026-03-13 00:20:22 -04:00
parent 58bf80189d
commit e5141238fb
25 changed files with 940 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/server/api/app.ts
View File

@@ -18,6 +18,7 @@ import type {
} from '@/lib/types';
import { auth } from '@/lib/auth';
import { requireAuthenticatedSession } from '@/lib/server/auth-session';
import { getLatestFinancialIngestionSchemaStatus } from '@/lib/server/db/financial-ingestion-schema';
import { asErrorMessage, jsonError } from '@/lib/server/http';
import { buildPortfolioSummary } from '@/lib/server/portfolio';
import {
@@ -391,16 +392,36 @@ export const app = new Elysia({ prefix: '/api' })
getTaskQueueSnapshot(),
checkWorkflowBackend()
]);
const ingestionSchema = getLatestFinancialIngestionSchemaStatus();
const ingestionSchemaPayload = ingestionSchema
? {
ok: ingestionSchema.ok,
mode: ingestionSchema.mode,
missingIndexes: ingestionSchema.missingIndexes,
duplicateGroups: ingestionSchema.duplicateGroups,
lastCheckedAt: ingestionSchema.lastCheckedAt
}
: {
ok: false,
mode: 'failed' as const,
missingIndexes: [],
duplicateGroups: 0,
lastCheckedAt: new Date().toISOString()
};
const schemaHealthy = ingestionSchema?.ok ?? false;
if (!workflowBackend.ok) {
if (!workflowBackend.ok || !schemaHealthy) {
return Response.json({
status: 'degraded',
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
database: {
ingestionSchema: ingestionSchemaPayload
},
workflow: {
ok: false,
reason: workflowBackend.reason
ok: workflowBackend.ok,
...(workflowBackend.ok ? {} : { reason: workflowBackend.reason })
}
}, { status: 503 });
}
@@ -410,6 +431,9 @@ export const app = new Elysia({ prefix: '/api' })
version: '4.0.0',
timestamp: new Date().toISOString(),
queue,
database: {
ingestionSchema: ingestionSchemaPayload
},
workflow: {
ok: true
}
@@ -1366,12 +1390,13 @@ export const app = new Elysia({ prefix: '/api' })
return jsonError('ticker is required');
}
const [filings, holding, watchlistItem, liveQuote, priceHistory, journalPreview, memo, secProfile] = await Promise.all([
const [filings, holding, watchlistItem, liveQuote, priceHistory, benchmarkHistory, journalPreview, memo, secProfile] = await Promise.all([
listFilingsRecords({ ticker, limit: 40 }),
getHoldingByTicker(session.user.id, ticker),
getWatchlistItemByTicker(session.user.id, ticker),
getQuote(ticker),
getPriceHistory(ticker),
getPriceHistory('^GSPC'),
listResearchJournalEntries(session.user.id, ticker, 6),
getResearchMemoByTicker(session.user.id, ticker),
getSecCompanyProfile(ticker)
@@ -1478,6 +1503,7 @@ export const app = new Elysia({ prefix: '/api' })
quote: liveQuote,
position: holding,
priceHistory,
benchmarkHistory,
financials,
filings: redactedFilings.slice(0, 20),
aiReports,

130
lib/server/api/task-workflow-hybrid.e2e.test.ts
View File

@@ -74,11 +74,44 @@ function resetDbSingletons() {
const globalState = globalThis as typeof globalThis & {
__fiscalSqliteClient?: { close?: () => void };
__fiscalDrizzleDb?: unknown;
__financialIngestionSchemaStatus?: unknown;
};
globalState.__fiscalSqliteClient?.close?.();
globalState.__fiscalSqliteClient = undefined;
globalState.__fiscalDrizzleDb = undefined;
globalState.__financialIngestionSchemaStatus = undefined;
}
function setFinancialIngestionSchemaStatus(input: {
ok: boolean;
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
missingIndexes?: string[];
duplicateGroups?: number;
}) {
const globalState = globalThis as typeof globalThis & {
__financialIngestionSchemaStatus?: {
ok: boolean;
mode: 'healthy' | 'repaired' | 'drifted' | 'failed';
requestedMode: 'auto' | 'check-only' | 'off';
missingIndexes: string[];
duplicateGroups: number;
lastCheckedAt: string;
repair: null;
error: string | null;
};
};
globalState.__financialIngestionSchemaStatus = {
ok: input.ok,
mode: input.mode,
requestedMode: 'auto',
missingIndexes: input.missingIndexes ?? [],
duplicateGroups: input.duplicateGroups ?? 0,
lastCheckedAt: new Date().toISOString(),
repair: null,
error: input.ok ? null : 'schema drift injected by test'
};
}
function applySqlMigrations(client: { exec: (query: string) => void }) {
@@ -250,6 +283,10 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
runStatuses.clear();
runCounter = 0;
workflowBackendHealthy = true;
setFinancialIngestionSchemaStatus({
ok: true,
mode: 'healthy'
});
});
it('queues multiple analyze jobs and suppresses duplicate in-flight analyze jobs', async () => {
@@ -808,9 +845,100 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') {
const healthy = await jsonRequest('GET', '/api/health');
expect(healthy.response.status).toBe(200);
expect((healthy.json as { status: string; workflow: { ok: boolean } }).status).toBe('ok');
expect((healthy.json as {
status: string;
workflow: { ok: boolean };
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).status).toBe('ok');
expect((healthy.json as { status: string; workflow: { ok: boolean } }).workflow.ok).toBe(true);
expect((healthy.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.ok).toBe(true);
expect((healthy.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
};
};
}).database.ingestionSchema.mode).toBe('healthy');
setFinancialIngestionSchemaStatus({
ok: false,
mode: 'drifted',
missingIndexes: ['company_financial_bundle_uidx'],
duplicateGroups: 1
});
const schemaDrifted = await jsonRequest('GET', '/api/health');
expect(schemaDrifted.response.status).toBe(503);
expect((schemaDrifted.json as {
status: string;
workflow: { ok: boolean };
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).status).toBe('degraded');
expect((schemaDrifted.json as {
workflow: { ok: boolean };
}).workflow.ok).toBe(true);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.ok).toBe(false);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
ok: boolean;
mode: string;
missingIndexes: string[];
duplicateGroups: number;
};
};
}).database.ingestionSchema.mode).toBe('drifted');
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
missingIndexes: string[];
};
};
}).database.ingestionSchema.missingIndexes).toEqual(['company_financial_bundle_uidx']);
expect((schemaDrifted.json as {
database: {
ingestionSchema: {
duplicateGroups: number;
};
};
}).database.ingestionSchema.duplicateGroups).toBe(1);
setFinancialIngestionSchemaStatus({
ok: true,
mode: 'healthy'
});
workflowBackendHealthy = false;
const degraded = await jsonRequest('GET', '/api/health');
expect(degraded.response.status).toBe(503);

7
lib/server/db/index.ts
View File

@@ -3,6 +3,10 @@ import { dirname, join } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { load as loadSqliteVec } from 'sqlite-vec';
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from './financial-ingestion-schema';
import { schema } from './schema';
type AppDrizzleDb = ReturnType<typeof createDb>;
@@ -564,6 +568,9 @@ export function getSqliteClient() {
client.exec('PRAGMA busy_timeout = 5000;');
loadSqliteExtensions(client);
ensureLocalSqliteSchema(client);
ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
});
ensureSearchVirtualTables(client);
globalThis.__fiscalSqliteClient = client;

165
lib/server/financial-taxonomy.test.ts
View File

@@ -1586,6 +1586,99 @@ describe('financial taxonomy internals', () => {
expect(merged[0]?.provenanceType).toBe('taxonomy');
});
it('builds faithful rows when persisted statement rows are missing sourceFactIds', () => {
const malformedSnapshot = {
...createSnapshot({
filingId: 19,
filingType: '10-K',
filingDate: '2026-02-20',
statement: 'income',
periods: [
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
]
}),
statement_rows: {
income: [{
...createRow({
key: 'revenue',
label: 'Revenue',
statement: 'income',
values: { '2025-fy': 123_000_000 }
}),
sourceFactIds: undefined
} as unknown as TaxonomyStatementRow],
balance: [],
cash_flow: [],
equity: [],
comprehensive_income: []
}
} satisfies FilingTaxonomySnapshotRecord;
const rows = __financialTaxonomyInternals.buildRows(
[malformedSnapshot],
'income',
new Set(['2025-fy'])
);
expect(rows).toHaveLength(1);
expect(rows[0]?.key).toBe('revenue');
expect(rows[0]?.sourceFactIds).toEqual([]);
});
it('aggregates persisted surface rows when legacy snapshots are missing source arrays', () => {
const snapshot = {
...createSnapshot({
filingId: 20,
filingType: '10-K',
filingDate: '2026-02-21',
statement: 'income',
periods: [
{ id: '2025-fy', periodStart: '2025-01-01', periodEnd: '2025-12-31', periodLabel: '2025 FY' }
]
}),
surface_rows: {
income: [{
key: 'revenue',
label: 'Revenue',
category: 'revenue',
templateSection: 'statement',
order: 10,
unit: 'currency',
values: { '2025-fy': 123_000_000 },
sourceConcepts: undefined,
sourceRowKeys: undefined,
sourceFactIds: undefined,
formulaKey: null,
hasDimensions: false,
resolvedSourceRowKeys: {},
statement: 'income',
detailCount: 0,
resolutionMethod: 'direct',
confidence: 'high',
warningCodes: []
} as unknown as FilingTaxonomySnapshotRecord['surface_rows']['income'][number]],
balance: [],
cash_flow: [],
equity: [],
comprehensive_income: []
}
} satisfies FilingTaxonomySnapshotRecord;
const rows = __financialTaxonomyInternals.aggregateSurfaceRows({
snapshots: [snapshot],
statement: 'income',
selectedPeriodIds: new Set(['2025-fy'])
});
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({
key: 'revenue',
sourceConcepts: [],
sourceRowKeys: [],
sourceFactIds: []
});
});
it('builds normalization metadata from snapshot fiscal pack and counts', () => {
const snapshot = {
...createSnapshot({
@@ -1610,11 +1703,81 @@ describe('financial taxonomy internals', () => {
} satisfies FilingTaxonomySnapshotRecord;
expect(__financialTaxonomyInternals.buildNormalizationMetadata([snapshot])).toEqual({
parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: 'bank_lender',
parserVersion: '0.1.0',
surfaceRowCount: 5,
detailRowCount: 3,
kpiRowCount: 2,
unmappedRowCount: 4,
materialUnmappedRowCount: 1
materialUnmappedRowCount: 1,
warnings: []
});
});
it('aggregates normalization counts and warning codes across snapshots while using the latest parser identity', () => {
const olderSnapshot = {
...createSnapshot({
filingId: 17,
filingType: '10-K',
filingDate: '2025-02-13',
statement: 'income',
periods: [
{ id: '2024-fy', periodStart: '2024-01-01', periodEnd: '2024-12-31', periodLabel: '2024 FY' }
]
}),
parser_engine: 'fiscal-xbrl',
parser_version: '0.9.0',
fiscal_pack: 'core',
normalization_summary: {
surfaceRowCount: 4,
detailRowCount: 2,
kpiRowCount: 1,
unmappedRowCount: 3,
materialUnmappedRowCount: 1,
warnings: ['balance_residual_detected', 'income_sparse_mapping']
}
} satisfies FilingTaxonomySnapshotRecord;
const latestSnapshot = {
...createSnapshot({
filingId: 18,
filingType: '10-Q',
filingDate: '2026-02-13',
statement: 'income',
periods: [
{ id: '2025-q4', periodStart: '2025-10-01', periodEnd: '2025-12-31', periodLabel: '2025 Q4' }
]
}),
parser_engine: 'fiscal-xbrl',
parser_version: '1.1.0',
fiscal_pack: 'bank_lender',
normalization_summary: {
surfaceRowCount: 6,
detailRowCount: 5,
kpiRowCount: 4,
unmappedRowCount: 2,
materialUnmappedRowCount: 0,
warnings: ['income_sparse_mapping', 'unmapped_cash_flow_bridge']
}
} satisfies FilingTaxonomySnapshotRecord;
expect(__financialTaxonomyInternals.buildNormalizationMetadata([olderSnapshot, latestSnapshot])).toEqual({
parserEngine: 'fiscal-xbrl',
regime: 'us-gaap',
fiscalPack: 'bank_lender',
parserVersion: '1.1.0',
surfaceRowCount: 10,
detailRowCount: 7,
kpiRowCount: 5,
unmappedRowCount: 5,
materialUnmappedRowCount: 1,
warnings: [
'balance_residual_detected',
'income_sparse_mapping',
'unmapped_cash_flow_bridge'
]
});
});

82
lib/server/financial-taxonomy.ts
View File

@@ -177,7 +177,7 @@ function buildKpiDimensionBreakdown(input: {
continue;
}
const matchedFacts = input.facts.filter((fact) => row.sourceFactIds.includes(fact.id));
const matchedFacts = input.facts.filter((fact) => (row.sourceFactIds ?? []).includes(fact.id));
if (matchedFacts.length === 0) {
continue;
}
@@ -213,9 +213,9 @@ function latestPeriodDate(period: FinancialStatementPeriod) {
function cloneStructuredKpiRow(row: StructuredKpiRow): StructuredKpiRow {
return {
...row,
values: { ...row.values },
sourceConcepts: [...row.sourceConcepts],
sourceFactIds: [...row.sourceFactIds]
values: { ...(row.values ?? {}) },
sourceConcepts: [...(row.sourceConcepts ?? [])],
sourceFactIds: [...(row.sourceFactIds ?? [])]
};
}
@@ -230,7 +230,7 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
continue;
}
for (const [periodId, value] of Object.entries(row.values)) {
for (const [periodId, value] of Object.entries(row.values ?? {})) {
const hasExistingValue = Object.prototype.hasOwnProperty.call(existing.values, periodId)
&& existing.values[periodId] !== null;
if (!hasExistingValue) {
@@ -238,9 +238,9 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
}
}
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])]
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...(row.sourceConcepts ?? [])])]
.sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])]
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...(row.sourceFactIds ?? [])])]
.sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.segment ??= row.segment;
@@ -260,11 +260,16 @@ function mergeStructuredKpiRowsByPriority(groups: StructuredKpiRow[][]) {
function emptyNormalizationMetadata(): NormalizationMetadata {
return {
parserEngine: 'unknown',
regime: 'unknown',
fiscalPack: null,
parserVersion: '0.0.0',
surfaceRowCount: 0,
detailRowCount: 0,
kpiRowCount: 0,
unmappedRowCount: 0,
materialUnmappedRowCount: 0
materialUnmappedRowCount: 0,
warnings: []
};
}
@@ -277,9 +282,22 @@ function buildNormalizationMetadata(
}
return {
parserEngine: latestSnapshot.parser_engine,
regime: latestSnapshot.taxonomy_regime,
fiscalPack: latestSnapshot.fiscal_pack,
parserVersion: latestSnapshot.parser_version,
surfaceRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.surfaceRowCount ?? 0),
0
),
detailRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.detailRowCount ?? 0),
0
),
kpiRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.kpiRowCount ?? 0),
0
),
unmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.unmappedRowCount ?? 0),
0
@@ -287,7 +305,9 @@ function buildNormalizationMetadata(
materialUnmappedRowCount: snapshots.reduce(
(sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0),
0
)
),
warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))]
.sort((left, right) => left.localeCompare(right))
};
}
@@ -329,8 +349,12 @@ function aggregateSurfaceRows(input: {
for (const snapshot of input.snapshots) {
const rows = snapshot.surface_rows?.[input.statement] ?? [];
for (const row of rows) {
const sourceConcepts = row.sourceConcepts ?? [];
const sourceRowKeys = row.sourceRowKeys ?? [];
const sourceFactIds = row.sourceFactIds ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
const filteredResolvedSourceRowKeys = Object.fromEntries(
Object.entries(row.resolvedSourceRowKeys ?? {}).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
@@ -345,9 +369,9 @@ function aggregateSurfaceRows(input: {
...row,
values: filteredValues,
resolvedSourceRowKeys: filteredResolvedSourceRowKeys,
sourceConcepts: [...row.sourceConcepts],
sourceRowKeys: [...row.sourceRowKeys],
sourceFactIds: [...row.sourceFactIds],
sourceConcepts: [...sourceConcepts],
sourceRowKeys: [...sourceRowKeys],
sourceFactIds: [...sourceFactIds],
warningCodes: row.warningCodes ? [...row.warningCodes] : undefined
});
continue;
@@ -365,9 +389,9 @@ function aggregateSurfaceRows(input: {
}
}
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...row.sourceRowKeys])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceRowKeys = [...new Set([...existing.sourceRowKeys, ...sourceRowKeys])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
existing.order = Math.min(existing.order, row.order);
existing.detailCount = Math.max(existing.detailCount ?? 0, row.detailCount ?? 0);
@@ -406,8 +430,11 @@ function aggregateDetailRows(input: {
}
for (const row of rows) {
const sourceFactIds = row.sourceFactIds ?? [];
const dimensionsSummary = row.dimensionsSummary ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -418,8 +445,8 @@ function aggregateDetailRows(input: {
bucket.set(row.key, {
...row,
values: filteredValues,
sourceFactIds: [...row.sourceFactIds],
dimensionsSummary: [...row.dimensionsSummary]
sourceFactIds: [...sourceFactIds],
dimensionsSummary: [...dimensionsSummary]
});
continue;
}
@@ -430,8 +457,8 @@ function aggregateDetailRows(input: {
}
}
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...row.dimensionsSummary])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.dimensionsSummary = [...new Set([...existing.dimensionsSummary, ...dimensionsSummary])].sort((left, right) => left.localeCompare(right));
existing.isExtension = existing.isExtension || row.isExtension;
existing.residualFlag = existing.residualFlag || row.residualFlag;
}
@@ -521,8 +548,11 @@ function aggregatePersistedKpiRows(input: {
for (const snapshot of input.snapshots) {
for (const row of snapshot.kpi_rows ?? []) {
const sourceConcepts = row.sourceConcepts ?? [];
const sourceFactIds = row.sourceFactIds ?? [];
const rowValues = row.values ?? {};
const filteredValues = Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => input.selectedPeriodIds.has(periodId))
);
if (!rowHasValues(filteredValues)) {
continue;
@@ -533,8 +563,8 @@ function aggregatePersistedKpiRows(input: {
rowMap.set(row.key, {
...row,
values: filteredValues,
sourceConcepts: [...row.sourceConcepts],
sourceFactIds: [...row.sourceFactIds]
sourceConcepts: [...sourceConcepts],
sourceFactIds: [...sourceFactIds]
});
continue;
}
@@ -543,8 +573,8 @@ function aggregatePersistedKpiRows(input: {
...existing.values,
...filteredValues
};
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...row.sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...row.sourceFactIds])].sort((left, right) => left - right);
existing.sourceConcepts = [...new Set([...existing.sourceConcepts, ...sourceConcepts])].sort((left, right) => left.localeCompare(right));
existing.sourceFactIds = [...new Set([...existing.sourceFactIds, ...sourceFactIds])].sort((left, right) => left - right);
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
}
}

15
lib/server/financials/cadence.ts
View File

@@ -282,17 +282,20 @@ export function buildRows(
const rows = snapshot.statement_rows?.[statement] ?? [];
for (const row of rows) {
const rowValues = row.values ?? {};
const rowUnits = row.units ?? {};
const sourceFactIds = row.sourceFactIds ?? [];
const existing = rowMap.get(row.key);
if (!existing) {
rowMap.set(row.key, {
...row,
values: Object.fromEntries(
Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
Object.entries(rowValues).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
units: Object.fromEntries(
Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
Object.entries(rowUnits).filter(([periodId]) => selectedPeriodIds.has(periodId))
),
sourceFactIds: [...row.sourceFactIds]
sourceFactIds: [...sourceFactIds]
});
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
@@ -308,19 +311,19 @@ export function buildRows(
existing.parentKey = row.parentKey;
}
for (const [periodId, value] of Object.entries(row.values)) {
for (const [periodId, value] of Object.entries(rowValues)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
existing.values[periodId] = value;
}
}
for (const [periodId, unit] of Object.entries(row.units)) {
for (const [periodId, unit] of Object.entries(rowUnits)) {
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
existing.units[periodId] = unit;
}
}
for (const factId of row.sourceFactIds) {
for (const factId of sourceFactIds) {
if (!existing.sourceFactIds.includes(factId)) {
existing.sourceFactIds.push(factId);
}

24
lib/server/prices.ts
View File

@@ -1,5 +1,9 @@
const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
function buildYahooChartUrl(ticker: string, params: string) {
return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
}
function fallbackQuote(ticker: string) {
const normalized = ticker.trim().toUpperCase();
let hash = 0;
@@ -15,7 +19,7 @@ export async function getQuote(ticker: string): Promise<number> {
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -47,7 +51,7 @@ export async function getQuoteOrNull(ticker: string): Promise<number | null> {
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=1d`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -87,7 +91,7 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
}
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1d&range=10y`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=20y'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -143,7 +147,7 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
const normalizedTicker = ticker.trim().toUpperCase();
try {
const response = await fetch(`${YAHOO_BASE}/${normalizedTicker}?interval=1wk&range=1y`, {
const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1wk&range=20y'), {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
},
@@ -195,11 +199,13 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
const now = Date.now();
const base = fallbackQuote(normalizedTicker);
return Array.from({ length: 26 }, (_, index) => {
const step = 25 - index;
const date = new Date(now - step * 14 * 24 * 60 * 60 * 1000).toISOString();
const wave = Math.sin(index / 3.5) * 0.05;
const trend = (index - 13) * 0.006;
const totalWeeks = 20 * 52;
return Array.from({ length: totalWeeks }, (_, index) => {
const step = (totalWeeks - 1) - index;
const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
const wave = Math.sin(index / 8) * 0.06;
const trend = (index - totalWeeks / 2) * 0.0009;
const close = Math.max(base * (1 + wave + trend), 1);
return {

53
lib/server/repos/company-financial-bundles.ts
View File

@@ -3,7 +3,8 @@ import type {
FinancialCadence,
FinancialSurfaceKind
} from '@/lib/types';
import { db } from '@/lib/server/db';
import { db, getSqliteClient } from '@/lib/server/db';
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import { companyFinancialBundle } from '@/lib/server/db/schema';
export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;
@@ -64,34 +65,38 @@ export async function upsertCompanyFinancialBundle(input: {
}) {
const now = new Date().toISOString();
const [saved] = await db
.insert(companyFinancialBundle)
.values({
ticker: input.ticker.trim().toUpperCase(),
surface_kind: input.surfaceKind,
cadence: input.cadence,
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
created_at: now,
updated_at: now
})
.onConflictDoUpdate({
target: [
companyFinancialBundle.ticker,
companyFinancialBundle.surface_kind,
companyFinancialBundle.cadence
],
set: {
const [saved] = await withFinancialIngestionSchemaRetry({
client: getSqliteClient(),
context: 'company-financial-bundle-upsert',
operation: async () => await db
.insert(companyFinancialBundle)
.values({
ticker: input.ticker.trim().toUpperCase(),
surface_kind: input.surfaceKind,
cadence: input.cadence,
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
created_at: now,
updated_at: now
}
})
.returning();
})
.onConflictDoUpdate({
target: [
companyFinancialBundle.ticker,
companyFinancialBundle.surface_kind,
companyFinancialBundle.cadence
],
set: {
bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
source_snapshot_ids: input.sourceSnapshotIds,
source_signature: input.sourceSignature,
payload: input.payload,
updated_at: now
}
})
.returning()
});
return toBundleRecord(saved);
}

77
lib/server/repos/filing-taxonomy.ts
View File

@@ -11,7 +11,8 @@ import type {
TaxonomyFactRow,
TaxonomyStatementRow
} from '@/lib/types';
import { db } from '@/lib/server/db';
import { db, getSqliteClient } from '@/lib/server/db';
import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema';
import {
filingTaxonomyAsset,
filingTaxonomyConcept,
@@ -552,38 +553,13 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
const now = new Date().toISOString();
const [saved] = await db
.insert(filingTaxonomySnapshot)
.values({
filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
parser_engine: input.parser_engine,
parser_version: input.parser_version,
taxonomy_regime: input.taxonomy_regime,
fiscal_pack: input.fiscal_pack,
periods: input.periods,
faithful_rows: input.faithful_rows,
statement_rows: input.statement_rows,
surface_rows: input.surface_rows,
detail_rows: input.detail_rows,
kpi_rows: input.kpi_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
normalization_summary: input.normalization_summary,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
created_at: now,
updated_at: now
})
.onConflictDoUpdate({
target: filingTaxonomySnapshot.filing_id,
set: {
const [saved] = await withFinancialIngestionSchemaRetry({
client: getSqliteClient(),
context: 'filing-taxonomy-snapshot-upsert',
operation: async () => await db
.insert(filingTaxonomySnapshot)
.values({
filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
@@ -606,10 +582,39 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
created_at: now,
updated_at: now
}
})
.returning();
})
.onConflictDoUpdate({
target: filingTaxonomySnapshot.filing_id,
set: {
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
parser_engine: input.parser_engine,
parser_version: input.parser_version,
taxonomy_regime: input.taxonomy_regime,
fiscal_pack: input.fiscal_pack,
periods: input.periods,
faithful_rows: input.faithful_rows,
statement_rows: input.statement_rows,
surface_rows: input.surface_rows,
detail_rows: input.detail_rows,
kpi_rows: input.kpi_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
normalization_summary: input.normalization_summary,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
updated_at: now
}
})
.returning()
});
const snapshotId = saved.id;