Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Add history window controls and expand taxonomy pack support

Browse Source 
- add 3Y/5Y/10Y financial history filtering and reorganize normalization details UI
- add new fiscal taxonomy surface/income bridge/KPI packs and update Rust taxonomy loading
- auto-detect Homebrew SQLite for native `sqlite-vec` in local dev/e2e with docs and env guidance
This commit is contained in:
francy51
2026-03-18 23:40:28 -04:00
parent f8426c4dde
commit 17de3dd72d
102 changed files with 14978 additions and 1316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
lib/server/db/index.ts
View File

@@ -1,19 +1,19 @@
import { mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { load as loadSqliteVec } from 'sqlite-vec';
import { mkdirSync } from "node:fs";
import { dirname } from "node:path";
import { Database } from "bun:sqlite";
import { drizzle } from "drizzle-orm/bun-sqlite";
import { load as loadSqliteVec } from "sqlite-vec";
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from './financial-ingestion-schema';
import { schema } from './schema';
resolveFinancialSchemaRepairMode,
} from "./financial-ingestion-schema";
import { schema } from "./schema";
import {
ensureLocalSqliteSchema,
hasColumn,
hasTable,
TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS
} from './sqlite-schema-compat';
TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS,
} from "./sqlite-schema-compat";
type AppDrizzleDb = ReturnType<typeof createDb>;
@@ -25,15 +25,15 @@ declare global {
}
function getDatabasePath() {
const raw = process.env.DATABASE_URL?.trim() || 'file:data/fiscal.sqlite';
let databasePath = raw.startsWith('file:') ? raw.slice(5) : raw;
const raw = process.env.DATABASE_URL?.trim() || "file:data/fiscal.sqlite";
let databasePath = raw.startsWith("file:") ? raw.slice(5) : raw;
if (databasePath.startsWith('///')) {
if (databasePath.startsWith("///")) {
databasePath = databasePath.slice(2);
}
if (!databasePath) {
throw new Error('DATABASE_URL must point to a SQLite file path.');
throw new Error("DATABASE_URL must point to a SQLite file path.");
}
return databasePath;
@@ -48,7 +48,7 @@ function configureCustomSqliteRuntime() {
}
const customSqlitePath = process.env.SQLITE_CUSTOM_LIB_PATH?.trim();
if (process.platform === 'darwin' && customSqlitePath) {
if (process.platform === "darwin" && customSqlitePath) {
Database.setCustomSQLite(customSqlitePath);
}
@@ -56,9 +56,11 @@ function configureCustomSqliteRuntime() {
}
function loadSqliteExtensions(client: Database) {
try {
const customVectorExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH?.trim();
const customVectorExtensionPath =
process.env.SQLITE_VEC_EXTENSION_PATH?.trim();
const customSqlitePath = process.env.SQLITE_CUSTOM_LIB_PATH?.trim();
try {
if (customVectorExtensionPath) {
client.loadExtension(customVectorExtensionPath);
} else {
@@ -69,8 +71,18 @@ function loadSqliteExtensions(client: Database) {
} catch (error) {
vectorExtensionStatus.set(client, false);
const reason = error instanceof Error ? error.message : 'Unknown sqlite extension error';
console.warn(`[sqlite] sqlite-vec unavailable, falling back to table-backed vector storage: ${reason}`);
const reason =
error instanceof Error ? error.message : "Unknown sqlite extension error";
if (customSqlitePath || customVectorExtensionPath) {
console.warn(
`[sqlite] sqlite-vec native extension load failed (SQLITE_CUSTOM_LIB_PATH=${customSqlitePath ?? "unset"}, SQLITE_VEC_EXTENSION_PATH=${customVectorExtensionPath ?? "package default"}). Falling back to table-backed vector storage: ${reason}`,
);
return;
}
console.warn(
`[sqlite] sqlite-vec unavailable, falling back to table-backed vector storage: ${reason}`,
);
}
}
@@ -98,18 +110,18 @@ function ensureSearchVirtualTables(client: Database) {
if (isVectorExtensionLoaded(client)) {
client.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_vec\` USING vec0(
\`chunk_id\` integer PRIMARY KEY,
\`embedding\` float[256],
\`scope\` text,
\`user_id\` text,
\`source_kind\` text,
\`ticker\` text,
\`accession_number\` text,
\`filing_date\` text,
+\`document_id\` integer,
+\`chunk_index\` integer,
+\`citation_label\` text
CREATE VIRTUAL TABLE IF NOT EXISTS search_chunk_vec USING vec0(
chunk_id integer PRIMARY KEY,
embedding float[256],
scope text,
user_id text,
source_kind text,
ticker text,
accession_number text,
filing_date text,
+document_id integer,
+chunk_index integer,
+citation_label text
);
`);
return;
@@ -130,17 +142,19 @@ function ensureSearchVirtualTables(client: Database) {
\`citation_label\` text NOT NULL
);
`);
client.exec('CREATE INDEX IF NOT EXISTS `search_chunk_vec_lookup_idx` ON `search_chunk_vec` (`scope`, `user_id`, `source_kind`, `ticker`);');
client.exec(
"CREATE INDEX IF NOT EXISTS `search_chunk_vec_lookup_idx` ON `search_chunk_vec` (`scope`, `user_id`, `source_kind`, `ticker`);",
);
}
function verifyCriticalSchema(client: Database) {
if (!hasTable(client, 'filing_taxonomy_snapshot')) {
if (!hasTable(client, "filing_taxonomy_snapshot")) {
return;
}
const missingColumns: string[] = [];
for (const columnName of TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS) {
if (!hasColumn(client, 'filing_taxonomy_snapshot', columnName)) {
if (!hasColumn(client, "filing_taxonomy_snapshot", columnName)) {
missingColumns.push(columnName);
}
}
@@ -148,8 +162,8 @@ function verifyCriticalSchema(client: Database) {
if (missingColumns.length > 0) {
throw new Error(
`[db] CRITICAL: Database schema is incompatible. ` +
`filing_taxonomy_snapshot is missing columns: ${missingColumns.join(', ')}. ` +
`Delete the database file and restart to rebuild schema.`
`filing_taxonomy_snapshot is missing columns: ${missingColumns.join(", ")}. ` +
`Delete the database file and restart to rebuild schema.`,
);
}
}
@@ -159,19 +173,21 @@ export function getSqliteClient() {
configureCustomSqliteRuntime();
const databasePath = getDatabasePath();
if (databasePath !== ':memory:') {
if (databasePath !== ":memory:") {
mkdirSync(dirname(databasePath), { recursive: true });
}
const client = new Database(databasePath, { create: true });
client.exec('PRAGMA foreign_keys = ON;');
client.exec('PRAGMA journal_mode = WAL;');
client.exec('PRAGMA busy_timeout = 5000;');
client.exec("PRAGMA foreign_keys = ON;");
client.exec("PRAGMA journal_mode = WAL;");
client.exec("PRAGMA busy_timeout = 5000;");
loadSqliteExtensions(client);
ensureLocalSqliteSchema(client);
verifyCriticalSchema(client);
ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(process.env.FINANCIAL_SCHEMA_REPAIR_MODE)
mode: resolveFinancialSchemaRepairMode(
process.env.FINANCIAL_SCHEMA_REPAIR_MODE,
),
});
ensureSearchVirtualTables(client);
@@ -200,5 +216,5 @@ export const __dbInternals = {
hasTable,
isVectorExtensionLoaded,
loadSqliteExtensions,
verifyCriticalSchema
verifyCriticalSchema,
};