import { mkdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { Database } from 'bun:sqlite'; import { drizzle } from 'drizzle-orm/bun-sqlite'; import { load as loadSqliteVec } from 'sqlite-vec'; import { schema } from './schema'; type AppDrizzleDb = ReturnType; declare global { // eslint-disable-next-line no-var var __fiscalSqliteClient: Database | undefined; // eslint-disable-next-line no-var var __fiscalDrizzleDb: AppDrizzleDb | undefined; } function getDatabasePath() { const raw = process.env.DATABASE_URL?.trim() || 'file:data/fiscal.sqlite'; let databasePath = raw.startsWith('file:') ? raw.slice(5) : raw; if (databasePath.startsWith('///')) { databasePath = databasePath.slice(2); } if (!databasePath) { throw new Error('DATABASE_URL must point to a SQLite file path.'); } return databasePath; } function hasTable(client: Database, tableName: string) { const row = client .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') .get('table', tableName) as { name: string } | null; return row !== null; } function hasColumn(client: Database, tableName: string, columnName: string) { if (!hasTable(client, tableName)) { return false; } const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; return rows.some((row) => row.name === columnName); } function applySqlFile(client: Database, fileName: string) { const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); client.exec(sql); } let customSqliteConfigured = false; const vectorExtensionStatus = new WeakMap(); function configureCustomSqliteRuntime() { if (customSqliteConfigured) { return; } const customSqlitePath = process.env.SQLITE_CUSTOM_LIB_PATH?.trim(); if (process.platform === 'darwin' && customSqlitePath) { Database.setCustomSQLite(customSqlitePath); } customSqliteConfigured = true; } function loadSqliteExtensions(client: Database) { try { const customVectorExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH?.trim(); if (customVectorExtensionPath) { client.loadExtension(customVectorExtensionPath); } else { loadSqliteVec(client); } vectorExtensionStatus.set(client, true); } catch (error) { vectorExtensionStatus.set(client, false); const reason = error instanceof Error ? error.message : 'Unknown sqlite extension error'; console.warn(`[sqlite] sqlite-vec unavailable, falling back to table-backed vector storage: ${reason}`); } } function isVectorExtensionLoaded(client: Database) { return vectorExtensionStatus.get(client) ?? false; } function ensureLocalSqliteSchema(client: Database) { if (!hasTable(client, 'filing_statement_snapshot')) { applySqlFile(client, '0001_glossy_statement_snapshots.sql'); } if (hasTable(client, 'task_run')) { const missingTaskColumns: Array<{ name: string; sql: string }> = [ { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" }, { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' }, { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' }, { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' }, { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' } ]; for (const column of missingTaskColumns) { if (!hasColumn(client, 'task_run', column.name)) { client.exec(column.sql); } } } if (!hasTable(client, 'task_stage_event')) { applySqlFile(client, '0003_task_stage_event_timeline.sql'); } if (hasTable(client, 'watchlist_item')) { const missingWatchlistColumns: Array<{ name: string; sql: string }> = [ { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' }, { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' }, { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" }, { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" }, { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" }, { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' } ]; for (const column of missingWatchlistColumns) { if (!hasColumn(client, 'watchlist_item', column.name)) { client.exec(column.sql); } } client.exec(` UPDATE \`watchlist_item\` SET \`status\` = CASE WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog' ELSE \`status\` END, \`priority\` = CASE WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium' ELSE \`priority\` END, \`updated_at\` = CASE WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP) ELSE \`updated_at\` END; `); client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);'); } if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) { client.exec('ALTER TABLE `holding` ADD `company_name` text;'); } if (!hasTable(client, 'filing_taxonomy_snapshot')) { applySqlFile(client, '0005_financial_taxonomy_v3.sql'); } if (!hasTable(client, 'company_financial_bundle')) { applySqlFile(client, '0007_company_financial_bundles.sql'); } if (!hasTable(client, 'research_journal_entry')) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_journal_entry\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, \`user_id\` text NOT NULL, \`ticker\` text NOT NULL, \`accession_number\` text, \`entry_type\` text NOT NULL, \`title\` text, \`body_markdown\` text NOT NULL, \`metadata\` text, \`created_at\` text NOT NULL, \`updated_at\` text NOT NULL, FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade ); `); client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);'); client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);'); } if (!hasTable(client, 'search_document')) { applySqlFile(client, '0008_search_rag.sql'); } } function ensureSearchVirtualTables(client: Database) { client.exec(` CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_fts\` USING fts5( \`chunk_text\`, \`citation_label\`, \`heading_path\`, \`chunk_id\` UNINDEXED, \`document_id\` UNINDEXED, \`chunk_index\` UNINDEXED, \`scope\` UNINDEXED, \`user_id\` UNINDEXED, \`source_kind\` UNINDEXED, \`ticker\` UNINDEXED, \`accession_number\` UNINDEXED, \`filing_date\` UNINDEXED ); `); if (isVectorExtensionLoaded(client)) { client.exec(` CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_vec\` USING vec0( \`chunk_id\` integer PRIMARY KEY, \`embedding\` float[256], \`scope\` text, \`user_id\` text, \`source_kind\` text, \`ticker\` text, \`accession_number\` text, \`filing_date\` text, +\`document_id\` integer, +\`chunk_index\` integer, +\`citation_label\` text ); `); return; } client.exec(` CREATE TABLE IF NOT EXISTS \`search_chunk_vec\` ( \`chunk_id\` integer PRIMARY KEY NOT NULL, \`embedding\` text NOT NULL, \`scope\` text NOT NULL, \`user_id\` text, \`source_kind\` text NOT NULL, \`ticker\` text, \`accession_number\` text, \`filing_date\` text, \`document_id\` integer NOT NULL, \`chunk_index\` integer NOT NULL, \`citation_label\` text NOT NULL ); `); client.exec('CREATE INDEX IF NOT EXISTS `search_chunk_vec_lookup_idx` ON `search_chunk_vec` (`scope`, `user_id`, `source_kind`, `ticker`);'); } export function getSqliteClient() { if (!globalThis.__fiscalSqliteClient) { configureCustomSqliteRuntime(); const databasePath = getDatabasePath(); if (databasePath !== ':memory:') { mkdirSync(dirname(databasePath), { recursive: true }); } const client = new Database(databasePath, { create: true }); client.exec('PRAGMA foreign_keys = ON;'); client.exec('PRAGMA journal_mode = WAL;'); client.exec('PRAGMA busy_timeout = 5000;'); loadSqliteExtensions(client); ensureLocalSqliteSchema(client); ensureSearchVirtualTables(client); globalThis.__fiscalSqliteClient = client; } return globalThis.__fiscalSqliteClient; } function createDb() { return drizzle(getSqliteClient(), { schema }); } export const db = globalThis.__fiscalDrizzleDb ?? createDb(); if (!globalThis.__fiscalDrizzleDb) { globalThis.__fiscalDrizzleDb = db; } export const __dbInternals = { configureCustomSqliteRuntime, ensureLocalSqliteSchema, ensureSearchVirtualTables, getDatabasePath, hasColumn, hasTable, isVectorExtensionLoaded, loadSqliteExtensions };