diff --git a/lib/server/db/index.test.ts b/lib/server/db/index.test.ts index 1983ae8..32fd6ea 100644 --- a/lib/server/db/index.test.ts +++ b/lib/server/db/index.test.ts @@ -38,7 +38,34 @@ describe('sqlite schema compatibility bootstrap', () => { expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true); expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true); expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true); expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true); expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true); @@ -57,4 +84,119 @@ describe('sqlite schema compatibility bootstrap', () => { client.close(); }); + + it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => { + const client = new Database(':memory:'); + client.exec('PRAGMA foreign_keys = ON;'); + + applyMigration(client, '0000_cold_silver_centurion.sql'); + applyMigration(client, '0005_financial_taxonomy_v3.sql'); + + client.exec(` + INSERT INTO \`filing\` ( + \`ticker\`, + \`filing_type\`, + \`filing_date\`, + \`accession_number\`, + \`cik\`, + \`company_name\`, + \`created_at\`, + \`updated_at\` + ) VALUES ( + 'AAPL', + '10-K', + '2024-09-28', + '0000320193-24-000001', + '0000320193', + 'Apple Inc.', + '2024-10-30T00:00:00.000Z', + '2024-10-30T00:00:00.000Z' + ); + `); + + const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; + + client.exec(` + INSERT INTO \`filing_taxonomy_snapshot\` ( + \`filing_id\`, + \`ticker\`, + \`filing_date\`, + \`filing_type\`, + \`parse_status\`, + \`source\`, + \`statement_rows\`, + \`created_at\`, + \`updated_at\` + ) VALUES ( + 1, + 'AAPL', + '2024-09-28', + '10-K', + 'success', + 'xbrl_instance', + '${statementRows}', + '2024-10-30T00:00:00.000Z', + '2024-10-30T00:00:00.000Z' + ); + `); + + __dbInternals.ensureLocalSqliteSchema(client); + __dbInternals.ensureLocalSqliteSchema(client); + + const row = client.query(` + SELECT + \`parser_engine\`, + \`parser_version\`, + \`taxonomy_regime\`, + \`faithful_rows\`, + \`surface_rows\`, + \`detail_rows\`, + \`kpi_rows\`, + \`normalization_summary\` + FROM \`filing_taxonomy_snapshot\` + WHERE \`filing_id\` = 1 + `).get() as { + parser_engine: string; + parser_version: string; + taxonomy_regime: string; + faithful_rows: string | null; + surface_rows: string | null; + detail_rows: string | null; + kpi_rows: string | null; + normalization_summary: string | null; + }; + + expect(row.parser_engine).toBe('fiscal-xbrl'); + expect(row.parser_version).toBe('unknown'); + expect(row.taxonomy_regime).toBe('unknown'); + expect(row.faithful_rows).toBe(statementRows); + expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'); + expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'); + expect(row.kpi_rows).toBe('[]'); + expect(row.normalization_summary).toBeNull(); + + client.close(); + }); + + it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => { + const client = new Database(':memory:'); + client.exec('PRAGMA foreign_keys = ON;'); + + applyMigration(client, '0000_cold_silver_centurion.sql'); + applyMigration(client, '0005_financial_taxonomy_v3.sql'); + client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';"); + + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false); + + __dbInternals.ensureLocalSqliteSchema(client); + + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); + expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); + expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); + + client.close(); + }); }); diff --git a/lib/server/db/index.ts b/lib/server/db/index.ts index 597ba76..f88ab89 100644 --- a/lib/server/db/index.ts +++ b/lib/server/db/index.ts @@ -1,5 +1,5 @@ -import { mkdirSync, readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import { mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; import { Database } from 'bun:sqlite'; import { drizzle } from 'drizzle-orm/bun-sqlite'; import { load as loadSqliteVec } from 'sqlite-vec'; @@ -8,6 +8,11 @@ import { resolveFinancialSchemaRepairMode } from './financial-ingestion-schema'; import { schema } from './schema'; +import { + ensureLocalSqliteSchema, + hasColumn, + hasTable +} from './sqlite-schema-compat'; type AppDrizzleDb = ReturnType; @@ -33,37 +38,6 @@ function getDatabasePath() { return databasePath; } -function hasTable(client: Database, tableName: string) { - const row = client - .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') - .get('table', tableName) as { name: string } | null; - - return row !== null; -} - -function hasColumn(client: Database, tableName: string, columnName: string) { - if (!hasTable(client, tableName)) { - return false; - } - - const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; - return rows.some((row) => row.name === columnName); -} - -function applySqlFile(client: Database, fileName: string) { - const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); - client.exec(sql); -} - -function applyBaseSchemaCompat(client: Database) { - const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8') - .replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `') - .replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `') - .replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `'); - - client.exec(sql); -} - let customSqliteConfigured = false; const vectorExtensionStatus = new WeakMap(); @@ -103,405 +77,6 @@ function isVectorExtensionLoaded(client: Database) { return vectorExtensionStatus.get(client) ?? false; } -function ensureResearchWorkspaceSchema(client: Database) { - if (!hasTable(client, 'research_artifact')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`research_artifact\` ( - \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - \`user_id\` text NOT NULL, - \`organization_id\` text, - \`ticker\` text NOT NULL, - \`accession_number\` text, - \`kind\` text NOT NULL, - \`source\` text NOT NULL DEFAULT 'user', - \`subtype\` text, - \`title\` text, - \`summary\` text, - \`body_markdown\` text, - \`search_text\` text, - \`visibility_scope\` text NOT NULL DEFAULT 'private', - \`tags\` text, - \`metadata\` text, - \`file_name\` text, - \`mime_type\` text, - \`file_size_bytes\` integer, - \`storage_path\` text, - \`created_at\` text NOT NULL, - \`updated_at\` text NOT NULL, - FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade, - FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null - ); - `); - } - - if (!hasTable(client, 'research_memo')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`research_memo\` ( - \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - \`user_id\` text NOT NULL, - \`organization_id\` text, - \`ticker\` text NOT NULL, - \`rating\` text, - \`conviction\` text, - \`time_horizon_months\` integer, - \`packet_title\` text, - \`packet_subtitle\` text, - \`thesis_markdown\` text NOT NULL DEFAULT '', - \`variant_view_markdown\` text NOT NULL DEFAULT '', - \`catalysts_markdown\` text NOT NULL DEFAULT '', - \`risks_markdown\` text NOT NULL DEFAULT '', - \`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '', - \`next_actions_markdown\` text NOT NULL DEFAULT '', - \`created_at\` text NOT NULL, - \`updated_at\` text NOT NULL, - FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade, - FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null - ); - `); - } - - if (!hasTable(client, 'research_memo_evidence')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` ( - \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - \`memo_id\` integer NOT NULL, - \`artifact_id\` integer NOT NULL, - \`section\` text NOT NULL, - \`annotation\` text, - \`sort_order\` integer NOT NULL DEFAULT 0, - \`created_at\` text NOT NULL, - FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade, - FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade - ); - `); - } - - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);'); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);'); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);'); - client.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5( - artifact_id UNINDEXED, - user_id UNINDEXED, - ticker UNINDEXED, - title, - summary, - body_markdown, - search_text, - tags_text - ); - `); - - client.exec(` - INSERT INTO \`research_artifact\` ( - \`user_id\`, - \`organization_id\`, - \`ticker\`, - \`accession_number\`, - \`kind\`, - \`source\`, - \`subtype\`, - \`title\`, - \`summary\`, - \`body_markdown\`, - \`search_text\`, - \`visibility_scope\`, - \`tags\`, - \`metadata\`, - \`created_at\`, - \`updated_at\` - ) - SELECT - r.\`user_id\`, - NULL, - r.\`ticker\`, - r.\`accession_number\`, - CASE - WHEN r.\`entry_type\` = 'status_change' THEN 'status_change' - ELSE 'note' - END, - CASE - WHEN r.\`entry_type\` = 'status_change' THEN 'system' - ELSE 'user' - END, - r.\`entry_type\`, - r.\`title\`, - CASE - WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL - ELSE SUBSTR(r.\`body_markdown\`, 1, 280) - END, - r.\`body_markdown\`, - r.\`body_markdown\`, - 'private', - NULL, - r.\`metadata\`, - r.\`created_at\`, - r.\`updated_at\` - FROM \`research_journal_entry\` r - WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry') - AND NOT EXISTS ( - SELECT 1 - FROM \`research_artifact\` a - WHERE a.\`user_id\` = r.\`user_id\` - AND a.\`ticker\` = r.\`ticker\` - AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '') - AND a.\`kind\` = CASE - WHEN r.\`entry_type\` = 'status_change' THEN 'status_change' - ELSE 'note' - END - AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '') - AND a.\`created_at\` = r.\`created_at\` - ); - `); - - client.exec(` - INSERT INTO \`research_artifact\` ( - \`user_id\`, - \`organization_id\`, - \`ticker\`, - \`accession_number\`, - \`kind\`, - \`source\`, - \`subtype\`, - \`title\`, - \`summary\`, - \`body_markdown\`, - \`search_text\`, - \`visibility_scope\`, - \`tags\`, - \`metadata\`, - \`created_at\`, - \`updated_at\` - ) - SELECT - w.\`user_id\`, - NULL, - f.\`ticker\`, - f.\`accession_number\`, - 'ai_report', - 'system', - 'filing_analysis', - f.\`filing_type\` || ' AI memo', - COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), - 'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) || - 'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) || - COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), - COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), - 'private', - NULL, - json_object( - 'provider', json_extract(f.\`analysis\`, '$.provider'), - 'model', json_extract(f.\`analysis\`, '$.model'), - 'filingType', f.\`filing_type\`, - 'filingDate', f.\`filing_date\` - ), - f.\`created_at\`, - f.\`updated_at\` - FROM \`filing\` f - JOIN \`watchlist_item\` w - ON w.\`ticker\` = f.\`ticker\` - WHERE f.\`analysis\` IS NOT NULL - AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> '' - AND NOT EXISTS ( - SELECT 1 - FROM \`research_artifact\` a - WHERE a.\`user_id\` = w.\`user_id\` - AND a.\`ticker\` = f.\`ticker\` - AND a.\`accession_number\` = f.\`accession_number\` - AND a.\`kind\` = 'ai_report' - ); - `); - - client.exec('DELETE FROM `research_artifact_fts`;'); - client.exec(` - INSERT INTO \`research_artifact_fts\` ( - \`artifact_id\`, - \`user_id\`, - \`ticker\`, - \`title\`, - \`summary\`, - \`body_markdown\`, - \`search_text\`, - \`tags_text\` - ) - SELECT - \`id\`, - \`user_id\`, - \`ticker\`, - COALESCE(\`title\`, ''), - COALESCE(\`summary\`, ''), - COALESCE(\`body_markdown\`, ''), - COALESCE(\`search_text\`, ''), - CASE - WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN '' - ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '') - END - FROM \`research_artifact\`; - `); -} - -function ensureLocalSqliteSchema(client: Database) { - const missingBaseSchema = [ - 'filing', - 'watchlist_item', - 'holding', - 'task_run', - 'portfolio_insight' - ].some((tableName) => !hasTable(client, tableName)); - - if (missingBaseSchema) { - applyBaseSchemaCompat(client); - } - - if (!hasTable(client, 'user')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`user\` ( - \`id\` text PRIMARY KEY NOT NULL, - \`name\` text NOT NULL, - \`email\` text NOT NULL, - \`emailVerified\` integer NOT NULL DEFAULT 0, - \`image\` text, - \`createdAt\` integer NOT NULL, - \`updatedAt\` integer NOT NULL, - \`role\` text, - \`banned\` integer DEFAULT 0, - \`banReason\` text, - \`banExpires\` integer - ); - `); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);'); - } - - if (!hasTable(client, 'organization')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`organization\` ( - \`id\` text PRIMARY KEY NOT NULL, - \`name\` text NOT NULL, - \`slug\` text NOT NULL, - \`logo\` text, - \`createdAt\` integer NOT NULL, - \`metadata\` text - ); - `); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);'); - } - - if (!hasTable(client, 'filing_statement_snapshot')) { - applySqlFile(client, '0001_glossy_statement_snapshots.sql'); - } - - if (hasTable(client, 'task_run')) { - const missingTaskColumns: Array<{ name: string; sql: string }> = [ - { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" }, - { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' }, - { name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' }, - { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' }, - { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' }, - { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' } - ]; - - for (const column of missingTaskColumns) { - if (!hasColumn(client, 'task_run', column.name)) { - client.exec(column.sql); - } - } - } - - if (!hasTable(client, 'task_stage_event')) { - applySqlFile(client, '0003_task_stage_event_timeline.sql'); - } - - if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) { - client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;'); - } - - client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);'); - - if (hasTable(client, 'watchlist_item')) { - const missingWatchlistColumns: Array<{ name: string; sql: string }> = [ - { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' }, - { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' }, - { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" }, - { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" }, - { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" }, - { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' } - ]; - - for (const column of missingWatchlistColumns) { - if (!hasColumn(client, 'watchlist_item', column.name)) { - client.exec(column.sql); - } - } - - client.exec(` - UPDATE \`watchlist_item\` - SET - \`status\` = CASE - WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog' - ELSE \`status\` - END, - \`priority\` = CASE - WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium' - ELSE \`priority\` - END, - \`updated_at\` = CASE - WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP) - ELSE \`updated_at\` - END; - `); - - client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);'); - } - - if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) { - client.exec('ALTER TABLE `holding` ADD `company_name` text;'); - } - - if (!hasTable(client, 'filing_taxonomy_snapshot')) { - applySqlFile(client, '0005_financial_taxonomy_v3.sql'); - } - - if (!hasTable(client, 'company_financial_bundle')) { - applySqlFile(client, '0007_company_financial_bundles.sql'); - } - - if (!hasTable(client, 'company_overview_cache')) { - applySqlFile(client, '0012_company_overview_cache.sql'); - } - - if (!hasTable(client, 'research_journal_entry')) { - client.exec(` - CREATE TABLE IF NOT EXISTS \`research_journal_entry\` ( - \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - \`user_id\` text NOT NULL, - \`ticker\` text NOT NULL, - \`accession_number\` text, - \`entry_type\` text NOT NULL, - \`title\` text, - \`body_markdown\` text NOT NULL, - \`metadata\` text, - \`created_at\` text NOT NULL, - \`updated_at\` text NOT NULL, - FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade - ); - `); - client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);'); - } - - if (!hasTable(client, 'search_document')) { - applySqlFile(client, '0008_search_rag.sql'); - } - - ensureResearchWorkspaceSchema(client); -} - function ensureSearchVirtualTables(client: Database) { client.exec(` CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_fts\` USING fts5( diff --git a/lib/server/db/sqlite-schema-compat.ts b/lib/server/db/sqlite-schema-compat.ts new file mode 100644 index 0000000..22f580d --- /dev/null +++ b/lib/server/db/sqlite-schema-compat.ts @@ -0,0 +1,533 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Database } from 'bun:sqlite'; + +const DEFAULT_SURFACE_ROWS_JSON = '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; +const DEFAULT_DETAIL_ROWS_JSON = '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'; + +type MissingColumnDefinition = { + name: string; + sql: string; +}; + +export function hasTable(client: Database, tableName: string) { + const row = client + .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') + .get('table', tableName) as { name: string } | null; + + return row !== null; +} + +export function hasColumn(client: Database, tableName: string, columnName: string) { + if (!hasTable(client, tableName)) { + return false; + } + + const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; + return rows.some((row) => row.name === columnName); +} + +export function applySqlFile(client: Database, fileName: string) { + const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); + client.exec(sql); +} + +export function applyBaseSchemaCompat(client: Database) { + const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8') + .replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `') + .replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `') + .replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `'); + + client.exec(sql); +} + +function ensureColumns(client: Database, tableName: string, columns: MissingColumnDefinition[]) { + if (!hasTable(client, tableName)) { + return; + } + + for (const column of columns) { + if (!hasColumn(client, tableName, column.name)) { + client.exec(column.sql); + } + } +} + +function ensureResearchWorkspaceSchema(client: Database) { + if (!hasTable(client, 'research_artifact')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`research_artifact\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`user_id\` text NOT NULL, + \`organization_id\` text, + \`ticker\` text NOT NULL, + \`accession_number\` text, + \`kind\` text NOT NULL, + \`source\` text NOT NULL DEFAULT 'user', + \`subtype\` text, + \`title\` text, + \`summary\` text, + \`body_markdown\` text, + \`search_text\` text, + \`visibility_scope\` text NOT NULL DEFAULT 'private', + \`tags\` text, + \`metadata\` text, + \`file_name\` text, + \`mime_type\` text, + \`file_size_bytes\` integer, + \`storage_path\` text, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL, + FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null + ); + `); + } + + if (!hasTable(client, 'research_memo')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`research_memo\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`user_id\` text NOT NULL, + \`organization_id\` text, + \`ticker\` text NOT NULL, + \`rating\` text, + \`conviction\` text, + \`time_horizon_months\` integer, + \`packet_title\` text, + \`packet_subtitle\` text, + \`thesis_markdown\` text NOT NULL DEFAULT '', + \`variant_view_markdown\` text NOT NULL DEFAULT '', + \`catalysts_markdown\` text NOT NULL DEFAULT '', + \`risks_markdown\` text NOT NULL DEFAULT '', + \`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '', + \`next_actions_markdown\` text NOT NULL DEFAULT '', + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL, + FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null + ); + `); + } + + if (!hasTable(client, 'research_memo_evidence')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`memo_id\` integer NOT NULL, + \`artifact_id\` integer NOT NULL, + \`section\` text NOT NULL, + \`annotation\` text, + \`sort_order\` integer NOT NULL DEFAULT 0, + \`created_at\` text NOT NULL, + FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade + ); + `); + } + + client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);'); + client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);'); + client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);'); + client.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5( + artifact_id UNINDEXED, + user_id UNINDEXED, + ticker UNINDEXED, + title, + summary, + body_markdown, + search_text, + tags_text + ); + `); + + client.exec(` + INSERT INTO \`research_artifact\` ( + \`user_id\`, + \`organization_id\`, + \`ticker\`, + \`accession_number\`, + \`kind\`, + \`source\`, + \`subtype\`, + \`title\`, + \`summary\`, + \`body_markdown\`, + \`search_text\`, + \`visibility_scope\`, + \`tags\`, + \`metadata\`, + \`created_at\`, + \`updated_at\` + ) + SELECT + r.\`user_id\`, + NULL, + r.\`ticker\`, + r.\`accession_number\`, + CASE + WHEN r.\`entry_type\` = 'status_change' THEN 'status_change' + ELSE 'note' + END, + CASE + WHEN r.\`entry_type\` = 'status_change' THEN 'system' + ELSE 'user' + END, + r.\`entry_type\`, + r.\`title\`, + CASE + WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL + ELSE SUBSTR(r.\`body_markdown\`, 1, 280) + END, + r.\`body_markdown\`, + r.\`body_markdown\`, + 'private', + NULL, + r.\`metadata\`, + r.\`created_at\`, + r.\`updated_at\` + FROM \`research_journal_entry\` r + WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry') + AND NOT EXISTS ( + SELECT 1 + FROM \`research_artifact\` a + WHERE a.\`user_id\` = r.\`user_id\` + AND a.\`ticker\` = r.\`ticker\` + AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '') + AND a.\`kind\` = CASE + WHEN r.\`entry_type\` = 'status_change' THEN 'status_change' + ELSE 'note' + END + AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '') + AND a.\`created_at\` = r.\`created_at\` + ); + `); + + client.exec(` + INSERT INTO \`research_artifact\` ( + \`user_id\`, + \`organization_id\`, + \`ticker\`, + \`accession_number\`, + \`kind\`, + \`source\`, + \`subtype\`, + \`title\`, + \`summary\`, + \`body_markdown\`, + \`search_text\`, + \`visibility_scope\`, + \`tags\`, + \`metadata\`, + \`created_at\`, + \`updated_at\` + ) + SELECT + w.\`user_id\`, + NULL, + f.\`ticker\`, + f.\`accession_number\`, + 'ai_report', + 'system', + 'filing_analysis', + f.\`filing_type\` || ' AI memo', + COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), + 'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) || + 'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) || + COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), + COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')), + 'private', + NULL, + json_object( + 'provider', json_extract(f.\`analysis\`, '$.provider'), + 'model', json_extract(f.\`analysis\`, '$.model'), + 'filingType', f.\`filing_type\`, + 'filingDate', f.\`filing_date\` + ), + f.\`created_at\`, + f.\`updated_at\` + FROM \`filing\` f + JOIN \`watchlist_item\` w + ON w.\`ticker\` = f.\`ticker\` + WHERE f.\`analysis\` IS NOT NULL + AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> '' + AND NOT EXISTS ( + SELECT 1 + FROM \`research_artifact\` a + WHERE a.\`user_id\` = w.\`user_id\` + AND a.\`ticker\` = f.\`ticker\` + AND a.\`accession_number\` = f.\`accession_number\` + AND a.\`kind\` = 'ai_report' + ); + `); + + client.exec('DELETE FROM `research_artifact_fts`;'); + client.exec(` + INSERT INTO \`research_artifact_fts\` ( + \`artifact_id\`, + \`user_id\`, + \`ticker\`, + \`title\`, + \`summary\`, + \`body_markdown\`, + \`search_text\`, + \`tags_text\` + ) + SELECT + \`id\`, + \`user_id\`, + \`ticker\`, + COALESCE(\`title\`, ''), + COALESCE(\`summary\`, ''), + COALESCE(\`body_markdown\`, ''), + COALESCE(\`search_text\`, ''), + CASE + WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN '' + ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '') + END + FROM \`research_artifact\`; + `); +} + +function ensureTaxonomySnapshotCompat(client: Database) { + ensureColumns(client, 'filing_taxonomy_snapshot', [ + { name: 'parser_engine', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'fiscal-xbrl';" }, + { name: 'parser_version', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text NOT NULL DEFAULT 'unknown';" }, + { name: 'taxonomy_regime', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text NOT NULL DEFAULT 'unknown';" }, + { name: 'fiscal_pack', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;' }, + { name: 'faithful_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;' }, + { name: 'surface_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;' }, + { name: 'detail_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;' }, + { name: 'kpi_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;' }, + { name: 'normalization_summary', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;' } + ]); + + if (!hasTable(client, 'filing_taxonomy_snapshot')) { + return; + } + + client.exec(` + UPDATE \`filing_taxonomy_snapshot\` + SET + \`faithful_rows\` = COALESCE(\`faithful_rows\`, \`statement_rows\`), + \`surface_rows\` = COALESCE(\`surface_rows\`, '${DEFAULT_SURFACE_ROWS_JSON}'), + \`detail_rows\` = COALESCE(\`detail_rows\`, '${DEFAULT_DETAIL_ROWS_JSON}'), + \`kpi_rows\` = COALESCE(\`kpi_rows\`, '[]'); + `); +} + +function ensureTaxonomyContextCompat(client: Database) { + if (!hasTable(client, 'filing_taxonomy_context')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`filing_taxonomy_context\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`snapshot_id\` integer NOT NULL, + \`context_id\` text NOT NULL, + \`entity_identifier\` text, + \`entity_scheme\` text, + \`period_start\` text, + \`period_end\` text, + \`period_instant\` text, + \`segment_json\` text, + \`scenario_json\` text, + \`created_at\` text NOT NULL, + FOREIGN KEY (\`snapshot_id\`) REFERENCES \`filing_taxonomy_snapshot\`(\`id\`) ON UPDATE no action ON DELETE cascade + ); + `); + } + + client.exec('CREATE INDEX IF NOT EXISTS `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);'); + client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);'); +} + +function ensureTaxonomyConceptCompat(client: Database) { + ensureColumns(client, 'filing_taxonomy_concept', [ + { name: 'balance', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;' }, + { name: 'period_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;' }, + { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;' }, + { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;' }, + { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;' }, + { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;' }, + { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;' }, + { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;' }, + { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer NOT NULL DEFAULT false;' } + ]); +} + +function ensureTaxonomyFactCompat(client: Database) { + ensureColumns(client, 'filing_taxonomy_fact', [ + { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;' }, + { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;' }, + { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;' }, + { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;' }, + { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;' }, + { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;' }, + { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer NOT NULL DEFAULT false;' }, + { name: 'precision', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;' }, + { name: 'nil', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer NOT NULL DEFAULT false;' } + ]); +} + +function ensureTaxonomyCompat(client: Database) { + ensureTaxonomySnapshotCompat(client); + ensureTaxonomyContextCompat(client); + ensureTaxonomyConceptCompat(client); + ensureTaxonomyFactCompat(client); +} + +export function ensureLocalSqliteSchema(client: Database) { + const missingBaseSchema = [ + 'filing', + 'watchlist_item', + 'holding', + 'task_run', + 'portfolio_insight' + ].some((tableName) => !hasTable(client, tableName)); + + if (missingBaseSchema) { + applyBaseSchemaCompat(client); + } + + if (!hasTable(client, 'user')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`user\` ( + \`id\` text PRIMARY KEY NOT NULL, + \`name\` text NOT NULL, + \`email\` text NOT NULL, + \`emailVerified\` integer NOT NULL DEFAULT 0, + \`image\` text, + \`createdAt\` integer NOT NULL, + \`updatedAt\` integer NOT NULL, + \`role\` text, + \`banned\` integer DEFAULT 0, + \`banReason\` text, + \`banExpires\` integer + ); + `); + client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);'); + } + + if (!hasTable(client, 'organization')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`organization\` ( + \`id\` text PRIMARY KEY NOT NULL, + \`name\` text NOT NULL, + \`slug\` text NOT NULL, + \`logo\` text, + \`createdAt\` integer NOT NULL, + \`metadata\` text + ); + `); + client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);'); + } + + if (!hasTable(client, 'filing_statement_snapshot')) { + applySqlFile(client, '0001_glossy_statement_snapshots.sql'); + } + + ensureColumns(client, 'task_run', [ + { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" }, + { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' }, + { name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' }, + { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' }, + { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' }, + { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' } + ]); + + if (!hasTable(client, 'task_stage_event')) { + applySqlFile(client, '0003_task_stage_event_timeline.sql'); + } + + if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) { + client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;'); + } + + client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);'); + + ensureColumns(client, 'watchlist_item', [ + { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' }, + { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' }, + { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" }, + { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" }, + { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" }, + { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' } + ]); + + if (hasTable(client, 'watchlist_item')) { + client.exec(` + UPDATE \`watchlist_item\` + SET + \`status\` = CASE + WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog' + ELSE \`status\` + END, + \`priority\` = CASE + WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium' + ELSE \`priority\` + END, + \`updated_at\` = CASE + WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP) + ELSE \`updated_at\` + END; + `); + + client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);'); + } + + if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) { + client.exec('ALTER TABLE `holding` ADD `company_name` text;'); + } + + if (!hasTable(client, 'filing_taxonomy_snapshot')) { + applySqlFile(client, '0005_financial_taxonomy_v3.sql'); + } + ensureTaxonomyCompat(client); + + if (!hasTable(client, 'company_financial_bundle')) { + applySqlFile(client, '0007_company_financial_bundles.sql'); + } + + if (!hasTable(client, 'company_overview_cache')) { + applySqlFile(client, '0012_company_overview_cache.sql'); + } + + if (!hasTable(client, 'research_journal_entry')) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`research_journal_entry\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`user_id\` text NOT NULL, + \`ticker\` text NOT NULL, + \`accession_number\` text, + \`entry_type\` text NOT NULL, + \`title\` text, + \`body_markdown\` text NOT NULL, + \`metadata\` text, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL, + FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade + ); + `); + client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);'); + client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);'); + } + + if (!hasTable(client, 'search_document')) { + applySqlFile(client, '0008_search_rag.sql'); + } + + ensureResearchWorkspaceSchema(client); +} + +export const __sqliteSchemaCompatInternals = { + applyBaseSchemaCompat, + applySqlFile, + hasColumn, + hasTable +}; diff --git a/scripts/dev.ts b/scripts/dev.ts index b87e1be..9bf1e82 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -7,6 +7,7 @@ import { ensureFinancialIngestionSchemaHealthy, resolveFinancialSchemaRepairMode } from '../lib/server/db/financial-ingestion-schema'; +import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat'; import { buildLocalDevConfig, resolveSqlitePath } from './dev-env'; type DrizzleJournal = { @@ -135,6 +136,7 @@ if (!initializedDatabase && databasePath && databasePath !== ':memory:') { try { client.exec('PRAGMA foreign_keys = ON;'); + ensureLocalSqliteSchema(client); const repairResult = ensureFinancialIngestionSchemaHealthy(client, { mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE) }); diff --git a/scripts/e2e-prepare.test.ts b/scripts/e2e-prepare.test.ts new file mode 100644 index 0000000..45abb06 --- /dev/null +++ b/scripts/e2e-prepare.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Database } from 'bun:sqlite'; +import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema'; +import { hasColumn, hasTable } from '../lib/server/db/sqlite-schema-compat'; +import { prepareE2eDatabase } from './e2e-prepare'; + +describe('prepareE2eDatabase', () => { + it('bootstraps a fresh e2e database with the current taxonomy schema shape', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'fiscal-e2e-prepare-')); + const databasePath = join(tempDir, 'e2e.sqlite'); + const workflowDataDir = join(tempDir, 'workflow-data'); + + try { + prepareE2eDatabase({ databasePath, workflowDataDir }); + + const client = new Database(databasePath, { create: true }); + + try { + client.exec('PRAGMA foreign_keys = ON;'); + + expect(hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); + expect(hasTable(client, 'filing_taxonomy_context')).toBe(true); + + const health = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' }); + expect(health.ok).toBe(true); + } finally { + client.close(); + } + } finally { + rmSync(tempDir, { force: true, recursive: true }); + } + }); +}); diff --git a/scripts/e2e-prepare.ts b/scripts/e2e-prepare.ts index ff571fc..d6ae231 100644 --- a/scripts/e2e-prepare.ts +++ b/scripts/e2e-prepare.ts @@ -1,46 +1,38 @@ -import { mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { mkdirSync, rmSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { Database } from 'bun:sqlite'; - -const MIGRATION_FILES = [ - '0000_cold_silver_centurion.sql', - '0001_glossy_statement_snapshots.sql', - '0002_workflow_task_projection_metadata.sql', - '0003_task_stage_event_timeline.sql', - '0004_watchlist_company_taxonomy.sql', - '0005_financial_taxonomy_v3.sql', - '0006_coverage_journal_tracking.sql', - '0007_company_financial_bundles.sql', - '0008_research_workspace.sql' -] as const; +import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema'; +import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat'; export const E2E_DATABASE_PATH = join(process.cwd(), 'data', 'e2e.sqlite'); export const E2E_WORKFLOW_DATA_DIR = join(process.cwd(), '.workflow-data', 'e2e'); +type PrepareE2eDatabaseOptions = { + databasePath?: string; + workflowDataDir?: string; +}; + function removeFileIfPresent(path: string) { rmSync(path, { force: true }); } -function applyMigrations(database: Database) { - for (const file of MIGRATION_FILES) { - const sql = readFileSync(join(process.cwd(), 'drizzle', file), 'utf8'); - database.exec(sql); - } -} +export function prepareE2eDatabase(options: PrepareE2eDatabaseOptions = {}) { + const databasePath = options.databasePath ?? E2E_DATABASE_PATH; + const workflowDataDir = options.workflowDataDir ?? E2E_WORKFLOW_DATA_DIR; -export function prepareE2eDatabase() { - mkdirSync(dirname(E2E_DATABASE_PATH), { recursive: true }); + mkdirSync(dirname(databasePath), { recursive: true }); - removeFileIfPresent(E2E_DATABASE_PATH); - removeFileIfPresent(`${E2E_DATABASE_PATH}-shm`); - removeFileIfPresent(`${E2E_DATABASE_PATH}-wal`); - rmSync(E2E_WORKFLOW_DATA_DIR, { force: true, recursive: true }); + removeFileIfPresent(databasePath); + removeFileIfPresent(`${databasePath}-shm`); + removeFileIfPresent(`${databasePath}-wal`); + rmSync(workflowDataDir, { force: true, recursive: true }); - const database = new Database(E2E_DATABASE_PATH, { create: true }); + const database = new Database(databasePath, { create: true }); try { database.exec('PRAGMA foreign_keys = ON;'); - applyMigrations(database); + ensureLocalSqliteSchema(database); + ensureFinancialIngestionSchemaHealthy(database, { mode: 'auto' }); } finally { database.close(); }