import { describe, expect, it } from 'bun:test'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { Database } from 'bun:sqlite'; import { __dbInternals } from './index'; function applyMigration(client: Database, fileName: string) { const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); client.exec(sql); } describe('sqlite schema compatibility bootstrap', () => { it('adds missing watchlist columns and taxonomy tables for older local databases', () => { const client = new Database(':memory:'); client.exec('PRAGMA foreign_keys = ON;'); applyMigration(client, '0000_cold_silver_centurion.sql'); applyMigration(client, '0001_glossy_statement_snapshots.sql'); applyMigration(client, '0002_workflow_task_projection_metadata.sql'); applyMigration(client, '0003_task_stage_event_timeline.sql'); applyMigration(client, '0009_task_notification_context.sql'); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(false); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(false); expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(false); expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(false); expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(false); expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(false); expect(__dbInternals.hasTable(client, 'research_memo')).toBe(false); __dbInternals.ensureLocalSqliteSchema(client); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(true); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'tags')).toBe(true); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(true); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'priority')).toBe(true); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'updated_at')).toBe(true); expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true); expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true); expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true); expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true); expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true); expect(__dbInternals.hasTable(client, 'search_document')).toBe(true); expect(__dbInternals.hasTable(client, 'search_chunk')).toBe(true); expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(true); expect(__dbInternals.hasTable(client, 'research_memo')).toBe(true); expect(__dbInternals.hasTable(client, 'research_memo_evidence')).toBe(true); expect(__dbInternals.hasTable(client, 'company_overview_cache')).toBe(true); __dbInternals.loadSqliteExtensions(client); __dbInternals.ensureSearchVirtualTables(client); expect(__dbInternals.hasTable(client, 'search_chunk_fts')).toBe(true); expect(__dbInternals.hasTable(client, 'search_chunk_vec')).toBe(true); client.close(); }); it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => { const client = new Database(':memory:'); client.exec('PRAGMA foreign_keys = ON;'); applyMigration(client, '0000_cold_silver_centurion.sql'); applyMigration(client, '0005_financial_taxonomy_v3.sql'); client.exec(` INSERT INTO \`filing\` ( \`ticker\`, \`filing_type\`, \`filing_date\`, \`accession_number\`, \`cik\`, \`company_name\`, \`created_at\`, \`updated_at\` ) VALUES ( 'AAPL', '10-K', '2024-09-28', '0000320193-24-000001', '0000320193', 'Apple Inc.', '2024-10-30T00:00:00.000Z', '2024-10-30T00:00:00.000Z' ); `); const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; client.exec(` INSERT INTO \`filing_taxonomy_snapshot\` ( \`filing_id\`, \`ticker\`, \`filing_date\`, \`filing_type\`, \`parse_status\`, \`source\`, \`statement_rows\`, \`created_at\`, \`updated_at\` ) VALUES ( 1, 'AAPL', '2024-09-28', '10-K', 'success', 'xbrl_instance', '${statementRows}', '2024-10-30T00:00:00.000Z', '2024-10-30T00:00:00.000Z' ); `); __dbInternals.ensureLocalSqliteSchema(client); __dbInternals.ensureLocalSqliteSchema(client); const row = client.query(` SELECT \`parser_engine\`, \`parser_version\`, \`taxonomy_regime\`, \`faithful_rows\`, \`surface_rows\`, \`detail_rows\`, \`kpi_rows\`, \`normalization_summary\` FROM \`filing_taxonomy_snapshot\` WHERE \`filing_id\` = 1 `).get() as { parser_engine: string; parser_version: string; taxonomy_regime: string; faithful_rows: string | null; surface_rows: string | null; detail_rows: string | null; kpi_rows: string | null; normalization_summary: string | null; }; expect(row.parser_engine).toBe('fiscal-xbrl'); expect(row.parser_version).toBe('unknown'); expect(row.taxonomy_regime).toBe('unknown'); expect(row.faithful_rows).toBe(statementRows); expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'); expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'); expect(row.kpi_rows).toBe('[]'); expect(row.normalization_summary).toBeNull(); client.close(); }); it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => { const client = new Database(':memory:'); client.exec('PRAGMA foreign_keys = ON;'); applyMigration(client, '0000_cold_silver_centurion.sql'); applyMigration(client, '0005_financial_taxonomy_v3.sql'); client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';"); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false); expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false); __dbInternals.ensureLocalSqliteSchema(client); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); client.close(); }); it('throws on missing parser_engine column when verifyCriticalSchema is called', () => { const client = new Database(':memory:'); client.exec('PRAGMA foreign_keys = ON;'); applyMigration(client, '0000_cold_silver_centurion.sql'); applyMigration(client, '0005_financial_taxonomy_v3.sql'); expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(false); expect(() => __dbInternals.verifyCriticalSchema(client)).toThrow( /filing_taxonomy_snapshot is missing columns: parser_engine/ ); client.close(); }); it('verifyCriticalSchema passes when all required columns exist', () => { const client = new Database(':memory:'); client.exec('PRAGMA foreign_keys = ON;'); applyMigration(client, '0000_cold_silver_centurion.sql'); applyMigration(client, '0005_financial_taxonomy_v3.sql'); __dbInternals.ensureLocalSqliteSchema(client); expect(() => __dbInternals.verifyCriticalSchema(client)).not.toThrow(); client.close(); }); });