Fix SQLite taxonomy schema bootstrap drift

lib/server/db/index.test.ts

@@ -38,7 +38,34 @@ describe('sqlite schema compatibility bootstrap', () => {
     expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true);
     expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true);
     expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
+    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
     expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true);
     expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true);
     expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true);
     expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true);
@@ -57,4 +84,119 @@ describe('sqlite schema compatibility bootstrap', () => {
 
     client.close();
   });
+
+  it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => {
+    const client = new Database(':memory:');
+    client.exec('PRAGMA foreign_keys = ON;');
+
+    applyMigration(client, '0000_cold_silver_centurion.sql');
+    applyMigration(client, '0005_financial_taxonomy_v3.sql');
+
+    client.exec(`
+      INSERT INTO \`filing\` (
+        \`ticker\`,
+        \`filing_type\`,
+        \`filing_date\`,
+        \`accession_number\`,
+        \`cik\`,
+        \`company_name\`,
+        \`created_at\`,
+        \`updated_at\`
+      ) VALUES (
+        'AAPL',
+        '10-K',
+        '2024-09-28',
+        '0000320193-24-000001',
+        '0000320193',
+        'Apple Inc.',
+        '2024-10-30T00:00:00.000Z',
+        '2024-10-30T00:00:00.000Z'
+      );
+    `);
+
+    const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';
+
+    client.exec(`
+      INSERT INTO \`filing_taxonomy_snapshot\` (
+        \`filing_id\`,
+        \`ticker\`,
+        \`filing_date\`,
+        \`filing_type\`,
+        \`parse_status\`,
+        \`source\`,
+        \`statement_rows\`,
+        \`created_at\`,
+        \`updated_at\`
+      ) VALUES (
+        1,
+        'AAPL',
+        '2024-09-28',
+        '10-K',
+        'success',
+        'xbrl_instance',
+        '${statementRows}',
+        '2024-10-30T00:00:00.000Z',
+        '2024-10-30T00:00:00.000Z'
+      );
+    `);
+
+    __dbInternals.ensureLocalSqliteSchema(client);
+    __dbInternals.ensureLocalSqliteSchema(client);
+
+    const row = client.query(`
+      SELECT
+        \`parser_engine\`,
+        \`parser_version\`,
+        \`taxonomy_regime\`,
+        \`faithful_rows\`,
+        \`surface_rows\`,
+        \`detail_rows\`,
+        \`kpi_rows\`,
+        \`normalization_summary\`
+      FROM \`filing_taxonomy_snapshot\`
+      WHERE \`filing_id\` = 1
+    `).get() as {
+      parser_engine: string;
+      parser_version: string;
+      taxonomy_regime: string;
+      faithful_rows: string | null;
+      surface_rows: string | null;
+      detail_rows: string | null;
+      kpi_rows: string | null;
+      normalization_summary: string | null;
+    };
+
+    expect(row.parser_engine).toBe('fiscal-xbrl');
+    expect(row.parser_version).toBe('unknown');
+    expect(row.taxonomy_regime).toBe('unknown');
+    expect(row.faithful_rows).toBe(statementRows);
+    expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}');
+    expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}');
+    expect(row.kpi_rows).toBe('[]');
+    expect(row.normalization_summary).toBeNull();
+
+    client.close();
+  });
+
+  it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => {
+    const client = new Database(':memory:');
+    client.exec('PRAGMA foreign_keys = ON;');
+
+    applyMigration(client, '0000_cold_silver_centurion.sql');
+    applyMigration(client, '0005_financial_taxonomy_v3.sql');
+    client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';");
+
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false);
+    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false);
+
+    __dbInternals.ensureLocalSqliteSchema(client);
+
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
+    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
+    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
+
+    client.close();
+  });
 });

lib/server/db/index.ts

@@ -1,5 +1,5 @@
-import { mkdirSync, readFileSync } from 'node:fs';
+import { mkdirSync } from 'node:fs';
-import { dirname, join } from 'node:path';
+import { dirname } from 'node:path';
 import { Database } from 'bun:sqlite';
 import { drizzle } from 'drizzle-orm/bun-sqlite';
 import { load as loadSqliteVec } from 'sqlite-vec';
@@ -8,6 +8,11 @@ import {
   resolveFinancialSchemaRepairMode
 } from './financial-ingestion-schema';
 import { schema } from './schema';
+import {
+  ensureLocalSqliteSchema,
+  hasColumn,
+  hasTable
+} from './sqlite-schema-compat';
 
 type AppDrizzleDb = ReturnType<typeof createDb>;
 
@@ -33,37 +38,6 @@ function getDatabasePath() {
   return databasePath;
 }
 
-function hasTable(client: Database, tableName: string) {
-  const row = client
-    .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
-    .get('table', tableName) as { name: string } | null;
-
-  return row !== null;
-}
-
-function hasColumn(client: Database, tableName: string, columnName: string) {
-  if (!hasTable(client, tableName)) {
-    return false;
-  }
-
-  const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>;
-  return rows.some((row) => row.name === columnName);
-}
-
-function applySqlFile(client: Database, fileName: string) {
-  const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8');
-  client.exec(sql);
-}
-
-function applyBaseSchemaCompat(client: Database) {
-  const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
-    .replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
-    .replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
-    .replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
-
-  client.exec(sql);
-}
-
 let customSqliteConfigured = false;
 const vectorExtensionStatus = new WeakMap<Database, boolean>();
 
@@ -103,405 +77,6 @@ function isVectorExtensionLoaded(client: Database) {
   return vectorExtensionStatus.get(client) ?? false;
 }
 
-function ensureResearchWorkspaceSchema(client: Database) {
-  if (!hasTable(client, 'research_artifact')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`research_artifact\` (
-        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-        \`user_id\` text NOT NULL,
-        \`organization_id\` text,
-        \`ticker\` text NOT NULL,
-        \`accession_number\` text,
-        \`kind\` text NOT NULL,
-        \`source\` text NOT NULL DEFAULT 'user',
-        \`subtype\` text,
-        \`title\` text,
-        \`summary\` text,
-        \`body_markdown\` text,
-        \`search_text\` text,
-        \`visibility_scope\` text NOT NULL DEFAULT 'private',
-        \`tags\` text,
-        \`metadata\` text,
-        \`file_name\` text,
-        \`mime_type\` text,
-        \`file_size_bytes\` integer,
-        \`storage_path\` text,
-        \`created_at\` text NOT NULL,
-        \`updated_at\` text NOT NULL,
-        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
-        FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
-      );
-    `);
-  }
-
-  if (!hasTable(client, 'research_memo')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`research_memo\` (
-        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-        \`user_id\` text NOT NULL,
-        \`organization_id\` text,
-        \`ticker\` text NOT NULL,
-        \`rating\` text,
-        \`conviction\` text,
-        \`time_horizon_months\` integer,
-        \`packet_title\` text,
-        \`packet_subtitle\` text,
-        \`thesis_markdown\` text NOT NULL DEFAULT '',
-        \`variant_view_markdown\` text NOT NULL DEFAULT '',
-        \`catalysts_markdown\` text NOT NULL DEFAULT '',
-        \`risks_markdown\` text NOT NULL DEFAULT '',
-        \`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
-        \`next_actions_markdown\` text NOT NULL DEFAULT '',
-        \`created_at\` text NOT NULL,
-        \`updated_at\` text NOT NULL,
-        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
-        FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
-      );
-    `);
-  }
-
-  if (!hasTable(client, 'research_memo_evidence')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
-        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-        \`memo_id\` integer NOT NULL,
-        \`artifact_id\` integer NOT NULL,
-        \`section\` text NOT NULL,
-        \`annotation\` text,
-        \`sort_order\` integer NOT NULL DEFAULT 0,
-        \`created_at\` text NOT NULL,
-        FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
-        FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
-      );
-    `);
-  }
-
-  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
-  client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
-  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
-  client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
-  client.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
-      artifact_id UNINDEXED,
-      user_id UNINDEXED,
-      ticker UNINDEXED,
-      title,
-      summary,
-      body_markdown,
-      search_text,
-      tags_text
-    );
-  `);
-
-  client.exec(`
-    INSERT INTO \`research_artifact\` (
-      \`user_id\`,
-      \`organization_id\`,
-      \`ticker\`,
-      \`accession_number\`,
-      \`kind\`,
-      \`source\`,
-      \`subtype\`,
-      \`title\`,
-      \`summary\`,
-      \`body_markdown\`,
-      \`search_text\`,
-      \`visibility_scope\`,
-      \`tags\`,
-      \`metadata\`,
-      \`created_at\`,
-      \`updated_at\`
-    )
-    SELECT
-      r.\`user_id\`,
-      NULL,
-      r.\`ticker\`,
-      r.\`accession_number\`,
-      CASE
-        WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
-        ELSE 'note'
-      END,
-      CASE
-        WHEN r.\`entry_type\` = 'status_change' THEN 'system'
-        ELSE 'user'
-      END,
-      r.\`entry_type\`,
-      r.\`title\`,
-      CASE
-        WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
-        ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
-      END,
-      r.\`body_markdown\`,
-      r.\`body_markdown\`,
-      'private',
-      NULL,
-      r.\`metadata\`,
-      r.\`created_at\`,
-      r.\`updated_at\`
-    FROM \`research_journal_entry\` r
-    WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
-      AND NOT EXISTS (
-        SELECT 1
-        FROM \`research_artifact\` a
-        WHERE a.\`user_id\` = r.\`user_id\`
-          AND a.\`ticker\` = r.\`ticker\`
-          AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
-          AND a.\`kind\` = CASE
-            WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
-            ELSE 'note'
-          END
-          AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
-          AND a.\`created_at\` = r.\`created_at\`
-      );
-  `);
-
-  client.exec(`
-    INSERT INTO \`research_artifact\` (
-      \`user_id\`,
-      \`organization_id\`,
-      \`ticker\`,
-      \`accession_number\`,
-      \`kind\`,
-      \`source\`,
-      \`subtype\`,
-      \`title\`,
-      \`summary\`,
-      \`body_markdown\`,
-      \`search_text\`,
-      \`visibility_scope\`,
-      \`tags\`,
-      \`metadata\`,
-      \`created_at\`,
-      \`updated_at\`
-    )
-    SELECT
-      w.\`user_id\`,
-      NULL,
-      f.\`ticker\`,
-      f.\`accession_number\`,
-      'ai_report',
-      'system',
-      'filing_analysis',
-      f.\`filing_type\` || ' AI memo',
-      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
-      'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
-      'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
-      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
-      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
-      'private',
-      NULL,
-      json_object(
-        'provider', json_extract(f.\`analysis\`, '$.provider'),
-        'model', json_extract(f.\`analysis\`, '$.model'),
-        'filingType', f.\`filing_type\`,
-        'filingDate', f.\`filing_date\`
-      ),
-      f.\`created_at\`,
-      f.\`updated_at\`
-    FROM \`filing\` f
-    JOIN \`watchlist_item\` w
-      ON w.\`ticker\` = f.\`ticker\`
-    WHERE f.\`analysis\` IS NOT NULL
-      AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
-      AND NOT EXISTS (
-        SELECT 1
-        FROM \`research_artifact\` a
-        WHERE a.\`user_id\` = w.\`user_id\`
-          AND a.\`ticker\` = f.\`ticker\`
-          AND a.\`accession_number\` = f.\`accession_number\`
-          AND a.\`kind\` = 'ai_report'
-      );
-  `);
-
-  client.exec('DELETE FROM `research_artifact_fts`;');
-  client.exec(`
-    INSERT INTO \`research_artifact_fts\` (
-      \`artifact_id\`,
-      \`user_id\`,
-      \`ticker\`,
-      \`title\`,
-      \`summary\`,
-      \`body_markdown\`,
-      \`search_text\`,
-      \`tags_text\`
-    )
-    SELECT
-      \`id\`,
-      \`user_id\`,
-      \`ticker\`,
-      COALESCE(\`title\`, ''),
-      COALESCE(\`summary\`, ''),
-      COALESCE(\`body_markdown\`, ''),
-      COALESCE(\`search_text\`, ''),
-      CASE
-        WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
-        ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
-      END
-    FROM \`research_artifact\`;
-  `);
-}
-
-function ensureLocalSqliteSchema(client: Database) {
-  const missingBaseSchema = [
-    'filing',
-    'watchlist_item',
-    'holding',
-    'task_run',
-    'portfolio_insight'
-  ].some((tableName) => !hasTable(client, tableName));
-
-  if (missingBaseSchema) {
-    applyBaseSchemaCompat(client);
-  }
-
-  if (!hasTable(client, 'user')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`user\` (
-        \`id\` text PRIMARY KEY NOT NULL,
-        \`name\` text NOT NULL,
-        \`email\` text NOT NULL,
-        \`emailVerified\` integer NOT NULL DEFAULT 0,
-        \`image\` text,
-        \`createdAt\` integer NOT NULL,
-        \`updatedAt\` integer NOT NULL,
-        \`role\` text,
-        \`banned\` integer DEFAULT 0,
-        \`banReason\` text,
-        \`banExpires\` integer
-      );
-    `);
-    client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
-  }
-
-  if (!hasTable(client, 'organization')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`organization\` (
-        \`id\` text PRIMARY KEY NOT NULL,
-        \`name\` text NOT NULL,
-        \`slug\` text NOT NULL,
-        \`logo\` text,
-        \`createdAt\` integer NOT NULL,
-        \`metadata\` text
-      );
-    `);
-    client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
-  }
-
-  if (!hasTable(client, 'filing_statement_snapshot')) {
-    applySqlFile(client, '0001_glossy_statement_snapshots.sql');
-  }
-
-  if (hasTable(client, 'task_run')) {
-    const missingTaskColumns: Array<{ name: string; sql: string }> = [
-      { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" },
-      { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' },
-      { name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' },
-      { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' },
-      { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' },
-      { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' }
-    ];
-
-    for (const column of missingTaskColumns) {
-      if (!hasColumn(client, 'task_run', column.name)) {
-        client.exec(column.sql);
-      }
-    }
-  }
-
-  if (!hasTable(client, 'task_stage_event')) {
-    applySqlFile(client, '0003_task_stage_event_timeline.sql');
-  }
-
-  if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) {
-    client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;');
-  }
-
-  client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);');
-
-  if (hasTable(client, 'watchlist_item')) {
-    const missingWatchlistColumns: Array<{ name: string; sql: string }> = [
-      { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' },
-      { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' },
-      { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" },
-      { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" },
-      { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" },
-      { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' }
-    ];
-
-    for (const column of missingWatchlistColumns) {
-      if (!hasColumn(client, 'watchlist_item', column.name)) {
-        client.exec(column.sql);
-      }
-    }
-
-    client.exec(`
-      UPDATE \`watchlist_item\`
-      SET
-        \`status\` = CASE
-          WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog'
-          ELSE \`status\`
-        END,
-        \`priority\` = CASE
-          WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium'
-          ELSE \`priority\`
-        END,
-        \`updated_at\` = CASE
-          WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP)
-          ELSE \`updated_at\`
-        END;
-    `);
-
-    client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);');
-  }
-
-  if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) {
-    client.exec('ALTER TABLE `holding` ADD `company_name` text;');
-  }
-
-  if (!hasTable(client, 'filing_taxonomy_snapshot')) {
-    applySqlFile(client, '0005_financial_taxonomy_v3.sql');
-  }
-
-  if (!hasTable(client, 'company_financial_bundle')) {
-    applySqlFile(client, '0007_company_financial_bundles.sql');
-  }
-
-  if (!hasTable(client, 'company_overview_cache')) {
-    applySqlFile(client, '0012_company_overview_cache.sql');
-  }
-
-  if (!hasTable(client, 'research_journal_entry')) {
-    client.exec(`
-      CREATE TABLE IF NOT EXISTS \`research_journal_entry\` (
-        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-        \`user_id\` text NOT NULL,
-        \`ticker\` text NOT NULL,
-        \`accession_number\` text,
-        \`entry_type\` text NOT NULL,
-        \`title\` text,
-        \`body_markdown\` text NOT NULL,
-        \`metadata\` text,
-        \`created_at\` text NOT NULL,
-        \`updated_at\` text NOT NULL,
-        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade
-      );
-    `);
-    client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);');
-    client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);');
-  }
-
-  if (!hasTable(client, 'search_document')) {
-    applySqlFile(client, '0008_search_rag.sql');
-  }
-
-  ensureResearchWorkspaceSchema(client);
-}
-
 function ensureSearchVirtualTables(client: Database) {
   client.exec(`
     CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_fts\` USING fts5(

lib/server/db/sqlite-schema-compat.ts

@@ -0,0 +1,533 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Database } from 'bun:sqlite';
+
+const DEFAULT_SURFACE_ROWS_JSON = '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';
+const DEFAULT_DETAIL_ROWS_JSON = '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}';
+
+type MissingColumnDefinition = {
+  name: string;
+  sql: string;
+};
+
+export function hasTable(client: Database, tableName: string) {
+  const row = client
+    .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
+    .get('table', tableName) as { name: string } | null;
+
+  return row !== null;
+}
+
+export function hasColumn(client: Database, tableName: string, columnName: string) {
+  if (!hasTable(client, tableName)) {
+    return false;
+  }
+
+  const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>;
+  return rows.some((row) => row.name === columnName);
+}
+
+export function applySqlFile(client: Database, fileName: string) {
+  const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8');
+  client.exec(sql);
+}
+
+export function applyBaseSchemaCompat(client: Database) {
+  const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
+    .replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
+    .replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
+    .replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
+
+  client.exec(sql);
+}
+
+function ensureColumns(client: Database, tableName: string, columns: MissingColumnDefinition[]) {
+  if (!hasTable(client, tableName)) {
+    return;
+  }
+
+  for (const column of columns) {
+    if (!hasColumn(client, tableName, column.name)) {
+      client.exec(column.sql);
+    }
+  }
+}
+
+function ensureResearchWorkspaceSchema(client: Database) {
+  if (!hasTable(client, 'research_artifact')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`research_artifact\` (
+        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        \`user_id\` text NOT NULL,
+        \`organization_id\` text,
+        \`ticker\` text NOT NULL,
+        \`accession_number\` text,
+        \`kind\` text NOT NULL,
+        \`source\` text NOT NULL DEFAULT 'user',
+        \`subtype\` text,
+        \`title\` text,
+        \`summary\` text,
+        \`body_markdown\` text,
+        \`search_text\` text,
+        \`visibility_scope\` text NOT NULL DEFAULT 'private',
+        \`tags\` text,
+        \`metadata\` text,
+        \`file_name\` text,
+        \`mime_type\` text,
+        \`file_size_bytes\` integer,
+        \`storage_path\` text,
+        \`created_at\` text NOT NULL,
+        \`updated_at\` text NOT NULL,
+        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
+        FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
+      );
+    `);
+  }
+
+  if (!hasTable(client, 'research_memo')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`research_memo\` (
+        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        \`user_id\` text NOT NULL,
+        \`organization_id\` text,
+        \`ticker\` text NOT NULL,
+        \`rating\` text,
+        \`conviction\` text,
+        \`time_horizon_months\` integer,
+        \`packet_title\` text,
+        \`packet_subtitle\` text,
+        \`thesis_markdown\` text NOT NULL DEFAULT '',
+        \`variant_view_markdown\` text NOT NULL DEFAULT '',
+        \`catalysts_markdown\` text NOT NULL DEFAULT '',
+        \`risks_markdown\` text NOT NULL DEFAULT '',
+        \`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
+        \`next_actions_markdown\` text NOT NULL DEFAULT '',
+        \`created_at\` text NOT NULL,
+        \`updated_at\` text NOT NULL,
+        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
+        FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
+      );
+    `);
+  }
+
+  if (!hasTable(client, 'research_memo_evidence')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
+        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        \`memo_id\` integer NOT NULL,
+        \`artifact_id\` integer NOT NULL,
+        \`section\` text NOT NULL,
+        \`annotation\` text,
+        \`sort_order\` integer NOT NULL DEFAULT 0,
+        \`created_at\` text NOT NULL,
+        FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
+        FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
+      );
+    `);
+  }
+
+  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
+  client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
+  client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
+  client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
+  client.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
+      artifact_id UNINDEXED,
+      user_id UNINDEXED,
+      ticker UNINDEXED,
+      title,
+      summary,
+      body_markdown,
+      search_text,
+      tags_text
+    );
+  `);
+
+  client.exec(`
+    INSERT INTO \`research_artifact\` (
+      \`user_id\`,
+      \`organization_id\`,
+      \`ticker\`,
+      \`accession_number\`,
+      \`kind\`,
+      \`source\`,
+      \`subtype\`,
+      \`title\`,
+      \`summary\`,
+      \`body_markdown\`,
+      \`search_text\`,
+      \`visibility_scope\`,
+      \`tags\`,
+      \`metadata\`,
+      \`created_at\`,
+      \`updated_at\`
+    )
+    SELECT
+      r.\`user_id\`,
+      NULL,
+      r.\`ticker\`,
+      r.\`accession_number\`,
+      CASE
+        WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
+        ELSE 'note'
+      END,
+      CASE
+        WHEN r.\`entry_type\` = 'status_change' THEN 'system'
+        ELSE 'user'
+      END,
+      r.\`entry_type\`,
+      r.\`title\`,
+      CASE
+        WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
+        ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
+      END,
+      r.\`body_markdown\`,
+      r.\`body_markdown\`,
+      'private',
+      NULL,
+      r.\`metadata\`,
+      r.\`created_at\`,
+      r.\`updated_at\`
+    FROM \`research_journal_entry\` r
+    WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
+      AND NOT EXISTS (
+        SELECT 1
+        FROM \`research_artifact\` a
+        WHERE a.\`user_id\` = r.\`user_id\`
+          AND a.\`ticker\` = r.\`ticker\`
+          AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
+          AND a.\`kind\` = CASE
+            WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
+            ELSE 'note'
+          END
+          AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
+          AND a.\`created_at\` = r.\`created_at\`
+      );
+  `);
+
+  client.exec(`
+    INSERT INTO \`research_artifact\` (
+      \`user_id\`,
+      \`organization_id\`,
+      \`ticker\`,
+      \`accession_number\`,
+      \`kind\`,
+      \`source\`,
+      \`subtype\`,
+      \`title\`,
+      \`summary\`,
+      \`body_markdown\`,
+      \`search_text\`,
+      \`visibility_scope\`,
+      \`tags\`,
+      \`metadata\`,
+      \`created_at\`,
+      \`updated_at\`
+    )
+    SELECT
+      w.\`user_id\`,
+      NULL,
+      f.\`ticker\`,
+      f.\`accession_number\`,
+      'ai_report',
+      'system',
+      'filing_analysis',
+      f.\`filing_type\` || ' AI memo',
+      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
+      'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
+      'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
+      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
+      COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
+      'private',
+      NULL,
+      json_object(
+        'provider', json_extract(f.\`analysis\`, '$.provider'),
+        'model', json_extract(f.\`analysis\`, '$.model'),
+        'filingType', f.\`filing_type\`,
+        'filingDate', f.\`filing_date\`
+      ),
+      f.\`created_at\`,
+      f.\`updated_at\`
+    FROM \`filing\` f
+    JOIN \`watchlist_item\` w
+      ON w.\`ticker\` = f.\`ticker\`
+    WHERE f.\`analysis\` IS NOT NULL
+      AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
+      AND NOT EXISTS (
+        SELECT 1
+        FROM \`research_artifact\` a
+        WHERE a.\`user_id\` = w.\`user_id\`
+          AND a.\`ticker\` = f.\`ticker\`
+          AND a.\`accession_number\` = f.\`accession_number\`
+          AND a.\`kind\` = 'ai_report'
+      );
+  `);
+
+  client.exec('DELETE FROM `research_artifact_fts`;');
+  client.exec(`
+    INSERT INTO \`research_artifact_fts\` (
+      \`artifact_id\`,
+      \`user_id\`,
+      \`ticker\`,
+      \`title\`,
+      \`summary\`,
+      \`body_markdown\`,
+      \`search_text\`,
+      \`tags_text\`
+    )
+    SELECT
+      \`id\`,
+      \`user_id\`,
+      \`ticker\`,
+      COALESCE(\`title\`, ''),
+      COALESCE(\`summary\`, ''),
+      COALESCE(\`body_markdown\`, ''),
+      COALESCE(\`search_text\`, ''),
+      CASE
+        WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
+        ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
+      END
+    FROM \`research_artifact\`;
+  `);
+}
+
+function ensureTaxonomySnapshotCompat(client: Database) {
+  ensureColumns(client, 'filing_taxonomy_snapshot', [
+    { name: 'parser_engine', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'fiscal-xbrl';" },
+    { name: 'parser_version', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text NOT NULL DEFAULT 'unknown';" },
+    { name: 'taxonomy_regime', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text NOT NULL DEFAULT 'unknown';" },
+    { name: 'fiscal_pack', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;' },
+    { name: 'faithful_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;' },
+    { name: 'surface_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;' },
+    { name: 'detail_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;' },
+    { name: 'kpi_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;' },
+    { name: 'normalization_summary', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;' }
+  ]);
+
+  if (!hasTable(client, 'filing_taxonomy_snapshot')) {
+    return;
+  }
+
+  client.exec(`
+    UPDATE \`filing_taxonomy_snapshot\`
+    SET
+      \`faithful_rows\` = COALESCE(\`faithful_rows\`, \`statement_rows\`),
+      \`surface_rows\` = COALESCE(\`surface_rows\`, '${DEFAULT_SURFACE_ROWS_JSON}'),
+      \`detail_rows\` = COALESCE(\`detail_rows\`, '${DEFAULT_DETAIL_ROWS_JSON}'),
+      \`kpi_rows\` = COALESCE(\`kpi_rows\`, '[]');
+  `);
+}
+
+function ensureTaxonomyContextCompat(client: Database) {
+  if (!hasTable(client, 'filing_taxonomy_context')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`filing_taxonomy_context\` (
+        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        \`snapshot_id\` integer NOT NULL,
+        \`context_id\` text NOT NULL,
+        \`entity_identifier\` text,
+        \`entity_scheme\` text,
+        \`period_start\` text,
+        \`period_end\` text,
+        \`period_instant\` text,
+        \`segment_json\` text,
+        \`scenario_json\` text,
+        \`created_at\` text NOT NULL,
+        FOREIGN KEY (\`snapshot_id\`) REFERENCES \`filing_taxonomy_snapshot\`(\`id\`) ON UPDATE no action ON DELETE cascade
+      );
+    `);
+  }
+
+  client.exec('CREATE INDEX IF NOT EXISTS `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);');
+  client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);');
+}
+
+function ensureTaxonomyConceptCompat(client: Database) {
+  ensureColumns(client, 'filing_taxonomy_concept', [
+    { name: 'balance', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;' },
+    { name: 'period_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;' },
+    { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;' },
+    { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;' },
+    { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;' },
+    { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;' },
+    { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;' },
+    { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;' },
+    { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer NOT NULL DEFAULT false;' }
+  ]);
+}
+
+function ensureTaxonomyFactCompat(client: Database) {
+  ensureColumns(client, 'filing_taxonomy_fact', [
+    { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;' },
+    { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;' },
+    { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;' },
+    { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;' },
+    { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;' },
+    { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;' },
+    { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer NOT NULL DEFAULT false;' },
+    { name: 'precision', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;' },
+    { name: 'nil', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer NOT NULL DEFAULT false;' }
+  ]);
+}
+
+function ensureTaxonomyCompat(client: Database) {
+  ensureTaxonomySnapshotCompat(client);
+  ensureTaxonomyContextCompat(client);
+  ensureTaxonomyConceptCompat(client);
+  ensureTaxonomyFactCompat(client);
+}
+
+export function ensureLocalSqliteSchema(client: Database) {
+  const missingBaseSchema = [
+    'filing',
+    'watchlist_item',
+    'holding',
+    'task_run',
+    'portfolio_insight'
+  ].some((tableName) => !hasTable(client, tableName));
+
+  if (missingBaseSchema) {
+    applyBaseSchemaCompat(client);
+  }
+
+  if (!hasTable(client, 'user')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`user\` (
+        \`id\` text PRIMARY KEY NOT NULL,
+        \`name\` text NOT NULL,
+        \`email\` text NOT NULL,
+        \`emailVerified\` integer NOT NULL DEFAULT 0,
+        \`image\` text,
+        \`createdAt\` integer NOT NULL,
+        \`updatedAt\` integer NOT NULL,
+        \`role\` text,
+        \`banned\` integer DEFAULT 0,
+        \`banReason\` text,
+        \`banExpires\` integer
+      );
+    `);
+    client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
+  }
+
+  if (!hasTable(client, 'organization')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`organization\` (
+        \`id\` text PRIMARY KEY NOT NULL,
+        \`name\` text NOT NULL,
+        \`slug\` text NOT NULL,
+        \`logo\` text,
+        \`createdAt\` integer NOT NULL,
+        \`metadata\` text
+      );
+    `);
+    client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
+  }
+
+  if (!hasTable(client, 'filing_statement_snapshot')) {
+    applySqlFile(client, '0001_glossy_statement_snapshots.sql');
+  }
+
+  ensureColumns(client, 'task_run', [
+    { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" },
+    { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' },
+    { name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' },
+    { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' },
+    { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' },
+    { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' }
+  ]);
+
+  if (!hasTable(client, 'task_stage_event')) {
+    applySqlFile(client, '0003_task_stage_event_timeline.sql');
+  }
+
+  if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) {
+    client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;');
+  }
+
+  client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);');
+
+  ensureColumns(client, 'watchlist_item', [
+    { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' },
+    { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' },
+    { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" },
+    { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" },
+    { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" },
+    { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' }
+  ]);
+
+  if (hasTable(client, 'watchlist_item')) {
+    client.exec(`
+      UPDATE \`watchlist_item\`
+      SET
+        \`status\` = CASE
+          WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog'
+          ELSE \`status\`
+        END,
+        \`priority\` = CASE
+          WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium'
+          ELSE \`priority\`
+        END,
+        \`updated_at\` = CASE
+          WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP)
+          ELSE \`updated_at\`
+        END;
+    `);
+
+    client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);');
+  }
+
+  if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) {
+    client.exec('ALTER TABLE `holding` ADD `company_name` text;');
+  }
+
+  if (!hasTable(client, 'filing_taxonomy_snapshot')) {
+    applySqlFile(client, '0005_financial_taxonomy_v3.sql');
+  }
+  ensureTaxonomyCompat(client);
+
+  if (!hasTable(client, 'company_financial_bundle')) {
+    applySqlFile(client, '0007_company_financial_bundles.sql');
+  }
+
+  if (!hasTable(client, 'company_overview_cache')) {
+    applySqlFile(client, '0012_company_overview_cache.sql');
+  }
+
+  if (!hasTable(client, 'research_journal_entry')) {
+    client.exec(`
+      CREATE TABLE IF NOT EXISTS \`research_journal_entry\` (
+        \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        \`user_id\` text NOT NULL,
+        \`ticker\` text NOT NULL,
+        \`accession_number\` text,
+        \`entry_type\` text NOT NULL,
+        \`title\` text,
+        \`body_markdown\` text NOT NULL,
+        \`metadata\` text,
+        \`created_at\` text NOT NULL,
+        \`updated_at\` text NOT NULL,
+        FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade
+      );
+    `);
+    client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);');
+    client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);');
+  }
+
+  if (!hasTable(client, 'search_document')) {
+    applySqlFile(client, '0008_search_rag.sql');
+  }
+
+  ensureResearchWorkspaceSchema(client);
+}
+
+export const __sqliteSchemaCompatInternals = {
+  applyBaseSchemaCompat,
+  applySqlFile,
+  hasColumn,
+  hasTable
+};

scripts/dev.ts

@@ -7,6 +7,7 @@ import {
   ensureFinancialIngestionSchemaHealthy,
   resolveFinancialSchemaRepairMode
 } from '../lib/server/db/financial-ingestion-schema';
+import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat';
 import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
 
 type DrizzleJournal = {
@@ -135,6 +136,7 @@ if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
 
   try {
     client.exec('PRAGMA foreign_keys = ON;');
+    ensureLocalSqliteSchema(client);
     const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
       mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
     });

scripts/e2e-prepare.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Database } from 'bun:sqlite';
+import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema';
+import { hasColumn, hasTable } from '../lib/server/db/sqlite-schema-compat';
+import { prepareE2eDatabase } from './e2e-prepare';
+
+describe('prepareE2eDatabase', () => {
+  it('bootstraps a fresh e2e database with the current taxonomy schema shape', () => {
+    const tempDir = mkdtempSync(join(tmpdir(), 'fiscal-e2e-prepare-'));
+    const databasePath = join(tempDir, 'e2e.sqlite');
+    const workflowDataDir = join(tempDir, 'workflow-data');
+
+    try {
+      prepareE2eDatabase({ databasePath, workflowDataDir });
+
+      const client = new Database(databasePath, { create: true });
+
+      try {
+        client.exec('PRAGMA foreign_keys = ON;');
+
+        expect(hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
+        expect(hasTable(client, 'filing_taxonomy_context')).toBe(true);
+
+        const health = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
+        expect(health.ok).toBe(true);
+      } finally {
+        client.close();
+      }
+    } finally {
+      rmSync(tempDir, { force: true, recursive: true });
+    }
+  });
+});

scripts/e2e-prepare.ts

@@ -1,46 +1,38 @@
-import { mkdirSync, readFileSync, rmSync } from 'node:fs';
+import { mkdirSync, rmSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { Database } from 'bun:sqlite';
-
+import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema';
-const MIGRATION_FILES = [
+import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat';
-  '0000_cold_silver_centurion.sql',
-  '0001_glossy_statement_snapshots.sql',
-  '0002_workflow_task_projection_metadata.sql',
-  '0003_task_stage_event_timeline.sql',
-  '0004_watchlist_company_taxonomy.sql',
-  '0005_financial_taxonomy_v3.sql',
-  '0006_coverage_journal_tracking.sql',
-  '0007_company_financial_bundles.sql',
-  '0008_research_workspace.sql'
-] as const;
 
 export const E2E_DATABASE_PATH = join(process.cwd(), 'data', 'e2e.sqlite');
 export const E2E_WORKFLOW_DATA_DIR = join(process.cwd(), '.workflow-data', 'e2e');
 
+type PrepareE2eDatabaseOptions = {
+  databasePath?: string;
+  workflowDataDir?: string;
+};
+
 function removeFileIfPresent(path: string) {
   rmSync(path, { force: true });
 }
 
-function applyMigrations(database: Database) {
+export function prepareE2eDatabase(options: PrepareE2eDatabaseOptions = {}) {
-  for (const file of MIGRATION_FILES) {
+  const databasePath = options.databasePath ?? E2E_DATABASE_PATH;
-    const sql = readFileSync(join(process.cwd(), 'drizzle', file), 'utf8');
+  const workflowDataDir = options.workflowDataDir ?? E2E_WORKFLOW_DATA_DIR;
-    database.exec(sql);
-  }
-}
 
-export function prepareE2eDatabase() {
+  mkdirSync(dirname(databasePath), { recursive: true });
-  mkdirSync(dirname(E2E_DATABASE_PATH), { recursive: true });
 
-  removeFileIfPresent(E2E_DATABASE_PATH);
+  removeFileIfPresent(databasePath);
-  removeFileIfPresent(`${E2E_DATABASE_PATH}-shm`);
+  removeFileIfPresent(`${databasePath}-shm`);
-  removeFileIfPresent(`${E2E_DATABASE_PATH}-wal`);
+  removeFileIfPresent(`${databasePath}-wal`);
-  rmSync(E2E_WORKFLOW_DATA_DIR, { force: true, recursive: true });
+  rmSync(workflowDataDir, { force: true, recursive: true });
 
-  const database = new Database(E2E_DATABASE_PATH, { create: true });
+  const database = new Database(databasePath, { create: true });
 
   try {
     database.exec('PRAGMA foreign_keys = ON;');
-    applyMigrations(database);
+    ensureLocalSqliteSchema(database);
+    ensureFinancialIngestionSchemaHealthy(database, { mode: 'auto' });
   } finally {
     database.close();
   }