Neon-Desk/lib/server/db/index.test.ts

import { describe, expect, it } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { Database } from 'bun:sqlite';
import { __dbInternals } from './index';

function applyMigration(client: Database, fileName: string) {
  const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8');
  client.exec(sql);
}

describe('sqlite schema compatibility bootstrap', () => {
  it('adds missing watchlist columns and taxonomy tables for older local databases', () => {
    const client = new Database(':memory:');
    client.exec('PRAGMA foreign_keys = ON;');

    applyMigration(client, '0000_cold_silver_centurion.sql');
    applyMigration(client, '0001_glossy_statement_snapshots.sql');
    applyMigration(client, '0002_workflow_task_projection_metadata.sql');
    applyMigration(client, '0003_task_stage_event_timeline.sql');
    applyMigration(client, '0009_task_notification_context.sql');

    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(false);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(false);
    expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(false);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(false);
    expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(false);
    expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(false);
    expect(__dbInternals.hasTable(client, 'research_memo')).toBe(false);

    __dbInternals.ensureLocalSqliteSchema(client);

    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'tags')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'priority')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'updated_at')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true);
    expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true);
    expect(__dbInternals.hasTable(client, 'search_document')).toBe(true);
    expect(__dbInternals.hasTable(client, 'search_chunk')).toBe(true);
    expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(true);
    expect(__dbInternals.hasTable(client, 'research_memo')).toBe(true);
    expect(__dbInternals.hasTable(client, 'research_memo_evidence')).toBe(true);
    expect(__dbInternals.hasTable(client, 'company_overview_cache')).toBe(true);

    __dbInternals.loadSqliteExtensions(client);
    __dbInternals.ensureSearchVirtualTables(client);

    expect(__dbInternals.hasTable(client, 'search_chunk_fts')).toBe(true);
    expect(__dbInternals.hasTable(client, 'search_chunk_vec')).toBe(true);

    client.close();
  });

  it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => {
    const client = new Database(':memory:');
    client.exec('PRAGMA foreign_keys = ON;');

    applyMigration(client, '0000_cold_silver_centurion.sql');
    applyMigration(client, '0005_financial_taxonomy_v3.sql');

    client.exec(`
      INSERT INTO \`filing\` (
        \`ticker\`,
        \`filing_type\`,
        \`filing_date\`,
        \`accession_number\`,
        \`cik\`,
        \`company_name\`,
        \`created_at\`,
        \`updated_at\`
      ) VALUES (
        'AAPL',
        '10-K',
        '2024-09-28',
        '0000320193-24-000001',
        '0000320193',
        'Apple Inc.',
        '2024-10-30T00:00:00.000Z',
        '2024-10-30T00:00:00.000Z'
      );
    `);

    const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';

    client.exec(`
      INSERT INTO \`filing_taxonomy_snapshot\` (
        \`filing_id\`,
        \`ticker\`,
        \`filing_date\`,
        \`filing_type\`,
        \`parse_status\`,
        \`source\`,
        \`statement_rows\`,
        \`created_at\`,
        \`updated_at\`
      ) VALUES (
        1,
        'AAPL',
        '2024-09-28',
        '10-K',
        'success',
        'xbrl_instance',
        '${statementRows}',
        '2024-10-30T00:00:00.000Z',
        '2024-10-30T00:00:00.000Z'
      );
    `);

    __dbInternals.ensureLocalSqliteSchema(client);
    __dbInternals.ensureLocalSqliteSchema(client);

    const row = client.query(`
      SELECT
        \`parser_engine\`,
        \`parser_version\`,
        \`taxonomy_regime\`,
        \`faithful_rows\`,
        \`surface_rows\`,
        \`detail_rows\`,
        \`kpi_rows\`,
        \`normalization_summary\`
      FROM \`filing_taxonomy_snapshot\`
      WHERE \`filing_id\` = 1
    `).get() as {
      parser_engine: string;
      parser_version: string;
      taxonomy_regime: string;
      faithful_rows: string | null;
      surface_rows: string | null;
      detail_rows: string | null;
      kpi_rows: string | null;
      normalization_summary: string | null;
    };

    expect(row.parser_engine).toBe('fiscal-xbrl');
    expect(row.parser_version).toBe('unknown');
    expect(row.taxonomy_regime).toBe('unknown');
    expect(row.faithful_rows).toBe(statementRows);
    expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}');
    expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}');
    expect(row.kpi_rows).toBe('[]');
    expect(row.normalization_summary).toBeNull();

    client.close();
  });

  it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => {
    const client = new Database(':memory:');
    client.exec('PRAGMA foreign_keys = ON;');

    applyMigration(client, '0000_cold_silver_centurion.sql');
    applyMigration(client, '0005_financial_taxonomy_v3.sql');
    client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';");

    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false);

    __dbInternals.ensureLocalSqliteSchema(client);

    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
    expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);

    client.close();
  });

  it('throws on missing parser_engine column when verifyCriticalSchema is called', () => {
    const client = new Database(':memory:');
    client.exec('PRAGMA foreign_keys = ON;');

    applyMigration(client, '0000_cold_silver_centurion.sql');
    applyMigration(client, '0005_financial_taxonomy_v3.sql');

    expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
    expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(false);

    expect(() => __dbInternals.verifyCriticalSchema(client)).toThrow(
      /filing_taxonomy_snapshot is missing columns: parser_engine/
    );

    client.close();
  });

  it('verifyCriticalSchema passes when all required columns exist', () => {
    const client = new Database(':memory:');
    client.exec('PRAGMA foreign_keys = ON;');

    applyMigration(client, '0000_cold_silver_centurion.sql');
    applyMigration(client, '0005_financial_taxonomy_v3.sql');

    __dbInternals.ensureLocalSqliteSchema(client);

    expect(() => __dbInternals.verifyCriticalSchema(client)).not.toThrow();

    client.close();
  });
});