Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix SQLite taxonomy schema bootstrap drift

Browse Source
This commit is contained in:
francy51
2026-03-14 19:00:29 -04:00
parent f5730597f4
commit b735b864d2
6 changed files with 739 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
lib/server/db/index.test.ts
View File

@@ -38,7 +38,34 @@ describe('sqlite schema compatibility bootstrap', () => {
expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true);
expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true);
expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true);
expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true);
@@ -57,4 +84,119 @@ describe('sqlite schema compatibility bootstrap', () => {
client.close();
});
it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => {
const client = new Database(':memory:');
client.exec('PRAGMA foreign_keys = ON;');
applyMigration(client, '0000_cold_silver_centurion.sql');
applyMigration(client, '0005_financial_taxonomy_v3.sql');
client.exec(`
INSERT INTO \`filing\` (
\`ticker\`,
\`filing_type\`,
\`filing_date\`,
\`accession_number\`,
\`cik\`,
\`company_name\`,
\`created_at\`,
\`updated_at\`
) VALUES (
'AAPL',
'10-K',
'2024-09-28',
'0000320193-24-000001',
'0000320193',
'Apple Inc.',
'2024-10-30T00:00:00.000Z',
'2024-10-30T00:00:00.000Z'
);
`);
const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';
client.exec(`
INSERT INTO \`filing_taxonomy_snapshot\` (
\`filing_id\`,
\`ticker\`,
\`filing_date\`,
\`filing_type\`,
\`parse_status\`,
\`source\`,
\`statement_rows\`,
\`created_at\`,
\`updated_at\`
) VALUES (
1,
'AAPL',
'2024-09-28',
'10-K',
'success',
'xbrl_instance',
'${statementRows}',
'2024-10-30T00:00:00.000Z',
'2024-10-30T00:00:00.000Z'
);
`);
__dbInternals.ensureLocalSqliteSchema(client);
__dbInternals.ensureLocalSqliteSchema(client);
const row = client.query(`
SELECT
\`parser_engine\`,
\`parser_version\`,
\`taxonomy_regime\`,
\`faithful_rows\`,
\`surface_rows\`,
\`detail_rows\`,
\`kpi_rows\`,
\`normalization_summary\`
FROM \`filing_taxonomy_snapshot\`
WHERE \`filing_id\` = 1
`).get() as {
parser_engine: string;
parser_version: string;
taxonomy_regime: string;
faithful_rows: string | null;
surface_rows: string | null;
detail_rows: string | null;
kpi_rows: string | null;
normalization_summary: string | null;
};
expect(row.parser_engine).toBe('fiscal-xbrl');
expect(row.parser_version).toBe('unknown');
expect(row.taxonomy_regime).toBe('unknown');
expect(row.faithful_rows).toBe(statementRows);
expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}');
expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}');
expect(row.kpi_rows).toBe('[]');
expect(row.normalization_summary).toBeNull();
client.close();
});
it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => {
const client = new Database(':memory:');
client.exec('PRAGMA foreign_keys = ON;');
applyMigration(client, '0000_cold_silver_centurion.sql');
applyMigration(client, '0005_financial_taxonomy_v3.sql');
client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';");
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false);
__dbInternals.ensureLocalSqliteSchema(client);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
client.close();
});
});

439
lib/server/db/index.ts
View File

@@ -1,5 +1,5 @@
import { mkdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import { Database } from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import { load as loadSqliteVec } from 'sqlite-vec';
@@ -8,6 +8,11 @@ import {
resolveFinancialSchemaRepairMode
} from './financial-ingestion-schema';
import { schema } from './schema';
import {
ensureLocalSqliteSchema,
hasColumn,
hasTable
} from './sqlite-schema-compat';
type AppDrizzleDb = ReturnType<typeof createDb>;
@@ -33,37 +38,6 @@ function getDatabasePath() {
return databasePath;
}
function hasTable(client: Database, tableName: string) {
const row = client
.query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
.get('table', tableName) as { name: string } | null;
return row !== null;
}
function hasColumn(client: Database, tableName: string, columnName: string) {
if (!hasTable(client, tableName)) {
return false;
}
const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>;
return rows.some((row) => row.name === columnName);
}
function applySqlFile(client: Database, fileName: string) {
const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8');
client.exec(sql);
}
function applyBaseSchemaCompat(client: Database) {
const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
.replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
.replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
.replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
client.exec(sql);
}
let customSqliteConfigured = false;
const vectorExtensionStatus = new WeakMap<Database, boolean>();
@@ -103,405 +77,6 @@ function isVectorExtensionLoaded(client: Database) {
return vectorExtensionStatus.get(client) ?? false;
}
function ensureResearchWorkspaceSchema(client: Database) {
if (!hasTable(client, 'research_artifact')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_artifact\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`accession_number\` text,
\`kind\` text NOT NULL,
\`source\` text NOT NULL DEFAULT 'user',
\`subtype\` text,
\`title\` text,
\`summary\` text,
\`body_markdown\` text,
\`search_text\` text,
\`visibility_scope\` text NOT NULL DEFAULT 'private',
\`tags\` text,
\`metadata\` text,
\`file_name\` text,
\`mime_type\` text,
\`file_size_bytes\` integer,
\`storage_path\` text,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`rating\` text,
\`conviction\` text,
\`time_horizon_months\` integer,
\`packet_title\` text,
\`packet_subtitle\` text,
\`thesis_markdown\` text NOT NULL DEFAULT '',
\`variant_view_markdown\` text NOT NULL DEFAULT '',
\`catalysts_markdown\` text NOT NULL DEFAULT '',
\`risks_markdown\` text NOT NULL DEFAULT '',
\`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
\`next_actions_markdown\` text NOT NULL DEFAULT '',
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo_evidence')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`memo_id\` integer NOT NULL,
\`artifact_id\` integer NOT NULL,
\`section\` text NOT NULL,
\`annotation\` text,
\`sort_order\` integer NOT NULL DEFAULT 0,
\`created_at\` text NOT NULL,
FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
}
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
client.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
artifact_id UNINDEXED,
user_id UNINDEXED,
ticker UNINDEXED,
title,
summary,
body_markdown,
search_text,
tags_text
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
r.\`user_id\`,
NULL,
r.\`ticker\`,
r.\`accession_number\`,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'system'
ELSE 'user'
END,
r.\`entry_type\`,
r.\`title\`,
CASE
WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
END,
r.\`body_markdown\`,
r.\`body_markdown\`,
'private',
NULL,
r.\`metadata\`,
r.\`created_at\`,
r.\`updated_at\`
FROM \`research_journal_entry\` r
WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = r.\`user_id\`
AND a.\`ticker\` = r.\`ticker\`
AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
AND a.\`kind\` = CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END
AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
AND a.\`created_at\` = r.\`created_at\`
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
w.\`user_id\`,
NULL,
f.\`ticker\`,
f.\`accession_number\`,
'ai_report',
'system',
'filing_analysis',
f.\`filing_type\` || ' AI memo',
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'private',
NULL,
json_object(
'provider', json_extract(f.\`analysis\`, '$.provider'),
'model', json_extract(f.\`analysis\`, '$.model'),
'filingType', f.\`filing_type\`,
'filingDate', f.\`filing_date\`
),
f.\`created_at\`,
f.\`updated_at\`
FROM \`filing\` f
JOIN \`watchlist_item\` w
ON w.\`ticker\` = f.\`ticker\`
WHERE f.\`analysis\` IS NOT NULL
AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = w.\`user_id\`
AND a.\`ticker\` = f.\`ticker\`
AND a.\`accession_number\` = f.\`accession_number\`
AND a.\`kind\` = 'ai_report'
);
`);
client.exec('DELETE FROM `research_artifact_fts`;');
client.exec(`
INSERT INTO \`research_artifact_fts\` (
\`artifact_id\`,
\`user_id\`,
\`ticker\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`tags_text\`
)
SELECT
\`id\`,
\`user_id\`,
\`ticker\`,
COALESCE(\`title\`, ''),
COALESCE(\`summary\`, ''),
COALESCE(\`body_markdown\`, ''),
COALESCE(\`search_text\`, ''),
CASE
WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
END
FROM \`research_artifact\`;
`);
}
function ensureLocalSqliteSchema(client: Database) {
const missingBaseSchema = [
'filing',
'watchlist_item',
'holding',
'task_run',
'portfolio_insight'
].some((tableName) => !hasTable(client, tableName));
if (missingBaseSchema) {
applyBaseSchemaCompat(client);
}
if (!hasTable(client, 'user')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`user\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`email\` text NOT NULL,
\`emailVerified\` integer NOT NULL DEFAULT 0,
\`image\` text,
\`createdAt\` integer NOT NULL,
\`updatedAt\` integer NOT NULL,
\`role\` text,
\`banned\` integer DEFAULT 0,
\`banReason\` text,
\`banExpires\` integer
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
}
if (!hasTable(client, 'organization')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`organization\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`slug\` text NOT NULL,
\`logo\` text,
\`createdAt\` integer NOT NULL,
\`metadata\` text
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
}
if (!hasTable(client, 'filing_statement_snapshot')) {
applySqlFile(client, '0001_glossy_statement_snapshots.sql');
}
if (hasTable(client, 'task_run')) {
const missingTaskColumns: Array<{ name: string; sql: string }> = [
{ name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" },
{ name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' },
{ name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' },
{ name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' },
{ name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' },
{ name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' }
];
for (const column of missingTaskColumns) {
if (!hasColumn(client, 'task_run', column.name)) {
client.exec(column.sql);
}
}
}
if (!hasTable(client, 'task_stage_event')) {
applySqlFile(client, '0003_task_stage_event_timeline.sql');
}
if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) {
client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;');
}
client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);');
if (hasTable(client, 'watchlist_item')) {
const missingWatchlistColumns: Array<{ name: string; sql: string }> = [
{ name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' },
{ name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' },
{ name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" },
{ name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" },
{ name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" },
{ name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' }
];
for (const column of missingWatchlistColumns) {
if (!hasColumn(client, 'watchlist_item', column.name)) {
client.exec(column.sql);
}
}
client.exec(`
UPDATE \`watchlist_item\`
SET
\`status\` = CASE
WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog'
ELSE \`status\`
END,
\`priority\` = CASE
WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium'
ELSE \`priority\`
END,
\`updated_at\` = CASE
WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP)
ELSE \`updated_at\`
END;
`);
client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);');
}
if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) {
client.exec('ALTER TABLE `holding` ADD `company_name` text;');
}
if (!hasTable(client, 'filing_taxonomy_snapshot')) {
applySqlFile(client, '0005_financial_taxonomy_v3.sql');
}
if (!hasTable(client, 'company_financial_bundle')) {
applySqlFile(client, '0007_company_financial_bundles.sql');
}
if (!hasTable(client, 'company_overview_cache')) {
applySqlFile(client, '0012_company_overview_cache.sql');
}
if (!hasTable(client, 'research_journal_entry')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_journal_entry\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`ticker\` text NOT NULL,
\`accession_number\` text,
\`entry_type\` text NOT NULL,
\`title\` text,
\`body_markdown\` text NOT NULL,
\`metadata\` text,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);');
}
if (!hasTable(client, 'search_document')) {
applySqlFile(client, '0008_search_rag.sql');
}
ensureResearchWorkspaceSchema(client);
}
function ensureSearchVirtualTables(client: Database) {
client.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS \`search_chunk_fts\` USING fts5(

533
lib/server/db/sqlite-schema-compat.ts Normal file
View File

@@ -0,0 +1,533 @@
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import type { Database } from 'bun:sqlite';
const DEFAULT_SURFACE_ROWS_JSON = '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';
const DEFAULT_DETAIL_ROWS_JSON = '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}';
type MissingColumnDefinition = {
name: string;
sql: string;
};
export function hasTable(client: Database, tableName: string) {
const row = client
.query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
.get('table', tableName) as { name: string } | null;
return row !== null;
}
export function hasColumn(client: Database, tableName: string, columnName: string) {
if (!hasTable(client, tableName)) {
return false;
}
const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>;
return rows.some((row) => row.name === columnName);
}
export function applySqlFile(client: Database, fileName: string) {
const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8');
client.exec(sql);
}
export function applyBaseSchemaCompat(client: Database) {
const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
.replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
.replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
.replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
client.exec(sql);
}
function ensureColumns(client: Database, tableName: string, columns: MissingColumnDefinition[]) {
if (!hasTable(client, tableName)) {
return;
}
for (const column of columns) {
if (!hasColumn(client, tableName, column.name)) {
client.exec(column.sql);
}
}
}
function ensureResearchWorkspaceSchema(client: Database) {
if (!hasTable(client, 'research_artifact')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_artifact\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`accession_number\` text,
\`kind\` text NOT NULL,
\`source\` text NOT NULL DEFAULT 'user',
\`subtype\` text,
\`title\` text,
\`summary\` text,
\`body_markdown\` text,
\`search_text\` text,
\`visibility_scope\` text NOT NULL DEFAULT 'private',
\`tags\` text,
\`metadata\` text,
\`file_name\` text,
\`mime_type\` text,
\`file_size_bytes\` integer,
\`storage_path\` text,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`rating\` text,
\`conviction\` text,
\`time_horizon_months\` integer,
\`packet_title\` text,
\`packet_subtitle\` text,
\`thesis_markdown\` text NOT NULL DEFAULT '',
\`variant_view_markdown\` text NOT NULL DEFAULT '',
\`catalysts_markdown\` text NOT NULL DEFAULT '',
\`risks_markdown\` text NOT NULL DEFAULT '',
\`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
\`next_actions_markdown\` text NOT NULL DEFAULT '',
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo_evidence')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`memo_id\` integer NOT NULL,
\`artifact_id\` integer NOT NULL,
\`section\` text NOT NULL,
\`annotation\` text,
\`sort_order\` integer NOT NULL DEFAULT 0,
\`created_at\` text NOT NULL,
FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
}
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
client.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
artifact_id UNINDEXED,
user_id UNINDEXED,
ticker UNINDEXED,
title,
summary,
body_markdown,
search_text,
tags_text
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
r.\`user_id\`,
NULL,
r.\`ticker\`,
r.\`accession_number\`,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'system'
ELSE 'user'
END,
r.\`entry_type\`,
r.\`title\`,
CASE
WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
END,
r.\`body_markdown\`,
r.\`body_markdown\`,
'private',
NULL,
r.\`metadata\`,
r.\`created_at\`,
r.\`updated_at\`
FROM \`research_journal_entry\` r
WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = r.\`user_id\`
AND a.\`ticker\` = r.\`ticker\`
AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
AND a.\`kind\` = CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END
AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
AND a.\`created_at\` = r.\`created_at\`
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
w.\`user_id\`,
NULL,
f.\`ticker\`,
f.\`accession_number\`,
'ai_report',
'system',
'filing_analysis',
f.\`filing_type\` || ' AI memo',
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'private',
NULL,
json_object(
'provider', json_extract(f.\`analysis\`, '$.provider'),
'model', json_extract(f.\`analysis\`, '$.model'),
'filingType', f.\`filing_type\`,
'filingDate', f.\`filing_date\`
),
f.\`created_at\`,
f.\`updated_at\`
FROM \`filing\` f
JOIN \`watchlist_item\` w
ON w.\`ticker\` = f.\`ticker\`
WHERE f.\`analysis\` IS NOT NULL
AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = w.\`user_id\`
AND a.\`ticker\` = f.\`ticker\`
AND a.\`accession_number\` = f.\`accession_number\`
AND a.\`kind\` = 'ai_report'
);
`);
client.exec('DELETE FROM `research_artifact_fts`;');
client.exec(`
INSERT INTO \`research_artifact_fts\` (
\`artifact_id\`,
\`user_id\`,
\`ticker\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`tags_text\`
)
SELECT
\`id\`,
\`user_id\`,
\`ticker\`,
COALESCE(\`title\`, ''),
COALESCE(\`summary\`, ''),
COALESCE(\`body_markdown\`, ''),
COALESCE(\`search_text\`, ''),
CASE
WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
END
FROM \`research_artifact\`;
`);
}
function ensureTaxonomySnapshotCompat(client: Database) {
ensureColumns(client, 'filing_taxonomy_snapshot', [
{ name: 'parser_engine', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'fiscal-xbrl';" },
{ name: 'parser_version', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text NOT NULL DEFAULT 'unknown';" },
{ name: 'taxonomy_regime', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text NOT NULL DEFAULT 'unknown';" },
{ name: 'fiscal_pack', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;' },
{ name: 'faithful_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;' },
{ name: 'surface_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;' },
{ name: 'detail_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;' },
{ name: 'kpi_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;' },
{ name: 'normalization_summary', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;' }
]);
if (!hasTable(client, 'filing_taxonomy_snapshot')) {
return;
}
client.exec(`
UPDATE \`filing_taxonomy_snapshot\`
SET
\`faithful_rows\` = COALESCE(\`faithful_rows\`, \`statement_rows\`),
\`surface_rows\` = COALESCE(\`surface_rows\`, '${DEFAULT_SURFACE_ROWS_JSON}'),
\`detail_rows\` = COALESCE(\`detail_rows\`, '${DEFAULT_DETAIL_ROWS_JSON}'),
\`kpi_rows\` = COALESCE(\`kpi_rows\`, '[]');
`);
}
function ensureTaxonomyContextCompat(client: Database) {
if (!hasTable(client, 'filing_taxonomy_context')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`filing_taxonomy_context\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`snapshot_id\` integer NOT NULL,
\`context_id\` text NOT NULL,
\`entity_identifier\` text,
\`entity_scheme\` text,
\`period_start\` text,
\`period_end\` text,
\`period_instant\` text,
\`segment_json\` text,
\`scenario_json\` text,
\`created_at\` text NOT NULL,
FOREIGN KEY (\`snapshot_id\`) REFERENCES \`filing_taxonomy_snapshot\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
}
client.exec('CREATE INDEX IF NOT EXISTS `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);');
}
function ensureTaxonomyConceptCompat(client: Database) {
ensureColumns(client, 'filing_taxonomy_concept', [
{ name: 'balance', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;' },
{ name: 'period_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;' },
{ name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;' },
{ name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;' },
{ name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;' },
{ name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;' },
{ name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;' },
{ name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;' },
{ name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer NOT NULL DEFAULT false;' }
]);
}
function ensureTaxonomyFactCompat(client: Database) {
ensureColumns(client, 'filing_taxonomy_fact', [
{ name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;' },
{ name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;' },
{ name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;' },
{ name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;' },
{ name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;' },
{ name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;' },
{ name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer NOT NULL DEFAULT false;' },
{ name: 'precision', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;' },
{ name: 'nil', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer NOT NULL DEFAULT false;' }
]);
}
function ensureTaxonomyCompat(client: Database) {
ensureTaxonomySnapshotCompat(client);
ensureTaxonomyContextCompat(client);
ensureTaxonomyConceptCompat(client);
ensureTaxonomyFactCompat(client);
}
export function ensureLocalSqliteSchema(client: Database) {
const missingBaseSchema = [
'filing',
'watchlist_item',
'holding',
'task_run',
'portfolio_insight'
].some((tableName) => !hasTable(client, tableName));
if (missingBaseSchema) {
applyBaseSchemaCompat(client);
}
if (!hasTable(client, 'user')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`user\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`email\` text NOT NULL,
\`emailVerified\` integer NOT NULL DEFAULT 0,
\`image\` text,
\`createdAt\` integer NOT NULL,
\`updatedAt\` integer NOT NULL,
\`role\` text,
\`banned\` integer DEFAULT 0,
\`banReason\` text,
\`banExpires\` integer
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
}
if (!hasTable(client, 'organization')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`organization\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`slug\` text NOT NULL,
\`logo\` text,
\`createdAt\` integer NOT NULL,
\`metadata\` text
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
}
if (!hasTable(client, 'filing_statement_snapshot')) {
applySqlFile(client, '0001_glossy_statement_snapshots.sql');
}
ensureColumns(client, 'task_run', [
{ name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" },
{ name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' },
{ name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' },
{ name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' },
{ name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' },
{ name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' }
]);
if (!hasTable(client, 'task_stage_event')) {
applySqlFile(client, '0003_task_stage_event_timeline.sql');
}
if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) {
client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;');
}
client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);');
ensureColumns(client, 'watchlist_item', [
{ name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' },
{ name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' },
{ name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" },
{ name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" },
{ name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" },
{ name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' }
]);
if (hasTable(client, 'watchlist_item')) {
client.exec(`
UPDATE \`watchlist_item\`
SET
\`status\` = CASE
WHEN \`status\` IS NULL OR TRIM(\`status\`) = '' THEN 'backlog'
ELSE \`status\`
END,
\`priority\` = CASE
WHEN \`priority\` IS NULL OR TRIM(\`priority\`) = '' THEN 'medium'
ELSE \`priority\`
END,
\`updated_at\` = CASE
WHEN \`updated_at\` IS NULL OR TRIM(\`updated_at\`) = '' THEN COALESCE(NULLIF(\`created_at\`, ''), CURRENT_TIMESTAMP)
ELSE \`updated_at\`
END;
`);
client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);');
}
if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) {
client.exec('ALTER TABLE `holding` ADD `company_name` text;');
}
if (!hasTable(client, 'filing_taxonomy_snapshot')) {
applySqlFile(client, '0005_financial_taxonomy_v3.sql');
}
ensureTaxonomyCompat(client);
if (!hasTable(client, 'company_financial_bundle')) {
applySqlFile(client, '0007_company_financial_bundles.sql');
}
if (!hasTable(client, 'company_overview_cache')) {
applySqlFile(client, '0012_company_overview_cache.sql');
}
if (!hasTable(client, 'research_journal_entry')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_journal_entry\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`ticker\` text NOT NULL,
\`accession_number\` text,
\`entry_type\` text NOT NULL,
\`title\` text,
\`body_markdown\` text NOT NULL,
\`metadata\` text,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);');
}
if (!hasTable(client, 'search_document')) {
applySqlFile(client, '0008_search_rag.sql');
}
ensureResearchWorkspaceSchema(client);
}
export const __sqliteSchemaCompatInternals = {
applyBaseSchemaCompat,
applySqlFile,
hasColumn,
hasTable
};

2
scripts/dev.ts
View File

@@ -7,6 +7,7 @@ import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from '../lib/server/db/financial-ingestion-schema';
import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat';
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
type DrizzleJournal = {
@@ -135,6 +136,7 @@ if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
try {
client.exec('PRAGMA foreign_keys = ON;');
ensureLocalSqliteSchema(client);
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
});

36
scripts/e2e-prepare.test.ts Normal file
View File

@@ -0,0 +1,36 @@
import { describe, expect, it } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Database } from 'bun:sqlite';
import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema';
import { hasColumn, hasTable } from '../lib/server/db/sqlite-schema-compat';
import { prepareE2eDatabase } from './e2e-prepare';
describe('prepareE2eDatabase', () => {
it('bootstraps a fresh e2e database with the current taxonomy schema shape', () => {
const tempDir = mkdtempSync(join(tmpdir(), 'fiscal-e2e-prepare-'));
const databasePath = join(tempDir, 'e2e.sqlite');
const workflowDataDir = join(tempDir, 'workflow-data');
try {
prepareE2eDatabase({ databasePath, workflowDataDir });
const client = new Database(databasePath, { create: true });
try {
client.exec('PRAGMA foreign_keys = ON;');
expect(hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
expect(hasTable(client, 'filing_taxonomy_context')).toBe(true);
const health = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
expect(health.ok).toBe(true);
} finally {
client.close();
}
} finally {
rmSync(tempDir, { force: true, recursive: true });
}
});
});

46
scripts/e2e-prepare.ts
View File

@@ -1,46 +1,38 @@
import { mkdirSync, readFileSync, rmSync } from 'node:fs';
import { mkdirSync, rmSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { Database } from 'bun:sqlite';
const MIGRATION_FILES = [
'0000_cold_silver_centurion.sql',
'0001_glossy_statement_snapshots.sql',
'0002_workflow_task_projection_metadata.sql',
'0003_task_stage_event_timeline.sql',
'0004_watchlist_company_taxonomy.sql',
'0005_financial_taxonomy_v3.sql',
'0006_coverage_journal_tracking.sql',
'0007_company_financial_bundles.sql',
'0008_research_workspace.sql'
] as const;
import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema';
import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat';
export const E2E_DATABASE_PATH = join(process.cwd(), 'data', 'e2e.sqlite');
export const E2E_WORKFLOW_DATA_DIR = join(process.cwd(), '.workflow-data', 'e2e');
type PrepareE2eDatabaseOptions = {
databasePath?: string;
workflowDataDir?: string;
};
function removeFileIfPresent(path: string) {
rmSync(path, { force: true });
}
function applyMigrations(database: Database) {
for (const file of MIGRATION_FILES) {
const sql = readFileSync(join(process.cwd(), 'drizzle', file), 'utf8');
database.exec(sql);
}
}
export function prepareE2eDatabase(options: PrepareE2eDatabaseOptions = {}) {
const databasePath = options.databasePath ?? E2E_DATABASE_PATH;
const workflowDataDir = options.workflowDataDir ?? E2E_WORKFLOW_DATA_DIR;
export function prepareE2eDatabase() {
mkdirSync(dirname(E2E_DATABASE_PATH), { recursive: true });
mkdirSync(dirname(databasePath), { recursive: true });
removeFileIfPresent(E2E_DATABASE_PATH);
removeFileIfPresent(`${E2E_DATABASE_PATH}-shm`);
removeFileIfPresent(`${E2E_DATABASE_PATH}-wal`);
rmSync(E2E_WORKFLOW_DATA_DIR, { force: true, recursive: true });
removeFileIfPresent(databasePath);
removeFileIfPresent(`${databasePath}-shm`);
removeFileIfPresent(`${databasePath}-wal`);
rmSync(workflowDataDir, { force: true, recursive: true });
const database = new Database(E2E_DATABASE_PATH, { create: true });
const database = new Database(databasePath, { create: true });
try {
database.exec('PRAGMA foreign_keys = ON;');
applyMigrations(database);
ensureLocalSqliteSchema(database);
ensureFinancialIngestionSchemaHealthy(database, { mode: 'auto' });
} finally {
database.close();
}