Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix SQLite taxonomy schema bootstrap drift

Browse Source
This commit is contained in:
francy51
2026-03-14 19:00:29 -04:00
parent f5730597f4
commit b735b864d2
6 changed files with 739 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
lib/server/db/index.test.ts
View File

@@ -38,7 +38,34 @@ describe('sqlite schema compatibility bootstrap', () => {
expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true);
expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true);
expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true);
expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true);
@@ -57,4 +84,119 @@ describe('sqlite schema compatibility bootstrap', () => {
client.close();
});
it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => {
const client = new Database(':memory:');
client.exec('PRAGMA foreign_keys = ON;');
applyMigration(client, '0000_cold_silver_centurion.sql');
applyMigration(client, '0005_financial_taxonomy_v3.sql');
client.exec(`
INSERT INTO \`filing\` (
\`ticker\`,
\`filing_type\`,
\`filing_date\`,
\`accession_number\`,
\`cik\`,
\`company_name\`,
\`created_at\`,
\`updated_at\`
) VALUES (
'AAPL',
'10-K',
'2024-09-28',
'0000320193-24-000001',
'0000320193',
'Apple Inc.',
'2024-10-30T00:00:00.000Z',
'2024-10-30T00:00:00.000Z'
);
`);
const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}';
client.exec(`
INSERT INTO \`filing_taxonomy_snapshot\` (
\`filing_id\`,
\`ticker\`,
\`filing_date\`,
\`filing_type\`,
\`parse_status\`,
\`source\`,
\`statement_rows\`,
\`created_at\`,
\`updated_at\`
) VALUES (
1,
'AAPL',
'2024-09-28',
'10-K',
'success',
'xbrl_instance',
'${statementRows}',
'2024-10-30T00:00:00.000Z',
'2024-10-30T00:00:00.000Z'
);
`);
__dbInternals.ensureLocalSqliteSchema(client);
__dbInternals.ensureLocalSqliteSchema(client);
const row = client.query(`
SELECT
\`parser_engine\`,
\`parser_version\`,
\`taxonomy_regime\`,
\`faithful_rows\`,
\`surface_rows\`,
\`detail_rows\`,
\`kpi_rows\`,
\`normalization_summary\`
FROM \`filing_taxonomy_snapshot\`
WHERE \`filing_id\` = 1
`).get() as {
parser_engine: string;
parser_version: string;
taxonomy_regime: string;
faithful_rows: string | null;
surface_rows: string | null;
detail_rows: string | null;
kpi_rows: string | null;
normalization_summary: string | null;
};
expect(row.parser_engine).toBe('fiscal-xbrl');
expect(row.parser_version).toBe('unknown');
expect(row.taxonomy_regime).toBe('unknown');
expect(row.faithful_rows).toBe(statementRows);
expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}');
expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}');
expect(row.kpi_rows).toBe('[]');
expect(row.normalization_summary).toBeNull();
client.close();
});
it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => {
const client = new Database(':memory:');
client.exec('PRAGMA foreign_keys = ON;');
applyMigration(client, '0000_cold_silver_centurion.sql');
applyMigration(client, '0005_financial_taxonomy_v3.sql');
client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';");
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false);
__dbInternals.ensureLocalSqliteSchema(client);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true);
expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true);
client.close();
});
});