Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Add untracked chart and schema files

Browse Source
This commit is contained in:
francy51
2026-03-13 00:11:59 -04:00
parent 8a8c4f7177
commit 01199d489a
16 changed files with 1794 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
lib/server/db/financial-ingestion-schema.test.ts Normal file
View File

@@ -0,0 +1,180 @@
import { describe, expect, it } from 'bun:test';
import { Database } from 'bun:sqlite';
import {
ensureFinancialIngestionSchemaHealthy,
inspectFinancialIngestionSchema,
withFinancialIngestionSchemaRetry
} from './financial-ingestion-schema';
function createBundleTable(client: Database) {
client.exec(`
CREATE TABLE \`company_financial_bundle\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`ticker\` text NOT NULL,
\`surface_kind\` text NOT NULL,
\`cadence\` text NOT NULL,
\`bundle_version\` integer NOT NULL,
\`source_snapshot_ids\` text NOT NULL,
\`source_signature\` text NOT NULL,
\`payload\` text NOT NULL,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL
);
`);
}
function createSnapshotTable(client: Database) {
client.exec(`
CREATE TABLE \`filing_taxonomy_snapshot\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`filing_id\` integer NOT NULL,
\`ticker\` text NOT NULL,
\`filing_date\` text NOT NULL,
\`filing_type\` text NOT NULL,
\`parse_status\` text NOT NULL,
\`updated_at\` text NOT NULL
);
`);
}
function createMetricValidationTable(client: Database) {
client.exec(`
CREATE TABLE \`filing_taxonomy_metric_validation\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`snapshot_id\` integer NOT NULL,
\`metric_key\` text NOT NULL,
\`status\` text NOT NULL,
\`updated_at\` text NOT NULL
);
`);
}
function createHealthyIndexes(client: Database) {
client.exec('CREATE UNIQUE INDEX `company_financial_bundle_uidx` ON `company_financial_bundle` (`ticker`,`surface_kind`,`cadence`);');
client.exec('CREATE INDEX `company_financial_bundle_ticker_idx` ON `company_financial_bundle` (`ticker`,`updated_at`);');
client.exec('CREATE UNIQUE INDEX `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);');
client.exec('CREATE INDEX `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);');
client.exec('CREATE INDEX `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);');
client.exec('CREATE UNIQUE INDEX `filing_taxonomy_metric_validation_uidx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`metric_key`);');
client.exec('CREATE INDEX `filing_taxonomy_metric_validation_snapshot_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`);');
client.exec('CREATE INDEX `filing_taxonomy_metric_validation_status_idx` ON `filing_taxonomy_metric_validation` (`snapshot_id`,`status`);');
}
describe('financial ingestion schema repair', () => {
it('reports a healthy schema when all critical indexes exist', () => {
const client = new Database(':memory:');
try {
createBundleTable(client);
createSnapshotTable(client);
createMetricValidationTable(client);
createHealthyIndexes(client);
const report = inspectFinancialIngestionSchema(client);
expect(report.ok).toBe(true);
expect(report.missingIndexes).toEqual([]);
expect(report.duplicateGroups).toBe(0);
} finally {
client.close();
}
});
it('repairs missing company financial bundle indexes and dedupes rows', () => {
const client = new Database(':memory:');
try {
createBundleTable(client);
createSnapshotTable(client);
createMetricValidationTable(client);
createHealthyIndexes(client);
client.exec('DROP INDEX `company_financial_bundle_uidx`;');
client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;');
client.exec(`
INSERT INTO \`company_financial_bundle\` (
\`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\`
) VALUES
('MSFT', 'income_statement', 'annual', 14, '[]', 'old', '{}', '2026-03-12T10:00:00.000Z', '2026-03-12T10:00:00.000Z'),
('MSFT', 'income_statement', 'annual', 14, '[]', 'new', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z');
`);
const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
const rows = client.query('SELECT `source_signature` FROM `company_financial_bundle`').all() as Array<{ source_signature: string }>;
const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>;
expect(result.ok).toBe(true);
expect(result.mode).toBe('repaired');
expect(rows).toEqual([{ source_signature: 'new' }]);
expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true);
expect(indexes.some((row) => row.name === 'company_financial_bundle_ticker_idx')).toBe(true);
} finally {
client.close();
}
});
it('dedupes filing taxonomy snapshots and clears bundle cache before recreating indexes', () => {
const client = new Database(':memory:');
try {
createBundleTable(client);
createSnapshotTable(client);
createMetricValidationTable(client);
createHealthyIndexes(client);
client.exec('DROP INDEX `filing_taxonomy_snapshot_filing_uidx`;');
client.exec(`
INSERT INTO \`filing_taxonomy_snapshot\` (
\`filing_id\`, \`ticker\`, \`filing_date\`, \`filing_type\`, \`parse_status\`, \`updated_at\`
) VALUES
(10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T10:00:00.000Z'),
(10, 'MSFT', '2026-03-12', '10-K', 'ready', '2026-03-12T11:00:00.000Z');
`);
client.exec(`
INSERT INTO \`company_financial_bundle\` (
\`ticker\`, \`surface_kind\`, \`cadence\`, \`bundle_version\`, \`source_snapshot_ids\`, \`source_signature\`, \`payload\`, \`created_at\`, \`updated_at\`
) VALUES
('MSFT', 'income_statement', 'annual', 14, '[1,2]', 'cached', '{}', '2026-03-12T11:00:00.000Z', '2026-03-12T11:00:00.000Z');
`);
const result = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' });
const snapshotCount = client.query('SELECT COUNT(*) AS count FROM `filing_taxonomy_snapshot`').get() as { count: number };
const bundleCount = client.query('SELECT COUNT(*) AS count FROM `company_financial_bundle`').get() as { count: number };
expect(result.ok).toBe(true);
expect(result.repair?.snapshotDuplicateRowsDeleted).toBe(1);
expect(result.repair?.bundleCacheCleared).toBe(true);
expect(snapshotCount.count).toBe(1);
expect(bundleCount.count).toBe(0);
} finally {
client.close();
}
});
it('retries once after repairing the schema for ON CONFLICT drift errors', async () => {
const client = new Database(':memory:');
try {
createBundleTable(client);
createSnapshotTable(client);
createMetricValidationTable(client);
createHealthyIndexes(client);
client.exec('DROP INDEX `company_financial_bundle_uidx`;');
client.exec('DROP INDEX `company_financial_bundle_ticker_idx`;');
let calls = 0;
const result = await withFinancialIngestionSchemaRetry({
client,
context: 'test-retry',
operation: async () => {
calls += 1;
if (calls === 1) {
throw new Error('ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint');
}
return 'ok';
}
});
expect(result).toBe('ok');
expect(calls).toBe(2);
const indexes = client.query('PRAGMA index_list(`company_financial_bundle`)').all() as Array<{ name: string }>;
expect(indexes.some((row) => row.name === 'company_financial_bundle_uidx')).toBe(true);
} finally {
client.close();
}
});
});