Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Add history window controls and expand taxonomy pack support

Browse Source 
- add 3Y/5Y/10Y financial history filtering and reorganize normalization details UI
- add new fiscal taxonomy surface/income bridge/KPI packs and update Rust taxonomy loading
- auto-detect Homebrew SQLite for native `sqlite-vec` in local dev/e2e with docs and env guidance
This commit is contained in:
francy51
2026-03-18 23:40:28 -04:00
parent f8426c4dde
commit 17de3dd72d
102 changed files with 14978 additions and 1316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
scripts/dev.ts
View File

@@ -1,14 +1,15 @@
import { spawn } from 'node:child_process';
import { mkdirSync, readFileSync } from 'node:fs';
import { Database } from 'bun:sqlite';
import { createServer } from 'node:net';
import { dirname, join } from 'node:path';
import { spawn } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { Database } from "bun:sqlite";
import { createServer } from "node:net";
import { dirname, join } from "node:path";
import {
ensureFinancialIngestionSchemaHealthy,
resolveFinancialSchemaRepairMode
} from '../lib/server/db/financial-ingestion-schema';
import { ensureLocalSqliteSchema } from '../lib/server/db/sqlite-schema-compat';
import { buildLocalDevConfig, resolveSqlitePath } from './dev-env';
resolveFinancialSchemaRepairMode,
} from "../lib/server/db/financial-ingestion-schema";
import { ensureLocalSqliteSchema } from "../lib/server/db/sqlite-schema-compat";
import { buildLocalDevConfig, resolveSqlitePath } from "./dev-env";
import { applyLocalSqliteVectorEnv } from "./sqlite-vector-env";
type DrizzleJournal = {
entries: Array<{ tag: string }>;
@@ -28,7 +29,7 @@ async function isPortAvailable(port: number, host: string) {
return await new Promise<boolean>((resolve) => {
const server = createServer();
server.once('error', () => resolve(false));
server.once("error", () => resolve(false));
server.listen(port, host, () => {
server.close(() => resolve(true));
});
@@ -44,28 +45,39 @@ async function pickLocalPort(host: string) {
}
}
throw new Error(`Unable to find an open local dev port from: ${candidatePorts.join(', ')}`);
throw new Error(
`Unable to find an open local dev port from: ${candidatePorts.join(", ")}`,
);
}
function hasTable(database: Database, tableName: string) {
const row = database
.query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1')
.get('table', tableName) as { name: string } | null;
try {
const row = database
.query("SELECT name FROM sqlite_master WHERE type='table' AND name = ?")
.get(tableName) as { name: string } | null;
return row !== null;
return row !== null;
} catch (e) {
return false;
}
}
function readMigrationFiles() {
const journal = JSON.parse(
readFileSync(join(process.cwd(), 'drizzle', 'meta', '_journal.json'), 'utf8')
readFileSync(
join(process.cwd(), "drizzle", "meta", "_journal.json"),
"utf8",
),
) as DrizzleJournal;
return journal.entries.map((entry) => join(process.cwd(), 'drizzle', `${entry.tag}.sql`));
return journal.entries.map((entry) =>
join(process.cwd(), "drizzle", `${entry.tag}.sql`),
);
}
function bootstrapFreshDatabase(databaseUrl: string) {
const databasePath = resolveSqlitePath(databaseUrl);
if (!databasePath || databasePath === ':memory:') {
if (!databasePath || databasePath === ":memory:") {
return false;
}
@@ -74,16 +86,16 @@ function bootstrapFreshDatabase(databaseUrl: string) {
const database = new Database(databasePath, { create: true });
try {
database.exec('PRAGMA foreign_keys = ON;');
database.exec("PRAGMA foreign_keys = ON;");
const existingCoreTables = [
'user',
'filing',
'watchlist_item',
'filing_statement_snapshot',
'filing_taxonomy_snapshot',
'task_run',
'company_financial_bundle'
"user",
"filing",
"watchlist_item",
"filing_statement_snapshot",
"filing_taxonomy_snapshot",
"task_run",
"company_financial_bundle",
];
if (existingCoreTables.some((tableName) => hasTable(database, tableName))) {
@@ -91,7 +103,7 @@ function bootstrapFreshDatabase(databaseUrl: string) {
}
for (const migrationFile of readMigrationFiles()) {
database.exec(readFileSync(migrationFile, 'utf8'));
database.exec(readFileSync(migrationFile, "utf8"));
}
return true;
@@ -102,7 +114,9 @@ function bootstrapFreshDatabase(databaseUrl: string) {
function exitFromResult(result: ExitResult) {
if (result.signal) {
process.exit(result.signal === 'SIGINT' || result.signal === 'SIGTERM' ? 0 : 1);
process.exit(
result.signal === "SIGINT" || result.signal === "SIGTERM" ? 0 : 1,
);
return;
}
@@ -110,49 +124,52 @@ function exitFromResult(result: ExitResult) {
}
const explicitPort = trim(process.env.PORT) || trim(process.env.APP_PORT);
const bindHost = trim(process.env.HOSTNAME) || trim(process.env.HOST) || '127.0.0.1';
const resolvedPort = explicitPort || await pickLocalPort(bindHost);
const bindHost =
trim(process.env.HOSTNAME) || trim(process.env.HOST) || "127.0.0.1";
const resolvedPort = explicitPort || (await pickLocalPort(bindHost));
const config = buildLocalDevConfig({
...process.env,
HOSTNAME: bindHost,
PORT: resolvedPort
PORT: resolvedPort,
});
const env = {
...config.env
const initialEnv = {
...config.env,
} as NodeJS.ProcessEnv;
const { config: sqliteVectorConfig, env } =
applyLocalSqliteVectorEnv(initialEnv);
delete env.NO_COLOR;
const databasePath = resolveSqlitePath(env.DATABASE_URL ?? '');
if (databasePath && databasePath !== ':memory:') {
const databasePath = resolveSqlitePath(env.DATABASE_URL ?? "");
if (databasePath && databasePath !== ":memory:") {
mkdirSync(dirname(databasePath), { recursive: true });
}
mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? '.workflow-data', { recursive: true });
mkdirSync(env.WORKFLOW_LOCAL_DATA_DIR ?? ".workflow-data", { recursive: true });
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? '');
const initializedDatabase = bootstrapFreshDatabase(env.DATABASE_URL ?? "");
if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
if (!initializedDatabase && databasePath && databasePath !== ":memory:") {
const client = new Database(databasePath, { create: true });
try {
client.exec('PRAGMA foreign_keys = ON;');
client.exec("PRAGMA foreign_keys = ON;");
ensureLocalSqliteSchema(client);
const repairResult = ensureFinancialIngestionSchemaHealthy(client, {
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE)
mode: resolveFinancialSchemaRepairMode(env.FINANCIAL_SCHEMA_REPAIR_MODE),
});
if (repairResult.mode === 'repaired') {
if (repairResult.mode === "repaired") {
console.info(
`[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(', ') || 'none'}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? 'yes' : 'no'})`
`[dev] repaired financial ingestion schema (missing indexes: ${repairResult.repair?.missingIndexesBefore.join(", ") || "none"}; duplicate groups resolved: ${repairResult.repair?.duplicateGroupsResolved ?? 0}; bundle cache cleared: ${repairResult.repair?.bundleCacheCleared ? "yes" : "no"})`,
);
} else if (repairResult.mode === 'drifted') {
} else if (repairResult.mode === "drifted") {
console.warn(
`[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(', ') || 'none'}; duplicate groups: ${repairResult.duplicateGroups})`
`[dev] financial ingestion schema drift detected (missing indexes: ${repairResult.missingIndexes.join(", ") || "none"}; duplicate groups: ${repairResult.duplicateGroups})`,
);
} else if (repairResult.mode === 'failed') {
} else if (repairResult.mode === "failed") {
console.warn(
`[dev] financial ingestion schema repair failed: ${repairResult.error ?? 'unknown error'}`
`[dev] financial ingestion schema repair failed: ${repairResult.error ?? "unknown error"}`,
);
}
} finally {
@@ -162,41 +179,69 @@ if (!initializedDatabase && databasePath && databasePath !== ':memory:') {
console.info(`[dev] local origin ${config.publicOrigin}`);
console.info(`[dev] sqlite ${env.DATABASE_URL}`);
console.info(`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`);
if (!explicitPort && resolvedPort !== '3000') {
console.info(`[dev] port 3000 is busy, using http://localhost:${resolvedPort} instead`);
console.info(
`[dev] workflow ${env.WORKFLOW_TARGET_WORLD} (${env.WORKFLOW_LOCAL_DATA_DIR})`,
);
if (sqliteVectorConfig.mode === "native") {
console.info(
`[dev] sqlite-vec native extension enabled (${sqliteVectorConfig.sqliteLibPath})`,
);
}
if (!explicitPort && resolvedPort !== "3000") {
console.info(
`[dev] port 3000 is busy, using http://localhost:${resolvedPort} instead`,
);
}
if (initializedDatabase) {
console.info('[dev] initialized the local SQLite schema from drizzle SQL files');
console.info(
"[dev] initialized the local SQLite schema from drizzle SQL files",
);
}
if (config.overrides.authOriginChanged) {
console.info('[dev] forcing Better Auth origin/trusted origins to the local origin');
console.info(
"[dev] forcing Better Auth origin/trusted origins to the local origin",
);
}
if (config.overrides.apiBaseChanged) {
console.info('[dev] forcing NEXT_PUBLIC_API_URL to same-origin for local dev');
console.info(
"[dev] forcing NEXT_PUBLIC_API_URL to same-origin for local dev",
);
}
if (config.overrides.databaseChanged) {
console.info('[dev] using a local SQLite database instead of the deployment path');
console.info(
"[dev] using a local SQLite database instead of the deployment path",
);
}
if (config.overrides.workflowChanged) {
console.info('[dev] forcing Workflow to the local runtime for local dev');
console.info("[dev] forcing Workflow to the local runtime for local dev");
}
if (config.overrides.secretFallbackUsed) {
console.info('[dev] using the built-in local Better Auth secret because BETTER_AUTH_SECRET is unset or still a placeholder');
console.info(
"[dev] using the built-in local Better Auth secret because BETTER_AUTH_SECRET is unset or still a placeholder",
);
}
const child = spawn(
'bun',
['--bun', 'next', 'dev', '--turbopack', '--hostname', config.bindHost, '--port', config.port],
"bun",
[
"--bun",
"next",
"dev",
"--turbopack",
"--hostname",
config.bindHost,
"--port",
config.port,
],
{
env,
stdio: 'inherit'
}
stdio: "inherit",
},
);
function forwardSignal(signal: NodeJS.Signals) {
@@ -205,9 +250,9 @@ function forwardSignal(signal: NodeJS.Signals) {
}
}
process.on('SIGINT', () => forwardSignal('SIGINT'));
process.on('SIGTERM', () => forwardSignal('SIGTERM'));
process.on("SIGINT", () => forwardSignal("SIGINT"));
process.on("SIGTERM", () => forwardSignal("SIGTERM"));
child.on('exit', (code, signal) => {
child.on("exit", (code, signal) => {
exitFromResult({ code, signal });
});