Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Add research workspace and graphing flows

Browse Source
This commit is contained in:
francy51
2026-03-07 16:52:35 -05:00
parent db01f207a5
commit 62bacdf104
37 changed files with 5494 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/server/db/index.test.ts
View File

@@ -24,6 +24,8 @@ describe('sqlite schema compatibility bootstrap', () => {
expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(false);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(false);
expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(false);
expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(false);
expect(__dbInternals.hasTable(client, 'research_memo')).toBe(false);
__dbInternals.ensureLocalSqliteSchema(client);
@@ -37,6 +39,9 @@ describe('sqlite schema compatibility bootstrap', () => {
expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true);
expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_memo')).toBe(true);
expect(__dbInternals.hasTable(client, 'research_memo_evidence')).toBe(true);
client.close();
});

299
lib/server/db/index.ts
View File

@@ -50,7 +50,304 @@ function applySqlFile(client: Database, fileName: string) {
client.exec(sql);
}
function applyBaseSchemaCompat(client: Database) {
const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
.replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
.replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
.replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
client.exec(sql);
}
function ensureResearchWorkspaceSchema(client: Database) {
if (!hasTable(client, 'research_artifact')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_artifact\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`accession_number\` text,
\`kind\` text NOT NULL,
\`source\` text NOT NULL DEFAULT 'user',
\`subtype\` text,
\`title\` text,
\`summary\` text,
\`body_markdown\` text,
\`search_text\` text,
\`visibility_scope\` text NOT NULL DEFAULT 'private',
\`tags\` text,
\`metadata\` text,
\`file_name\` text,
\`mime_type\` text,
\`file_size_bytes\` integer,
\`storage_path\` text,
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`user_id\` text NOT NULL,
\`organization_id\` text,
\`ticker\` text NOT NULL,
\`rating\` text,
\`conviction\` text,
\`time_horizon_months\` integer,
\`packet_title\` text,
\`packet_subtitle\` text,
\`thesis_markdown\` text NOT NULL DEFAULT '',
\`variant_view_markdown\` text NOT NULL DEFAULT '',
\`catalysts_markdown\` text NOT NULL DEFAULT '',
\`risks_markdown\` text NOT NULL DEFAULT '',
\`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
\`next_actions_markdown\` text NOT NULL DEFAULT '',
\`created_at\` text NOT NULL,
\`updated_at\` text NOT NULL,
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
);
`);
}
if (!hasTable(client, 'research_memo_evidence')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
\`memo_id\` integer NOT NULL,
\`artifact_id\` integer NOT NULL,
\`section\` text NOT NULL,
\`annotation\` text,
\`sort_order\` integer NOT NULL DEFAULT 0,
\`created_at\` text NOT NULL,
FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
);
`);
}
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
client.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
artifact_id UNINDEXED,
user_id UNINDEXED,
ticker UNINDEXED,
title,
summary,
body_markdown,
search_text,
tags_text
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
r.\`user_id\`,
NULL,
r.\`ticker\`,
r.\`accession_number\`,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END,
CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'system'
ELSE 'user'
END,
r.\`entry_type\`,
r.\`title\`,
CASE
WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
END,
r.\`body_markdown\`,
r.\`body_markdown\`,
'private',
NULL,
r.\`metadata\`,
r.\`created_at\`,
r.\`updated_at\`
FROM \`research_journal_entry\` r
WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = r.\`user_id\`
AND a.\`ticker\` = r.\`ticker\`
AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
AND a.\`kind\` = CASE
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
ELSE 'note'
END
AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
AND a.\`created_at\` = r.\`created_at\`
);
`);
client.exec(`
INSERT INTO \`research_artifact\` (
\`user_id\`,
\`organization_id\`,
\`ticker\`,
\`accession_number\`,
\`kind\`,
\`source\`,
\`subtype\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`visibility_scope\`,
\`tags\`,
\`metadata\`,
\`created_at\`,
\`updated_at\`
)
SELECT
w.\`user_id\`,
NULL,
f.\`ticker\`,
f.\`accession_number\`,
'ai_report',
'system',
'filing_analysis',
f.\`filing_type\` || ' AI memo',
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
'private',
NULL,
json_object(
'provider', json_extract(f.\`analysis\`, '$.provider'),
'model', json_extract(f.\`analysis\`, '$.model'),
'filingType', f.\`filing_type\`,
'filingDate', f.\`filing_date\`
),
f.\`created_at\`,
f.\`updated_at\`
FROM \`filing\` f
JOIN \`watchlist_item\` w
ON w.\`ticker\` = f.\`ticker\`
WHERE f.\`analysis\` IS NOT NULL
AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
AND NOT EXISTS (
SELECT 1
FROM \`research_artifact\` a
WHERE a.\`user_id\` = w.\`user_id\`
AND a.\`ticker\` = f.\`ticker\`
AND a.\`accession_number\` = f.\`accession_number\`
AND a.\`kind\` = 'ai_report'
);
`);
client.exec('DELETE FROM `research_artifact_fts`;');
client.exec(`
INSERT INTO \`research_artifact_fts\` (
\`artifact_id\`,
\`user_id\`,
\`ticker\`,
\`title\`,
\`summary\`,
\`body_markdown\`,
\`search_text\`,
\`tags_text\`
)
SELECT
\`id\`,
\`user_id\`,
\`ticker\`,
COALESCE(\`title\`, ''),
COALESCE(\`summary\`, ''),
COALESCE(\`body_markdown\`, ''),
COALESCE(\`search_text\`, ''),
CASE
WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
END
FROM \`research_artifact\`;
`);
}
function ensureLocalSqliteSchema(client: Database) {
const missingBaseSchema = [
'filing',
'watchlist_item',
'holding',
'task_run',
'portfolio_insight'
].some((tableName) => !hasTable(client, tableName));
if (missingBaseSchema) {
applyBaseSchemaCompat(client);
}
if (!hasTable(client, 'user')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`user\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`email\` text NOT NULL,
\`emailVerified\` integer NOT NULL DEFAULT 0,
\`image\` text,
\`createdAt\` integer NOT NULL,
\`updatedAt\` integer NOT NULL,
\`role\` text,
\`banned\` integer DEFAULT 0,
\`banReason\` text,
\`banExpires\` integer
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
}
if (!hasTable(client, 'organization')) {
client.exec(`
CREATE TABLE IF NOT EXISTS \`organization\` (
\`id\` text PRIMARY KEY NOT NULL,
\`name\` text NOT NULL,
\`slug\` text NOT NULL,
\`logo\` text,
\`createdAt\` integer NOT NULL,
\`metadata\` text
);
`);
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
}
if (!hasTable(client, 'filing_statement_snapshot')) {
applySqlFile(client, '0001_glossy_statement_snapshots.sql');
}
@@ -142,6 +439,8 @@ function ensureLocalSqliteSchema(client: Database) {
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);');
client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);');
}
ensureResearchWorkspaceSchema(client);
}
export function getSqliteClient() {

83
lib/server/db/schema.ts
View File

@@ -30,6 +30,18 @@ type TaxonomyMetricValidationStatus = 'not_run' | 'matched' | 'mismatch' | 'erro
type CoverageStatus = 'backlog' | 'active' | 'watch' | 'archive';
type CoveragePriority = 'low' | 'medium' | 'high';
type ResearchJournalEntryType = 'note' | 'filing_note' | 'status_change';
type ResearchArtifactKind = 'filing' | 'ai_report' | 'note' | 'upload' | 'memo_snapshot' | 'status_change';
type ResearchArtifactSource = 'system' | 'user';
type ResearchVisibilityScope = 'private' | 'organization';
type ResearchMemoRating = 'strong_buy' | 'buy' | 'hold' | 'sell';
type ResearchMemoConviction = 'low' | 'medium' | 'high';
type ResearchMemoSection =
| 'thesis'
| 'variant_view'
| 'catalysts'
| 'risks'
| 'disconfirming_evidence'
| 'next_actions';
type FinancialCadence = 'annual' | 'quarterly' | 'ltm';
type FinancialSurfaceKind =
| 'income_statement'
@@ -570,6 +582,72 @@ export const researchJournalEntry = sqliteTable('research_journal_entry', {
researchJournalAccessionIndex: index('research_journal_accession_idx').on(table.user_id, table.accession_number)
}));
export const researchArtifact = sqliteTable('research_artifact', {
id: integer('id').primaryKey({ autoIncrement: true }),
user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
organization_id: text('organization_id').references(() => organization.id, { onDelete: 'set null' }),
ticker: text('ticker').notNull(),
accession_number: text('accession_number'),
kind: text('kind').$type<ResearchArtifactKind>().notNull(),
source: text('source').$type<ResearchArtifactSource>().notNull().default('user'),
subtype: text('subtype'),
title: text('title'),
summary: text('summary'),
body_markdown: text('body_markdown'),
search_text: text('search_text'),
visibility_scope: text('visibility_scope').$type<ResearchVisibilityScope>().notNull().default('private'),
tags: text('tags', { mode: 'json' }).$type<string[]>(),
metadata: text('metadata', { mode: 'json' }).$type<Record<string, unknown> | null>(),
file_name: text('file_name'),
mime_type: text('mime_type'),
file_size_bytes: integer('file_size_bytes'),
storage_path: text('storage_path'),
created_at: text('created_at').notNull(),
updated_at: text('updated_at').notNull()
}, (table) => ({
researchArtifactTickerIndex: index('research_artifact_ticker_idx').on(table.user_id, table.ticker, table.updated_at),
researchArtifactKindIndex: index('research_artifact_kind_idx').on(table.user_id, table.kind, table.updated_at),
researchArtifactAccessionIndex: index('research_artifact_accession_idx').on(table.user_id, table.accession_number),
researchArtifactSourceIndex: index('research_artifact_source_idx').on(table.user_id, table.source, table.updated_at)
}));
export const researchMemo = sqliteTable('research_memo', {
id: integer('id').primaryKey({ autoIncrement: true }),
user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
organization_id: text('organization_id').references(() => organization.id, { onDelete: 'set null' }),
ticker: text('ticker').notNull(),
rating: text('rating').$type<ResearchMemoRating>(),
conviction: text('conviction').$type<ResearchMemoConviction>(),
time_horizon_months: integer('time_horizon_months'),
packet_title: text('packet_title'),
packet_subtitle: text('packet_subtitle'),
thesis_markdown: text('thesis_markdown').notNull().default(''),
variant_view_markdown: text('variant_view_markdown').notNull().default(''),
catalysts_markdown: text('catalysts_markdown').notNull().default(''),
risks_markdown: text('risks_markdown').notNull().default(''),
disconfirming_evidence_markdown: text('disconfirming_evidence_markdown').notNull().default(''),
next_actions_markdown: text('next_actions_markdown').notNull().default(''),
created_at: text('created_at').notNull(),
updated_at: text('updated_at').notNull()
}, (table) => ({
researchMemoTickerUnique: uniqueIndex('research_memo_ticker_uidx').on(table.user_id, table.ticker),
researchMemoUpdatedIndex: index('research_memo_updated_idx').on(table.user_id, table.updated_at)
}));
export const researchMemoEvidence = sqliteTable('research_memo_evidence', {
id: integer('id').primaryKey({ autoIncrement: true }),
memo_id: integer('memo_id').notNull().references(() => researchMemo.id, { onDelete: 'cascade' }),
artifact_id: integer('artifact_id').notNull().references(() => researchArtifact.id, { onDelete: 'cascade' }),
section: text('section').$type<ResearchMemoSection>().notNull(),
annotation: text('annotation'),
sort_order: integer('sort_order').notNull().default(0),
created_at: text('created_at').notNull()
}, (table) => ({
researchMemoEvidenceMemoIndex: index('research_memo_evidence_memo_idx').on(table.memo_id, table.section, table.sort_order),
researchMemoEvidenceArtifactIndex: index('research_memo_evidence_artifact_idx').on(table.artifact_id),
researchMemoEvidenceUnique: uniqueIndex('research_memo_evidence_unique_uidx').on(table.memo_id, table.artifact_id, table.section)
}));
export const authSchema = {
user,
session,
@@ -595,7 +673,10 @@ export const appSchema = {
taskRun,
taskStageEvent,
portfolioInsight,
researchJournalEntry
researchJournalEntry,
researchArtifact,
researchMemo,
researchMemoEvidence
};
export const schema = {