Merge branch 't3code/expand-research-management-plan'
# Conflicts: # app/analysis/page.tsx # app/watchlist/page.tsx # components/shell/app-shell.tsx # lib/api.ts # lib/query/options.ts # lib/server/api/app.ts # lib/server/db/index.test.ts # lib/server/db/index.ts # lib/server/db/schema.ts # lib/server/repos/research-journal.ts # lib/types.ts
This commit is contained in:
@@ -51,6 +51,15 @@ function applySqlFile(client: Database, fileName: string) {
|
||||
client.exec(sql);
|
||||
}
|
||||
|
||||
function applyBaseSchemaCompat(client: Database) {
|
||||
const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8')
|
||||
.replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `')
|
||||
.replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `')
|
||||
.replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `');
|
||||
|
||||
client.exec(sql);
|
||||
}
|
||||
|
||||
let customSqliteConfigured = false;
|
||||
const vectorExtensionStatus = new WeakMap<Database, boolean>();
|
||||
|
||||
@@ -90,7 +99,295 @@ function isVectorExtensionLoaded(client: Database) {
|
||||
return vectorExtensionStatus.get(client) ?? false;
|
||||
}
|
||||
|
||||
function ensureResearchWorkspaceSchema(client: Database) {
|
||||
if (!hasTable(client, 'research_artifact')) {
|
||||
client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS \`research_artifact\` (
|
||||
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
\`user_id\` text NOT NULL,
|
||||
\`organization_id\` text,
|
||||
\`ticker\` text NOT NULL,
|
||||
\`accession_number\` text,
|
||||
\`kind\` text NOT NULL,
|
||||
\`source\` text NOT NULL DEFAULT 'user',
|
||||
\`subtype\` text,
|
||||
\`title\` text,
|
||||
\`summary\` text,
|
||||
\`body_markdown\` text,
|
||||
\`search_text\` text,
|
||||
\`visibility_scope\` text NOT NULL DEFAULT 'private',
|
||||
\`tags\` text,
|
||||
\`metadata\` text,
|
||||
\`file_name\` text,
|
||||
\`mime_type\` text,
|
||||
\`file_size_bytes\` integer,
|
||||
\`storage_path\` text,
|
||||
\`created_at\` text NOT NULL,
|
||||
\`updated_at\` text NOT NULL,
|
||||
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
|
||||
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
if (!hasTable(client, 'research_memo')) {
|
||||
client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS \`research_memo\` (
|
||||
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
\`user_id\` text NOT NULL,
|
||||
\`organization_id\` text,
|
||||
\`ticker\` text NOT NULL,
|
||||
\`rating\` text,
|
||||
\`conviction\` text,
|
||||
\`time_horizon_months\` integer,
|
||||
\`packet_title\` text,
|
||||
\`packet_subtitle\` text,
|
||||
\`thesis_markdown\` text NOT NULL DEFAULT '',
|
||||
\`variant_view_markdown\` text NOT NULL DEFAULT '',
|
||||
\`catalysts_markdown\` text NOT NULL DEFAULT '',
|
||||
\`risks_markdown\` text NOT NULL DEFAULT '',
|
||||
\`disconfirming_evidence_markdown\` text NOT NULL DEFAULT '',
|
||||
\`next_actions_markdown\` text NOT NULL DEFAULT '',
|
||||
\`created_at\` text NOT NULL,
|
||||
\`updated_at\` text NOT NULL,
|
||||
FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade,
|
||||
FOREIGN KEY (\`organization_id\`) REFERENCES \`organization\`(\`id\`) ON UPDATE no action ON DELETE set null
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
if (!hasTable(client, 'research_memo_evidence')) {
|
||||
client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` (
|
||||
\`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
\`memo_id\` integer NOT NULL,
|
||||
\`artifact_id\` integer NOT NULL,
|
||||
\`section\` text NOT NULL,
|
||||
\`annotation\` text,
|
||||
\`sort_order\` integer NOT NULL DEFAULT 0,
|
||||
\`created_at\` text NOT NULL,
|
||||
FOREIGN KEY (\`memo_id\`) REFERENCES \`research_memo\`(\`id\`) ON UPDATE no action ON DELETE cascade,
|
||||
FOREIGN KEY (\`artifact_id\`) REFERENCES \`research_artifact\`(\`id\`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);');
|
||||
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);');
|
||||
client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);');
|
||||
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);');
|
||||
client.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5(
|
||||
artifact_id UNINDEXED,
|
||||
user_id UNINDEXED,
|
||||
ticker UNINDEXED,
|
||||
title,
|
||||
summary,
|
||||
body_markdown,
|
||||
search_text,
|
||||
tags_text
|
||||
);
|
||||
`);
|
||||
|
||||
client.exec(`
|
||||
INSERT INTO \`research_artifact\` (
|
||||
\`user_id\`,
|
||||
\`organization_id\`,
|
||||
\`ticker\`,
|
||||
\`accession_number\`,
|
||||
\`kind\`,
|
||||
\`source\`,
|
||||
\`subtype\`,
|
||||
\`title\`,
|
||||
\`summary\`,
|
||||
\`body_markdown\`,
|
||||
\`search_text\`,
|
||||
\`visibility_scope\`,
|
||||
\`tags\`,
|
||||
\`metadata\`,
|
||||
\`created_at\`,
|
||||
\`updated_at\`
|
||||
)
|
||||
SELECT
|
||||
r.\`user_id\`,
|
||||
NULL,
|
||||
r.\`ticker\`,
|
||||
r.\`accession_number\`,
|
||||
CASE
|
||||
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
|
||||
ELSE 'note'
|
||||
END,
|
||||
CASE
|
||||
WHEN r.\`entry_type\` = 'status_change' THEN 'system'
|
||||
ELSE 'user'
|
||||
END,
|
||||
r.\`entry_type\`,
|
||||
r.\`title\`,
|
||||
CASE
|
||||
WHEN r.\`body_markdown\` IS NULL OR TRIM(r.\`body_markdown\`) = '' THEN NULL
|
||||
ELSE SUBSTR(r.\`body_markdown\`, 1, 280)
|
||||
END,
|
||||
r.\`body_markdown\`,
|
||||
r.\`body_markdown\`,
|
||||
'private',
|
||||
NULL,
|
||||
r.\`metadata\`,
|
||||
r.\`created_at\`,
|
||||
r.\`updated_at\`
|
||||
FROM \`research_journal_entry\` r
|
||||
WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'research_journal_entry')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM \`research_artifact\` a
|
||||
WHERE a.\`user_id\` = r.\`user_id\`
|
||||
AND a.\`ticker\` = r.\`ticker\`
|
||||
AND IFNULL(a.\`accession_number\`, '') = IFNULL(r.\`accession_number\`, '')
|
||||
AND a.\`kind\` = CASE
|
||||
WHEN r.\`entry_type\` = 'status_change' THEN 'status_change'
|
||||
ELSE 'note'
|
||||
END
|
||||
AND IFNULL(a.\`title\`, '') = IFNULL(r.\`title\`, '')
|
||||
AND a.\`created_at\` = r.\`created_at\`
|
||||
);
|
||||
`);
|
||||
|
||||
client.exec(`
|
||||
INSERT INTO \`research_artifact\` (
|
||||
\`user_id\`,
|
||||
\`organization_id\`,
|
||||
\`ticker\`,
|
||||
\`accession_number\`,
|
||||
\`kind\`,
|
||||
\`source\`,
|
||||
\`subtype\`,
|
||||
\`title\`,
|
||||
\`summary\`,
|
||||
\`body_markdown\`,
|
||||
\`search_text\`,
|
||||
\`visibility_scope\`,
|
||||
\`tags\`,
|
||||
\`metadata\`,
|
||||
\`created_at\`,
|
||||
\`updated_at\`
|
||||
)
|
||||
SELECT
|
||||
w.\`user_id\`,
|
||||
NULL,
|
||||
f.\`ticker\`,
|
||||
f.\`accession_number\`,
|
||||
'ai_report',
|
||||
'system',
|
||||
'filing_analysis',
|
||||
f.\`filing_type\` || ' AI memo',
|
||||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
|
||||
'Stored AI memo for ' || f.\`company_name\` || ' (' || f.\`ticker\` || ').' || CHAR(10) ||
|
||||
'Accession: ' || f.\`accession_number\` || CHAR(10) || CHAR(10) ||
|
||||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
|
||||
COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights')),
|
||||
'private',
|
||||
NULL,
|
||||
json_object(
|
||||
'provider', json_extract(f.\`analysis\`, '$.provider'),
|
||||
'model', json_extract(f.\`analysis\`, '$.model'),
|
||||
'filingType', f.\`filing_type\`,
|
||||
'filingDate', f.\`filing_date\`
|
||||
),
|
||||
f.\`created_at\`,
|
||||
f.\`updated_at\`
|
||||
FROM \`filing\` f
|
||||
JOIN \`watchlist_item\` w
|
||||
ON w.\`ticker\` = f.\`ticker\`
|
||||
WHERE f.\`analysis\` IS NOT NULL
|
||||
AND TRIM(COALESCE(json_extract(f.\`analysis\`, '$.text'), json_extract(f.\`analysis\`, '$.legacyInsights'), '')) <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM \`research_artifact\` a
|
||||
WHERE a.\`user_id\` = w.\`user_id\`
|
||||
AND a.\`ticker\` = f.\`ticker\`
|
||||
AND a.\`accession_number\` = f.\`accession_number\`
|
||||
AND a.\`kind\` = 'ai_report'
|
||||
);
|
||||
`);
|
||||
|
||||
client.exec('DELETE FROM `research_artifact_fts`;');
|
||||
client.exec(`
|
||||
INSERT INTO \`research_artifact_fts\` (
|
||||
\`artifact_id\`,
|
||||
\`user_id\`,
|
||||
\`ticker\`,
|
||||
\`title\`,
|
||||
\`summary\`,
|
||||
\`body_markdown\`,
|
||||
\`search_text\`,
|
||||
\`tags_text\`
|
||||
)
|
||||
SELECT
|
||||
\`id\`,
|
||||
\`user_id\`,
|
||||
\`ticker\`,
|
||||
COALESCE(\`title\`, ''),
|
||||
COALESCE(\`summary\`, ''),
|
||||
COALESCE(\`body_markdown\`, ''),
|
||||
COALESCE(\`search_text\`, ''),
|
||||
CASE
|
||||
WHEN \`tags\` IS NULL OR TRIM(\`tags\`) = '' THEN ''
|
||||
ELSE REPLACE(REPLACE(REPLACE(\`tags\`, '[', ''), ']', ''), '\"', '')
|
||||
END
|
||||
FROM \`research_artifact\`;
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureLocalSqliteSchema(client: Database) {
|
||||
const missingBaseSchema = [
|
||||
'filing',
|
||||
'watchlist_item',
|
||||
'holding',
|
||||
'task_run',
|
||||
'portfolio_insight'
|
||||
].some((tableName) => !hasTable(client, tableName));
|
||||
|
||||
if (missingBaseSchema) {
|
||||
applyBaseSchemaCompat(client);
|
||||
}
|
||||
|
||||
if (!hasTable(client, 'user')) {
|
||||
client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS \`user\` (
|
||||
\`id\` text PRIMARY KEY NOT NULL,
|
||||
\`name\` text NOT NULL,
|
||||
\`email\` text NOT NULL,
|
||||
\`emailVerified\` integer NOT NULL DEFAULT 0,
|
||||
\`image\` text,
|
||||
\`createdAt\` integer NOT NULL,
|
||||
\`updatedAt\` integer NOT NULL,
|
||||
\`role\` text,
|
||||
\`banned\` integer DEFAULT 0,
|
||||
\`banReason\` text,
|
||||
\`banExpires\` integer
|
||||
);
|
||||
`);
|
||||
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);');
|
||||
}
|
||||
|
||||
if (!hasTable(client, 'organization')) {
|
||||
client.exec(`
|
||||
CREATE TABLE IF NOT EXISTS \`organization\` (
|
||||
\`id\` text PRIMARY KEY NOT NULL,
|
||||
\`name\` text NOT NULL,
|
||||
\`slug\` text NOT NULL,
|
||||
\`logo\` text,
|
||||
\`createdAt\` integer NOT NULL,
|
||||
\`metadata\` text
|
||||
);
|
||||
`);
|
||||
client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);');
|
||||
}
|
||||
|
||||
if (!hasTable(client, 'filing_statement_snapshot')) {
|
||||
applySqlFile(client, '0001_glossy_statement_snapshots.sql');
|
||||
}
|
||||
@@ -186,6 +483,8 @@ function ensureLocalSqliteSchema(client: Database) {
|
||||
if (!hasTable(client, 'search_document')) {
|
||||
applySqlFile(client, '0008_search_rag.sql');
|
||||
}
|
||||
|
||||
ensureResearchWorkspaceSchema(client);
|
||||
}
|
||||
|
||||
function ensureSearchVirtualTables(client: Database) {
|
||||
|
||||
Reference in New Issue
Block a user