Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Run playwright UI tests

Browse Source
This commit is contained in:
francy51
2026-03-06 14:40:43 -05:00
parent 610fce8db3
commit 8e62c66677
37 changed files with 4430 additions and 643 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/server/repos/filing-statements.ts
View File

@@ -16,6 +16,7 @@ export type FilingStatementSnapshotPeriod = {
filingId: number;
accessionNumber: string;
filingDate: string;
periodStart: string | null;
periodEnd: string | null;
filingType: '10-K' | '10-Q';
periodLabel: string;

676
lib/server/repos/filing-taxonomy.ts Normal file
View File

@@ -0,0 +1,676 @@
import { and, desc, eq, gte, inArray, lt, sql } from 'drizzle-orm';
import type { Filing, FinancialStatementKind, MetricValidationResult, TaxonomyDimensionMember, TaxonomyFactRow, TaxonomyStatementRow } from '@/lib/types';
import { db } from '@/lib/server/db';
import {
filingTaxonomyAsset,
filingTaxonomyConcept,
filingTaxonomyFact,
filingTaxonomyMetricValidation,
filingTaxonomySnapshot
} from '@/lib/server/db/schema';
export type FilingTaxonomyParseStatus = 'ready' | 'partial' | 'failed';
export type FilingTaxonomySource = 'xbrl_instance' | 'xbrl_instance_with_linkbase' | 'legacy_html_fallback';
export type FilingTaxonomyAssetType =
| 'instance'
| 'schema'
| 'presentation'
| 'label'
| 'calculation'
| 'definition'
| 'pdf'
| 'other';
export type FilingTaxonomyPeriod = {
id: string;
filingId: number;
accessionNumber: string;
filingDate: string;
periodStart: string | null;
periodEnd: string | null;
filingType: '10-K' | '10-Q';
periodLabel: string;
};
export type FilingTaxonomySnapshotRecord = {
id: number;
filing_id: number;
ticker: string;
filing_date: string;
filing_type: '10-K' | '10-Q';
parse_status: FilingTaxonomyParseStatus;
parse_error: string | null;
source: FilingTaxonomySource;
periods: FilingTaxonomyPeriod[];
statement_rows: Record<FinancialStatementKind, TaxonomyStatementRow[]>;
derived_metrics: Filing['metrics'];
validation_result: MetricValidationResult | null;
facts_count: number;
concepts_count: number;
dimensions_count: number;
created_at: string;
updated_at: string;
};
export type FilingTaxonomyAssetRecord = {
id: number;
snapshot_id: number;
asset_type: FilingTaxonomyAssetType;
name: string;
url: string;
size_bytes: number | null;
score: number | null;
is_selected: boolean;
created_at: string;
};
export type FilingTaxonomyConceptRecord = {
id: number;
snapshot_id: number;
concept_key: string;
qname: string;
namespace_uri: string;
local_name: string;
label: string | null;
is_extension: boolean;
statement_kind: FinancialStatementKind | null;
role_uri: string | null;
presentation_order: number | null;
presentation_depth: number | null;
parent_concept_key: string | null;
is_abstract: boolean;
created_at: string;
};
export type FilingTaxonomyFactRecord = {
id: number;
snapshot_id: number;
concept_key: string;
qname: string;
namespace_uri: string;
local_name: string;
statement_kind: FinancialStatementKind | null;
role_uri: string | null;
context_id: string;
unit: string | null;
decimals: string | null;
value_num: number;
period_start: string | null;
period_end: string | null;
period_instant: string | null;
dimensions: TaxonomyDimensionMember[];
is_dimensionless: boolean;
source_file: string | null;
created_at: string;
};
export type FilingTaxonomyMetricValidationRecord = {
id: number;
snapshot_id: number;
metric_key: keyof NonNullable<Filing['metrics']>;
taxonomy_value: number | null;
llm_value: number | null;
absolute_diff: number | null;
relative_diff: number | null;
status: 'not_run' | 'matched' | 'mismatch' | 'error';
evidence_pages: number[];
pdf_url: string | null;
provider: string | null;
model: string | null;
error: string | null;
created_at: string;
updated_at: string;
};
export type UpsertFilingTaxonomySnapshotInput = {
filing_id: number;
ticker: string;
filing_date: string;
filing_type: '10-K' | '10-Q';
parse_status: FilingTaxonomyParseStatus;
parse_error: string | null;
source: FilingTaxonomySource;
periods: FilingTaxonomyPeriod[];
statement_rows: Record<FinancialStatementKind, TaxonomyStatementRow[]>;
derived_metrics: Filing['metrics'];
validation_result: MetricValidationResult | null;
facts_count: number;
concepts_count: number;
dimensions_count: number;
assets: Array<{
asset_type: FilingTaxonomyAssetType;
name: string;
url: string;
size_bytes: number | null;
score: number | null;
is_selected: boolean;
}>;
concepts: Array<{
concept_key: string;
qname: string;
namespace_uri: string;
local_name: string;
label: string | null;
is_extension: boolean;
statement_kind: FinancialStatementKind | null;
role_uri: string | null;
presentation_order: number | null;
presentation_depth: number | null;
parent_concept_key: string | null;
is_abstract: boolean;
}>;
facts: Array<{
concept_key: string;
qname: string;
namespace_uri: string;
local_name: string;
statement_kind: FinancialStatementKind | null;
role_uri: string | null;
context_id: string;
unit: string | null;
decimals: string | null;
value_num: number;
period_start: string | null;
period_end: string | null;
period_instant: string | null;
dimensions: TaxonomyDimensionMember[];
is_dimensionless: boolean;
source_file: string | null;
}>;
metric_validations: Array<{
metric_key: keyof NonNullable<Filing['metrics']>;
taxonomy_value: number | null;
llm_value: number | null;
absolute_diff: number | null;
relative_diff: number | null;
status: 'not_run' | 'matched' | 'mismatch' | 'error';
evidence_pages: number[];
pdf_url: string | null;
provider: string | null;
model: string | null;
error: string | null;
}>;
};
function tenYearsAgoIso() {
const date = new Date();
date.setUTCFullYear(date.getUTCFullYear() - 10);
return date.toISOString().slice(0, 10);
}
function asNumber(value: unknown) {
if (typeof value === 'number') {
return Number.isFinite(value) ? value : null;
}
if (typeof value === 'string') {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
return null;
}
function asNumericText(value: number | null) {
if (value === null || !Number.isFinite(value)) {
return null;
}
return String(value);
}
function emptyStatementRows(): Record<FinancialStatementKind, TaxonomyStatementRow[]> {
return {
income: [],
balance: [],
cash_flow: [],
equity: [],
comprehensive_income: []
};
}
function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): FilingTaxonomySnapshotRecord {
return {
id: row.id,
filing_id: row.filing_id,
ticker: row.ticker,
filing_date: row.filing_date,
filing_type: row.filing_type,
parse_status: row.parse_status,
parse_error: row.parse_error,
source: row.source,
periods: row.periods ?? [],
statement_rows: row.statement_rows ?? emptyStatementRows(),
derived_metrics: row.derived_metrics ?? null,
validation_result: row.validation_result ?? null,
facts_count: row.facts_count,
concepts_count: row.concepts_count,
dimensions_count: row.dimensions_count,
created_at: row.created_at,
updated_at: row.updated_at
};
}
function toAssetRecord(row: typeof filingTaxonomyAsset.$inferSelect): FilingTaxonomyAssetRecord {
return {
id: row.id,
snapshot_id: row.snapshot_id,
asset_type: row.asset_type,
name: row.name,
url: row.url,
size_bytes: row.size_bytes,
score: asNumber(row.score),
is_selected: row.is_selected,
created_at: row.created_at
};
}
function toConceptRecord(row: typeof filingTaxonomyConcept.$inferSelect): FilingTaxonomyConceptRecord {
return {
id: row.id,
snapshot_id: row.snapshot_id,
concept_key: row.concept_key,
qname: row.qname,
namespace_uri: row.namespace_uri,
local_name: row.local_name,
label: row.label,
is_extension: row.is_extension,
statement_kind: row.statement_kind ?? null,
role_uri: row.role_uri,
presentation_order: asNumber(row.presentation_order),
presentation_depth: row.presentation_depth,
parent_concept_key: row.parent_concept_key,
is_abstract: row.is_abstract,
created_at: row.created_at
};
}
function toFactRecord(row: typeof filingTaxonomyFact.$inferSelect): FilingTaxonomyFactRecord {
const value = asNumber(row.value_num);
if (value === null) {
throw new Error(`Invalid value_num for taxonomy fact row ${row.id}`);
}
return {
id: row.id,
snapshot_id: row.snapshot_id,
concept_key: row.concept_key,
qname: row.qname,
namespace_uri: row.namespace_uri,
local_name: row.local_name,
statement_kind: row.statement_kind ?? null,
role_uri: row.role_uri,
context_id: row.context_id,
unit: row.unit,
decimals: row.decimals,
value_num: value,
period_start: row.period_start,
period_end: row.period_end,
period_instant: row.period_instant,
dimensions: row.dimensions,
is_dimensionless: row.is_dimensionless,
source_file: row.source_file,
created_at: row.created_at
};
}
function toMetricValidationRecord(row: typeof filingTaxonomyMetricValidation.$inferSelect): FilingTaxonomyMetricValidationRecord {
return {
id: row.id,
snapshot_id: row.snapshot_id,
metric_key: row.metric_key,
taxonomy_value: asNumber(row.taxonomy_value),
llm_value: asNumber(row.llm_value),
absolute_diff: asNumber(row.absolute_diff),
relative_diff: asNumber(row.relative_diff),
status: row.status,
evidence_pages: row.evidence_pages ?? [],
pdf_url: row.pdf_url,
provider: row.provider,
model: row.model,
error: row.error,
created_at: row.created_at,
updated_at: row.updated_at
};
}
export async function getFilingTaxonomySnapshotByFilingId(filingId: number) {
const [row] = await db
.select()
.from(filingTaxonomySnapshot)
.where(eq(filingTaxonomySnapshot.filing_id, filingId))
.limit(1);
return row ? toSnapshotRecord(row) : null;
}
export async function listFilingTaxonomyAssets(snapshotId: number) {
const rows = await db
.select()
.from(filingTaxonomyAsset)
.where(eq(filingTaxonomyAsset.snapshot_id, snapshotId))
.orderBy(desc(filingTaxonomyAsset.id));
return rows.map(toAssetRecord);
}
export async function listFilingTaxonomyConcepts(snapshotId: number) {
const rows = await db
.select()
.from(filingTaxonomyConcept)
.where(eq(filingTaxonomyConcept.snapshot_id, snapshotId))
.orderBy(desc(filingTaxonomyConcept.id));
return rows.map(toConceptRecord);
}
export async function listFilingTaxonomyFacts(snapshotId: number) {
const rows = await db
.select()
.from(filingTaxonomyFact)
.where(eq(filingTaxonomyFact.snapshot_id, snapshotId))
.orderBy(desc(filingTaxonomyFact.id));
return rows.map(toFactRecord);
}
export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
const rows = await db
.select()
.from(filingTaxonomyMetricValidation)
.where(eq(filingTaxonomyMetricValidation.snapshot_id, snapshotId))
.orderBy(desc(filingTaxonomyMetricValidation.id));
return rows.map(toMetricValidationRecord);
}
export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) {
const now = new Date().toISOString();
const [saved] = await db
.insert(filingTaxonomySnapshot)
.values({
filing_id: input.filing_id,
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
periods: input.periods,
statement_rows: input.statement_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
created_at: now,
updated_at: now
})
.onConflictDoUpdate({
target: filingTaxonomySnapshot.filing_id,
set: {
ticker: input.ticker,
filing_date: input.filing_date,
filing_type: input.filing_type,
parse_status: input.parse_status,
parse_error: input.parse_error,
source: input.source,
periods: input.periods,
statement_rows: input.statement_rows,
derived_metrics: input.derived_metrics,
validation_result: input.validation_result,
facts_count: input.facts_count,
concepts_count: input.concepts_count,
dimensions_count: input.dimensions_count,
updated_at: now
}
})
.returning();
const snapshotId = saved.id;
await db.delete(filingTaxonomyAsset).where(eq(filingTaxonomyAsset.snapshot_id, snapshotId));
await db.delete(filingTaxonomyConcept).where(eq(filingTaxonomyConcept.snapshot_id, snapshotId));
await db.delete(filingTaxonomyFact).where(eq(filingTaxonomyFact.snapshot_id, snapshotId));
await db.delete(filingTaxonomyMetricValidation).where(eq(filingTaxonomyMetricValidation.snapshot_id, snapshotId));
if (input.assets.length > 0) {
await db.insert(filingTaxonomyAsset).values(input.assets.map((asset) => ({
snapshot_id: snapshotId,
asset_type: asset.asset_type,
name: asset.name,
url: asset.url,
size_bytes: asset.size_bytes,
score: asNumericText(asset.score),
is_selected: asset.is_selected,
created_at: now
})));
}
if (input.concepts.length > 0) {
await db.insert(filingTaxonomyConcept).values(input.concepts.map((concept) => ({
snapshot_id: snapshotId,
concept_key: concept.concept_key,
qname: concept.qname,
namespace_uri: concept.namespace_uri,
local_name: concept.local_name,
label: concept.label,
is_extension: concept.is_extension,
statement_kind: concept.statement_kind,
role_uri: concept.role_uri,
presentation_order: asNumericText(concept.presentation_order),
presentation_depth: concept.presentation_depth,
parent_concept_key: concept.parent_concept_key,
is_abstract: concept.is_abstract,
created_at: now
})));
}
if (input.facts.length > 0) {
await db.insert(filingTaxonomyFact).values(input.facts.map((fact) => ({
snapshot_id: snapshotId,
concept_key: fact.concept_key,
qname: fact.qname,
namespace_uri: fact.namespace_uri,
local_name: fact.local_name,
statement_kind: fact.statement_kind,
role_uri: fact.role_uri,
context_id: fact.context_id,
unit: fact.unit,
decimals: fact.decimals,
value_num: String(fact.value_num),
period_start: fact.period_start,
period_end: fact.period_end,
period_instant: fact.period_instant,
dimensions: fact.dimensions,
is_dimensionless: fact.is_dimensionless,
source_file: fact.source_file,
created_at: now
})));
}
if (input.metric_validations.length > 0) {
await db.insert(filingTaxonomyMetricValidation).values(input.metric_validations.map((check) => ({
snapshot_id: snapshotId,
metric_key: check.metric_key,
taxonomy_value: asNumericText(check.taxonomy_value),
llm_value: asNumericText(check.llm_value),
absolute_diff: asNumericText(check.absolute_diff),
relative_diff: asNumericText(check.relative_diff),
status: check.status,
evidence_pages: check.evidence_pages,
pdf_url: check.pdf_url,
provider: check.provider,
model: check.model,
error: check.error,
created_at: now,
updated_at: now
})));
}
return toSnapshotRecord(saved);
}
export async function listFilingTaxonomySnapshotsByTicker(input: {
ticker: string;
window: '10y' | 'all';
limit?: number;
cursor?: string | null;
}) {
const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 40), 1), 120);
const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null;
const constraints = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())];
if (input.window === '10y') {
constraints.push(gte(filingTaxonomySnapshot.filing_date, tenYearsAgoIso()));
}
if (cursorId && Number.isFinite(cursorId) && cursorId > 0) {
constraints.push(lt(filingTaxonomySnapshot.id, cursorId));
}
const rows = await db
.select()
.from(filingTaxonomySnapshot)
.where(and(...constraints))
.orderBy(desc(filingTaxonomySnapshot.filing_date), desc(filingTaxonomySnapshot.id))
.limit(safeLimit + 1);
const hasMore = rows.length > safeLimit;
const usedRows = hasMore ? rows.slice(0, safeLimit) : rows;
const nextCursor = hasMore
? String(usedRows[usedRows.length - 1]?.id ?? '')
: null;
return {
snapshots: usedRows.map(toSnapshotRecord),
nextCursor
};
}
export async function countFilingTaxonomySnapshotStatuses(ticker: string) {
const rows = await db
.select({
status: filingTaxonomySnapshot.parse_status,
count: sql<string>`count(*)`
})
.from(filingTaxonomySnapshot)
.where(eq(filingTaxonomySnapshot.ticker, ticker.trim().toUpperCase()))
.groupBy(filingTaxonomySnapshot.parse_status);
return rows.reduce<Record<FilingTaxonomyParseStatus, number>>((acc, row) => {
acc[row.status] = Number(row.count);
return acc;
}, {
ready: 0,
partial: 0,
failed: 0
});
}
export async function listTaxonomyFactsByTicker(input: {
ticker: string;
window: '10y' | 'all';
statement?: FinancialStatementKind;
cursor?: string | null;
limit?: number;
}) {
const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 2000);
const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null;
const conditions = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())];
if (input.window === '10y') {
conditions.push(gte(filingTaxonomySnapshot.filing_date, tenYearsAgoIso()));
}
if (input.statement) {
conditions.push(eq(filingTaxonomyFact.statement_kind, input.statement));
}
if (cursorId && Number.isFinite(cursorId) && cursorId > 0) {
conditions.push(lt(filingTaxonomyFact.id, cursorId));
}
const rows = await db
.select({
id: filingTaxonomyFact.id,
snapshot_id: filingTaxonomyFact.snapshot_id,
filing_id: filingTaxonomySnapshot.filing_id,
filing_date: filingTaxonomySnapshot.filing_date,
statement_kind: filingTaxonomyFact.statement_kind,
role_uri: filingTaxonomyFact.role_uri,
concept_key: filingTaxonomyFact.concept_key,
qname: filingTaxonomyFact.qname,
namespace_uri: filingTaxonomyFact.namespace_uri,
local_name: filingTaxonomyFact.local_name,
value_num: filingTaxonomyFact.value_num,
context_id: filingTaxonomyFact.context_id,
unit: filingTaxonomyFact.unit,
decimals: filingTaxonomyFact.decimals,
period_start: filingTaxonomyFact.period_start,
period_end: filingTaxonomyFact.period_end,
period_instant: filingTaxonomyFact.period_instant,
dimensions: filingTaxonomyFact.dimensions,
is_dimensionless: filingTaxonomyFact.is_dimensionless,
source_file: filingTaxonomyFact.source_file
})
.from(filingTaxonomyFact)
.innerJoin(filingTaxonomySnapshot, eq(filingTaxonomyFact.snapshot_id, filingTaxonomySnapshot.id))
.where(and(...conditions))
.orderBy(desc(filingTaxonomyFact.id))
.limit(safeLimit + 1);
const hasMore = rows.length > safeLimit;
const used = hasMore ? rows.slice(0, safeLimit) : rows;
const nextCursor = hasMore ? String(used[used.length - 1]?.id ?? '') : null;
const facts: TaxonomyFactRow[] = used.map((row) => {
const value = asNumber(row.value_num);
if (value === null) {
throw new Error(`Invalid value_num in taxonomy fact ${row.id}`);
}
return {
id: row.id,
snapshotId: row.snapshot_id,
filingId: row.filing_id,
filingDate: row.filing_date,
statement: row.statement_kind,
roleUri: row.role_uri,
conceptKey: row.concept_key,
qname: row.qname,
namespaceUri: row.namespace_uri,
localName: row.local_name,
value,
contextId: row.context_id,
unit: row.unit,
decimals: row.decimals,
periodStart: row.period_start,
periodEnd: row.period_end,
periodInstant: row.period_instant,
dimensions: row.dimensions,
isDimensionless: row.is_dimensionless,
sourceFile: row.source_file
};
});
return {
facts,
nextCursor
};
}
export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) {
if (snapshotIds.length === 0) {
return [];
}
const rows = await db
.select()
.from(filingTaxonomyAsset)
.where(inArray(filingTaxonomyAsset.snapshot_id, snapshotIds))
.orderBy(desc(filingTaxonomyAsset.id));
return rows.map(toAssetRecord);
}

16
lib/server/repos/filings.ts
View File

@@ -170,3 +170,19 @@ export async function saveFilingAnalysis(
return updated ? toFiling(updated) : null;
}
export async function updateFilingMetricsById(
filingId: number,
metrics: Filing['metrics']
) {
const [updated] = await db
.update(filing)
.set({
metrics,
updated_at: new Date().toISOString()
})
.where(eq(filing.id, filingId))
.returning();
return updated ? toFiling(updated) : null;
}