Run playwright UI tests
This commit is contained in:
384
lib/server/financial-taxonomy.ts
Normal file
384
lib/server/financial-taxonomy.ts
Normal file
@@ -0,0 +1,384 @@
|
||||
import type {
|
||||
CompanyFinancialStatementsResponse,
|
||||
DimensionBreakdownRow,
|
||||
FinancialHistoryWindow,
|
||||
FinancialStatementKind,
|
||||
FinancialStatementPeriod,
|
||||
TaxonomyStatementRow
|
||||
} from '@/lib/types';
|
||||
import { listFilingsRecords } from '@/lib/server/repos/filings';
|
||||
import {
|
||||
countFilingTaxonomySnapshotStatuses,
|
||||
listFilingTaxonomySnapshotsByTicker,
|
||||
listTaxonomyFactsByTicker,
|
||||
type FilingTaxonomySnapshotRecord
|
||||
} from '@/lib/server/repos/filing-taxonomy';
|
||||
|
||||
type GetCompanyFinancialTaxonomyInput = {
|
||||
ticker: string;
|
||||
statement: FinancialStatementKind;
|
||||
window: FinancialHistoryWindow;
|
||||
includeDimensions: boolean;
|
||||
includeFacts: boolean;
|
||||
factsCursor?: string | null;
|
||||
factsLimit?: number;
|
||||
cursor?: string | null;
|
||||
limit?: number;
|
||||
v3Enabled: boolean;
|
||||
queuedSync: boolean;
|
||||
};
|
||||
|
||||
function safeTicker(input: string) {
|
||||
return input.trim().toUpperCase();
|
||||
}
|
||||
|
||||
function isFinancialForm(type: string): type is '10-K' | '10-Q' {
|
||||
return type === '10-K' || type === '10-Q';
|
||||
}
|
||||
|
||||
function parseEpoch(value: string | null) {
|
||||
if (!value) {
|
||||
return Number.NaN;
|
||||
}
|
||||
|
||||
return Date.parse(value);
|
||||
}
|
||||
|
||||
function periodSorter(left: FinancialStatementPeriod, right: FinancialStatementPeriod) {
|
||||
const leftDate = parseEpoch(left.periodEnd ?? left.filingDate);
|
||||
const rightDate = parseEpoch(right.periodEnd ?? right.filingDate);
|
||||
if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
|
||||
return leftDate - rightDate;
|
||||
}
|
||||
|
||||
return left.id.localeCompare(right.id);
|
||||
}
|
||||
|
||||
function isInstantPeriod(period: FinancialStatementPeriod) {
|
||||
return period.periodStart === null;
|
||||
}
|
||||
|
||||
function periodDurationDays(period: FinancialStatementPeriod) {
|
||||
if (!period.periodStart || !period.periodEnd) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const start = Date.parse(period.periodStart);
|
||||
const end = Date.parse(period.periodEnd);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Math.round((end - start) / 86_400_000) + 1;
|
||||
}
|
||||
|
||||
function preferredDurationDays(filingType: FinancialStatementPeriod['filingType']) {
|
||||
return filingType === '10-K' ? 365 : 90;
|
||||
}
|
||||
|
||||
function selectPrimaryPeriods(
|
||||
snapshots: FilingTaxonomySnapshotRecord[],
|
||||
statement: FinancialStatementKind
|
||||
) {
|
||||
const selectedByFilingId = new Map<number, FinancialStatementPeriod>();
|
||||
|
||||
for (const snapshot of snapshots) {
|
||||
const rows = snapshot.statement_rows?.[statement] ?? [];
|
||||
if (rows.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const usedPeriodIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
for (const periodId of Object.keys(row.values)) {
|
||||
usedPeriodIds.add(periodId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = (snapshot.periods ?? []).filter((period) => usedPeriodIds.has(period.id));
|
||||
if (candidates.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const selected = (() => {
|
||||
if (statement === 'balance') {
|
||||
const instantCandidates = candidates.filter(isInstantPeriod);
|
||||
return (instantCandidates.length > 0 ? instantCandidates : candidates)
|
||||
.sort((left, right) => periodSorter(right, left))[0] ?? null;
|
||||
}
|
||||
|
||||
const durationCandidates = candidates.filter((period) => !isInstantPeriod(period));
|
||||
if (durationCandidates.length === 0) {
|
||||
return candidates.sort((left, right) => periodSorter(right, left))[0] ?? null;
|
||||
}
|
||||
|
||||
const targetDays = preferredDurationDays(snapshot.filing_type);
|
||||
return durationCandidates.sort((left, right) => {
|
||||
const leftDate = parseEpoch(left.periodEnd ?? left.filingDate);
|
||||
const rightDate = parseEpoch(right.periodEnd ?? right.filingDate);
|
||||
if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
|
||||
return rightDate - leftDate;
|
||||
}
|
||||
|
||||
const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays);
|
||||
const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays);
|
||||
if (leftDistance !== rightDistance) {
|
||||
return leftDistance - rightDistance;
|
||||
}
|
||||
|
||||
return left.id.localeCompare(right.id);
|
||||
})[0] ?? null;
|
||||
})();
|
||||
|
||||
if (selected) {
|
||||
selectedByFilingId.set(selected.filingId, selected);
|
||||
}
|
||||
}
|
||||
|
||||
const periods = [...selectedByFilingId.values()].sort(periodSorter);
|
||||
return {
|
||||
periods,
|
||||
selectedPeriodIds: new Set(periods.map((period) => period.id)),
|
||||
periodByFilingId: new Map(periods.map((period) => [period.filingId, period]))
|
||||
};
|
||||
}
|
||||
|
||||
function buildPeriods(
|
||||
snapshots: FilingTaxonomySnapshotRecord[],
|
||||
statement: FinancialStatementKind
|
||||
) {
|
||||
return selectPrimaryPeriods(snapshots, statement).periods;
|
||||
}
|
||||
|
||||
function buildRows(
|
||||
snapshots: FilingTaxonomySnapshotRecord[],
|
||||
statement: FinancialStatementKind,
|
||||
selectedPeriodIds: Set<string>
|
||||
) {
|
||||
const rowMap = new Map<string, TaxonomyStatementRow>();
|
||||
|
||||
for (const snapshot of snapshots) {
|
||||
const rows = snapshot.statement_rows?.[statement] ?? [];
|
||||
|
||||
for (const row of rows) {
|
||||
const existing = rowMap.get(row.key);
|
||||
if (!existing) {
|
||||
rowMap.set(row.key, {
|
||||
...row,
|
||||
values: Object.fromEntries(
|
||||
Object.entries(row.values).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
),
|
||||
units: Object.fromEntries(
|
||||
Object.entries(row.units).filter(([periodId]) => selectedPeriodIds.has(periodId))
|
||||
),
|
||||
sourceFactIds: [...row.sourceFactIds]
|
||||
});
|
||||
if (Object.keys(rowMap.get(row.key)?.values ?? {}).length === 0) {
|
||||
rowMap.delete(row.key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
existing.hasDimensions = existing.hasDimensions || row.hasDimensions;
|
||||
existing.order = Math.min(existing.order, row.order);
|
||||
existing.depth = Math.min(existing.depth, row.depth);
|
||||
if (!existing.parentKey && row.parentKey) {
|
||||
existing.parentKey = row.parentKey;
|
||||
}
|
||||
|
||||
for (const [periodId, value] of Object.entries(row.values)) {
|
||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.values)) {
|
||||
existing.values[periodId] = value;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [periodId, unit] of Object.entries(row.units)) {
|
||||
if (selectedPeriodIds.has(periodId) && !(periodId in existing.units)) {
|
||||
existing.units[periodId] = unit;
|
||||
}
|
||||
}
|
||||
|
||||
for (const factId of row.sourceFactIds) {
|
||||
if (!existing.sourceFactIds.includes(factId)) {
|
||||
existing.sourceFactIds.push(factId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...rowMap.values()].sort((left, right) => {
|
||||
if (left.order !== right.order) {
|
||||
return left.order - right.order;
|
||||
}
|
||||
|
||||
return left.label.localeCompare(right.label);
|
||||
});
|
||||
}
|
||||
|
||||
function buildDimensionBreakdown(
|
||||
facts: Awaited<ReturnType<typeof listTaxonomyFactsByTicker>>['facts'],
|
||||
periods: FinancialStatementPeriod[]
|
||||
) {
|
||||
const periodByFilingId = new Map<number, FinancialStatementPeriod>();
|
||||
for (const period of periods) {
|
||||
periodByFilingId.set(period.filingId, period);
|
||||
}
|
||||
|
||||
const map = new Map<string, DimensionBreakdownRow[]>();
|
||||
|
||||
for (const fact of facts) {
|
||||
if (fact.dimensions.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const period = periodByFilingId.get(fact.filingId) ?? null;
|
||||
if (!period) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const matchesPeriod = period.periodStart
|
||||
? fact.periodStart === period.periodStart && fact.periodEnd === period.periodEnd
|
||||
: (fact.periodInstant ?? fact.periodEnd) === period.periodEnd;
|
||||
|
||||
if (!matchesPeriod) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const dimension of fact.dimensions) {
|
||||
const row: DimensionBreakdownRow = {
|
||||
rowKey: fact.conceptKey,
|
||||
concept: fact.qname,
|
||||
periodId: period.id,
|
||||
axis: dimension.axis,
|
||||
member: dimension.member,
|
||||
value: fact.value,
|
||||
unit: fact.unit
|
||||
};
|
||||
|
||||
const existing = map.get(fact.conceptKey);
|
||||
if (existing) {
|
||||
existing.push(row);
|
||||
} else {
|
||||
map.set(fact.conceptKey, [row]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return map.size > 0 ? Object.fromEntries(map.entries()) : null;
|
||||
}
|
||||
|
||||
function latestMetrics(snapshots: FilingTaxonomySnapshotRecord[]) {
|
||||
for (const snapshot of snapshots) {
|
||||
if (snapshot.derived_metrics) {
|
||||
return {
|
||||
taxonomy: snapshot.derived_metrics,
|
||||
validation: snapshot.validation_result
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
taxonomy: null,
|
||||
validation: null
|
||||
};
|
||||
}
|
||||
|
||||
export function defaultFinancialSyncLimit(window: FinancialHistoryWindow) {
|
||||
return window === 'all' ? 120 : 60;
|
||||
}
|
||||
|
||||
export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialTaxonomyInput): Promise<CompanyFinancialStatementsResponse> {
|
||||
const ticker = safeTicker(input.ticker);
|
||||
const snapshotResult = await listFilingTaxonomySnapshotsByTicker({
|
||||
ticker,
|
||||
window: input.window,
|
||||
limit: input.limit,
|
||||
cursor: input.cursor
|
||||
});
|
||||
|
||||
const statuses = await countFilingTaxonomySnapshotStatuses(ticker);
|
||||
const filings = await listFilingsRecords({
|
||||
ticker,
|
||||
limit: input.window === 'all' ? 250 : 120
|
||||
});
|
||||
|
||||
const financialFilings = filings.filter((filing) => isFinancialForm(filing.filing_type));
|
||||
const selection = selectPrimaryPeriods(snapshotResult.snapshots, input.statement);
|
||||
const periods = selection.periods;
|
||||
const rows = buildRows(snapshotResult.snapshots, input.statement, selection.selectedPeriodIds);
|
||||
|
||||
const factsResult = input.includeFacts
|
||||
? await listTaxonomyFactsByTicker({
|
||||
ticker,
|
||||
window: input.window,
|
||||
statement: input.statement,
|
||||
cursor: input.factsCursor,
|
||||
limit: input.factsLimit
|
||||
})
|
||||
: { facts: [], nextCursor: null };
|
||||
|
||||
const dimensionFacts = input.includeDimensions
|
||||
? await listTaxonomyFactsByTicker({
|
||||
ticker,
|
||||
window: input.window,
|
||||
statement: input.statement,
|
||||
limit: 1200
|
||||
})
|
||||
: { facts: [], nextCursor: null };
|
||||
|
||||
const latestFiling = filings[0] ?? null;
|
||||
const metrics = latestMetrics(snapshotResult.snapshots);
|
||||
const dimensionBreakdown = input.includeDimensions
|
||||
? buildDimensionBreakdown(dimensionFacts.facts, periods)
|
||||
: null;
|
||||
|
||||
const dimensionsCount = dimensionBreakdown
|
||||
? Object.values(dimensionBreakdown).reduce((total, entries) => total + entries.length, 0)
|
||||
: 0;
|
||||
|
||||
const factsCoverage = input.includeFacts
|
||||
? factsResult.facts.length
|
||||
: snapshotResult.snapshots.reduce((total, snapshot) => total + snapshot.facts_count, 0);
|
||||
|
||||
return {
|
||||
company: {
|
||||
ticker,
|
||||
companyName: latestFiling?.company_name ?? ticker,
|
||||
cik: latestFiling?.cik ?? null
|
||||
},
|
||||
statement: input.statement,
|
||||
window: input.window,
|
||||
periods,
|
||||
rows,
|
||||
nextCursor: snapshotResult.nextCursor,
|
||||
facts: input.includeFacts
|
||||
? {
|
||||
rows: factsResult.facts,
|
||||
nextCursor: factsResult.nextCursor
|
||||
}
|
||||
: null,
|
||||
coverage: {
|
||||
filings: periods.length,
|
||||
rows: rows.length,
|
||||
dimensions: dimensionsCount,
|
||||
facts: factsCoverage
|
||||
},
|
||||
dataSourceStatus: {
|
||||
enabled: input.v3Enabled,
|
||||
hydratedFilings: statuses.ready,
|
||||
partialFilings: statuses.partial,
|
||||
failedFilings: statuses.failed,
|
||||
pendingFilings: Math.max(0, financialFilings.length - statuses.ready - statuses.partial - statuses.failed),
|
||||
queuedSync: input.queuedSync
|
||||
},
|
||||
metrics,
|
||||
dimensionBreakdown
|
||||
};
|
||||
}
|
||||
|
||||
export const __financialTaxonomyInternals = {
|
||||
buildPeriods,
|
||||
isInstantPeriod,
|
||||
periodDurationDays,
|
||||
selectPrimaryPeriods
|
||||
};
|
||||
Reference in New Issue
Block a user