Run playwright UI tests
This commit is contained in:
@@ -13,12 +13,13 @@ import {
|
||||
getFilingByAccession,
|
||||
listFilingsRecords,
|
||||
saveFilingAnalysis,
|
||||
updateFilingMetricsById,
|
||||
upsertFilingsRecords
|
||||
} from '@/lib/server/repos/filings';
|
||||
import {
|
||||
getFilingStatementSnapshotByFilingId,
|
||||
upsertFilingStatementSnapshot
|
||||
} from '@/lib/server/repos/filing-statements';
|
||||
getFilingTaxonomySnapshotByFilingId,
|
||||
upsertFilingTaxonomySnapshot
|
||||
} from '@/lib/server/repos/filing-taxonomy';
|
||||
import {
|
||||
applyRefreshedPrices,
|
||||
listHoldingsForPriceRefresh,
|
||||
@@ -27,11 +28,10 @@ import {
|
||||
import { createPortfolioInsight } from '@/lib/server/repos/insights';
|
||||
import { updateTaskStage } from '@/lib/server/repos/tasks';
|
||||
import {
|
||||
fetchFilingMetricsForFilings,
|
||||
fetchPrimaryFilingText,
|
||||
fetchRecentFilings,
|
||||
hydrateFilingStatementSnapshot
|
||||
fetchRecentFilings
|
||||
} from '@/lib/server/sec';
|
||||
import { hydrateFilingTaxonomySnapshot } from '@/lib/server/taxonomy/engine';
|
||||
|
||||
const EXTRACTION_REQUIRED_KEYS = [
|
||||
'summary',
|
||||
@@ -88,6 +88,10 @@ const COMPANY_SPECIFIC_PATTERNS = [
|
||||
|
||||
type FilingMetricKey = keyof NonNullable<Filing['metrics']>;
|
||||
|
||||
function isFinancialMetricsForm(filingType: string): filingType is '10-K' | '10-Q' {
|
||||
return filingType === '10-K' || filingType === '10-Q';
|
||||
}
|
||||
|
||||
const METRIC_CHECK_PATTERNS: Array<{
|
||||
key: FilingMetricKey;
|
||||
label: string;
|
||||
@@ -120,10 +124,6 @@ const METRIC_CHECK_PATTERNS: Array<{
|
||||
}
|
||||
];
|
||||
|
||||
function isFinancialMetricsForm(form: Filing['filing_type']) {
|
||||
return form === '10-K' || form === '10-Q';
|
||||
}
|
||||
|
||||
function toTaskResult(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return { value };
|
||||
@@ -565,40 +565,6 @@ async function processSyncFilings(task: Task) {
|
||||
`Fetching up to ${limit} filings for ${ticker}${scopeLabel ? ` (${scopeLabel})` : ''}`
|
||||
);
|
||||
const filings = await fetchRecentFilings(ticker, limit);
|
||||
const metricsByAccession = new Map<string, Filing['metrics']>();
|
||||
const filingsByCik = new Map<string, typeof filings>();
|
||||
|
||||
for (const filing of filings) {
|
||||
const group = filingsByCik.get(filing.cik);
|
||||
if (group) {
|
||||
group.push(filing);
|
||||
continue;
|
||||
}
|
||||
|
||||
filingsByCik.set(filing.cik, [filing]);
|
||||
}
|
||||
|
||||
await setProjectionStage(task, 'sync.fetch_metrics', `Computing financial metrics for ${filings.length} filings`);
|
||||
for (const [cik, filingsForCik] of filingsByCik) {
|
||||
const filingsForFinancialMetrics = filingsForCik.filter((filing) => isFinancialMetricsForm(filing.filingType));
|
||||
if (filingsForFinancialMetrics.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metricsMap = await fetchFilingMetricsForFilings(
|
||||
cik,
|
||||
filingsForCik[0]?.ticker ?? ticker,
|
||||
filingsForFinancialMetrics.map((filing) => ({
|
||||
accessionNumber: filing.accessionNumber,
|
||||
filingDate: filing.filingDate,
|
||||
filingType: filing.filingType
|
||||
}))
|
||||
);
|
||||
|
||||
for (const [accessionNumber, metrics] of metricsMap.entries()) {
|
||||
metricsByAccession.set(accessionNumber, metrics);
|
||||
}
|
||||
}
|
||||
|
||||
await setProjectionStage(task, 'sync.persist_filings', 'Persisting filings and links');
|
||||
const saveResult = await upsertFilingsRecords(
|
||||
@@ -612,24 +578,24 @@ async function processSyncFilings(task: Task) {
|
||||
filing_url: filing.filingUrl,
|
||||
submission_url: filing.submissionUrl,
|
||||
primary_document: filing.primaryDocument,
|
||||
metrics: metricsByAccession.get(filing.accessionNumber) ?? null,
|
||||
metrics: null,
|
||||
links: filingLinks(filing)
|
||||
}))
|
||||
);
|
||||
|
||||
let statementSnapshotsHydrated = 0;
|
||||
let statementSnapshotsFailed = 0;
|
||||
let taxonomySnapshotsHydrated = 0;
|
||||
let taxonomySnapshotsFailed = 0;
|
||||
const hydrateCandidates = (await listFilingsRecords({
|
||||
ticker,
|
||||
limit: Math.min(Math.max(limit * 3, 40), STATEMENT_HYDRATION_MAX_FILINGS)
|
||||
}))
|
||||
.filter((filing): filing is Filing & { filing_type: '10-K' | '10-Q' } => {
|
||||
return filing.filing_type === '10-K' || filing.filing_type === '10-Q';
|
||||
return isFinancialMetricsForm(filing.filing_type);
|
||||
});
|
||||
|
||||
await setProjectionStage(task, 'sync.hydrate_statements', `Hydrating statement snapshots for ${hydrateCandidates.length} candidate filings`);
|
||||
await setProjectionStage(task, 'sync.discover_assets', `Discovering taxonomy assets for ${hydrateCandidates.length} candidate filings`);
|
||||
for (const filing of hydrateCandidates) {
|
||||
const existingSnapshot = await getFilingStatementSnapshotByFilingId(filing.id);
|
||||
const existingSnapshot = await getFilingTaxonomySnapshotByFilingId(filing.id);
|
||||
const shouldRefresh = !existingSnapshot
|
||||
|| Date.parse(existingSnapshot.updated_at) < Date.parse(filing.updated_at);
|
||||
|
||||
@@ -638,7 +604,8 @@ async function processSyncFilings(task: Task) {
|
||||
}
|
||||
|
||||
try {
|
||||
const snapshot = await hydrateFilingStatementSnapshot({
|
||||
await setProjectionStage(task, 'sync.extract_taxonomy', `Extracting XBRL taxonomy for ${filing.accession_number}`);
|
||||
const snapshot = await hydrateFilingTaxonomySnapshot({
|
||||
filingId: filing.id,
|
||||
ticker: filing.ticker,
|
||||
cik: filing.cik,
|
||||
@@ -646,27 +613,50 @@ async function processSyncFilings(task: Task) {
|
||||
filingDate: filing.filing_date,
|
||||
filingType: filing.filing_type,
|
||||
filingUrl: filing.filing_url,
|
||||
primaryDocument: filing.primary_document ?? null,
|
||||
metrics: filing.metrics
|
||||
primaryDocument: filing.primary_document ?? null
|
||||
});
|
||||
|
||||
await upsertFilingStatementSnapshot(snapshot);
|
||||
statementSnapshotsHydrated += 1;
|
||||
await setProjectionStage(task, 'sync.normalize_taxonomy', `Materializing statements for ${filing.accession_number}`);
|
||||
await setProjectionStage(task, 'sync.derive_metrics', `Deriving taxonomy metrics for ${filing.accession_number}`);
|
||||
await setProjectionStage(task, 'sync.validate_pdf_metrics', `Validating metrics via PDF + LLM for ${filing.accession_number}`);
|
||||
await setProjectionStage(task, 'sync.persist_taxonomy', `Persisting taxonomy snapshot for ${filing.accession_number}`);
|
||||
|
||||
await upsertFilingTaxonomySnapshot(snapshot);
|
||||
await updateFilingMetricsById(filing.id, snapshot.derived_metrics);
|
||||
taxonomySnapshotsHydrated += 1;
|
||||
} catch (error) {
|
||||
await upsertFilingStatementSnapshot({
|
||||
const now = new Date().toISOString();
|
||||
await upsertFilingTaxonomySnapshot({
|
||||
filing_id: filing.id,
|
||||
ticker: filing.ticker,
|
||||
filing_date: filing.filing_date,
|
||||
filing_type: filing.filing_type,
|
||||
period_end: filing.filing_date,
|
||||
statement_bundle: null,
|
||||
standardized_bundle: null,
|
||||
dimension_bundle: null,
|
||||
parse_status: 'failed',
|
||||
parse_error: error instanceof Error ? error.message : 'Statement hydration failed',
|
||||
source: 'companyfacts_fallback'
|
||||
parse_error: error instanceof Error ? error.message : 'Taxonomy hydration failed',
|
||||
source: 'legacy_html_fallback',
|
||||
periods: [],
|
||||
statement_rows: {
|
||||
income: [],
|
||||
balance: [],
|
||||
cash_flow: [],
|
||||
equity: [],
|
||||
comprehensive_income: []
|
||||
},
|
||||
derived_metrics: filing.metrics ?? null,
|
||||
validation_result: {
|
||||
status: 'error',
|
||||
checks: [],
|
||||
validatedAt: now
|
||||
},
|
||||
facts_count: 0,
|
||||
concepts_count: 0,
|
||||
dimensions_count: 0,
|
||||
assets: [],
|
||||
concepts: [],
|
||||
facts: [],
|
||||
metric_validations: []
|
||||
});
|
||||
statementSnapshotsFailed += 1;
|
||||
taxonomySnapshotsFailed += 1;
|
||||
}
|
||||
|
||||
await Bun.sleep(STATEMENT_HYDRATION_DELAY_MS);
|
||||
@@ -679,8 +669,8 @@ async function processSyncFilings(task: Task) {
|
||||
fetched: filings.length,
|
||||
inserted: saveResult.inserted,
|
||||
updated: saveResult.updated,
|
||||
statementSnapshotsHydrated,
|
||||
statementSnapshotsFailed
|
||||
taxonomySnapshotsHydrated,
|
||||
taxonomySnapshotsFailed
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user