Run playwright UI tests
This commit is contained in:
336
lib/server/taxonomy/pdf-validation.ts
Normal file
336
lib/server/taxonomy/pdf-validation.ts
Normal file
@@ -0,0 +1,336 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import type { Filing, MetricValidationResult } from '@/lib/types';
|
||||
import { runAiAnalysis } from '@/lib/server/ai';
|
||||
import type { TaxonomyAsset, TaxonomyMetricValidationCheck } from '@/lib/server/taxonomy/types';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const METRIC_KEYS: Array<keyof NonNullable<Filing['metrics']>> = [
|
||||
'revenue',
|
||||
'netIncome',
|
||||
'totalAssets',
|
||||
'cash',
|
||||
'debt'
|
||||
];
|
||||
|
||||
function extractJsonCandidate(raw: string) {
|
||||
const fencedJson = raw.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1];
|
||||
const candidate = fencedJson ?? (() => {
|
||||
const start = raw.indexOf('{');
|
||||
const end = raw.lastIndexOf('}');
|
||||
return start >= 0 && end > start ? raw.slice(start, end + 1) : null;
|
||||
})();
|
||||
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function parseValidationPayload(raw: string) {
|
||||
const candidate = extractJsonCandidate(raw);
|
||||
if (!candidate) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(candidate) as Record<string, {
|
||||
value?: number | string | null;
|
||||
pages?: Array<number | string>;
|
||||
}>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function asNumber(value: unknown) {
|
||||
if (typeof value === 'number') {
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
const parsed = Number(value.replace(/[,\s]/g, ''));
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function asPageNumbers(raw: unknown): number[] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return raw
|
||||
.map((entry) => {
|
||||
if (typeof entry === 'number' && Number.isFinite(entry)) {
|
||||
return Math.trunc(entry);
|
||||
}
|
||||
|
||||
if (typeof entry === 'string') {
|
||||
const parsed = Number(entry);
|
||||
return Number.isFinite(parsed) ? Math.trunc(parsed) : Number.NaN;
|
||||
}
|
||||
|
||||
return Number.NaN;
|
||||
})
|
||||
.filter((entry) => Number.isFinite(entry) && entry > 0);
|
||||
}
|
||||
|
||||
function diffStatus(taxonomyValue: number | null, llmValue: number | null) {
|
||||
if (taxonomyValue === null && llmValue === null) {
|
||||
return {
|
||||
status: 'not_run' as const,
|
||||
absoluteDiff: null,
|
||||
relativeDiff: null
|
||||
};
|
||||
}
|
||||
|
||||
if (taxonomyValue === null || llmValue === null) {
|
||||
return {
|
||||
status: 'mismatch' as const,
|
||||
absoluteDiff: null,
|
||||
relativeDiff: null
|
||||
};
|
||||
}
|
||||
|
||||
const absoluteDiff = Math.abs(taxonomyValue - llmValue);
|
||||
const denominator = Math.max(Math.abs(taxonomyValue), 1);
|
||||
const relativeDiff = absoluteDiff / denominator;
|
||||
const tolerance = Math.max(1, Math.abs(taxonomyValue) * 0.005);
|
||||
|
||||
return {
|
||||
status: absoluteDiff <= tolerance ? 'matched' as const : 'mismatch' as const,
|
||||
absoluteDiff,
|
||||
relativeDiff
|
||||
};
|
||||
}
|
||||
|
||||
async function extractPdfText(url: string, fetchImpl: typeof fetch) {
|
||||
const response = await fetchImpl(url, {
|
||||
headers: {
|
||||
Accept: 'application/pdf, */*;q=0.8'
|
||||
},
|
||||
cache: 'no-store'
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`PDF request failed (${response.status})`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') ?? '';
|
||||
if (!/pdf/i.test(contentType) && !/\.pdf$/i.test(url)) {
|
||||
throw new Error(`Asset is not a PDF (${contentType || 'unknown content-type'})`);
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), 'fiscal-pdf-'));
|
||||
const pdfPath = join(tempRoot, 'source.pdf');
|
||||
|
||||
try {
|
||||
await writeFile(pdfPath, bytes);
|
||||
const { stdout } = await execFileAsync('pdftotext', ['-layout', '-enc', 'UTF-8', pdfPath, '-'], {
|
||||
maxBuffer: 16 * 1024 * 1024
|
||||
});
|
||||
|
||||
const text = stdout.trim();
|
||||
if (!text) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return text;
|
||||
} finally {
|
||||
await rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function validationPrompt(metrics: Filing['metrics'], pdfText: string) {
|
||||
const textSlice = pdfText.slice(0, 80_000);
|
||||
|
||||
return [
|
||||
'Extract numeric financial metrics from the provided financial statement PDF text.',
|
||||
`Taxonomy baseline metrics: ${JSON.stringify(metrics ?? {})}`,
|
||||
'Return ONLY JSON with keys revenue, netIncome, totalAssets, cash, debt.',
|
||||
'Each key must map to: {"value": number|null, "pages": [number]}.',
|
||||
'Use null when a metric is not found.',
|
||||
'PDF text follows:',
|
||||
textSlice
|
||||
].join('\n\n');
|
||||
}
|
||||
|
||||
function providerModelOrNull(value: string | undefined | null) {
|
||||
const normalized = value?.trim();
|
||||
return normalized && normalized.length > 0 ? normalized : null;
|
||||
}
|
||||
|
||||
export async function validateMetricsWithPdfLlm(input: {
|
||||
metrics: Filing['metrics'];
|
||||
assets: TaxonomyAsset[];
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<{
|
||||
validation_result: MetricValidationResult | null;
|
||||
metric_validations: TaxonomyMetricValidationCheck[];
|
||||
}> {
|
||||
const taxonomyMetrics = input.metrics ?? {
|
||||
revenue: null,
|
||||
netIncome: null,
|
||||
totalAssets: null,
|
||||
cash: null,
|
||||
debt: null
|
||||
};
|
||||
|
||||
const selectedPdf = input.assets.find((asset) => asset.asset_type === 'pdf' && asset.is_selected);
|
||||
if (!selectedPdf) {
|
||||
return {
|
||||
validation_result: {
|
||||
status: 'not_run',
|
||||
checks: [],
|
||||
validatedAt: null
|
||||
},
|
||||
metric_validations: []
|
||||
};
|
||||
}
|
||||
|
||||
const fetchImpl = input.fetchImpl ?? fetch;
|
||||
let pdfText: string | null = null;
|
||||
try {
|
||||
pdfText = await extractPdfText(selectedPdf.url, fetchImpl);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'PDF extraction failed';
|
||||
|
||||
const checks: TaxonomyMetricValidationCheck[] = METRIC_KEYS.map((metricKey) => ({
|
||||
metric_key: metricKey,
|
||||
taxonomy_value: taxonomyMetrics[metricKey],
|
||||
llm_value: null,
|
||||
absolute_diff: null,
|
||||
relative_diff: null,
|
||||
status: 'error',
|
||||
evidence_pages: [],
|
||||
pdf_url: selectedPdf.url,
|
||||
provider: null,
|
||||
model: null,
|
||||
error: message
|
||||
}));
|
||||
|
||||
return {
|
||||
validation_result: {
|
||||
status: 'error',
|
||||
checks: checks.map((check) => ({
|
||||
metricKey: check.metric_key,
|
||||
taxonomyValue: check.taxonomy_value,
|
||||
llmValue: check.llm_value,
|
||||
absoluteDiff: check.absolute_diff,
|
||||
relativeDiff: check.relative_diff,
|
||||
status: check.status,
|
||||
evidencePages: check.evidence_pages,
|
||||
pdfUrl: check.pdf_url,
|
||||
provider: check.provider,
|
||||
model: check.model,
|
||||
error: check.error
|
||||
})),
|
||||
validatedAt: new Date().toISOString()
|
||||
},
|
||||
metric_validations: checks
|
||||
};
|
||||
}
|
||||
|
||||
if (!pdfText) {
|
||||
return {
|
||||
validation_result: {
|
||||
status: 'not_run',
|
||||
checks: [],
|
||||
validatedAt: new Date().toISOString()
|
||||
},
|
||||
metric_validations: []
|
||||
};
|
||||
}
|
||||
|
||||
let parsed: Record<string, { value?: number | string | null; pages?: Array<number | string> }> | null = null;
|
||||
let provider: string | null = null;
|
||||
let model: string | null = null;
|
||||
let modelError: string | null = null;
|
||||
|
||||
try {
|
||||
const aiResult = await runAiAnalysis(validationPrompt(taxonomyMetrics, pdfText), undefined, {
|
||||
workload: 'extraction'
|
||||
});
|
||||
|
||||
provider = providerModelOrNull(aiResult.provider);
|
||||
model = providerModelOrNull(aiResult.model);
|
||||
parsed = parseValidationPayload(aiResult.text);
|
||||
if (!parsed) {
|
||||
modelError = 'LLM response did not contain valid JSON payload';
|
||||
}
|
||||
} catch (error) {
|
||||
modelError = error instanceof Error ? error.message : 'LLM validation failed';
|
||||
}
|
||||
|
||||
const validations: TaxonomyMetricValidationCheck[] = METRIC_KEYS.map((metricKey) => {
|
||||
const taxonomyValue = taxonomyMetrics[metricKey] ?? null;
|
||||
|
||||
if (!parsed) {
|
||||
return {
|
||||
metric_key: metricKey,
|
||||
taxonomy_value: taxonomyValue,
|
||||
llm_value: null,
|
||||
absolute_diff: null,
|
||||
relative_diff: null,
|
||||
status: modelError ? 'error' : 'not_run',
|
||||
evidence_pages: [],
|
||||
pdf_url: selectedPdf.url,
|
||||
provider,
|
||||
model,
|
||||
error: modelError
|
||||
};
|
||||
}
|
||||
|
||||
const entry = parsed[metricKey as string] ?? {};
|
||||
const llmValue = asNumber(entry.value);
|
||||
const pages = asPageNumbers(entry.pages);
|
||||
const diff = diffStatus(taxonomyValue, llmValue);
|
||||
|
||||
return {
|
||||
metric_key: metricKey,
|
||||
taxonomy_value: taxonomyValue,
|
||||
llm_value: llmValue,
|
||||
absolute_diff: diff.absoluteDiff,
|
||||
relative_diff: diff.relativeDiff,
|
||||
status: diff.status,
|
||||
evidence_pages: pages,
|
||||
pdf_url: selectedPdf.url,
|
||||
provider,
|
||||
model,
|
||||
error: null
|
||||
};
|
||||
});
|
||||
|
||||
const hasError = validations.some((entry) => entry.status === 'error');
|
||||
const hasMismatch = validations.some((entry) => entry.status === 'mismatch');
|
||||
|
||||
return {
|
||||
validation_result: {
|
||||
status: hasError ? 'error' : hasMismatch ? 'mismatch' : 'matched',
|
||||
checks: validations.map((check) => ({
|
||||
metricKey: check.metric_key,
|
||||
taxonomyValue: check.taxonomy_value,
|
||||
llmValue: check.llm_value,
|
||||
absoluteDiff: check.absolute_diff,
|
||||
relativeDiff: check.relative_diff,
|
||||
status: check.status,
|
||||
evidencePages: check.evidence_pages,
|
||||
pdfUrl: check.pdf_url,
|
||||
provider: check.provider,
|
||||
model: check.model,
|
||||
error: check.error
|
||||
})),
|
||||
validatedAt: new Date().toISOString()
|
||||
},
|
||||
metric_validations: validations
|
||||
};
|
||||
}
|
||||
|
||||
export const __pdfValidationInternals = {
|
||||
parseValidationPayload,
|
||||
diffStatus
|
||||
};
|
||||
Reference in New Issue
Block a user