Fix annual financial selector and QCOM standardization

2026-03-09 18:50:59 -04:00
parent 1a18ac825d
commit 9f972305e6
9 changed files with 3385 additions and 226 deletions
--- a/lib/server/financial-taxonomy.test.ts
+++ b/lib/server/financial-taxonomy.test.ts
--- a/lib/server/financial-taxonomy.ts
+++ b/lib/server/financial-taxonomy.ts
@@ -61,6 +61,7 @@ type GetCompanyFinancialsInput = {
 type StandardizedStatementBundlePayload = {
  rows: StandardizedFinancialRow[];
  trendSeries: CompanyFinancialStatementsResponse['trendSeries'];
+  categories: CompanyFinancialStatementsResponse['categories'];
 };

 type FilingDocumentRef = {
@@ -281,7 +282,8 @@ async function buildStatementSurfaceBundle(input: {
  if (!statement || (statement !== 'income' && statement !== 'balance' && statement !== 'cash_flow')) {
    return {
      rows: [],
-      trendSeries: []
+      trendSeries: [],
+      categories: []
    } satisfies StandardizedStatementBundlePayload;
  }

@@ -289,7 +291,7 @@ async function buildStatementSurfaceBundle(input: {
    rows: input.faithfulRows,
    statement,
    periods: input.periods,
-    facts: input.facts.filter((fact) => fact.statement === statement)
+    facts: input.facts
  });

  const payload = {
@@ -297,7 +299,8 @@ async function buildStatementSurfaceBundle(input: {
    trendSeries: buildTrendSeries({
      surfaceKind: input.surfaceKind,
      statementRows: standardizedRows
-    })
+    }),
+    categories: buildFinancialCategories(standardizedRows, input.surfaceKind)
  } satisfies StandardizedStatementBundlePayload;

  await writeFinancialBundle({
@@ -520,7 +523,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
    ticker,
    window: 'all',
    filingTypes: [...filingTypes],
-    limit: 2000
+    limit: 10000
  });

  if (isStatementSurface(input.surfaceKind)) {
@@ -546,12 +549,13 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
      : buildRows(selection.snapshots, statement, selection.selectedPeriodIds);

    const factsForStatement = allFacts.facts.filter((fact) => fact.statement === statement);
+    const factsForStandardization = allFacts.facts;
    const standardizedPayload = await buildStatementSurfaceBundle({
      surfaceKind: input.surfaceKind as Extract<FinancialSurfaceKind, 'income_statement' | 'balance_sheet' | 'cash_flow_statement'>,
      cadence: input.cadence,
      periods,
      faithfulRows,
-      facts: factsForStatement,
+      facts: factsForStandardization,
      snapshots: selection.snapshots
    });

@@ -561,7 +565,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
          rows: buildRows(selection.snapshots, statement, selection.selectedPeriodIds),
          statement: statement as Extract<FinancialStatementKind, 'income' | 'balance' | 'cash_flow'>,
          periods: selection.periods,
-          facts: factsForStatement
+          facts: factsForStandardization
        }),
        selection.periods,
        periods,
@@ -581,7 +585,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
      : { facts: [], nextCursor: null };

    const dimensionBreakdown = input.includeDimensions
-      ? buildDimensionBreakdown(factsForStatement, periods, faithfulRows, standardizedRows)
+      ? buildDimensionBreakdown(factsForStandardization, periods, faithfulRows, standardizedRows)
      : null;

    return {
@@ -605,7 +609,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
        surfaceKind: input.surfaceKind,
        statementRows: standardizedRows
      }),
-      categories: [],
+      categories: standardizedPayload.categories,
      availability: {
        adjusted: false,
        customMetrics: false
@@ -654,19 +658,19 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr
    rows: buildRows(incomeSelection.snapshots, 'income', incomeSelection.selectedPeriodIds),
    statement: 'income',
    periods: incomeSelection.periods,
-    facts: allFacts.facts.filter((fact) => fact.statement === 'income')
+    facts: allFacts.facts
  });
  const balanceQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({
    rows: buildRows(balanceSelection.snapshots, 'balance', balanceSelection.selectedPeriodIds),
    statement: 'balance',
    periods: balanceSelection.periods,
-    facts: allFacts.facts.filter((fact) => fact.statement === 'balance')
+    facts: allFacts.facts
  }), balanceSelection.periods, incomeSelection.periods);
  const cashFlowQuarterlyRows = rekeyRowsByFilingId(buildStandardizedRows({
    rows: buildRows(cashFlowSelection.snapshots, 'cash_flow', cashFlowSelection.selectedPeriodIds),
    statement: 'cash_flow',
    periods: cashFlowSelection.periods,
-    facts: allFacts.facts.filter((fact) => fact.statement === 'cash_flow')
+    facts: allFacts.facts
  }), cashFlowSelection.periods, incomeSelection.periods);

  const incomeRows = input.cadence === 'ltm'
--- a/lib/server/financials/bundles.ts
+++ b/lib/server/financials/bundles.ts
@@ -3,6 +3,7 @@ import type {
  FinancialSurfaceKind
 } from '@/lib/types';
 import {
+  CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
  getCompanyFinancialBundle,
  upsertCompanyFinancialBundle
 } from '@/lib/server/repos/company-financial-bundles';
@@ -28,7 +29,11 @@ export async function readCachedFinancialBundle(input: {
    cadence: input.cadence
  });

-  if (!cached || cached.source_signature !== sourceSignature) {
+  if (
+    !cached
+    || cached.bundle_version !== CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION
+    || cached.source_signature !== sourceSignature
+  ) {
    return null;
  }

--- a/lib/server/financials/cadence.ts
+++ b/lib/server/financials/cadence.ts
@@ -13,6 +13,11 @@ type PrimaryPeriodSelection = {
  snapshots: FilingTaxonomySnapshotRecord[];
 };

+type CandidateSelectionInput = {
+  rows: TaxonomyStatementRow[];
+  periods: FinancialStatementPeriod[];
+};
+
 function parseEpoch(value: string | null) {
  if (!value) {
    return Number.NaN;
@@ -54,6 +59,147 @@ function preferredDurationDays(filingType: FinancialStatementPeriod['filingType'
  return filingType === '10-K' ? 365 : 90;
 }

+function isPresentedStatementRow(row: TaxonomyStatementRow) {
+  return row.roleUri !== null && row.order !== Number.MAX_SAFE_INTEGER;
+}
+
+function isPlausiblePrimaryPeriod(period: FinancialStatementPeriod, filingDate: string) {
+  const filingEpoch = parseEpoch(filingDate);
+  const periodEndEpoch = parseEpoch(period.periodEnd ?? period.filingDate);
+  if (Number.isFinite(filingEpoch) && Number.isFinite(periodEndEpoch) && periodEndEpoch > filingEpoch) {
+    return false;
+  }
+
+  const periodStartEpoch = parseEpoch(period.periodStart);
+  if (Number.isFinite(filingEpoch) && Number.isFinite(periodStartEpoch) && periodStartEpoch > filingEpoch) {
+    return false;
+  }
+
+  return true;
+}
+
+function candidatePeriodsForRows(input: CandidateSelectionInput) {
+  const usedPeriodIds = new Set<string>();
+  for (const row of input.rows) {
+    for (const periodId of Object.keys(row.values)) {
+      usedPeriodIds.add(periodId);
+    }
+  }
+
+  return input.periods.filter((period) => usedPeriodIds.has(period.id));
+}
+
+function coverageScore(rows: TaxonomyStatementRow[], periodId: string) {
+  let count = 0;
+
+  for (const row of rows) {
+    if (periodId in row.values) {
+      count += 1;
+    }
+  }
+
+  return count;
+}
+
+function compareBalancePeriods(
+  left: FinancialStatementPeriod,
+  right: FinancialStatementPeriod,
+  rowCoverage: Map<string, number>
+) {
+  const leftInstant = isInstantPeriod(left) ? 1 : 0;
+  const rightInstant = isInstantPeriod(right) ? 1 : 0;
+  if (leftInstant !== rightInstant) {
+    return rightInstant - leftInstant;
+  }
+
+  const leftCoverage = rowCoverage.get(left.id) ?? 0;
+  const rightCoverage = rowCoverage.get(right.id) ?? 0;
+  if (leftCoverage !== rightCoverage) {
+    return rightCoverage - leftCoverage;
+  }
+
+  const leftDate = parseEpoch(left.periodEnd ?? left.filingDate);
+  const rightDate = parseEpoch(right.periodEnd ?? right.filingDate);
+  if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
+    return rightDate - leftDate;
+  }
+
+  return left.id.localeCompare(right.id);
+}
+
+function compareFlowPeriods(
+  left: FinancialStatementPeriod,
+  right: FinancialStatementPeriod,
+  rowCoverage: Map<string, number>,
+  targetDays: number,
+  preferLaterBeforeDistance: boolean
+) {
+  const leftDuration = isInstantPeriod(left) ? 0 : 1;
+  const rightDuration = isInstantPeriod(right) ? 0 : 1;
+  if (leftDuration !== rightDuration) {
+    return rightDuration - leftDuration;
+  }
+
+  const leftDate = parseEpoch(left.periodEnd ?? left.filingDate);
+  const rightDate = parseEpoch(right.periodEnd ?? right.filingDate);
+  if (preferLaterBeforeDistance && Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
+    return rightDate - leftDate;
+  }
+
+  const leftCoverage = rowCoverage.get(left.id) ?? 0;
+  const rightCoverage = rowCoverage.get(right.id) ?? 0;
+  if (leftCoverage !== rightCoverage) {
+    return rightCoverage - leftCoverage;
+  }
+
+  const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays);
+  const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays);
+  if (leftDistance !== rightDistance) {
+    return leftDistance - rightDistance;
+  }
+
+  if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
+    return rightDate - leftDate;
+  }
+
+  return left.id.localeCompare(right.id);
+}
+
+function chooseCandidates(snapshot: FilingTaxonomySnapshotRecord, rows: TaxonomyStatementRow[]) {
+  const periods = snapshot.periods ?? [];
+  const presentedRows = rows.filter(isPresentedStatementRow);
+  const plausiblePresented = candidatePeriodsForRows({
+    rows: presentedRows,
+    periods
+  }).filter((period) => isPlausiblePrimaryPeriod(period, snapshot.filing_date));
+  if (plausiblePresented.length > 0) {
+    return {
+      candidates: plausiblePresented,
+      coverageRows: presentedRows,
+      fallbackMode: false
+    };
+  }
+
+  const allCandidates = candidatePeriodsForRows({
+    rows,
+    periods
+  });
+  const plausibleAll = allCandidates.filter((period) => isPlausiblePrimaryPeriod(period, snapshot.filing_date));
+  if (plausibleAll.length > 0) {
+    return {
+      candidates: plausibleAll,
+      coverageRows: rows,
+      fallbackMode: true
+    };
+  }
+
+  return {
+    candidates: allCandidates,
+    coverageRows: rows,
+    fallbackMode: true
+  };
+}
+
 function selectPrimaryPeriodFromSnapshot(
  snapshot: FilingTaxonomySnapshotRecord,
  statement: FinancialStatementKind
@@ -63,45 +209,21 @@ function selectPrimaryPeriodFromSnapshot(
    return null;
  }

-  const usedPeriodIds = new Set<string>();
-  for (const row of rows) {
-    for (const periodId of Object.keys(row.values)) {
-      usedPeriodIds.add(periodId);
-    }
-  }
-
-  const candidates = (snapshot.periods ?? []).filter((period) => usedPeriodIds.has(period.id));
+  const { candidates, coverageRows, fallbackMode } = chooseCandidates(snapshot, rows);
  if (candidates.length === 0) {
    return null;
  }

-  if (statement === 'balance') {
-    const instantCandidates = candidates.filter(isInstantPeriod);
-    return (instantCandidates.length > 0 ? instantCandidates : candidates)
-      .sort((left, right) => periodSorter(right, left))[0] ?? null;
-  }
+  const rowCoverage = new Map<string, number>(
+    candidates.map((period) => [period.id, coverageScore(coverageRows, period.id)])
+  );

-  const durationCandidates = candidates.filter((period) => !isInstantPeriod(period));
-  if (durationCandidates.length === 0) {
-    return candidates.sort((left, right) => periodSorter(right, left))[0] ?? null;
+  if (statement === 'balance') {
+    return [...candidates].sort((left, right) => compareBalancePeriods(left, right, rowCoverage))[0] ?? null;
  }

  const targetDays = preferredDurationDays(snapshot.filing_type);
-  return durationCandidates.sort((left, right) => {
-    const leftDate = parseEpoch(left.periodEnd ?? left.filingDate);
-    const rightDate = parseEpoch(right.periodEnd ?? right.filingDate);
-    if (Number.isFinite(leftDate) && Number.isFinite(rightDate) && leftDate !== rightDate) {
-      return rightDate - leftDate;
-    }
-
-    const leftDistance = Math.abs((periodDurationDays(left) ?? targetDays) - targetDays);
-    const rightDistance = Math.abs((periodDurationDays(right) ?? targetDays) - targetDays);
-    if (leftDistance !== rightDistance) {
-      return leftDistance - rightDistance;
-    }
-
-    return left.id.localeCompare(right.id);
-  })[0] ?? null;
+  return [...candidates].sort((left, right) => compareFlowPeriods(left, right, rowCoverage, targetDays, fallbackMode))[0] ?? null;
 }

 function filingTypeForCadence(cadence: FinancialCadence) {
--- a/lib/server/financials/standard-template.ts
+++ b/lib/server/financials/standard-template.ts
--- a/lib/server/financials/standardize.ts
+++ b/lib/server/financials/standardize.ts
@@ -9,20 +9,30 @@ import type {
  TaxonomyStatementRow
 } from '@/lib/types';
 import {
-  CANONICAL_ROW_DEFINITIONS,
-  type CanonicalRowDefinition
-} from '@/lib/server/financials/canonical-definitions';
+  STANDARD_FINANCIAL_TEMPLATES,
+  type StandardTemplateRowDefinition,
+  type TemplateFormula
+} from '@/lib/server/financials/standard-template';

 function normalizeToken(value: string) {
  return value.trim().toLowerCase();
 }

+function tokenizeLabel(value: string) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter((token) => token.length > 0);
+}
+
 function valueOrNull(values: Record<string, number | null>, periodId: string) {
  return periodId in values ? values[periodId] : null;
 }

-function sumValues(values: Array<number | null>) {
-  if (values.some((value) => value === null)) {
+function sumValues(values: Array<number | null>, treatNullAsZero = false) {
+  if (!treatNullAsZero && values.some((value) => value === null)) {
    return null;
  }

@@ -45,21 +55,103 @@ function divideValues(left: number | null, right: number | null) {
  return left / right;
 }

-function matchesDefinition(row: TaxonomyStatementRow, definition: CanonicalRowDefinition) {
-  const rowLocalName = normalizeToken(row.localName);
-  if (definition.localNames?.some((localName) => normalizeToken(localName) === rowLocalName)) {
-    return true;
+type CandidateMatchKind = 'exact_local_name' | 'secondary_local_name' | 'label_phrase';
+
+type StatementRowCandidate = {
+  row: TaxonomyStatementRow;
+  matchKind: CandidateMatchKind;
+  aliasRank: number;
+  unit: FinancialUnit;
+  labelTokenCount: number;
+  matchedPhraseTokenCount: number;
+};
+
+type FactCandidate = {
+  fact: TaxonomyFactRow;
+  matchKind: Exclude<CandidateMatchKind, 'label_phrase'>;
+  aliasRank: number;
+  unit: FinancialUnit;
+};
+
+type ResolvedCandidate =
+  | {
+    sourceType: 'row';
+    matchKind: CandidateMatchKind;
+    aliasRank: number;
+    unit: FinancialUnit;
+    labelTokenCount: number;
+    matchedPhraseTokenCount: number;
+    row: TaxonomyStatementRow;
+  }
+  | {
+    sourceType: 'fact';
+    matchKind: Exclude<CandidateMatchKind, 'label_phrase'>;
+    aliasRank: number;
+    unit: FinancialUnit;
+      fact: TaxonomyFactRow;
+    };
+
+type DerivedRole = 'expense' | 'addback';
+
+type InternalRowMetadata = {
+  derivedRoleByPeriod: Record<string, DerivedRole | null>;
+};
+
+function resolvedCandidatesForPeriod(input: {
+  definition: StandardTemplateRowDefinition;
+  candidates: StatementRowCandidate[];
+  factCandidates: FactCandidate[];
+  period: FinancialStatementPeriod;
+}) {
+  const rowCandidates = input.candidates
+    .filter((candidate) => input.period.id in candidate.row.values && candidate.row.values[input.period.id] !== null)
+    .map((candidate) => ({
+      sourceType: 'row' as const,
+      ...candidate
+    }));
+  const factCandidates = input.factCandidates
+    .filter((candidate) => factMatchesPeriod(candidate.fact, input.period))
+    .map((candidate) => ({
+      sourceType: 'fact' as const,
+      ...candidate
+    }));
+
+  if (input.definition.selectionPolicy === 'aggregate_multiple_components') {
+    const aggregateCandidates = [...rowCandidates, ...factCandidates]
+      .sort((left, right) => compareResolvedCandidates(left, right, input.definition));
+    const dedupedCandidates: ResolvedCandidate[] = [];
+    const seenConcepts = new Set<string>();
+
+    for (const candidate of aggregateCandidates) {
+      const conceptKey = candidate.sourceType === 'row'
+        ? candidate.row.key
+        : candidate.fact.conceptKey;
+      if (seenConcepts.has(conceptKey)) {
+        continue;
      }

-  const label = normalizeToken(row.label);
-  return definition.labelIncludes?.some((token) => label.includes(normalizeToken(token))) ?? false;
+      seenConcepts.add(conceptKey);
+      dedupedCandidates.push(candidate);
    }

-function matchesDefinitionFact(fact: TaxonomyFactRow, definition: CanonicalRowDefinition) {
-  const localName = normalizeToken(fact.localName);
-  return definition.localNames?.some((entry) => normalizeToken(entry) === localName) ?? false;
+    return dedupedCandidates;
  }

+  const resolvedCandidate = [...rowCandidates, ...factCandidates]
+    .sort((left, right) => compareResolvedCandidates(left, right, input.definition))[0];
+
+  return resolvedCandidate ? [resolvedCandidate] : [];
+}
+
+const GLOBAL_EXCLUDE_LABEL_PHRASES = [
+  'pro forma',
+  'reconciliation',
+  'acquiree',
+  'business combination',
+  'assets acquired',
+  'liabilities assumed'
+] as const;
+
 function inferUnit(rawUnit: string | null, fallback: FinancialUnit) {
  const normalized = (rawUnit ?? '').toLowerCase();
  if (!normalized) {
@@ -81,6 +173,192 @@ function inferUnit(rawUnit: string | null, fallback: FinancialUnit) {
  return fallback;
 }

+function rowUnit(row: TaxonomyStatementRow, fallback: FinancialUnit) {
+  return inferUnit(Object.values(row.units)[0] ?? null, fallback);
+}
+
+function isUnitCompatible(expected: FinancialUnit, actual: FinancialUnit) {
+  if (expected === actual) {
+    return true;
+  }
+
+  if ((expected === 'percent' || expected === 'ratio') && (actual === 'percent' || actual === 'ratio')) {
+    return true;
+  }
+
+  return false;
+}
+
+function phraseTokens(phrase: string) {
+  return tokenizeLabel(phrase);
+}
+
+function labelContainsPhrase(labelTokens: string[], phrase: string) {
+  const target = phraseTokens(phrase);
+  if (target.length === 0 || target.length > labelTokens.length) {
+    return false;
+  }
+
+  for (let index = 0; index <= labelTokens.length - target.length; index += 1) {
+    let matched = true;
+    for (let offset = 0; offset < target.length; offset += 1) {
+      if (labelTokens[index + offset] !== target[offset]) {
+        matched = false;
+        break;
+      }
+    }
+
+    if (matched) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+function matchRank(matchKind: CandidateMatchKind) {
+  switch (matchKind) {
+    case 'exact_local_name':
+      return 0;
+    case 'secondary_local_name':
+      return 1;
+    case 'label_phrase':
+      return 2;
+  }
+}
+
+function aliasRank(localName: string, aliases: readonly string[] | undefined) {
+  const normalizedLocalName = normalizeToken(localName);
+  const matchIndex = (aliases ?? []).findIndex((alias) => normalizeToken(alias) === normalizedLocalName);
+  return matchIndex === -1 ? Number.MAX_SAFE_INTEGER : matchIndex;
+}
+
+function applySignTransform(value: number | null, transform: StandardTemplateRowDefinition['signTransform']) {
+  if (value === null || !transform) {
+    return value;
+  }
+
+  if (transform === 'invert') {
+    return value * -1;
+  }
+
+  return Math.abs(value);
+}
+
+function classifyStatementRowCandidate(
+  row: TaxonomyStatementRow,
+  definition: StandardTemplateRowDefinition
+) {
+  if (definition.selectionPolicy === 'formula_only') {
+    return null;
+  }
+
+  const rowLocalName = normalizeToken(row.localName);
+  if ((definition.matchers.excludeLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) {
+    return null;
+  }
+
+  const labelTokens = tokenizeLabel(row.label);
+  const excludedLabelPhrases = [
+    ...GLOBAL_EXCLUDE_LABEL_PHRASES,
+    ...(definition.matchers.excludeLabelPhrases ?? [])
+  ];
+  if (excludedLabelPhrases.some((phrase) => labelContainsPhrase(labelTokens, phrase))) {
+    return null;
+  }
+
+  const unit = rowUnit(row, definition.unit);
+  if (!isUnitCompatible(definition.unit, unit)) {
+    return null;
+  }
+
+  if ((definition.matchers.exactLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) {
+    return {
+      row,
+      matchKind: 'exact_local_name',
+      aliasRank: aliasRank(row.localName, definition.matchers.exactLocalNames),
+      unit,
+      labelTokenCount: labelTokens.length,
+      matchedPhraseTokenCount: 0
+    } satisfies StatementRowCandidate;
+  }
+
+  if ((definition.matchers.secondaryLocalNames ?? []).some((localName) => normalizeToken(localName) === rowLocalName)) {
+    return {
+      row,
+      matchKind: 'secondary_local_name',
+      aliasRank: aliasRank(row.localName, definition.matchers.secondaryLocalNames),
+      unit,
+      labelTokenCount: labelTokens.length,
+      matchedPhraseTokenCount: 0
+    } satisfies StatementRowCandidate;
+  }
+
+  const matchedPhrase = (definition.matchers.allowedLabelPhrases ?? [])
+    .map((phrase) => ({
+      phrase,
+      tokenCount: phraseTokens(phrase).length
+    }))
+    .filter(({ phrase }) => labelContainsPhrase(labelTokens, phrase))
+    .sort((left, right) => right.tokenCount - left.tokenCount)[0];
+
+  if (!matchedPhrase) {
+    return null;
+  }
+
+  if (row.hasDimensions) {
+    return null;
+  }
+
+  return {
+    row,
+    matchKind: 'label_phrase',
+    aliasRank: Number.MAX_SAFE_INTEGER,
+    unit,
+    labelTokenCount: labelTokens.length,
+    matchedPhraseTokenCount: matchedPhrase.tokenCount
+  } satisfies StatementRowCandidate;
+}
+
+function classifyFactCandidate(
+  fact: TaxonomyFactRow,
+  definition: StandardTemplateRowDefinition
+) {
+  if (!fact.isDimensionless) {
+    return null;
+  }
+
+  const localName = normalizeToken(fact.localName);
+  if ((definition.matchers.excludeLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) {
+    return null;
+  }
+
+  const unit = inferUnit(fact.unit ?? null, definition.unit);
+  if (!isUnitCompatible(definition.unit, unit)) {
+    return null;
+  }
+
+  if ((definition.matchers.exactLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) {
+    return {
+      fact,
+      matchKind: 'exact_local_name',
+      aliasRank: aliasRank(fact.localName, definition.matchers.exactLocalNames),
+      unit
+    } satisfies FactCandidate;
+  }
+
+  if ((definition.matchers.secondaryLocalNames ?? []).some((entry) => normalizeToken(entry) === localName)) {
+    return {
+      fact,
+      matchKind: 'secondary_local_name',
+      aliasRank: aliasRank(fact.localName, definition.matchers.secondaryLocalNames),
+      unit
+    } satisfies FactCandidate;
+  }
+
+  return null;
+}
+
 export function factMatchesPeriod(fact: TaxonomyFactRow, period: FinancialStatementPeriod) {
  if (period.periodStart) {
    return fact.periodStart === period.periodStart && fact.periodEnd === period.periodEnd;
@@ -89,61 +367,178 @@ export function factMatchesPeriod(fact: TaxonomyFactRow, period: FinancialStatem
  return (fact.periodInstant ?? fact.periodEnd) === period.periodEnd;
 }

-function buildCanonicalRow(
-  definition: CanonicalRowDefinition,
-  matches: TaxonomyStatementRow[],
-  facts: TaxonomyFactRow[],
-  periods: FinancialStatementPeriod[]
+function compareStatementRowCandidates(
+  left: StatementRowCandidate,
+  right: StatementRowCandidate,
+  definition: StandardTemplateRowDefinition
 ) {
-  const sortedMatches = [...matches].sort((left, right) => {
-    if (left.order !== right.order) {
-      return left.order - right.order;
+  const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind);
+  if (matchDelta !== 0) {
+    return matchDelta;
  }

-    return left.label.localeCompare(right.label);
-  });
-  const matchedFacts = facts.filter((fact) => matchesDefinitionFact(fact, definition) && fact.isDimensionless);
+  if (left.aliasRank !== right.aliasRank) {
+    return left.aliasRank - right.aliasRank;
+  }

+  if (left.row.hasDimensions !== right.row.hasDimensions) {
+    return left.row.hasDimensions ? 1 : -1;
+  }
+
+  if (definition.selectionPolicy === 'prefer_primary_statement_concept' && left.row.isExtension !== right.row.isExtension) {
+    return left.row.isExtension ? 1 : -1;
+  }
+
+  if (left.row.order !== right.row.order) {
+    return left.row.order - right.row.order;
+  }
+
+  if (left.matchedPhraseTokenCount !== right.matchedPhraseTokenCount) {
+    return right.matchedPhraseTokenCount - left.matchedPhraseTokenCount;
+  }
+
+  if (left.labelTokenCount !== right.labelTokenCount) {
+    return left.labelTokenCount - right.labelTokenCount;
+  }
+
+  return left.row.label.localeCompare(right.row.label);
+}
+
+function compareFactCandidates(left: FactCandidate, right: FactCandidate) {
+  const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind);
+  if (matchDelta !== 0) {
+    return matchDelta;
+  }
+
+  if (left.aliasRank !== right.aliasRank) {
+    return left.aliasRank - right.aliasRank;
+  }
+
+  return left.fact.qname.localeCompare(right.fact.qname);
+}
+
+function compareResolvedCandidates(
+  left: ResolvedCandidate,
+  right: ResolvedCandidate,
+  definition: StandardTemplateRowDefinition
+) {
+  const matchDelta = matchRank(left.matchKind) - matchRank(right.matchKind);
+  if (matchDelta !== 0) {
+    return matchDelta;
+  }
+
+  if (left.aliasRank !== right.aliasRank) {
+    return left.aliasRank - right.aliasRank;
+  }
+
+  if (left.sourceType === 'row' && right.sourceType === 'row') {
+    return compareStatementRowCandidates(left, right, definition);
+  }
+
+  if (left.sourceType === 'fact' && right.sourceType === 'fact') {
+    return compareFactCandidates(left, right);
+  }
+
+  if (left.sourceType === 'row' && right.sourceType === 'fact') {
+    return left.row.hasDimensions ? 1 : -1;
+  }
+
+  if (left.sourceType === 'fact' && right.sourceType === 'row') {
+    return right.row.hasDimensions ? -1 : 1;
+  }
+
+  return 0;
+}
+
+function buildTemplateRow(
+  definition: StandardTemplateRowDefinition,
+  candidates: StatementRowCandidate[],
+  factCandidates: FactCandidate[],
+  periods: FinancialStatementPeriod[]
+) {
  const sourceConcepts = new Set<string>();
  const sourceRowKeys = new Set<string>();
  const sourceFactIds = new Set<number>();
-  for (const row of sortedMatches) {
-    sourceConcepts.add(row.qname);
-    sourceRowKeys.add(row.key);
-    for (const factId of row.sourceFactIds) {
-      sourceFactIds.add(factId);
-    }
-  }
-
-  const values: Record<string, number | null> = {};
-  const resolvedSourceRowKeys: Record<string, string | null> = {};
+  const matchedRowKeys = new Set<string>();
+  const values: Record<string, number | null> = Object.fromEntries(periods.map((period) => [period.id, null]));
+  const resolvedSourceRowKeys: Record<string, string | null> = Object.fromEntries(periods.map((period) => [period.id, null]));
+  const metadata: InternalRowMetadata = {
+    derivedRoleByPeriod: Object.fromEntries(periods.map((period) => [period.id, null]))
+  };
  let unit = definition.unit;
+  let hasDimensions = false;

  for (const period of periods) {
-    const directMatch = sortedMatches.find((row) => period.id in row.values);
-    if (directMatch) {
-      values[period.id] = directMatch.values[period.id] ?? null;
-      unit = inferUnit(directMatch.units[period.id] ?? null, definition.unit);
-      resolvedSourceRowKeys[period.id] = directMatch.key;
+    const resolvedCandidates = resolvedCandidatesForPeriod({
+      definition,
+      candidates,
+      factCandidates,
+      period
+    });
+
+    if (resolvedCandidates.length === 0) {
      continue;
    }

-    const factMatch = matchedFacts.find((fact) => factMatchesPeriod(fact, period));
-    values[period.id] = factMatch?.value ?? null;
-    unit = inferUnit(factMatch?.unit ?? null, definition.unit);
-    resolvedSourceRowKeys[period.id] = factMatch?.conceptKey ?? null;
+    if (definition.key === 'depreciation_and_amortization') {
+      metadata.derivedRoleByPeriod[period.id] = resolvedCandidates.some((candidate) => {
+        const localName = candidate.sourceType === 'row'
+          ? candidate.row.localName
+          : candidate.fact.localName;
+        return normalizeToken(localName) === normalizeToken('CostOfGoodsAndServicesSoldDepreciationAndAmortization');
+      })
+        ? 'expense'
+        : 'addback';
+    }

-    if (factMatch) {
-      sourceConcepts.add(factMatch.qname);
-      sourceRowKeys.add(factMatch.conceptKey);
-      sourceFactIds.add(factMatch.id);
+    values[period.id] = definition.selectionPolicy === 'aggregate_multiple_components'
+      ? sumValues(resolvedCandidates.map((candidate) => {
+        if (candidate.sourceType === 'row') {
+          return applySignTransform(candidate.row.values[period.id] ?? null, definition.signTransform);
+        }
+
+        return applySignTransform(candidate.fact.value ?? null, definition.signTransform);
+      }))
+      : (() => {
+        const resolvedCandidate = resolvedCandidates[0]!;
+        if (resolvedCandidate.sourceType === 'row') {
+          return applySignTransform(resolvedCandidate.row.values[period.id] ?? null, definition.signTransform);
+        }
+
+        return applySignTransform(resolvedCandidate.fact.value ?? null, definition.signTransform);
+      })();
+    resolvedSourceRowKeys[period.id] = resolvedCandidates.length === 1
+      ? (resolvedCandidates[0]!.sourceType === 'row'
+        ? resolvedCandidates[0]!.row.key
+        : resolvedCandidates[0]!.fact.conceptKey ?? null)
+      : null;
+
+    for (const resolvedCandidate of resolvedCandidates) {
+      unit = resolvedCandidate.unit;
+
+      if (resolvedCandidate.sourceType === 'row') {
+        hasDimensions = hasDimensions || resolvedCandidate.row.hasDimensions;
+        matchedRowKeys.add(resolvedCandidate.row.key);
+        sourceConcepts.add(resolvedCandidate.row.qname);
+        sourceRowKeys.add(resolvedCandidate.row.key);
+        for (const factId of resolvedCandidate.row.sourceFactIds) {
+          sourceFactIds.add(factId);
+        }
+        continue;
+      }
+
+      sourceConcepts.add(resolvedCandidate.fact.qname);
+      sourceRowKeys.add(resolvedCandidate.fact.conceptKey);
+      sourceFactIds.add(resolvedCandidate.fact.id);
    }
  }

  return {
+    row: {
      key: definition.key,
      label: definition.label,
      category: definition.category,
+      templateSection: definition.category,
      order: definition.order,
      unit,
      values,
@@ -151,123 +546,138 @@ function buildCanonicalRow(
      sourceRowKeys: [...sourceRowKeys].sort((left, right) => left.localeCompare(right)),
      sourceFactIds: [...sourceFactIds].sort((left, right) => left - right),
      formulaKey: null,
-    hasDimensions: sortedMatches.some((row) => row.hasDimensions),
+      hasDimensions,
      resolvedSourceRowKeys
-  } satisfies StandardizedFinancialRow;
+    } satisfies StandardizedFinancialRow,
+    matchedRowKeys,
+    metadata
+  };
 }

-type FormulaDefinition = {
-  key: string;
-  formulaKey: string;
-  compute: (rowsByKey: Map<string, StandardizedFinancialRow>, periodId: string) => number | null;
-};
+function computeFormulaValue(
+  formula: TemplateFormula,
+  rowsByKey: Map<string, StandardizedFinancialRow>,
+  periodId: string
+) {
+  switch (formula.kind) {
+    case 'sum':
+      return sumValues(
+        formula.sourceKeys.map((key) => valueOrNull(rowsByKey.get(key)?.values ?? {}, periodId)),
+        formula.treatNullAsZero ?? false
+      );
+    case 'subtract':
+      return subtractValues(
+        valueOrNull(rowsByKey.get(formula.left)?.values ?? {}, periodId),
+        valueOrNull(rowsByKey.get(formula.right)?.values ?? {}, periodId)
+      );
+    case 'divide':
+      return divideValues(
+        valueOrNull(rowsByKey.get(formula.numerator)?.values ?? {}, periodId),
+        valueOrNull(rowsByKey.get(formula.denominator)?.values ?? {}, periodId)
+      );
+  }
+}

-const FORMULAS: Record<Extract<FinancialStatementKind, 'income' | 'balance' | 'cash_flow'>, FormulaDefinition[]> = {
-  income: [
-    {
-      key: 'gross_profit',
-      formulaKey: 'gross_profit',
-      compute: (rowsByKey, periodId) => subtractValues(
-        valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('cost_of_revenue')?.values ?? {}, periodId)
-      )
-    },
-    {
-      key: 'gross_margin',
-      formulaKey: 'gross_margin',
-      compute: (rowsByKey, periodId) => divideValues(
-        valueOrNull(rowsByKey.get('gross_profit')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId)
-      )
-    },
-    {
-      key: 'operating_margin',
-      formulaKey: 'operating_margin',
-      compute: (rowsByKey, periodId) => divideValues(
-        valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('revenue')?.values ?? {}, periodId)
-      )
-    },
-    {
-      key: 'effective_tax_rate',
-      formulaKey: 'effective_tax_rate',
-      compute: (rowsByKey, periodId) => divideValues(
-        valueOrNull(rowsByKey.get('income_tax_expense')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('pretax_income')?.values ?? {}, periodId)
-      )
-    },
-    {
-      key: 'ebitda',
-      formulaKey: 'ebitda',
-      compute: (rowsByKey, periodId) => sumValues([
-        valueOrNull(rowsByKey.get('operating_income')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('depreciation_and_amortization')?.values ?? {}, periodId)
-      ])
+function rowValueForPeriod(
+  rowsByKey: Map<string, StandardizedFinancialRow>,
+  key: string,
+  periodId: string
+) {
+  return valueOrNull(rowsByKey.get(key)?.values ?? {}, periodId);
 }
-  ],
-  balance: [
-    {
-      key: 'total_debt',
-      formulaKey: 'total_debt',
-      compute: (rowsByKey, periodId) => sumValues([
-        valueOrNull(rowsByKey.get('long_term_debt')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('current_debt')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('lease_liabilities')?.values ?? {}, periodId)
-      ])
-    },
-    {
-      key: 'net_cash_position',
-      formulaKey: 'net_cash_position',
-      compute: (rowsByKey, periodId) => subtractValues(
-        sumValues([
-          valueOrNull(rowsByKey.get('cash_and_equivalents')?.values ?? {}, periodId),
-          valueOrNull(rowsByKey.get('short_term_investments')?.values ?? {}, periodId)
-        ]),
-        valueOrNull(rowsByKey.get('total_debt')?.values ?? {}, periodId)
-      )
+
+function computeOperatingIncomeFallbackValue(
+  rowsByKey: Map<string, StandardizedFinancialRow>,
+  rowMetadataByKey: Map<string, InternalRowMetadata>,
+  periodId: string
+) {
+  const grossProfit = rowValueForPeriod(rowsByKey, 'gross_profit', periodId);
+  const sellingGeneralAndAdministrative = rowValueForPeriod(rowsByKey, 'selling_general_and_administrative', periodId);
+  const researchAndDevelopment = rowValueForPeriod(rowsByKey, 'research_and_development', periodId) ?? 0;
+  const depreciationAndAmortization = rowValueForPeriod(rowsByKey, 'depreciation_and_amortization', periodId);
+  const depreciationRole = rowMetadataByKey.get('depreciation_and_amortization')?.derivedRoleByPeriod[periodId] ?? null;
+
+  if (
+    depreciationRole === 'expense'
+    && grossProfit !== null
+    && sellingGeneralAndAdministrative !== null
+    && depreciationAndAmortization !== null
+  ) {
+    return grossProfit - sellingGeneralAndAdministrative - researchAndDevelopment - depreciationAndAmortization;
  }
-  ],
-  cash_flow: [
-    {
-      key: 'free_cash_flow',
-      formulaKey: 'free_cash_flow',
-      compute: (rowsByKey, periodId) => subtractValues(
-        valueOrNull(rowsByKey.get('operating_cash_flow')?.values ?? {}, periodId),
-        valueOrNull(rowsByKey.get('capital_expenditures')?.values ?? {}, periodId)
-      )
+
+  const pretaxIncome = rowValueForPeriod(rowsByKey, 'pretax_income', periodId);
+  if (pretaxIncome === null) {
+    return null;
+  }
+
+  const interestExpense = rowValueForPeriod(rowsByKey, 'interest_expense', periodId) ?? 0;
+  const interestIncome = rowValueForPeriod(rowsByKey, 'interest_income', periodId) ?? 0;
+  const otherNonOperatingIncome = rowValueForPeriod(rowsByKey, 'other_non_operating_income', periodId) ?? 0;
+
+  return pretaxIncome + interestExpense - interestIncome - otherNonOperatingIncome;
+}
+
+function computeFallbackValueForDefinition(
+  definition: StandardTemplateRowDefinition,
+  rowsByKey: Map<string, StandardizedFinancialRow>,
+  rowMetadataByKey: Map<string, InternalRowMetadata>,
+  periodId: string
+) {
+  if (definition.key === 'operating_income') {
+    return computeOperatingIncomeFallbackValue(rowsByKey, rowMetadataByKey, periodId);
+  }
+
+  if (!definition.fallbackFormula) {
+    return null;
+  }
+
+  return computeFormulaValue(definition.fallbackFormula, rowsByKey, periodId);
 }
-  ]
-};

 function applyFormulas(
  rowsByKey: Map<string, StandardizedFinancialRow>,
-  statement: Extract<FinancialStatementKind, 'income' | 'balance' | 'cash_flow'>,
+  rowMetadataByKey: Map<string, InternalRowMetadata>,
+  definitions: StandardTemplateRowDefinition[],
  periods: FinancialStatementPeriod[]
 ) {
-  for (const formula of FORMULAS[statement]) {
-    const target = rowsByKey.get(formula.key);
+  for (let pass = 0; pass < definitions.length; pass += 1) {
+    let changed = false;
+
+    for (const definition of definitions) {
+      if (!definition.fallbackFormula && definition.key !== 'operating_income') {
+        continue;
+      }
+
+      const target = rowsByKey.get(definition.key);
      if (!target) {
        continue;
      }

      let usedFormula = target.formulaKey !== null;
      for (const period of periods) {
-      if (target.values[period.id] !== null) {
+        if (definition.selectionPolicy !== 'formula_only' && target.values[period.id] !== null) {
          continue;
        }

-      const computed = formula.compute(rowsByKey, period.id);
+        const computed = computeFallbackValueForDefinition(definition, rowsByKey, rowMetadataByKey, period.id);
        if (computed === null) {
          continue;
        }

-      target.values[period.id] = computed;
+        target.values[period.id] = applySignTransform(computed, definition.signTransform);
        target.resolvedSourceRowKeys[period.id] = null;
        usedFormula = true;
+        changed = true;
      }

      if (usedFormula) {
-      target.formulaKey = formula.formulaKey;
+        target.formulaKey = definition.key;
+      }
+    }
+
+    if (!changed) {
+      break;
    }
  }
 }
@@ -278,31 +688,47 @@ export function buildStandardizedRows(input: {
  periods: FinancialStatementPeriod[];
  facts: TaxonomyFactRow[];
 }) {
-  const definitions = CANONICAL_ROW_DEFINITIONS[input.statement];
+  const definitions = STANDARD_FINANCIAL_TEMPLATES[input.statement];
  const rowsByKey = new Map<string, StandardizedFinancialRow>();
+  const rowMetadataByKey = new Map<string, InternalRowMetadata>();
  const matchedRowKeys = new Set<string>();

  for (const definition of definitions) {
-    const matches = input.rows.filter((row) => matchesDefinition(row, definition));
-    for (const row of matches) {
-      matchedRowKeys.add(row.key);
+    const candidates = input.rows
+      .map((row) => classifyStatementRowCandidate(row, definition))
+      .filter((candidate): candidate is StatementRowCandidate => candidate !== null);
+    const factCandidates = input.facts
+      .map((fact) => classifyFactCandidate(fact, definition))
+      .filter((candidate): candidate is FactCandidate => candidate !== null);
+    const templateRow = buildTemplateRow(definition, candidates, factCandidates, input.periods);
+
+    for (const rowKey of templateRow.matchedRowKeys) {
+      matchedRowKeys.add(rowKey);
    }

-    const canonical = buildCanonicalRow(definition, matches, input.facts, input.periods);
-    const hasAnyValue = Object.values(canonical.values).some((value) => value !== null);
-    if (hasAnyValue || definition.key.startsWith('gross_') || definition.key === 'operating_margin' || definition.key === 'effective_tax_rate' || definition.key === 'ebitda' || definition.key === 'total_debt' || definition.key === 'net_cash_position' || definition.key === 'free_cash_flow') {
-      rowsByKey.set(definition.key, canonical);
+    const hasAnyValue = Object.values(templateRow.row.values).some((value) => value !== null);
+    if (hasAnyValue || definition.fallbackFormula || definition.key === 'operating_income') {
+      rowsByKey.set(definition.key, templateRow.row);
+      rowMetadataByKey.set(definition.key, templateRow.metadata);
    }
  }

-  applyFormulas(rowsByKey, input.statement, input.periods);
+  applyFormulas(rowsByKey, rowMetadataByKey, definitions, input.periods);
+
+  const templateRows = definitions
+    .filter((definition) => definition.includeInOutput !== false)
+    .map((definition) => rowsByKey.get(definition.key))
+    .filter((row): row is StandardizedFinancialRow => row !== undefined);
+  const coveredTemplateSourceRowKeys = new Set(templateRows.flatMap((row) => row.sourceRowKeys));

  const unmatchedRows = input.rows
    .filter((row) => !matchedRowKeys.has(row.key))
+    .filter((row) => !(row.hasDimensions && coveredTemplateSourceRowKeys.has(row.key)))
    .map((row) => ({
      key: `other:${row.key}`,
      label: row.label,
      category: 'other',
+      templateSection: 'other',
      order: 10_000 + row.order,
      unit: inferUnit(Object.values(row.units)[0] ?? null, 'currency'),
      values: { ...row.values },
@@ -316,7 +742,7 @@ export function buildStandardizedRows(input: {
      )
    } satisfies StandardizedFinancialRow));

-  return [...rowsByKey.values(), ...unmatchedRows].sort((left, right) => {
+  return [...templateRows, ...unmatchedRows].sort((left, right) => {
    if (left.order !== right.order) {
      return left.order - right.order;
    }
--- a/lib/server/repos/company-financial-bundles.ts
+++ b/lib/server/repos/company-financial-bundles.ts
@@ -6,7 +6,7 @@ import type {
 import { db } from '@/lib/server/db';
 import { companyFinancialBundle } from '@/lib/server/db/schema';

-const BUNDLE_VERSION = 1;
+export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 14;

 export type CompanyFinancialBundleRecord = {
  id: number;
@@ -70,7 +70,7 @@ export async function upsertCompanyFinancialBundle(input: {
      ticker: input.ticker.trim().toUpperCase(),
      surface_kind: input.surfaceKind,
      cadence: input.cadence,
-      bundle_version: BUNDLE_VERSION,
+      bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
      source_snapshot_ids: input.sourceSnapshotIds,
      source_signature: input.sourceSignature,
      payload: input.payload,
@@ -84,7 +84,7 @@ export async function upsertCompanyFinancialBundle(input: {
        companyFinancialBundle.cadence
      ],
      set: {
-        bundle_version: BUNDLE_VERSION,
+        bundle_version: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION,
        source_snapshot_ids: input.sourceSnapshotIds,
        source_signature: input.sourceSignature,
        payload: input.payload,
@@ -103,5 +103,5 @@ export async function deleteCompanyFinancialBundlesForTicker(ticker: string) {
 }

 export const __companyFinancialBundlesInternals = {
-  BUNDLE_VERSION
+  BUNDLE_VERSION: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION
 };
--- a/lib/server/repos/filing-taxonomy.ts
+++ b/lib/server/repos/filing-taxonomy.ts
@@ -582,7 +582,7 @@ export async function listTaxonomyFactsByTicker(input: {
  cursor?: string | null;
  limit?: number;
 }) {
-  const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 2000);
+  const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 10000);
  const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null;
  const conditions = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())];

--- a/lib/types.ts
+++ b/lib/types.ts
@@ -398,6 +398,7 @@ export type DerivedFinancialRow = {
  key: string;
  label: string;
  category: FinancialCategory;
+  templateSection?: FinancialCategory;
  order: number;
  unit: FinancialUnit;
  values: Record<string, number | null>;