Automate issuer overlay creation from ticker searches

lib/server/repos/filing-taxonomy.ts

@@ -73,6 +73,7 @@ export type FilingTaxonomySnapshotRecord = {
   derived_metrics: Filing["metrics"];
   validation_result: MetricValidationResult | null;
   normalization_summary: NormalizationSummary | null;
+  issuer_overlay_revision_id: number | null;
   facts_count: number;
   concepts_count: number;
   dimensions_count: number;
@@ -80,7 +81,7 @@ export type FilingTaxonomySnapshotRecord = {
   updated_at: string;
 };
 
-export type FilingTaxonomyContextRecord = {
+type FilingTaxonomyContextRecord = {
   id: number;
   snapshot_id: number;
   context_id: string;
@@ -94,7 +95,7 @@ export type FilingTaxonomyContextRecord = {
   created_at: string;
 };
 
-export type FilingTaxonomyAssetRecord = {
+type FilingTaxonomyAssetRecord = {
   id: number;
   snapshot_id: number;
   asset_type: FilingTaxonomyAssetType;
@@ -133,7 +134,7 @@ export type FilingTaxonomyConceptRecord = {
   created_at: string;
 };
 
-export type FilingTaxonomyFactRecord = {
+type FilingTaxonomyFactRecord = {
   id: number;
   snapshot_id: number;
   concept_key: string;
@@ -164,7 +165,7 @@ export type FilingTaxonomyFactRecord = {
   created_at: string;
 };
 
-export type FilingTaxonomyMetricValidationRecord = {
+type FilingTaxonomyMetricValidationRecord = {
   id: number;
   snapshot_id: number;
   metric_key: keyof NonNullable<Filing["metrics"]>;
@@ -182,7 +183,7 @@ export type FilingTaxonomyMetricValidationRecord = {
   updated_at: string;
 };
 
-export type UpsertFilingTaxonomySnapshotInput = {
+type UpsertFilingTaxonomySnapshotInput = {
   filing_id: number;
   ticker: string;
   filing_date: string;
@@ -204,6 +205,7 @@ export type UpsertFilingTaxonomySnapshotInput = {
   derived_metrics: Filing["metrics"];
   validation_result: MetricValidationResult | null;
   normalization_summary: NormalizationSummary | null;
+  issuer_overlay_revision_id?: number | null;
   facts_count: number;
   concepts_count: number;
   dimensions_count: number;
@@ -294,6 +296,7 @@ const FINANCIAL_STATEMENT_KINDS = [
   "income",
   "balance",
   "cash_flow",
+  "disclosure",
   "equity",
   "comprehensive_income",
 ] as const satisfies FinancialStatementKind[];
@@ -351,6 +354,7 @@ function asStatementKind(value: unknown): FinancialStatementKind | null {
   return value === "income" ||
     value === "balance" ||
     value === "cash_flow" ||
+    value === "disclosure" ||
     value === "equity" ||
     value === "comprehensive_income"
     ? value
@@ -576,7 +580,9 @@ function normalizeSurfaceRows(
         if (
           normalizedStatement === "income" ||
           normalizedStatement === "balance" ||
-          normalizedStatement === "cash_flow"
+          normalizedStatement === "cash_flow" ||
+          normalizedStatement === "equity" ||
+          normalizedStatement === "disclosure"
         ) {
           normalizedRow.statement = normalizedStatement;
         }
@@ -856,6 +862,17 @@ function normalizeNormalizationSummary(value: unknown) {
       asNumber(
         row.materialUnmappedRowCount ?? row.material_unmapped_row_count,
       ) ?? 0,
+    residualPrimaryCount:
+      asNumber(row.residualPrimaryCount ?? row.residual_primary_count) ?? 0,
+    residualDisclosureCount:
+      asNumber(row.residualDisclosureCount ?? row.residual_disclosure_count) ??
+      0,
+    unsupportedConceptCount:
+      asNumber(row.unsupportedConceptCount ?? row.unsupported_concept_count) ??
+      0,
+    issuerOverlayMatchCount:
+      asNumber(row.issuerOverlayMatchCount ?? row.issuer_overlay_match_count) ??
+      0,
     warnings: normalizeStringArray(row.warnings),
   } satisfies NormalizationSummary;
 }
@@ -962,6 +979,7 @@ function toSnapshotRecord(
     derived_metrics: row.derived_metrics ?? null,
     validation_result: row.validation_result ?? null,
     normalization_summary: normalized.normalization_summary,
+    issuer_overlay_revision_id: row.issuer_overlay_revision_id ?? null,
     facts_count: row.facts_count,
     concepts_count: row.concepts_count,
     dimensions_count: row.dimensions_count,
@@ -1107,7 +1125,7 @@ export async function getFilingTaxonomySnapshotByFilingId(filingId: number) {
   return row ? toSnapshotRecord(row) : null;
 }
 
-export async function listFilingTaxonomyAssets(snapshotId: number) {
+async function listFilingTaxonomyAssets(snapshotId: number) {
   const rows = await db
     .select()
     .from(filingTaxonomyAsset)
@@ -1117,7 +1135,7 @@ export async function listFilingTaxonomyAssets(snapshotId: number) {
   return rows.map(toAssetRecord);
 }
 
-export async function listFilingTaxonomyContexts(snapshotId: number) {
+async function listFilingTaxonomyContexts(snapshotId: number) {
   const rows = await db
     .select()
     .from(filingTaxonomyContext)
@@ -1127,7 +1145,7 @@ export async function listFilingTaxonomyContexts(snapshotId: number) {
   return rows.map(toContextRecord);
 }
 
-export async function listFilingTaxonomyConcepts(snapshotId: number) {
+async function listFilingTaxonomyConcepts(snapshotId: number) {
   const rows = await db
     .select()
     .from(filingTaxonomyConcept)
@@ -1137,7 +1155,7 @@ export async function listFilingTaxonomyConcepts(snapshotId: number) {
   return rows.map(toConceptRecord);
 }
 
-export async function listFilingTaxonomyFacts(snapshotId: number) {
+async function listFilingTaxonomyFacts(snapshotId: number) {
   const rows = await db
     .select()
     .from(filingTaxonomyFact)
@@ -1147,7 +1165,7 @@ export async function listFilingTaxonomyFacts(snapshotId: number) {
   return rows.map(toFactRecord);
 }
 
-export async function listFilingTaxonomyMetricValidations(snapshotId: number) {
+async function listFilingTaxonomyMetricValidations(snapshotId: number) {
   const rows = await db
     .select()
     .from(filingTaxonomyMetricValidation)
@@ -1188,6 +1206,7 @@ export async function upsertFilingTaxonomySnapshot(
         derived_metrics: input.derived_metrics,
         validation_result: input.validation_result,
         normalization_summary: normalized.normalization_summary,
+        issuer_overlay_revision_id: input.issuer_overlay_revision_id ?? null,
         facts_count: input.facts_count,
         concepts_count: input.concepts_count,
         dimensions_count: input.dimensions_count,
@@ -1217,6 +1236,7 @@ export async function upsertFilingTaxonomySnapshot(
           derived_metrics: input.derived_metrics,
           validation_result: input.validation_result,
           normalization_summary: normalized.normalization_summary,
+          issuer_overlay_revision_id: input.issuer_overlay_revision_id ?? null,
           facts_count: input.facts_count,
           concepts_count: input.concepts_count,
           dimensions_count: input.dimensions_count,
@@ -1579,7 +1599,7 @@ export async function listTaxonomyFactsByTicker(input: {
   };
 }
 
-export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) {
+async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) {
   if (snapshotIds.length === 0) {
     return [];
   }
@@ -1593,6 +1613,22 @@ export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) {
   return rows.map(toAssetRecord);
 }
 
+export async function listFilingTaxonomyConceptsBySnapshotIds(
+  snapshotIds: number[],
+) {
+  if (snapshotIds.length === 0) {
+    return [];
+  }
+
+  const rows = await db
+    .select()
+    .from(filingTaxonomyConcept)
+    .where(inArray(filingTaxonomyConcept.snapshot_id, snapshotIds))
+    .orderBy(desc(filingTaxonomyConcept.id));
+
+  return rows.map(toConceptRecord);
+}
+
 export const __filingTaxonomyInternals = {
   normalizeFilingTaxonomySnapshotPayload,
   toSnapshotRecord,