Automate issuer overlay creation from ticker searches

lib/server/db/schema.ts

@@ -33,6 +33,7 @@ type TaxonomyMetricValidationStatus =
   | "matched"
   | "mismatch"
   | "error";
+type IssuerOverlayStatus = "empty" | "active" | "error";
 type CoverageStatus = "backlog" | "active" | "watch" | "archive";
 type CoveragePriority = "low" | "medium" | "high";
 type ResearchJournalEntryType = "note" | "filing_note" | "status_change";
@@ -65,6 +66,8 @@ type FinancialSurfaceKind =
   | "income_statement"
   | "balance_sheet"
   | "cash_flow_statement"
+  | "equity_statement"
+  | "disclosures"
   | "ratios"
   | "segments_kpis"
   | "adjusted"
@@ -103,6 +106,44 @@ type FinancialStatementKind =
   | "equity"
   | "comprehensive_income";
 
+export type IssuerOverlayDefinition = {
+  version: "fiscal-v1";
+  ticker: string;
+  pack: string | null;
+  mappings: Array<{
+    surface_key: string;
+    statement: FinancialStatementKind;
+    allowed_source_concepts: string[];
+    allowed_authoritative_concepts: string[];
+  }>;
+};
+
+export type IssuerOverlayStats = {
+  pack: string | null;
+  sampledSnapshotCount: number;
+  sampledSnapshotIds: number[];
+  acceptedMappingCount: number;
+  rejectedMappingCount: number;
+  publishedRevisionNumber: number | null;
+};
+
+export type IssuerOverlayDiagnostics = {
+  pack: string | null;
+  sampledSnapshotIds: number[];
+  acceptedMappings: Array<{
+    qname: string;
+    surface_key: string;
+    statement: FinancialStatementKind;
+    reason: "authoritative_match" | "local_name_match";
+    source_snapshot_ids: number[];
+  }>;
+  rejectedMappings: Array<{
+    qname: string;
+    reason: string;
+    source_snapshot_ids: number[];
+  }>;
+};
+
 type FilingStatementPeriod = {
   id: string;
   filingId: number;
@@ -208,7 +249,7 @@ type TaxonomySurfaceSnapshotRow = {
   formulaKey: string | null;
   hasDimensions: boolean;
   resolvedSourceRowKeys: Record<string, string | null>;
-  statement?: "income" | "balance" | "cash_flow";
+  statement?: "income" | "balance" | "cash_flow" | "equity" | "disclosure";
   detailCount?: number;
 };
 
@@ -270,6 +311,10 @@ type TaxonomyNormalizationSummary = {
   kpiRowCount: number;
   unmappedRowCount: number;
   materialUnmappedRowCount: number;
+  residualPrimaryCount: number;
+  residualDisclosureCount: number;
+  unsupportedConceptCount: number;
+  issuerOverlayMatchCount: number;
   warnings: string[];
 };
 
@@ -640,6 +685,7 @@ export const filingTaxonomySnapshot = sqliteTable(
     normalization_summary: text("normalization_summary", {
       mode: "json",
     }).$type<TaxonomyNormalizationSummary | null>(),
+    issuer_overlay_revision_id: integer("issuer_overlay_revision_id"),
     facts_count: integer("facts_count").notNull().default(0),
     concepts_count: integer("concepts_count").notNull().default(0),
     dimensions_count: integer("dimensions_count").notNull().default(0),
@@ -659,6 +705,65 @@ export const filingTaxonomySnapshot = sqliteTable(
   }),
 );
 
+export const issuerOverlayRevision = sqliteTable(
+  "issuer_overlay_revision",
+  {
+    id: integer("id").primaryKey({ autoIncrement: true }),
+    ticker: text("ticker").notNull(),
+    revision_number: integer("revision_number").notNull(),
+    definition_hash: text("definition_hash").notNull(),
+    definition_json: text("definition_json", {
+      mode: "json",
+    }).$type<IssuerOverlayDefinition>(),
+    diagnostics_json: text("diagnostics_json", {
+      mode: "json",
+    }).$type<IssuerOverlayDiagnostics | null>(),
+    source_snapshot_ids: text("source_snapshot_ids", {
+      mode: "json",
+    }).$type<number[]>(),
+    created_at: text("created_at").notNull(),
+  },
+  (table) => ({
+    issuerOverlayRevisionTickerRevisionUnique: uniqueIndex(
+      "issuer_overlay_revision_ticker_revision_uidx",
+    ).on(table.ticker, table.revision_number),
+    issuerOverlayRevisionTickerHashUnique: uniqueIndex(
+      "issuer_overlay_revision_ticker_hash_uidx",
+    ).on(table.ticker, table.definition_hash),
+    issuerOverlayRevisionTickerCreatedIndex: index(
+      "issuer_overlay_revision_ticker_created_idx",
+    ).on(table.ticker, table.created_at),
+  }),
+);
+
+export const issuerOverlay = sqliteTable(
+  "issuer_overlay",
+  {
+    ticker: text("ticker").primaryKey(),
+    status: text("status")
+      .$type<IssuerOverlayStatus>()
+      .notNull()
+      .default("empty"),
+    active_revision_id: integer("active_revision_id").references(
+      () => issuerOverlayRevision.id,
+      { onDelete: "set null" },
+    ),
+    last_built_at: text("last_built_at"),
+    last_error: text("last_error"),
+    stats_json: text("stats_json", {
+      mode: "json",
+    }).$type<IssuerOverlayStats | null>(),
+    created_at: text("created_at").notNull(),
+    updated_at: text("updated_at").notNull(),
+  },
+  (table) => ({
+    issuerOverlayStatusIndex: index("issuer_overlay_status_idx").on(
+      table.status,
+      table.updated_at,
+    ),
+  }),
+);
+
 export const filingTaxonomyContext = sqliteTable(
   "filing_taxonomy_context",
   {
@@ -1315,6 +1420,8 @@ export const appSchema = {
   filing,
   filingStatementSnapshot,
   filingTaxonomySnapshot,
+  issuerOverlay,
+  issuerOverlayRevision,
   filingTaxonomyAsset,
   filingTaxonomyConcept,
   filingTaxonomyFact,