Attempting to fix the Taxonomy. Stashing changes so far for worktree merge

scripts/backfill-taxonomy-snapshots.ts

@@ -5,6 +5,7 @@ import {
   normalizeFilingTaxonomySnapshotPayload,
   upsertFilingTaxonomySnapshot
 } from '@/lib/server/repos/filing-taxonomy';
+import { getIssuerOverlay } from '@/lib/server/repos/issuer-overlays';
 
 type ScriptOptions = {
   apply: boolean;
@@ -22,6 +23,17 @@ type ScriptSummary = {
   failed: number;
 };
 
+type ActiveOverlayState = {
+  definition: Awaited<ReturnType<typeof getIssuerOverlay>> extends infer T
+    ? T extends { active_revision: infer R | null }
+      ? R extends { definition_json: infer D }
+        ? D | null
+        : null
+      : null
+    : null;
+  revisionId: number | null;
+};
+
 type FilingRow = {
   id: number;
   ticker: string;
@@ -36,6 +48,28 @@ type FilingRow = {
 
 const REQUEST_DELAY_MS = 120;
 
+async function loadActiveOverlayState(
+  ticker: string,
+  cache: Map<string, Promise<ActiveOverlayState>>,
+) {
+  const normalizedTicker = ticker.trim().toUpperCase();
+  const existing = cache.get(normalizedTicker);
+  if (existing) {
+    return await existing;
+  }
+
+  const pending = (async (): Promise<ActiveOverlayState> => {
+    const overlay = await getIssuerOverlay(normalizedTicker);
+    return {
+      definition: overlay?.active_revision?.definition_json ?? null,
+      revisionId: overlay?.active_revision_id ?? null
+    };
+  })();
+  cache.set(normalizedTicker, pending);
+
+  return await pending;
+}
+
 function parseOptions(argv: string[]): ScriptOptions {
   const options: ScriptOptions = {
     apply: false,
@@ -147,6 +181,7 @@ async function loadFilings(options: ScriptOptions): Promise<FilingRow[]> {
 
 async function runBackfill(options: ScriptOptions): Promise<ScriptSummary> {
   const rows = await loadFilings(options);
+  const overlayCache = new Map<string, Promise<ActiveOverlayState>>();
   const summary: ScriptSummary = {
     scanned: 0,
     wouldWrite: 0,
@@ -164,8 +199,13 @@ async function runBackfill(options: ScriptOptions): Promise<ScriptSummary> {
     summary.scanned += 1;
     console.log(`[backfill-taxonomy-snapshots] [${index + 1}/${rows.length}] ${row.ticker} ${row.filingType} ${row.filingDate} ${row.accessionNumber}`);
 
+    const activeOverlay = await loadActiveOverlayState(row.ticker, overlayCache);
     const existing = await getFilingTaxonomySnapshotByFilingId(row.id);
-    const isFresh = existing && Date.parse(existing.updated_at) >= Date.parse(row.updatedAt);
+    const isFresh = Boolean(
+      existing
+      && Date.parse(existing.updated_at) >= Date.parse(row.updatedAt)
+      && (existing.issuer_overlay_revision_id ?? null) === activeOverlay.revisionId
+    );
 
     if (isFresh && !options.refresh) {
       summary.skippedFresh += 1;
@@ -181,7 +221,8 @@ async function runBackfill(options: ScriptOptions): Promise<ScriptSummary> {
         filingDate: row.filingDate,
         filingType: row.filingType,
         filingUrl: row.filingUrl,
-        primaryDocument: row.primaryDocument
+        primaryDocument: row.primaryDocument,
+        issuerOverlay: activeOverlay.definition
       });
 
       summary.wouldWrite += 1;
@@ -189,6 +230,7 @@ async function runBackfill(options: ScriptOptions): Promise<ScriptSummary> {
       if (options.apply) {
         const normalizedSnapshot = {
           ...snapshot,
+          issuer_overlay_revision_id: activeOverlay.revisionId,
           ...normalizeFilingTaxonomySnapshotPayload(snapshot)
         };
         await upsertFilingTaxonomySnapshot(normalizedSnapshot);