feat(taxonomy): add rust sidecar compact surface pipeline

drizzle/0010_taxonomy_surface_sidecar.sql

@@ -0,0 +1,79 @@
+ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text DEFAULT 'legacy-ts' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text DEFAULT '0.0.0' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text DEFAULT 'unknown' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;
+--> statement-breakpoint
+UPDATE `filing_taxonomy_snapshot`
+SET
+  `faithful_rows` = COALESCE(`faithful_rows`, `statement_rows`),
+  `surface_rows` = COALESCE(`surface_rows`, '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'),
+  `detail_rows` = COALESCE(`detail_rows`, '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'),
+  `kpi_rows` = COALESCE(`kpi_rows`, '[]');
+--> statement-breakpoint
+CREATE TABLE `filing_taxonomy_context` (
+  `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+  `snapshot_id` integer NOT NULL,
+  `context_id` text NOT NULL,
+  `entity_identifier` text,
+  `entity_scheme` text,
+  `period_start` text,
+  `period_end` text,
+  `period_instant` text,
+  `segment_json` text,
+  `scenario_json` text,
+  `created_at` text NOT NULL,
+  FOREIGN KEY (`snapshot_id`) REFERENCES `filing_taxonomy_snapshot`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE INDEX `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer DEFAULT false NOT NULL;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer DEFAULT false NOT NULL;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;
+--> statement-breakpoint
+ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer DEFAULT false NOT NULL;

drizzle/0011_remove_legacy_xbrl_defaults.sql

@@ -0,0 +1,100 @@
+PRAGMA foreign_keys=OFF;
+--> statement-breakpoint
+CREATE TABLE `__new_filing_taxonomy_snapshot` (
+  `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+  `filing_id` integer NOT NULL,
+  `ticker` text NOT NULL,
+  `filing_date` text NOT NULL,
+  `filing_type` text NOT NULL,
+  `parse_status` text NOT NULL,
+  `parse_error` text,
+  `source` text NOT NULL,
+  `parser_engine` text DEFAULT 'fiscal-xbrl' NOT NULL,
+  `parser_version` text DEFAULT 'unknown' NOT NULL,
+  `taxonomy_regime` text DEFAULT 'unknown' NOT NULL,
+  `fiscal_pack` text,
+  `periods` text,
+  `faithful_rows` text,
+  `statement_rows` text,
+  `surface_rows` text,
+  `detail_rows` text,
+  `kpi_rows` text,
+  `derived_metrics` text,
+  `validation_result` text,
+  `normalization_summary` text,
+  `facts_count` integer DEFAULT 0 NOT NULL,
+  `concepts_count` integer DEFAULT 0 NOT NULL,
+  `dimensions_count` integer DEFAULT 0 NOT NULL,
+  `created_at` text NOT NULL,
+  `updated_at` text NOT NULL,
+  FOREIGN KEY (`filing_id`) REFERENCES `filing`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+INSERT INTO `__new_filing_taxonomy_snapshot` (
+  `id`,
+  `filing_id`,
+  `ticker`,
+  `filing_date`,
+  `filing_type`,
+  `parse_status`,
+  `parse_error`,
+  `source`,
+  `parser_engine`,
+  `parser_version`,
+  `taxonomy_regime`,
+  `fiscal_pack`,
+  `periods`,
+  `faithful_rows`,
+  `statement_rows`,
+  `surface_rows`,
+  `detail_rows`,
+  `kpi_rows`,
+  `derived_metrics`,
+  `validation_result`,
+  `normalization_summary`,
+  `facts_count`,
+  `concepts_count`,
+  `dimensions_count`,
+  `created_at`,
+  `updated_at`
+)
+SELECT
+  `id`,
+  `filing_id`,
+  `ticker`,
+  `filing_date`,
+  `filing_type`,
+  `parse_status`,
+  `parse_error`,
+  `source`,
+  `parser_engine`,
+  `parser_version`,
+  `taxonomy_regime`,
+  `fiscal_pack`,
+  `periods`,
+  `faithful_rows`,
+  `statement_rows`,
+  `surface_rows`,
+  `detail_rows`,
+  `kpi_rows`,
+  `derived_metrics`,
+  `validation_result`,
+  `normalization_summary`,
+  `facts_count`,
+  `concepts_count`,
+  `dimensions_count`,
+  `created_at`,
+  `updated_at`
+FROM `filing_taxonomy_snapshot`;
+--> statement-breakpoint
+DROP TABLE `filing_taxonomy_snapshot`;
+--> statement-breakpoint
+ALTER TABLE `__new_filing_taxonomy_snapshot` RENAME TO `filing_taxonomy_snapshot`;
+--> statement-breakpoint
+CREATE UNIQUE INDEX `filing_taxonomy_snapshot_filing_uidx` ON `filing_taxonomy_snapshot` (`filing_id`);
+--> statement-breakpoint
+CREATE INDEX `filing_taxonomy_snapshot_ticker_date_idx` ON `filing_taxonomy_snapshot` (`ticker`,`filing_date`);
+--> statement-breakpoint
+CREATE INDEX `filing_taxonomy_snapshot_status_idx` ON `filing_taxonomy_snapshot` (`parse_status`);
+--> statement-breakpoint
+PRAGMA foreign_keys=ON;

drizzle/meta/_journal.json

@@ -71,6 +71,20 @@
       "when": 1773000000000,
       "tag": "0009_task_notification_context",
       "breakpoints": true
+    },
+    {
+      "idx": 10,
+      "version": "6",
+      "when": 1773090000000,
+      "tag": "0010_taxonomy_surface_sidecar",
+      "breakpoints": true
+    },
+    {
+      "idx": 11,
+      "version": "6",
+      "when": 1773180000000,
+      "tag": "0011_remove_legacy_xbrl_defaults",
+      "breakpoints": true
     }
   ]
 }