From 391d6d34ce4e3610d8acde85d2ff4b32372557d6 Mon Sep 17 00:00:00 2001 From: francy51 Date: Thu, 19 Mar 2026 20:44:58 -0400 Subject: [PATCH] Automate issuer overlay creation from ticker searches --- app/analysis/page.tsx | 5 + app/financials/page.tsx | 60 +- app/graphing/page.tsx | 8 +- app/search/page.tsx | 18 +- bun.lock | 109 ++- components/charts/utils/chart-colors.ts | 2 +- .../charts/utils/chart-data-transformers.ts | 8 +- components/dashboard/index-card-row.tsx | 2 +- components/financials/financials-toolbar.tsx | 2 +- components/financials/statement-matrix.tsx | 2 +- .../financials/statement-row-inspector.tsx | 6 +- .../notifications/task-stage-helpers.ts | 6 +- doc/issuer-overlay-automation.md | 63 ++ docs/architecture/financial-surfaces.md | 17 + docs/architecture/taxonomy.md | 27 +- drizzle/0015_issuer_overlay_automation.sql | 32 + hooks/use-api-queries.ts | 18 +- knip.json | 6 + lib/api.ts | 23 +- lib/financial-metrics.ts | 6 +- lib/financials/page-merge.test.ts | 4 + lib/financials/page-merge.ts | 2 +- lib/financials/statement-view-model.test.ts | 645 +++++++++++++----- lib/financials/statement-view-model.ts | 164 ++++- lib/graphing/catalog.ts | 24 +- lib/graphing/series.test.ts | 4 + lib/graphing/series.ts | 2 +- lib/query/options.ts | 6 +- lib/server/ai.ts | 4 +- lib/server/api/app.ts | 97 ++- .../api/task-workflow-hybrid.e2e.test.ts | 23 + lib/server/auth-session.ts | 4 +- lib/server/company-analysis.ts | 6 +- lib/server/db/financial-ingestion-schema.ts | 20 +- lib/server/db/index.test.ts | 11 + lib/server/db/schema.ts | 109 ++- lib/server/db/sqlite-schema-compat.ts | 58 +- lib/server/financial-statements.ts | 4 +- lib/server/financial-taxonomy.ts | 123 +++- lib/server/financials/bundles.ts | 2 +- lib/server/financials/cadence.ts | 10 +- lib/server/financials/kpi-registry.ts | 4 +- lib/server/financials/standardize.ts | 6 +- lib/server/financials/trend-series.ts | 6 + lib/server/issuer-overlays.ts | 575 ++++++++++++++++ lib/server/prices.ts | 6 +- lib/server/recent-developments.ts | 8 +- lib/server/repos/company-financial-bundles.ts | 4 +- lib/server/repos/company-overview-cache.ts | 6 +- lib/server/repos/filing-statements.ts | 8 +- lib/server/repos/filing-taxonomy.test.ts | 16 + lib/server/repos/filing-taxonomy.ts | 60 +- lib/server/repos/issuer-overlays.test.ts | 146 ++++ lib/server/repos/issuer-overlays.ts | 331 +++++++++ lib/server/repos/research-library.ts | 6 +- lib/server/repos/tasks.ts | 2 +- lib/server/sec-company-profile.ts | 2 +- lib/server/sec.ts | 4 +- lib/server/task-errors.ts | 2 +- lib/server/task-processors.outcomes.test.ts | 26 + lib/server/task-processors.ts | 520 +++++++++----- lib/server/taxonomy/engine.test.ts | 4 + lib/server/taxonomy/parser-client.test.ts | 62 ++ lib/server/taxonomy/parser-client.ts | 3 +- lib/server/taxonomy/types.ts | 22 +- lib/task-workflow.ts | 4 +- lib/types.ts | 26 +- package.json | 9 +- rust/fiscal-xbrl-core/src/lib.rs | 573 ++++++++++++++-- rust/fiscal-xbrl-core/src/surface_mapper.rs | 100 ++- rust/fiscal-xbrl-core/src/taxonomy_loader.rs | 286 +++++++- rust/taxonomy/crosswalk/us-gaap.json | 8 +- .../fiscal/v1/core.disclosure.surface.json | 389 +++++++++++ rust/taxonomy/fiscal/v1/core.surface.json | 117 +++- .../fiscal/v1/issuers/msft.surface.json | 184 +++++ scripts/dev-env.ts | 2 +- scripts/report-taxonomy-health.ts | 169 ++++- scripts/sqlite-vector-env.ts | 2 +- vitest.config.mts | 1 + 79 files changed, 4746 insertions(+), 695 deletions(-) create mode 100644 doc/issuer-overlay-automation.md create mode 100644 drizzle/0015_issuer_overlay_automation.sql create mode 100644 knip.json create mode 100644 lib/server/issuer-overlays.ts create mode 100644 lib/server/repos/issuer-overlays.test.ts create mode 100644 lib/server/repos/issuer-overlays.ts create mode 100644 rust/taxonomy/fiscal/v1/core.disclosure.surface.json create mode 100644 rust/taxonomy/fiscal/v1/issuers/msft.surface.json diff --git a/app/analysis/page.tsx b/app/analysis/page.tsx index 5908b0a..d7b9d09 100644 --- a/app/analysis/page.tsx +++ b/app/analysis/page.tsx @@ -15,6 +15,7 @@ import { ValuationFactsTable } from '@/components/analysis/valuation-facts-table import { Panel } from '@/components/ui/panel'; import { useAuthGuard } from '@/hooks/use-auth-guard'; import { useLinkPrefetch } from '@/hooks/use-link-prefetch'; +import { ensureTickerAutomation } from '@/lib/api'; import { buildGraphingHref } from '@/lib/graphing/catalog'; import { queryKeys } from '@/lib/query/keys'; import { companyAnalysisQueryOptions } from '@/lib/query/options'; @@ -119,6 +120,10 @@ function AnalysisPageContent() { return; } + void ensureTickerAutomation({ + ticker: normalized, + source: 'analysis' + }); setTicker(normalized); }} onRefresh={() => { diff --git a/app/financials/page.tsx b/app/financials/page.tsx index 0141f17..e1ae0d3 100644 --- a/app/financials/page.tsx +++ b/app/financials/page.tsx @@ -43,7 +43,7 @@ import { Input } from "@/components/ui/input"; import { Panel } from "@/components/ui/panel"; import { useAuthGuard } from "@/hooks/use-auth-guard"; import { useLinkPrefetch } from "@/hooks/use-link-prefetch"; -import { queueFilingSync } from "@/lib/api"; +import { ensureTickerAutomation, queueFilingSync } from "@/lib/api"; import { formatCurrencyByScale, formatFinancialStatementValue, @@ -104,6 +104,8 @@ const SURFACE_OPTIONS = [ { value: "income_statement", label: "Income" }, { value: "balance_sheet", label: "Balance" }, { value: "cash_flow_statement", label: "Cash Flow" }, + { value: "equity_statement", label: "Stockholders' Equity" }, + { value: "disclosures", label: "Disclosures" }, { value: "ratios", label: "Ratios" }, { value: "segments_kpis", label: "KPIs" }, ]; @@ -166,7 +168,9 @@ function isStatementSurfaceKind(surfaceKind: FinancialSurfaceKind) { return ( surfaceKind === "income_statement" || surfaceKind === "balance_sheet" || - surfaceKind === "cash_flow_statement" + surfaceKind === "cash_flow_statement" || + surfaceKind === "equity_statement" || + surfaceKind === "disclosures" ); } @@ -324,7 +328,11 @@ function buildStatementTreeDisplayValue(input: { scale: NumberScaleUnit; surfaceKind: Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >; }) { const current = statementRowValue(input.row, input.periodId); @@ -535,7 +543,9 @@ function FinancialsPageContent() { const statementSurface = surfaceKind === "income_statement" || surfaceKind === "balance_sheet" || - surfaceKind === "cash_flow_statement"; + surfaceKind === "cash_flow_statement" || + surfaceKind === "equity_statement" || + surfaceKind === "disclosures"; if (!statementSurface && displayMode !== "standardized") { setDisplayMode("standardized"); } @@ -551,6 +561,12 @@ function FinancialsPageContent() { } }, [displayMode, surfaceKind]); + useEffect(() => { + if (!financials?.displayModes.includes(displayMode)) { + setDisplayMode(financials?.defaultDisplayMode ?? "standardized"); + } + }, [displayMode, financials?.defaultDisplayMode, financials?.displayModes]); + const loadFinancials = useCallback( async (symbol: string, options?: LoadOptions) => { const normalizedTicker = symbol.trim().toUpperCase(); @@ -560,7 +576,9 @@ function FinancialsPageContent() { (surfaceKind === "segments_kpis" || surfaceKind === "income_statement" || surfaceKind === "balance_sheet" || - surfaceKind === "cash_flow_statement"); + surfaceKind === "cash_flow_statement" || + surfaceKind === "equity_statement" || + surfaceKind === "disclosures"); if (!options?.append) { setLoading(true); @@ -672,6 +690,8 @@ function FinancialsPageContent() { case "income_statement": case "balance_sheet": case "cash_flow_statement": + case "equity_statement": + case "disclosures": return displayMode === "faithful" ? (financials.statementRows?.faithful ?? []) : (financials.statementRows?.standardized ?? []); @@ -808,6 +828,15 @@ function FinancialsPageContent() { ); } + if (surfaceKind === "equity_statement") { + return ( + standardizedRows.find((row) => row.key === "total_equity") ?? + standardizedRows.find((row) => row.key === "retained_earnings") ?? + standardizedRows.find((row) => row.key === "common_stock_and_apic") ?? + null + ); + } + return null; }, [displayMode, financials?.statementRows, surfaceKind]); @@ -829,6 +858,11 @@ function FinancialsPageContent() { }, [trendSeries, visiblePeriods]); const controlSections = useMemo(() => { + const availableDisplayModeOptions = DISPLAY_MODE_OPTIONS.filter((option) => + financials?.displayModes + ? financials.displayModes.includes(option.value as FinancialDisplayMode) + : option.value === "standardized" || surfaceKind !== "equity_statement", + ); const sections: FinancialControlSection[] = [ { key: "surface", @@ -869,12 +903,14 @@ function FinancialsPageContent() { if ( surfaceKind === "income_statement" || surfaceKind === "balance_sheet" || - surfaceKind === "cash_flow_statement" + surfaceKind === "cash_flow_statement" || + surfaceKind === "equity_statement" || + surfaceKind === "disclosures" ) { sections.push({ key: "display", label: "Display", - options: DISPLAY_MODE_OPTIONS, + options: availableDisplayModeOptions, value: displayMode, onChange: (value) => { setDisplayMode(value as FinancialDisplayMode); @@ -894,7 +930,7 @@ function FinancialsPageContent() { }); return sections; - }, [cadence, displayMode, historyWindow, surfaceKind, valueScale]); + }, [cadence, displayMode, financials?.displayModes, historyWindow, surfaceKind, valueScale]); const toggleExpandedRow = useCallback((key: string) => { setExpandedRowKeys((current) => { @@ -1006,6 +1042,10 @@ function FinancialsPageContent() { if (!normalized) { return; } + void ensureTickerAutomation({ + ticker: normalized, + source: "financials", + }); setSelectedFlatRowKey(null); setSelectedRowRef(null); setExpandedRowKeys(new Set()); @@ -1464,7 +1504,9 @@ function FinancialsPageContent() { {(surfaceKind === "income_statement" || surfaceKind === "balance_sheet" || - surfaceKind === "cash_flow_statement") && ( + surfaceKind === "cash_flow_statement" || + surfaceKind === "equity_statement" || + surfaceKind === "disclosures") && (
diff --git a/app/graphing/page.tsx b/app/graphing/page.tsx index 335cab3..b475f90 100644 --- a/app/graphing/page.tsx +++ b/app/graphing/page.tsx @@ -36,7 +36,7 @@ import { type GraphingLatestValueRow, type GraphingSeriesPoint } from '@/lib/graphing/series'; -import { ApiError } from '@/lib/api'; +import { ApiError, ensureTickerAutomation } from '@/lib/api'; import { formatCurrencyByScale, formatPercent, @@ -422,6 +422,12 @@ function GraphingPageContent() { onSubmit={(event) => { event.preventDefault(); const nextTickers = normalizeGraphTickers(tickerInput); + if (nextTickers[0]) { + void ensureTickerAutomation({ + ticker: nextTickers[0], + source: 'graphing' + }); + } replaceGraphState({ tickers: nextTickers.length > 0 ? nextTickers : [...graphState.tickers] }); }} > diff --git a/app/search/page.tsx b/app/search/page.tsx index 2b7d9b2..970357c 100644 --- a/app/search/page.tsx +++ b/app/search/page.tsx @@ -10,7 +10,7 @@ import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Panel } from '@/components/ui/panel'; import { useAuthGuard } from '@/hooks/use-auth-guard'; -import { getSearchAnswer } from '@/lib/api'; +import { ensureTickerAutomation, getSearchAnswer } from '@/lib/api'; import { searchQueryOptions } from '@/lib/query/options'; import type { SearchAnswerResponse, SearchResult, SearchSource } from '@/lib/types'; @@ -113,6 +113,13 @@ function SearchPageContent() { return; } + if (ticker.trim()) { + void ensureTickerAutomation({ + ticker, + source: 'search' + }); + } + startAnswerTransition(() => { setError(null); getSearchAnswer({ @@ -141,8 +148,15 @@ function SearchPageContent() { className="space-y-3" onSubmit={(event) => { event.preventDefault(); + const normalizedTicker = tickerInput.trim().toUpperCase(); + if (normalizedTicker) { + void ensureTickerAutomation({ + ticker: normalizedTicker, + source: 'search' + }); + } setQuery(queryInput.trim()); - setTicker(tickerInput.trim().toUpperCase()); + setTicker(normalizedTicker); setAnswer(null); }} > diff --git a/bun.lock b/bun.lock index 0e86961..ad0a6ea 100644 --- a/bun.lock +++ b/bun.lock @@ -32,12 +32,13 @@ }, "devDependencies": { "@playwright/test": "^1.58.2", - "@types/node": "^25.3.5", + "@types/node": "^25.5.0", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "autoprefixer": "^10.4.27", "bun-types": "^1.3.10", "drizzle-kit": "^0.31.9", + "knip": "^5.88.1", "postcss": "^8.5.8", "prettier": "^3.6.2", "tailwindcss": "^4.2.1", @@ -147,8 +148,12 @@ "@elysiajs/eden": ["@elysiajs/eden@1.4.8", "", { "peerDependencies": { "elysia": ">=1.4.19" } }, "sha512-a7oct2kFa49tH+GawZtSUCZR2rQgucNYgGLz8alXUqb4IrU3PASA0T4zXJw4MhdV1Xb6vyiR7p7kkqJcVjgbkA=="], + "@emnapi/core": ["@emnapi/core@1.9.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.0", "tslib": "^2.4.0" } }, "sha512-mukuNALVsoix/w1BJwFzwXBN/dHeejQtuVzcDsfOEsdpCumXb/E9j8w11h5S54tT1xhifGfbbSm/ICrObRb3KA=="], + "@emnapi/runtime": ["@emnapi/runtime@1.8.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg=="], + "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg=="], + "@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="], "@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="], @@ -339,6 +344,8 @@ "@napi-rs/nice-win32-x64-msvc": ["@napi-rs/nice-win32-x64-msvc@1.1.1", "", { "os": "win32", "cpu": "x64" }, "sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ=="], + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.1", "", { "dependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1", "@tybys/wasm-util": "^0.10.1" } }, "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A=="], + "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="], "@nestjs/common": ["@nestjs/common@11.1.14", "", { "dependencies": { "file-type": "21.3.0", "iterare": "1.2.1", "load-esm": "1.0.3", "tslib": "2.8.1", "uid": "2.0.2" }, "peerDependencies": { "class-transformer": ">=0.4.1", "class-validator": ">=0.13.2", "reflect-metadata": "^0.1.12 || ^0.2.0", "rxjs": "^7.1.0" }, "optionalPeers": ["class-transformer", "class-validator"] }, "sha512-IN/tlqd7Nl9gl6f0jsWEuOrQDaCI9vHzxv0fisHysfBQzfQIkqlv5A7w4Qge02BUQyczXT9HHPgHtWHCxhjRng=="], @@ -367,6 +374,12 @@ "@noble/hashes": ["@noble/hashes@2.0.1", "", {}, "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], + + "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], + + "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@nuxt/kit": ["@nuxt/kit@4.3.1", "", { "dependencies": { "c12": "^3.3.3", "consola": "^3.4.2", "defu": "^6.1.4", "destr": "^2.0.5", "errx": "^0.1.0", "exsolve": "^1.0.8", "ignore": "^7.0.5", "jiti": "^2.6.1", "klona": "^2.0.6", "mlly": "^1.8.0", "ohash": "^2.0.11", "pathe": "^2.0.3", "pkg-types": "^2.3.0", "rc9": "^3.0.0", "scule": "^1.3.0", "semver": "^7.7.4", "tinyglobby": "^0.2.15", "ufo": "^1.6.3", "unctx": "^2.5.0", "untyped": "^2.0.0" } }, "sha512-UjBFt72dnpc+83BV3OIbCT0YHLevJtgJCHpxMX0YRKWLDhhbcDdUse87GtsQBrjvOzK7WUNUYLDS/hQLYev5rA=="], "@nuxt/opencollective": ["@nuxt/opencollective@0.4.1", "", { "dependencies": { "consola": "^3.2.3" }, "bin": { "opencollective": "bin/opencollective.js" } }, "sha512-GXD3wy50qYbxCJ652bDrDzgMr3NFEkIS374+IgFQKkCvk9yiYcLvX2XDYr7UyQxf4wK0e+yqDYRubZ0DtOxnmQ=="], @@ -377,6 +390,46 @@ "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], + "@oxc-resolver/binding-android-arm-eabi": ["@oxc-resolver/binding-android-arm-eabi@11.19.1", "", { "os": "android", "cpu": "arm" }, "sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg=="], + + "@oxc-resolver/binding-android-arm64": ["@oxc-resolver/binding-android-arm64@11.19.1", "", { "os": "android", "cpu": "arm64" }, "sha512-oolbkRX+m7Pq2LNjr/kKgYeC7bRDMVTWPgxBGMjSpZi/+UskVo4jsMU3MLheZV55jL6c3rNelPl4oD60ggYmqA=="], + + "@oxc-resolver/binding-darwin-arm64": ["@oxc-resolver/binding-darwin-arm64@11.19.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-nUC6d2i3R5B12sUW4O646qD5cnMXf2oBGPLIIeaRfU9doJRORAbE2SGv4eW6rMqhD+G7nf2Y8TTJTLiiO3Q/dQ=="], + + "@oxc-resolver/binding-darwin-x64": ["@oxc-resolver/binding-darwin-x64@11.19.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-cV50vE5+uAgNcFa3QY1JOeKDSkM/9ReIcc/9wn4TavhW/itkDGrXhw9jaKnkQnGbjJ198Yh5nbX/Gr2mr4Z5jQ=="], + + "@oxc-resolver/binding-freebsd-x64": ["@oxc-resolver/binding-freebsd-x64@11.19.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-xZOQiYGFxtk48PBKff+Zwoym7ScPAIVp4c14lfLxizO2LTTTJe5sx9vQNGrBymrf/vatSPNMD4FgsaaRigPkqw=="], + + "@oxc-resolver/binding-linux-arm-gnueabihf": ["@oxc-resolver/binding-linux-arm-gnueabihf@11.19.1", "", { "os": "linux", "cpu": "arm" }, "sha512-lXZYWAC6kaGe/ky2su94e9jN9t6M0/6c+GrSlCqL//XO1cxi5lpAhnJYdyrKfm0ZEr/c7RNyAx3P7FSBcBd5+A=="], + + "@oxc-resolver/binding-linux-arm-musleabihf": ["@oxc-resolver/binding-linux-arm-musleabihf@11.19.1", "", { "os": "linux", "cpu": "arm" }, "sha512-veG1kKsuK5+t2IsO9q0DErYVSw2azvCVvWHnfTOS73WE0STdLLB7Q1bB9WR+yHPQM76ASkFyRbogWo1GR1+WbQ=="], + + "@oxc-resolver/binding-linux-arm64-gnu": ["@oxc-resolver/binding-linux-arm64-gnu@11.19.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-heV2+jmXyYnUrpUXSPugqWDRpnsQcDm2AX4wzTuvgdlZfoNYO0O3W2AVpJYaDn9AG4JdM6Kxom8+foE7/BcSig=="], + + "@oxc-resolver/binding-linux-arm64-musl": ["@oxc-resolver/binding-linux-arm64-musl@11.19.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-jvo2Pjs1c9KPxMuMPIeQsgu0mOJF9rEb3y3TdpsrqwxRM+AN6/nDDwv45n5ZrUnQMsdBy5gIabioMKnQfWo9ew=="], + + "@oxc-resolver/binding-linux-ppc64-gnu": ["@oxc-resolver/binding-linux-ppc64-gnu@11.19.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-vLmdNxWCdN7Uo5suays6A/+ywBby2PWBBPXctWPg5V0+eVuzsJxgAn6MMB4mPlshskYbppjpN2Zg83ArHze9gQ=="], + + "@oxc-resolver/binding-linux-riscv64-gnu": ["@oxc-resolver/binding-linux-riscv64-gnu@11.19.1", "", { "os": "linux", "cpu": "none" }, "sha512-/b+WgR+VTSBxzgOhDO7TlMXC1ufPIMR6Vj1zN+/x+MnyXGW7prTLzU9eW85Aj7Th7CCEG9ArCbTeqxCzFWdg2w=="], + + "@oxc-resolver/binding-linux-riscv64-musl": ["@oxc-resolver/binding-linux-riscv64-musl@11.19.1", "", { "os": "linux", "cpu": "none" }, "sha512-YlRdeWb9j42p29ROh+h4eg/OQ3dTJlpHSa+84pUM9+p6i3djtPz1q55yLJhgW9XfDch7FN1pQ/Vd6YP+xfRIuw=="], + + "@oxc-resolver/binding-linux-s390x-gnu": ["@oxc-resolver/binding-linux-s390x-gnu@11.19.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-EDpafVOQWF8/MJynsjOGFThcqhRHy417sRyLfQmeiamJ8qVhSKAn2Dn2VVKUGCjVB9C46VGjhNo7nOPUi1x6uA=="], + + "@oxc-resolver/binding-linux-x64-gnu": ["@oxc-resolver/binding-linux-x64-gnu@11.19.1", "", { "os": "linux", "cpu": "x64" }, "sha512-NxjZe+rqWhr+RT8/Ik+5ptA3oz7tUw361Wa5RWQXKnfqwSSHdHyrw6IdcTfYuml9dM856AlKWZIUXDmA9kkiBQ=="], + + "@oxc-resolver/binding-linux-x64-musl": ["@oxc-resolver/binding-linux-x64-musl@11.19.1", "", { "os": "linux", "cpu": "x64" }, "sha512-cM/hQwsO3ReJg5kR+SpI69DMfvNCp+A/eVR4b4YClE5bVZwz8rh2Nh05InhwI5HR/9cArbEkzMjcKgTHS6UaNw=="], + + "@oxc-resolver/binding-openharmony-arm64": ["@oxc-resolver/binding-openharmony-arm64@11.19.1", "", { "os": "none", "cpu": "arm64" }, "sha512-QF080IowFB0+9Rh6RcD19bdgh49BpQHUW5TajG1qvWHvmrQznTZZjYlgE2ltLXyKY+qs4F/v5xuX1XS7Is+3qA=="], + + "@oxc-resolver/binding-wasm32-wasi": ["@oxc-resolver/binding-wasm32-wasi@11.19.1", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.1.1" }, "cpu": "none" }, "sha512-w8UCKhX826cP/ZLokXDS6+milN8y4X7zidsAttEdWlVoamTNf6lhBJldaWr3ukTDiye7s4HRcuPEPOXNC432Vg=="], + + "@oxc-resolver/binding-win32-arm64-msvc": ["@oxc-resolver/binding-win32-arm64-msvc@11.19.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-nJ4AsUVZrVKwnU/QRdzPCCrO0TrabBqgJ8pJhXITdZGYOV28TIYystV1VFLbQ7DtAcaBHpocT5/ZJnF78YJPtQ=="], + + "@oxc-resolver/binding-win32-ia32-msvc": ["@oxc-resolver/binding-win32-ia32-msvc@11.19.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-EW+ND5q2Tl+a3pH81l1QbfgbF3HmqgwLfDfVithRFheac8OTcnbXt/JxqD2GbDkb7xYEqy1zNaVFRr3oeG8npA=="], + + "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.19.1", "", { "os": "win32", "cpu": "x64" }, "sha512-6hIU3RQu45B+VNTY4Ru8ppFwjVS/S5qwYyGhBotmjxfEKk41I2DlGtRfGJndZ5+6lneE2pwloqunlOyZuX/XAw=="], + "@playwright/test": ["@playwright/test@1.58.2", "", { "dependencies": { "playwright": "1.58.2" }, "bin": { "playwright": "cli.js" } }, "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA=="], "@prisma/client": ["@prisma/client@7.4.2", "", { "dependencies": { "@prisma/client-runtime-utils": "7.4.2" }, "peerDependencies": { "prisma": "*", "typescript": ">=5.4.0" }, "optionalPeers": ["prisma", "typescript"] }, "sha512-ts2mu+cQHriAhSxngO3StcYubBGTWDtu/4juZhXCUKOwgh26l+s4KD3vT2kMUzFyrYnll9u/3qWrtzRv9CGWzA=="], @@ -617,6 +670,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], "@types/d3-array": ["@types/d3-array@3.2.2", "", {}, "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="], @@ -649,7 +704,7 @@ "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], - "@types/node": ["@types/node@25.3.5", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA=="], + "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], "@types/pg": ["@types/pg@8.16.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ=="], @@ -813,6 +868,8 @@ "brace-expansion": ["brace-expansion@5.0.3", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA=="], + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], + "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": "cli.js" }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], "bson": ["bson@7.2.0", "", {}, "sha512-YCEo7KjMlbNlyHhz7zAZNDpIpQbd+wOEHJYezv0nMYTn4x31eIUM2yomNNubclAt63dObUzKHWsBLJ9QcZNSnQ=="], @@ -1071,12 +1128,18 @@ "fast-fifo": ["fast-fifo@1.3.2", "", {}, "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], + "fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="], "fast-xml-builder": ["fast-xml-builder@1.0.0", "", {}, "sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ=="], "fast-xml-parser": ["fast-xml-parser@5.4.1", "", { "dependencies": { "fast-xml-builder": "^1.0.0", "strnum": "^2.1.2" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-BQ30U1mKkvXQXXkAGcuyUA/GA26oEB7NzOtsxCDtyu62sjGw5QraKFhx2Em3WQNjPw9PG6MQ9yuIIgkSDfGu5A=="], + "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + + "fd-package-json": ["fd-package-json@2.0.0", "", { "dependencies": { "walk-up-path": "^4.0.0" } }, "sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], @@ -1091,6 +1154,8 @@ "filenamify": ["filenamify@6.0.0", "", { "dependencies": { "filename-reserved-regex": "^3.0.0" } }, "sha512-vqIlNogKeyD3yzrm0yhRMQg8hOVwYcYRfjEoODd49iCprMn4HL85gK3HcykQE53EPIpX3HcAbGA5ELQv216dAQ=="], + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + "finalhandler": ["finalhandler@1.3.2", "", { "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "statuses": "~2.0.2", "unpipe": "~1.0.0" } }, "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg=="], "find-up": ["find-up@7.0.0", "", { "dependencies": { "locate-path": "^7.2.0", "path-exists": "^5.0.0", "unicorn-magic": "^0.1.0" } }, "sha512-YyZM99iHrqLKjmt4LJDj58KI+fYyufRLBSYcqycxf//KpBk9FoewoGX0450m9nB44qrZnovzC2oeP5hUibxc/g=="], @@ -1101,6 +1166,8 @@ "form-data-encoder": ["form-data-encoder@2.1.4", "", {}, "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw=="], + "formatly": ["formatly@0.3.0", "", { "dependencies": { "fd-package-json": "^2.0.0" }, "bin": { "formatly": "bin/index.mjs" } }, "sha512-9XNj/o4wrRFyhSMJOvsuyMwy8aUfBaZ1VrqHVfohyXf0Sw0e+yfKG+xZaY3arGCOMdwFsqObtzVOc1gU9KiT9w=="], + "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], @@ -1135,6 +1202,8 @@ "giget": ["giget@2.0.0", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.0", "defu": "^6.1.4", "node-fetch-native": "^1.6.6", "nypm": "^0.6.0", "pathe": "^2.0.3" }, "bin": { "giget": "dist/cli.mjs" } }, "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA=="], + "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + "glob-to-regexp": ["glob-to-regexp@0.4.1", "", {}, "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -1199,12 +1268,18 @@ "is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], "is-interactive": ["is-interactive@2.0.0", "", {}, "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ=="], + "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + "is-plain-obj": ["is-plain-obj@1.1.0", "", {}, "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg=="], "is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="], @@ -1249,6 +1324,8 @@ "klona": ["klona@2.0.6", "", {}, "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA=="], + "knip": ["knip@5.88.1", "", { "dependencies": { "@nodelib/fs.walk": "^1.2.3", "fast-glob": "^3.3.3", "formatly": "^0.3.0", "jiti": "^2.6.0", "minimist": "^1.2.8", "oxc-resolver": "^11.19.1", "picocolors": "^1.1.1", "picomatch": "^4.0.1", "smol-toml": "^1.5.2", "strip-json-comments": "5.0.3", "unbash": "^2.2.0", "yaml": "^2.8.2", "zod": "^4.1.11" }, "peerDependencies": { "@types/node": ">=18", "typescript": ">=5.0.4 <7" }, "bin": { "knip": "bin/knip.js", "knip-bun": "bin/knip-bun.js" } }, "sha512-tpy5o7zu1MjawVkLPuahymVJekYY3kYjvzcoInhIchgePxTlo+api90tBv2KfhAIe5uXh+mez1tAfmbv8/TiZg=="], + "knitwork": ["knitwork@1.3.0", "", {}, "sha512-4LqMNoONzR43B1W0ek0fhXMsDNW/zxa1NdFAVMY+k28pgZLovR4G3PB5MrpTxCy1QaZCqNoiaKPr5w5qZHfSNw=="], "kysely": ["kysely@0.28.11", "", {}, "sha512-zpGIFg0HuoC893rIjYX1BETkVWdDnzTzF5e0kWXJFg5lE0k1/LfNWBejrcnOFu8Q2Rfq/hTDTU7XLUM8QOrpzg=="], @@ -1315,8 +1392,12 @@ "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + "methods": ["methods@1.1.2", "", {}, "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="], + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + "mime": ["mime@1.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], "mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], @@ -1331,6 +1412,8 @@ "minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="], + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + "mixpart": ["mixpart@0.0.5", "", {}, "sha512-TpWi9/2UIr7VWCVAM7NB4WR4yOglAetBkuKfxs3K0vFcUukqAaW1xsgX0v1gNGiDKzYhPHFcHgarC7jmnaOy4w=="], "mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="], @@ -1387,6 +1470,8 @@ "os-paths": ["os-paths@4.4.0", "", {}, "sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg=="], + "oxc-resolver": ["oxc-resolver@11.19.1", "", { "optionalDependencies": { "@oxc-resolver/binding-android-arm-eabi": "11.19.1", "@oxc-resolver/binding-android-arm64": "11.19.1", "@oxc-resolver/binding-darwin-arm64": "11.19.1", "@oxc-resolver/binding-darwin-x64": "11.19.1", "@oxc-resolver/binding-freebsd-x64": "11.19.1", "@oxc-resolver/binding-linux-arm-gnueabihf": "11.19.1", "@oxc-resolver/binding-linux-arm-musleabihf": "11.19.1", "@oxc-resolver/binding-linux-arm64-gnu": "11.19.1", "@oxc-resolver/binding-linux-arm64-musl": "11.19.1", "@oxc-resolver/binding-linux-ppc64-gnu": "11.19.1", "@oxc-resolver/binding-linux-riscv64-gnu": "11.19.1", "@oxc-resolver/binding-linux-riscv64-musl": "11.19.1", "@oxc-resolver/binding-linux-s390x-gnu": "11.19.1", "@oxc-resolver/binding-linux-x64-gnu": "11.19.1", "@oxc-resolver/binding-linux-x64-musl": "11.19.1", "@oxc-resolver/binding-openharmony-arm64": "11.19.1", "@oxc-resolver/binding-wasm32-wasi": "11.19.1", "@oxc-resolver/binding-win32-arm64-msvc": "11.19.1", "@oxc-resolver/binding-win32-ia32-msvc": "11.19.1", "@oxc-resolver/binding-win32-x64-msvc": "11.19.1" } }, "sha512-qE/CIg/spwrTBFt5aKmwe3ifeDdLfA2NESN30E42X/lII5ClF8V7Wt6WIJhcGZjp0/Q+nQ+9vgxGk//xZNX2hg=="], + "p-cancelable": ["p-cancelable@3.0.0", "", {}, "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="], "p-limit": ["p-limit@4.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ=="], @@ -1479,6 +1564,8 @@ "qs": ["qs@6.14.2", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], + "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], @@ -1527,12 +1614,16 @@ "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + "rollup": ["rollup@4.59.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.59.0", "@rollup/rollup-android-arm64": "4.59.0", "@rollup/rollup-darwin-arm64": "4.59.0", "@rollup/rollup-darwin-x64": "4.59.0", "@rollup/rollup-freebsd-arm64": "4.59.0", "@rollup/rollup-freebsd-x64": "4.59.0", "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", "@rollup/rollup-linux-arm-musleabihf": "4.59.0", "@rollup/rollup-linux-arm64-gnu": "4.59.0", "@rollup/rollup-linux-arm64-musl": "4.59.0", "@rollup/rollup-linux-loong64-gnu": "4.59.0", "@rollup/rollup-linux-loong64-musl": "4.59.0", "@rollup/rollup-linux-ppc64-gnu": "4.59.0", "@rollup/rollup-linux-ppc64-musl": "4.59.0", "@rollup/rollup-linux-riscv64-gnu": "4.59.0", "@rollup/rollup-linux-riscv64-musl": "4.59.0", "@rollup/rollup-linux-s390x-gnu": "4.59.0", "@rollup/rollup-linux-x64-gnu": "4.59.0", "@rollup/rollup-linux-x64-musl": "4.59.0", "@rollup/rollup-openbsd-x64": "4.59.0", "@rollup/rollup-openharmony-arm64": "4.59.0", "@rollup/rollup-win32-arm64-msvc": "4.59.0", "@rollup/rollup-win32-ia32-msvc": "4.59.0", "@rollup/rollup-win32-x64-gnu": "4.59.0", "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg=="], "rou3": ["rou3@0.7.12", "", {}, "sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg=="], "run-applescript": ["run-applescript@7.1.0", "", {}, "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q=="], + "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + "rxjs": ["rxjs@7.8.2", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA=="], "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], @@ -1583,6 +1674,8 @@ "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], + "smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="], + "sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="], "sort-keys": ["sort-keys@1.1.2", "", { "dependencies": { "is-plain-obj": "^1.0.0" } }, "sha512-vzn8aSqKgytVik0iwdBEi+zevbTYZogewTUM6dtpmGwEcdzbub/TX4bCzRhebDCRC3QzXgJsLRKB2V/Oof7HXg=="], @@ -1631,6 +1724,8 @@ "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="], + "strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="], + "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], "strnum": ["strnum@2.1.2", "", {}, "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ=="], @@ -1669,6 +1764,8 @@ "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="], + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], @@ -1691,6 +1788,8 @@ "ulid": ["ulid@3.0.1", "", { "bin": { "ulid": "dist/cli.js" } }, "sha512-dPJyqPzx8preQhqq24bBG1YNkvigm87K8kVEHCD+ruZg24t6IFEFv00xMWfxcC4djmFtiTLdFuADn4+DOz6R7Q=="], + "unbash": ["unbash@2.2.0", "", {}, "sha512-X2wH19RAPZE3+ldGicOkoj/SIA83OIxcJ6Cuaw23hf8Xc6fQpvZXY0SftE2JgS0QhYLUG4uwodSI3R53keyh7w=="], + "unbzip2-stream": ["unbzip2-stream@1.4.3", "", { "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" } }, "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg=="], "unctx": ["unctx@2.5.0", "", { "dependencies": { "acorn": "^8.15.0", "estree-walker": "^3.0.3", "magic-string": "^0.30.21", "unplugin": "^2.3.11" } }, "sha512-p+Rz9x0R7X+CYDkT+Xg8/GhpcShTlU8n+cf9OtOEf7zEQsNcCZO1dPKNRDqvUTaq+P32PMMkxWHwfrxkqfqAYg=="], @@ -1727,6 +1826,8 @@ "vitest": ["vitest@3.2.4", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.4", "@vitest/mocker": "3.2.4", "@vitest/pretty-format": "^3.2.4", "@vitest/runner": "3.2.4", "@vitest/snapshot": "3.2.4", "@vitest/spy": "3.2.4", "@vitest/utils": "3.2.4", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "picomatch": "^4.0.2", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.14", "tinypool": "^1.1.1", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", "vite-node": "3.2.4", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.2.4", "@vitest/ui": "3.2.4", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A=="], + "walk-up-path": ["walk-up-path@4.0.0", "", {}, "sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A=="], + "watchpack": ["watchpack@2.5.1", "", { "dependencies": { "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" } }, "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg=="], "wcwidth": ["wcwidth@1.0.1", "", { "dependencies": { "defaults": "^1.0.3" } }, "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg=="], @@ -1767,6 +1868,8 @@ "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], @@ -1937,6 +2040,8 @@ "log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "mlly/pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], "next/caniuse-lite": ["caniuse-lite@1.0.30001770", "", {}, "sha512-x/2CLQ1jHENRbHg5PSId2sXq1CIO1CISvwWAj027ltMVG2UNgW+w9oH2+HzgEIRFembL8bUlXtfbBHR1fCg2xw=="], diff --git a/components/charts/utils/chart-colors.ts b/components/charts/utils/chart-colors.ts index 405c369..81cc074 100644 --- a/components/charts/utils/chart-colors.ts +++ b/components/charts/utils/chart-colors.ts @@ -31,7 +31,7 @@ export function getPriceChangeColor(change: number): string { * Convert CSS variable to computed color value * Used for chart export since html-to-image can't render CSS variables */ -export function cssVarToColor(cssVar: string): string { +function cssVarToColor(cssVar: string): string { if (typeof window === 'undefined') return cssVar; // If it's already a color value, return as-is diff --git a/components/charts/utils/chart-data-transformers.ts b/components/charts/utils/chart-data-transformers.ts index f306ccd..1029b41 100644 --- a/components/charts/utils/chart-data-transformers.ts +++ b/components/charts/utils/chart-data-transformers.ts @@ -59,7 +59,7 @@ export function isPriceData(data: ChartDataPoint): data is { date: string; price * Normalize data to ensure consistent structure * Converts price data to OHLCV-like structure for candlestick charts */ -export function normalizeChartData(data: T[]): T[] { +function normalizeChartData(data: T[]): T[] { if (!data || data.length === 0) return []; // Sort by date ascending @@ -72,7 +72,7 @@ export function normalizeChartData(data: T[]): T[] { * Sample data for performance with large datasets * Keeps every Nth point when dataset is too large */ -export function sampleData( +function sampleData( data: T[], maxPoints: number = 1000 ): T[] { @@ -96,7 +96,7 @@ export function sampleData( /** * Calculate min/max values for Y-axis domain */ -export function calculateYAxisDomain( +function calculateYAxisDomain( data: T[], padding: number = 0.1 ): [number, number] { @@ -126,7 +126,7 @@ export function calculateYAxisDomain( /** * Calculate volume max for volume indicator Y-axis */ -export function calculateVolumeMax(data: T[]): number { +function calculateVolumeMax(data: T[]): number { if (data.length === 0 || !isOHLCVData(data[0])) return 0; return Math.max(...data.map(d => (isOHLCVData(d) ? d.volume : 0))); diff --git a/components/dashboard/index-card-row.tsx b/components/dashboard/index-card-row.tsx index 0556fd6..2e602a8 100644 --- a/components/dashboard/index-card-row.tsx +++ b/components/dashboard/index-card-row.tsx @@ -10,7 +10,7 @@ type IndexCardProps = { positive?: boolean; }; -export type { IndexCardProps }; +; type IndexCardRowProps = { cards: IndexCardProps[]; diff --git a/components/financials/financials-toolbar.tsx b/components/financials/financials-toolbar.tsx index 3cb6a6a..ecec2db 100644 --- a/components/financials/financials-toolbar.tsx +++ b/components/financials/financials-toolbar.tsx @@ -27,7 +27,7 @@ export type FinancialControlSection = { onChange: (value: string) => void; }; -export type FinancialsToolbarProps = { +type FinancialsToolbarProps = { sections: FinancialControlSection[]; searchValue: string; onSearchChange: (value: string) => void; diff --git a/components/financials/statement-matrix.tsx b/components/financials/statement-matrix.tsx index 83ff40f..d170321 100644 --- a/components/financials/statement-matrix.tsx +++ b/components/financials/statement-matrix.tsx @@ -58,7 +58,7 @@ function surfaceBadges(node: Extract) { badges.push({ label: "Low confidence", tone: "warning" }); } - const detailCount = node.row.detailCount ?? node.directDetailCount; + const detailCount = node.directDetailCount; if (detailCount > 0) { badges.push({ label: `${detailCount} details`, tone: "default" }); } diff --git a/components/financials/statement-row-inspector.tsx b/components/financials/statement-row-inspector.tsx index 05c00cf..6f47748 100644 --- a/components/financials/statement-row-inspector.tsx +++ b/components/financials/statement-row-inspector.tsx @@ -16,7 +16,11 @@ type StatementRowInspectorProps = { periods: FinancialStatementPeriod[]; surfaceKind: Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >; renderValue: ( row: SurfaceFinancialRow | DetailFinancialRow, diff --git a/components/notifications/task-stage-helpers.ts b/components/notifications/task-stage-helpers.ts index 0774fa3..5a0fdd4 100644 --- a/components/notifications/task-stage-helpers.ts +++ b/components/notifications/task-stage-helpers.ts @@ -1,8 +1,8 @@ export { buildStageTimeline, - fallbackStageProgress, + stageLabel, - taskStageOrder, + taskTypeLabel, - type StageTimelineItem + } from '@/lib/task-workflow'; diff --git a/doc/issuer-overlay-automation.md b/doc/issuer-overlay-automation.md new file mode 100644 index 0000000..f53faa0 --- /dev/null +++ b/doc/issuer-overlay-automation.md @@ -0,0 +1,63 @@ +# Issuer Overlay Automation + +## Overview + +Issuer overlays are now runtime data, not just checked-in JSON files. + +The system creates or updates a global overlay for a ticker after an explicit user ticker submit triggers filing sync. Overlay generation is additive-only: + +- it extends existing canonical surfaces +- it never synthesizes brand-new surfaces +- it never auto-deletes prior mappings + +## Trigger Flow + +1. An explicit ticker submit calls `POST /api/tickers/ensure`. +2. The endpoint ensures an `issuer_overlay` row exists for the ticker. +3. If the ticker has no ready taxonomy snapshots, no overlay build history, or its latest ready snapshot was hydrated with an older overlay revision, the endpoint queues `sync_filings`. +4. `sync_filings` hydrates taxonomy snapshots with the current active overlay revision. +5. The task then generates a new overlay candidate from stored residual extension concepts. +6. If a new overlay revision is published, the task immediately rehydrates recent filings with that revision before queueing search indexing. + +## Storage + +Two tables back the feature: + +- `issuer_overlay`: one row per ticker with status, active revision, error state, and summary stats +- `issuer_overlay_revision`: immutable overlay revisions with normalized definition JSON and diagnostics + +Each `filing_taxonomy_snapshot` also stores `issuer_overlay_revision_id` so the sync pipeline can detect when a filing needs rehydration because the overlay changed. + +## Generator Rules + +The generator samples up to 12 ready `10-K`/`10-Q` snapshots from the last 3 years and chooses the most common detected fiscal pack in that window. + +It only promotes a concept when all of the following are true: + +- the concept is an extension concept +- the concept is still residual/unmapped in the stored taxonomy output +- the concept is not abstract +- the concept has a consistent statement kind across filings +- the concept appears in at least 2 distinct filings +- the concept can be mapped uniquely to an existing canonical surface by: + - authoritative concept local-name match, or + - exact local-name match against canonical source/authoritative concepts + +Accepted concepts are appended to `allowed_source_concepts` for the resolved surface in the runtime overlay definition. + +## Merge Order + +The hydrator merges surfaces in this order: + +1. pack primary/disclosure +2. core primary/disclosure +3. optional static issuer overlay file +4. optional runtime issuer overlay + +Runtime overlays therefore override static issuer overlays for the same surface when both are present. + +## Failure Handling + +- Generator failures set the overlay status to `error` and preserve the prior active revision. +- Empty builds set the status to `empty` only when no active revision exists. +- Search indexing is queued only after the final hydration pass, so downstream search reflects the overlay-adjusted snapshot state. diff --git a/docs/architecture/financial-surfaces.md b/docs/architecture/financial-surfaces.md index 80704d1..ff98b0b 100644 --- a/docs/architecture/financial-surfaces.md +++ b/docs/architecture/financial-surfaces.md @@ -45,6 +45,15 @@ As of Issue #26, the financial statement mapping architecture follows a **Rust-f ## Source of Truth +## Statement Admission + +Primary statements are admitted conservatively: + +- presentation-role classification is authoritative +- no-role fallback only admits concepts that match a primary statement taxonomy surface +- disclosure-only concepts stay in raw persisted facts/concepts and must not appear as primary statement residuals +- `equity_statement` is a first-class surfaced statement, but it is faithful-first and only has minimal standardized coverage in this pass + ### Authoritative Sources (Edit These) 1. **`rust/taxonomy/fiscal/v1/core.surface.json`** - Defines all surface keys, labels, categories, orders, and formulas @@ -138,6 +147,14 @@ See `rust/taxonomy/fiscal/v1/core.surface.json` for complete list. ### Cash Flow Statement See `rust/taxonomy/fiscal/v1/core.surface.json` for complete list. +### Stockholders' Equity Statement + +`equity_statement` is exposed in the API/UI as a primary statement surface. + +- faithful rows come from snapshot `statement_rows.equity` +- standardized rows currently cover only a minimal canonical set +- disclosure-style equity concepts are intentionally excluded from primary statement admission + ## Related Files - `rust/fiscal-xbrl-core/src/surface_mapper.rs` - Surface resolution logic - `rust/fiscal-xbrl-core/src/taxonomy_loader.rs` - JSON loading diff --git a/docs/architecture/taxonomy.md b/docs/architecture/taxonomy.md index 3a30f65..0bc637c 100644 --- a/docs/architecture/taxonomy.md +++ b/docs/architecture/taxonomy.md @@ -10,7 +10,7 @@ The taxonomy system defines all financial surfaces, computed ratios, and KPIs us ┌─────────────────────────────────────────────────────────────────┐ │ rust/taxonomy/fiscal/v1/ │ │ │ -│ core.surface.json - Income/Balance/Cash Flow surfaces │ +│ core.surface.json - Income/Balance/Cash Flow/Equity │ │ core.computed.json - Ratio definitions │ │ core.kpis.json - Sector-specific KPIs │ │ core.income-bridge.json - Income statement mapping rules │ @@ -130,7 +130,7 @@ Generated TypeScript statement catalogs are built from the deduped union of core | Field | Type | Description | | ------------------------- | -------- | ------------------------------------------------------------------------------------------ | | `surface_key` | string | Unique identifier (snake_case) | -| `statement` | enum | `income`, `balance`, `cash_flow`, `equity`, `comprehensive_income` | +| `statement` | enum | `income`, `balance`, `cash_flow`, `equity`, `comprehensive_income`, `disclosure` | | `label` | string | Human-readable label | | `category` | string | Grouping category | | `order` | number | Display order | @@ -215,6 +215,29 @@ This ensures consistency across packs while allowing sector-specific income stat Auto-classification remains conservative. Pack selection uses concept and role scoring, then falls back to `core` when the top match is weak or ambiguous. +## Issuer Overlay Automation + +Issuer overlays now support a runtime, database-backed path in addition to checked-in JSON files. Explicit user ticker submits enqueue filing sync through `POST /api/tickers/ensure`; the sync task hydrates filings with the current overlay revision, generates additive issuer mappings from residual extension concepts, and immediately rehydrates recent filings when a new overlay revision is published. + +Automation is intentionally conservative: + +- it only extends existing canonical surfaces +- it does not synthesize new surfaces +- it does not auto-delete prior mappings + +Runtime overlay merge order is: + +1. pack primary/disclosure +2. core primary/disclosure +3. static issuer overlay file +4. runtime issuer overlay + +No-role statement admission is taxonomy-aware: + +- primary statement admission is allowed only when a concept matches a primary statement surface +- disclosure-only concepts are excluded from surfaced primary statements +- explicit overlap handling exists for shared balance/equity concepts such as `StockholdersEquity` and `LiabilitiesAndStockholdersEquity` + ## Build Pipeline ```bash diff --git a/drizzle/0015_issuer_overlay_automation.sql b/drizzle/0015_issuer_overlay_automation.sql new file mode 100644 index 0000000..5dcf832 --- /dev/null +++ b/drizzle/0015_issuer_overlay_automation.sql @@ -0,0 +1,32 @@ +CREATE TABLE `issuer_overlay_revision` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `ticker` text NOT NULL, + `revision_number` integer NOT NULL, + `definition_hash` text NOT NULL, + `definition_json` text NOT NULL, + `diagnostics_json` text, + `source_snapshot_ids` text NOT NULL DEFAULT '[]', + `created_at` text NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `issuer_overlay_revision_ticker_revision_uidx` ON `issuer_overlay_revision` (`ticker`,`revision_number`); +--> statement-breakpoint +CREATE UNIQUE INDEX `issuer_overlay_revision_ticker_hash_uidx` ON `issuer_overlay_revision` (`ticker`,`definition_hash`); +--> statement-breakpoint +CREATE INDEX `issuer_overlay_revision_ticker_created_idx` ON `issuer_overlay_revision` (`ticker`,`created_at`); +--> statement-breakpoint +CREATE TABLE `issuer_overlay` ( + `ticker` text PRIMARY KEY NOT NULL, + `status` text NOT NULL DEFAULT 'empty', + `active_revision_id` integer, + `last_built_at` text, + `last_error` text, + `stats_json` text, + `created_at` text NOT NULL, + `updated_at` text NOT NULL, + FOREIGN KEY (`active_revision_id`) REFERENCES `issuer_overlay_revision`(`id`) ON UPDATE no action ON DELETE set null +); +--> statement-breakpoint +CREATE INDEX `issuer_overlay_status_idx` ON `issuer_overlay` (`status`,`updated_at`); +--> statement-breakpoint +ALTER TABLE `filing_taxonomy_snapshot` ADD `issuer_overlay_revision_id` integer REFERENCES `issuer_overlay_revision`(`id`) ON UPDATE no action ON DELETE set null; diff --git a/hooks/use-api-queries.ts b/hooks/use-api-queries.ts index a5b058f..d95c04c 100644 --- a/hooks/use-api-queries.ts +++ b/hooks/use-api-queries.ts @@ -14,56 +14,56 @@ import { watchlistQueryOptions } from '@/lib/query/options'; -export function useCompanyAnalysisQuery(ticker: string, enabled = true) { +function useCompanyAnalysisQuery(ticker: string, enabled = true) { return useQuery({ ...companyAnalysisQueryOptions(ticker), enabled: enabled && ticker.trim().length > 0 }); } -export function useFilingsQuery(input: { ticker?: string; limit?: number }, enabled = true) { +function useFilingsQuery(input: { ticker?: string; limit?: number }, enabled = true) { return useQuery({ ...filingsQueryOptions(input), enabled }); } -export function useAiReportQuery(accessionNumber: string, enabled = true) { +function useAiReportQuery(accessionNumber: string, enabled = true) { return useQuery({ ...aiReportQueryOptions(accessionNumber), enabled: enabled && accessionNumber.trim().length > 0 }); } -export function useWatchlistQuery(enabled = true) { +function useWatchlistQuery(enabled = true) { return useQuery({ ...watchlistQueryOptions(), enabled }); } -export function useHoldingsQuery(enabled = true) { +function useHoldingsQuery(enabled = true) { return useQuery({ ...holdingsQueryOptions(), enabled }); } -export function usePortfolioSummaryQuery(enabled = true) { +function usePortfolioSummaryQuery(enabled = true) { return useQuery({ ...portfolioSummaryQueryOptions(), enabled }); } -export function useLatestPortfolioInsightQuery(enabled = true) { +function useLatestPortfolioInsightQuery(enabled = true) { return useQuery({ ...latestPortfolioInsightQueryOptions(), enabled }); } -export function useTaskQuery(taskId: string, enabled = true) { +function useTaskQuery(taskId: string, enabled = true) { return useQuery({ ...taskQueryOptions(taskId), enabled: enabled && taskId.length > 0 @@ -77,7 +77,7 @@ export function useTaskTimelineQuery(taskId: string, enabled = true) { }); } -export function useRecentTasksQuery(limit = 20, enabled = true) { +function useRecentTasksQuery(limit = 20, enabled = true) { return useQuery({ ...recentTasksQueryOptions(limit), enabled diff --git a/knip.json b/knip.json new file mode 100644 index 0000000..c9201d3 --- /dev/null +++ b/knip.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://unpkg.com/knip@5/schema.json", + "tags": [ + "-lintignore" + ] +} diff --git a/lib/api.ts b/lib/api.ts index 008e669..91addff 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -31,6 +31,7 @@ import type { Task, TaskStatus, TaskTimeline, + TickerAutomationSource, User, WatchlistItem } from './types'; @@ -147,7 +148,7 @@ async function requestJson(input: { return payload as T; } -export async function getMe() { +async function getMe() { const result = await client.api.me.get(); return await unwrapData<{ user: User }>(result, 'Unable to fetch session'); } @@ -202,7 +203,7 @@ export async function listResearchJournal(ticker: string) { return await unwrapData<{ entries: ResearchJournalEntry[] }>(result, 'Unable to fetch research journal'); } -export async function createResearchJournalEntry(input: { +async function createResearchJournalEntry(input: { ticker: string; accessionNumber?: string; entryType: ResearchJournalEntryType; @@ -422,7 +423,7 @@ export async function getResearchPacket(ticker: string) { }, 'Unable to fetch research packet'); } -export async function updateResearchJournalEntry(id: number, input: { +async function updateResearchJournalEntry(id: number, input: { title?: string; bodyMarkdown?: string; metadata?: Record; @@ -434,7 +435,7 @@ export async function updateResearchJournalEntry(id: number, input: { }, 'Unable to update journal entry'); } -export async function deleteResearchJournalEntry(id: number) { +async function deleteResearchJournalEntry(id: number) { const result = await client.api.research.journal[id].delete(); return await unwrapData<{ success: boolean }>(result, 'Unable to delete journal entry'); } @@ -570,6 +571,20 @@ export async function getCompanyAnalysis(ticker: string, options?: { refresh?: b return await unwrapData<{ analysis: CompanyAnalysis }>(result, 'Unable to fetch company analysis'); } +export async function ensureTickerAutomation(input: { + ticker: string; + source: TickerAutomationSource; +}) { + return await requestJson<{ queued: boolean; task: Task | null }>({ + path: '/api/tickers/ensure', + method: 'POST', + body: { + ticker: input.ticker.trim().toUpperCase(), + source: input.source + } + }, 'Unable to ensure ticker automation'); +} + export async function getCompanyFinancialStatements(input: { ticker: string; surfaceKind: FinancialSurfaceKind; diff --git a/lib/financial-metrics.ts b/lib/financial-metrics.ts index 8986840..5797fbb 100644 --- a/lib/financial-metrics.ts +++ b/lib/financial-metrics.ts @@ -19,7 +19,7 @@ export type GraphableFinancialSurfaceKind = Extract< 'income_statement' | 'balance_sheet' | 'cash_flow_statement' | 'ratios' >; -export type StatementMetricDefinition = { +type StatementMetricDefinition = { key: string; label: string; category: string; @@ -27,7 +27,7 @@ export type StatementMetricDefinition = { unit: FinancialUnit; }; -export type RatioMetricDefinition = { +type RatioMetricDefinition = { key: string; label: string; category: string; @@ -84,4 +84,4 @@ export const CASH_FLOW_STATEMENT_METRIC_DEFINITIONS: StatementMetricDefinition[] export const RATIO_DEFINITIONS: RatioMetricDefinition[] = ALL_COMPUTED.map(computedToRatioMetric); -export { RATIO_CATEGORIES, type RatioCategory } from '@/lib/generated'; +export { RATIO_CATEGORIES, } from '@/lib/generated'; diff --git a/lib/financials/page-merge.test.ts b/lib/financials/page-merge.test.ts index c9a5021..d15ce10 100644 --- a/lib/financials/page-merge.test.ts +++ b/lib/financials/page-merge.test.ts @@ -57,6 +57,10 @@ function createResponse(partial: Partial): C kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: [] }, dimensionBreakdown: null, diff --git a/lib/financials/page-merge.ts b/lib/financials/page-merge.ts index 9595316..362b9ef 100644 --- a/lib/financials/page-merge.ts +++ b/lib/financials/page-merge.ts @@ -2,7 +2,7 @@ import type { CompanyFinancialStatementsResponse } from '@/lib/types'; -export function mergeDetailMaps( +function mergeDetailMaps( base: CompanyFinancialStatementsResponse['statementDetails'], next: CompanyFinancialStatementsResponse['statementDetails'] ) { diff --git a/lib/financials/statement-view-model.test.ts b/lib/financials/statement-view-model.test.ts index 2286ec0..e1f8e7d 100644 --- a/lib/financials/statement-view-model.test.ts +++ b/lib/financials/statement-view-model.test.ts @@ -1,317 +1,598 @@ -import { describe, expect, it } from 'bun:test'; +import { describe, expect, it } from "bun:test"; import { buildStatementTree, - resolveStatementSelection -} from '@/lib/financials/statement-view-model'; -import type { DetailFinancialRow, SurfaceFinancialRow } from '@/lib/types'; + resolveStatementSelection, +} from "@/lib/financials/statement-view-model"; +import type { DetailFinancialRow, SurfaceFinancialRow } from "@/lib/types"; -function createSurfaceRow(input: Partial & Pick): SurfaceFinancialRow { +function createSurfaceRow( + input: Partial & + Pick, +): SurfaceFinancialRow { return { key: input.key, label: input.label, - category: input.category ?? 'revenue', + category: input.category ?? "revenue", order: input.order ?? 10, - unit: input.unit ?? 'currency', + unit: input.unit ?? "currency", values: input.values, sourceConcepts: input.sourceConcepts ?? [input.key], sourceRowKeys: input.sourceRowKeys ?? [input.key], sourceFactIds: input.sourceFactIds ?? [1], formulaKey: input.formulaKey ?? null, hasDimensions: input.hasDimensions ?? false, - resolvedSourceRowKeys: input.resolvedSourceRowKeys ?? Object.fromEntries(Object.keys(input.values).map((periodId) => [periodId, input.key])), - statement: input.statement ?? 'income', + resolvedSourceRowKeys: + input.resolvedSourceRowKeys ?? + Object.fromEntries( + Object.keys(input.values).map((periodId) => [periodId, input.key]), + ), + statement: input.statement ?? "income", detailCount: input.detailCount, resolutionMethod: input.resolutionMethod, confidence: input.confidence, - warningCodes: input.warningCodes + warningCodes: input.warningCodes, }; } -function createDetailRow(input: Partial & Pick): DetailFinancialRow { +function createDetailRow( + input: Partial & + Pick, +): DetailFinancialRow { return { key: input.key, parentSurfaceKey: input.parentSurfaceKey, label: input.label, conceptKey: input.conceptKey ?? input.key, qname: input.qname ?? `us-gaap:${input.key}`, - namespaceUri: input.namespaceUri ?? 'http://fasb.org/us-gaap/2024', + namespaceUri: input.namespaceUri ?? "http://fasb.org/us-gaap/2024", localName: input.localName ?? input.key, - unit: input.unit ?? 'USD', + unit: input.unit ?? "USD", values: input.values, sourceFactIds: input.sourceFactIds ?? [100], isExtension: input.isExtension ?? false, dimensionsSummary: input.dimensionsSummary ?? [], - residualFlag: input.residualFlag ?? false + residualFlag: input.residualFlag ?? false, }; } -describe('statement view model', () => { - const categories = [{ key: 'opex', label: 'Operating Expenses', count: 4 }]; +describe("statement view model", () => { + const categories = [{ key: "opex", label: "Operating Expenses", count: 4 }]; - it('builds a root-only tree when there are no configured children or details', () => { + it("builds a root-only tree when there are no configured children or details", () => { const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows: [ - createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } }) + createSurfaceRow({ + key: "revenue", + label: "Revenue", + category: "revenue", + values: { p1: 100 }, + }), ], statementDetails: null, categories: [], - searchQuery: '', - expandedRowKeys: new Set() + searchQuery: "", + expandedRowKeys: new Set(), }); expect(model.sections).toHaveLength(1); expect(model.sections[0]?.nodes[0]).toMatchObject({ - kind: 'surface', - row: { key: 'revenue' }, - expandable: false + kind: "surface", + row: { key: "revenue" }, + expandable: false, }); }); - it('nests the operating expense child surfaces under the parent row', () => { + it("nests the operating expense child surfaces under the parent row", () => { const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows: [ - createSurfaceRow({ key: 'operating_expenses', label: 'Operating Expenses', category: 'opex', order: 10, values: { p1: 40 } }), - createSurfaceRow({ key: 'selling_general_and_administrative', label: 'SG&A', category: 'opex', order: 20, values: { p1: 20 } }), - createSurfaceRow({ key: 'research_and_development', label: 'Research Expense', category: 'opex', order: 30, values: { p1: 12 } }), - createSurfaceRow({ key: 'other_operating_expense', label: 'Other Expense', category: 'opex', order: 40, values: { p1: 8 } }) + createSurfaceRow({ + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "selling_general_and_administrative", + label: "SG&A", + category: "opex", + order: 20, + values: { p1: 20 }, + }), + createSurfaceRow({ + key: "research_and_development", + label: "Research Expense", + category: "opex", + order: 30, + values: { p1: 12 }, + }), + createSurfaceRow({ + key: "other_operating_expense", + label: "Other Expense", + category: "opex", + order: 40, + values: { p1: 8 }, + }), ], statementDetails: null, categories, - searchQuery: '', - expandedRowKeys: new Set(['operating_expenses']) + searchQuery: "", + expandedRowKeys: new Set(["operating_expenses"]), }); const parent = model.sections[0]?.nodes[0]; - expect(parent?.kind).toBe('surface'); - const childKeys = parent?.kind === 'surface' - ? parent.children.map((node) => node.row.key) - : []; + expect(parent?.kind).toBe("surface"); + const childKeys = + parent?.kind === "surface" + ? parent.children.map((node) => node.row.key) + : []; expect(childKeys).toEqual([ - 'selling_general_and_administrative', - 'research_and_development', - 'other_operating_expense' + "selling_general_and_administrative", + "research_and_development", + "other_operating_expense", ]); }); - it('nests raw detail rows under the matching child surface row', () => { + it("nests raw detail rows under the matching child surface row", () => { const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows: [ - createSurfaceRow({ key: 'operating_expenses', label: 'Operating Expenses', category: 'opex', order: 10, values: { p1: 40 } }), - createSurfaceRow({ key: 'selling_general_and_administrative', label: 'SG&A', category: 'opex', order: 20, values: { p1: 20 } }) + createSurfaceRow({ + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "selling_general_and_administrative", + label: "SG&A", + category: "opex", + order: 20, + values: { p1: 20 }, + }), ], statementDetails: { selling_general_and_administrative: [ - createDetailRow({ key: 'corporate_sga', label: 'Corporate SG&A', parentSurfaceKey: 'selling_general_and_administrative', values: { p1: 20 } }) - ] + createDetailRow({ + key: "corporate_sga", + label: "Corporate SG&A", + parentSurfaceKey: "selling_general_and_administrative", + values: { p1: 20 }, + }), + ], }, categories, - searchQuery: '', - expandedRowKeys: new Set(['operating_expenses', 'selling_general_and_administrative']) + searchQuery: "", + expandedRowKeys: new Set([ + "operating_expenses", + "selling_general_and_administrative", + ]), }); const child = model.sections[0]?.nodes[0]; - expect(child?.kind).toBe('surface'); - const sgaNode = child?.kind === 'surface' ? child.children[0] : null; - expect(sgaNode?.kind).toBe('surface'); - const detailNode = sgaNode?.kind === 'surface' ? sgaNode.children[0] : null; + expect(child?.kind).toBe("surface"); + const sgaNode = child?.kind === "surface" ? child.children[0] : null; + expect(sgaNode?.kind).toBe("surface"); + const detailNode = sgaNode?.kind === "surface" ? sgaNode.children[0] : null; expect(detailNode).toMatchObject({ - kind: 'detail', - row: { key: 'corporate_sga' } + kind: "detail", + row: { key: "corporate_sga" }, }); }); - it('auto-expands the parent chain when search matches a child surface or detail row', () => { - const rows = [ - createSurfaceRow({ key: 'operating_expenses', label: 'Operating Expenses', category: 'opex', order: 10, values: { p1: 40 } }), - createSurfaceRow({ key: 'research_and_development', label: 'Research Expense', category: 'opex', order: 20, values: { p1: 12 } }) - ]; - - const childSearch = buildStatementTree({ - surfaceKind: 'income_statement', - rows, - statementDetails: null, - categories, - searchQuery: 'research', - expandedRowKeys: new Set() - }); - - expect(childSearch.autoExpandedKeys.has('operating_expenses')).toBe(true); - expect(childSearch.sections[0]?.nodes[0]?.kind === 'surface' && childSearch.sections[0]?.nodes[0].expanded).toBe(true); - - const detailSearch = buildStatementTree({ - surfaceKind: 'income_statement', - rows, - statementDetails: { - research_and_development: [ - createDetailRow({ key: 'ai_lab_expense', label: 'AI Lab Expense', parentSurfaceKey: 'research_and_development', values: { p1: 12 } }) - ] - }, - categories, - searchQuery: 'ai lab', - expandedRowKeys: new Set() - }); - - const parent = detailSearch.sections[0]?.nodes[0]; - const child = parent?.kind === 'surface' ? parent.children[0] : null; - expect(parent?.kind === 'surface' && parent.expanded).toBe(true); - expect(child?.kind === 'surface' && child.expanded).toBe(true); - }); - - it('does not throw when legacy surface rows are missing source arrays', () => { - const malformedRow = { - ...createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } }), - sourceConcepts: undefined, - sourceRowKeys: undefined - } as unknown as SurfaceFinancialRow; - - const model = buildStatementTree({ - surfaceKind: 'income_statement', - rows: [malformedRow], - statementDetails: null, - categories: [], - searchQuery: 'revenue', - expandedRowKeys: new Set() - }); - - expect(model.sections[0]?.nodes[0]).toMatchObject({ - kind: 'surface', - row: { key: 'revenue' } - }); - }); - - it('keeps not meaningful rows visible and resolves selections for surface and detail nodes', () => { + it("dedupes raw detail rows that are already covered by a child surface concept", () => { const rows = [ createSurfaceRow({ - key: 'gross_profit', - label: 'Gross Profit', - category: 'profit', - values: { p1: null }, - resolutionMethod: 'not_meaningful', - warningCodes: ['gross_profit_not_meaningful_bank_pack'] - }) + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "research_and_development", + label: "Research Expense", + category: "opex", + order: 20, + values: { p1: 12 }, + sourceConcepts: ["us-gaap:ResearchAndDevelopmentExpense"], + }), ]; - const details = { - gross_profit: [ - createDetailRow({ key: 'gp_unmapped', label: 'Gross Profit residual', parentSurfaceKey: 'gross_profit', values: { p1: null }, residualFlag: true }) - ] + const statementDetails = { + operating_expenses: [ + createDetailRow({ + key: "rnd_raw", + label: "Research And Development Expense", + parentSurfaceKey: "operating_expenses", + conceptKey: "research_and_development_expense", + qname: "us-gaap:ResearchAndDevelopmentExpense", + localName: "ResearchAndDevelopmentExpense", + values: { p1: 12 }, + }), + createDetailRow({ + key: "general_admin", + label: "General And Administrative Expense", + parentSurfaceKey: "operating_expenses", + conceptKey: "general_and_administrative_expense", + qname: "us-gaap:GeneralAndAdministrativeExpense", + localName: "GeneralAndAdministrativeExpense", + values: { p1: 7 }, + }), + ], }; const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows, - statementDetails: details, + statementDetails, + categories, + searchQuery: "", + expandedRowKeys: new Set(["operating_expenses"]), + }); + + const parent = model.sections[0]?.nodes[0]; + expect(parent?.kind).toBe("surface"); + if (parent?.kind !== "surface") { + throw new Error("expected surface node"); + } + + expect(parent.directDetailCount).toBe(1); + expect(parent.children.map((node) => node.row.key)).toEqual([ + "research_and_development", + "general_admin", + ]); + expect(parent.children.some((node) => node.row.key === "rnd_raw")).toBe( + false, + ); + expect(model.totalNodeCount).toBe(3); + }); + + it("returns the deduped detail rows for a selected surface", () => { + const rows = [ + createSurfaceRow({ + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "research_and_development", + label: "Research Expense", + category: "opex", + order: 20, + values: { p1: 12 }, + sourceConcepts: ["us-gaap:ResearchAndDevelopmentExpense"], + }), + ]; + const statementDetails = { + operating_expenses: [ + createDetailRow({ + key: "rnd_raw", + label: "Research And Development Expense", + parentSurfaceKey: "operating_expenses", + conceptKey: "research_and_development_expense", + qname: "us-gaap:ResearchAndDevelopmentExpense", + localName: "ResearchAndDevelopmentExpense", + values: { p1: 12 }, + }), + createDetailRow({ + key: "general_admin", + label: "General And Administrative Expense", + parentSurfaceKey: "operating_expenses", + conceptKey: "general_and_administrative_expense", + qname: "us-gaap:GeneralAndAdministrativeExpense", + localName: "GeneralAndAdministrativeExpense", + values: { p1: 7 }, + }), + ], + }; + + const selection = resolveStatementSelection({ + surfaceKind: "income_statement", + rows, + statementDetails, + selection: { kind: "surface", key: "operating_expenses" }, + }); + + expect(selection).toMatchObject({ + kind: "surface", + detailRows: [{ key: "general_admin" }], + }); + expect( + selection?.kind === "surface" && + selection.detailRows.some((row) => row.key === "rnd_raw"), + ).toBe(false); + }); + + it("auto-expands the parent chain when search matches a child surface or detail row", () => { + const rows = [ + createSurfaceRow({ + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "research_and_development", + label: "Research Expense", + category: "opex", + order: 20, + values: { p1: 12 }, + }), + ]; + + const childSearch = buildStatementTree({ + surfaceKind: "income_statement", + rows, + statementDetails: null, + categories, + searchQuery: "research", + expandedRowKeys: new Set(), + }); + + expect(childSearch.autoExpandedKeys.has("operating_expenses")).toBe(true); + expect( + childSearch.sections[0]?.nodes[0]?.kind === "surface" && + childSearch.sections[0]?.nodes[0].expanded, + ).toBe(true); + + const detailSearch = buildStatementTree({ + surfaceKind: "income_statement", + rows, + statementDetails: { + research_and_development: [ + createDetailRow({ + key: "ai_lab_expense", + label: "AI Lab Expense", + parentSurfaceKey: "research_and_development", + values: { p1: 12 }, + }), + ], + }, + categories, + searchQuery: "ai lab", + expandedRowKeys: new Set(), + }); + + const parent = detailSearch.sections[0]?.nodes[0]; + const child = parent?.kind === "surface" ? parent.children[0] : null; + expect(parent?.kind === "surface" && parent.expanded).toBe(true); + expect(child?.kind === "surface" && child.expanded).toBe(true); + }); + + it("searches hidden duplicate concepts through the mapped child surface without reintroducing the raw detail row", () => { + const rows = [ + createSurfaceRow({ + key: "operating_expenses", + label: "Operating Expenses", + category: "opex", + order: 10, + values: { p1: 40 }, + }), + createSurfaceRow({ + key: "research_and_development", + label: "Research Expense", + category: "opex", + order: 20, + values: { p1: 12 }, + sourceConcepts: ["us-gaap:ResearchAndDevelopmentExpense"], + }), + ]; + + const model = buildStatementTree({ + surfaceKind: "income_statement", + rows, + statementDetails: { + operating_expenses: [ + createDetailRow({ + key: "rnd_raw", + label: "Research And Development Expense", + parentSurfaceKey: "operating_expenses", + conceptKey: "research_and_development_expense", + qname: "us-gaap:ResearchAndDevelopmentExpense", + localName: "ResearchAndDevelopmentExpense", + values: { p1: 12 }, + }), + ], + }, + categories, + searchQuery: "researchanddevelopmentexpense", + expandedRowKeys: new Set(), + }); + + const parent = model.sections[0]?.nodes[0]; + expect(parent?.kind === "surface" && parent.expanded).toBe(true); + expect( + parent?.kind === "surface" && parent.children.map((node) => node.row.key), + ).toEqual(["research_and_development"]); + }); + + it("does not throw when legacy surface rows are missing source arrays", () => { + const malformedRow = { + ...createSurfaceRow({ + key: "revenue", + label: "Revenue", + category: "revenue", + values: { p1: 100 }, + }), + sourceConcepts: undefined, + sourceRowKeys: undefined, + } as unknown as SurfaceFinancialRow; + + const model = buildStatementTree({ + surfaceKind: "income_statement", + rows: [malformedRow], + statementDetails: null, categories: [], - searchQuery: '', - expandedRowKeys: new Set(['gross_profit']) + searchQuery: "revenue", + expandedRowKeys: new Set(), }); expect(model.sections[0]?.nodes[0]).toMatchObject({ - kind: 'surface', - row: { key: 'gross_profit', resolutionMethod: 'not_meaningful' } - }); - - const surfaceSelection = resolveStatementSelection({ - surfaceKind: 'income_statement', - rows, - statementDetails: details, - selection: { kind: 'surface', key: 'gross_profit' } - }); - expect(surfaceSelection?.kind).toBe('surface'); - - const detailSelection = resolveStatementSelection({ - surfaceKind: 'income_statement', - rows, - statementDetails: details, - selection: { kind: 'detail', key: 'gp_unmapped', parentKey: 'gross_profit' } - }); - expect(detailSelection).toMatchObject({ - kind: 'detail', - row: { key: 'gp_unmapped', residualFlag: true } + kind: "surface", + row: { key: "revenue" }, }); }); - it('renders unmapped detail rows in a dedicated residual section and counts them', () => { + it("keeps not meaningful rows visible and resolves selections for surface and detail nodes", () => { + const rows = [ + createSurfaceRow({ + key: "gross_profit", + label: "Gross Profit", + category: "profit", + values: { p1: null }, + resolutionMethod: "not_meaningful", + warningCodes: ["gross_profit_not_meaningful_bank_pack"], + }), + ]; + const details = { + gross_profit: [ + createDetailRow({ + key: "gp_unmapped", + label: "Gross Profit residual", + parentSurfaceKey: "gross_profit", + values: { p1: null }, + residualFlag: true, + }), + ], + }; + const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", + rows, + statementDetails: details, + categories: [], + searchQuery: "", + expandedRowKeys: new Set(["gross_profit"]), + }); + + expect(model.sections[0]?.nodes[0]).toMatchObject({ + kind: "surface", + row: { key: "gross_profit", resolutionMethod: "not_meaningful" }, + }); + + const surfaceSelection = resolveStatementSelection({ + surfaceKind: "income_statement", + rows, + statementDetails: details, + selection: { kind: "surface", key: "gross_profit" }, + }); + expect(surfaceSelection?.kind).toBe("surface"); + + const detailSelection = resolveStatementSelection({ + surfaceKind: "income_statement", + rows, + statementDetails: details, + selection: { + kind: "detail", + key: "gp_unmapped", + parentKey: "gross_profit", + }, + }); + expect(detailSelection).toMatchObject({ + kind: "detail", + row: { key: "gp_unmapped", residualFlag: true }, + }); + }); + + it("renders unmapped detail rows in a dedicated residual section and counts them", () => { + const model = buildStatementTree({ + surfaceKind: "income_statement", rows: [ - createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } }) + createSurfaceRow({ + key: "revenue", + label: "Revenue", + category: "revenue", + values: { p1: 100 }, + }), ], statementDetails: { unmapped: [ createDetailRow({ - key: 'unmapped_other_income', - label: 'Other income residual', - parentSurfaceKey: 'unmapped', + key: "unmapped_other_income", + label: "Other income residual", + parentSurfaceKey: "unmapped", values: { p1: 5 }, - residualFlag: true - }) - ] + residualFlag: true, + }), + ], }, categories: [], - searchQuery: '', - expandedRowKeys: new Set() + searchQuery: "", + expandedRowKeys: new Set(), }); expect(model.sections).toHaveLength(2); expect(model.sections[1]).toMatchObject({ - key: 'unmapped_residual', - label: 'Unmapped / Residual' + key: "unmapped_residual", + label: "Unmapped / Residual", }); expect(model.sections[1]?.nodes[0]).toMatchObject({ - kind: 'detail', - row: { key: 'unmapped_other_income', parentSurfaceKey: 'unmapped' } + kind: "detail", + row: { key: "unmapped_other_income", parentSurfaceKey: "unmapped" }, }); expect(model.visibleNodeCount).toBe(2); expect(model.totalNodeCount).toBe(2); }); - it('matches search and resolves selection for unmapped detail rows without a real parent surface', () => { + it("matches search and resolves selection for unmapped detail rows without a real parent surface", () => { const rows = [ - createSurfaceRow({ key: 'revenue', label: 'Revenue', category: 'revenue', values: { p1: 100 } }) + createSurfaceRow({ + key: "revenue", + label: "Revenue", + category: "revenue", + values: { p1: 100 }, + }), ]; const statementDetails = { unmapped: [ createDetailRow({ - key: 'unmapped_fx_gain', - label: 'FX gain residual', - parentSurfaceKey: 'unmapped', + key: "unmapped_fx_gain", + label: "FX gain residual", + parentSurfaceKey: "unmapped", values: { p1: 2 }, - residualFlag: true - }) - ] + residualFlag: true, + }), + ], }; const model = buildStatementTree({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows, statementDetails, categories: [], - searchQuery: 'fx gain', - expandedRowKeys: new Set() + searchQuery: "fx gain", + expandedRowKeys: new Set(), }); expect(model.sections).toHaveLength(1); expect(model.sections[0]).toMatchObject({ - key: 'unmapped_residual', - label: 'Unmapped / Residual' + key: "unmapped_residual", + label: "Unmapped / Residual", }); expect(model.visibleNodeCount).toBe(1); expect(model.totalNodeCount).toBe(2); const selection = resolveStatementSelection({ - surfaceKind: 'income_statement', + surfaceKind: "income_statement", rows, statementDetails, - selection: { kind: 'detail', key: 'unmapped_fx_gain', parentKey: 'unmapped' } + selection: { + kind: "detail", + key: "unmapped_fx_gain", + parentKey: "unmapped", + }, }); expect(selection).toMatchObject({ - kind: 'detail', - row: { key: 'unmapped_fx_gain', parentSurfaceKey: 'unmapped' }, - parentSurfaceRow: null + kind: "detail", + row: { key: "unmapped_fx_gain", parentSurfaceKey: "unmapped" }, + parentSurfaceRow: null, }); }); }); diff --git a/lib/financials/statement-view-model.ts b/lib/financials/statement-view-model.ts index ce7239c..0609c5a 100644 --- a/lib/financials/statement-view-model.ts +++ b/lib/financials/statement-view-model.ts @@ -10,7 +10,11 @@ const SURFACE_CHILDREN: Partial< Record< Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >, Record > @@ -110,6 +114,8 @@ const SURFACE_CHILDREN: Partial< "other_financing_activities", ], }, + equity_statement: {}, + disclosures: {}, }; export type StatementInspectorSelection = { @@ -151,7 +157,7 @@ export type StatementTreeSection = { nodes: StatementTreeNode[]; }; -export type StatementTreeModel = { +type StatementTreeModel = { sections: StatementTreeSection[]; autoExpandedKeys: Set; visibleNodeCount: number; @@ -184,7 +190,11 @@ const UNMAPPED_SECTION_LABEL = "Unmapped / Residual"; function surfaceConfigForKind( surfaceKind: Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >, ) { return SURFACE_CHILDREN[surfaceKind] ?? {}; @@ -198,6 +208,101 @@ function normalize(value: string) { return value.trim().toLowerCase(); } +function normalizeConceptIdentity(value: string | null | undefined) { + if (!value) { + return null; + } + + const trimmed = value.trim().toLowerCase(); + if (trimmed.length === 0) { + return null; + } + + const withoutNamespace = trimmed.includes(":") + ? (trimmed.split(":").pop() ?? trimmed) + : trimmed; + const normalized = withoutNamespace.replace(/[\s_-]+/g, ""); + + return normalized.length > 0 ? normalized : null; +} + +function surfaceConceptIdentities(row: SurfaceFinancialRow) { + const identities = new Set(); + + const addIdentity = (value: string | null | undefined) => { + const normalized = normalizeConceptIdentity(value); + if (normalized) { + identities.add(normalized); + } + }; + + addIdentity(row.key); + + for (const sourceConcept of row.sourceConcepts ?? []) { + addIdentity(sourceConcept); + } + + for (const sourceRowKey of row.sourceRowKeys ?? []) { + addIdentity(sourceRowKey); + } + + for (const resolvedSourceRowKey of Object.values( + row.resolvedSourceRowKeys ?? {}, + )) { + addIdentity(resolvedSourceRowKey); + } + + return identities; +} + +function detailConceptIdentities(row: DetailFinancialRow) { + const identities = new Set(); + + const addIdentity = (value: string | null | undefined) => { + const normalized = normalizeConceptIdentity(value); + if (normalized) { + identities.add(normalized); + } + }; + + addIdentity(row.key); + addIdentity(row.conceptKey); + addIdentity(row.qname); + addIdentity(row.localName); + + return identities; +} + +function dedupeDetailRowsAgainstChildSurfaces( + detailRows: DetailFinancialRow[], + childSurfaceRows: SurfaceFinancialRow[], +) { + if (detailRows.length === 0 || childSurfaceRows.length === 0) { + return detailRows; + } + + const childConceptIdentities = new Set(); + for (const childSurfaceRow of childSurfaceRows) { + for (const identity of surfaceConceptIdentities(childSurfaceRow)) { + childConceptIdentities.add(identity); + } + } + + if (childConceptIdentities.size === 0) { + return detailRows; + } + + return detailRows.filter((detailRow) => { + for (const identity of detailConceptIdentities(detailRow)) { + if (childConceptIdentities.has(identity)) { + return false; + } + } + + return true; + }); +} + function searchTextForSurface(row: SurfaceFinancialRow) { return [ row.label, @@ -283,7 +388,11 @@ function countNodes(nodes: StatementTreeNode[]) { export function buildStatementTree(input: { surfaceKind: Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >; rows: SurfaceFinancialRow[]; statementDetails: SurfaceDetailMap | null; @@ -316,8 +425,9 @@ export function buildStatementTree(input: { Boolean(candidate), ) .sort(sortSurfaceRows); - const detailRows = [...(input.statementDetails?.[row.key] ?? [])].sort( - sortDetailRows, + const detailRows = dedupeDetailRowsAgainstChildSurfaces( + [...(input.statementDetails?.[row.key] ?? [])].sort(sortDetailRows), + childSurfaceRows, ); const childSurfaceNodes = childSurfaceRows .map((childRow) => buildSurfaceNode(childRow, level + 1)) @@ -389,6 +499,22 @@ export function buildStatementTree(input: { .map((row) => buildSurfaceNode(row, 0)) .filter((node): node is StatementTreeSurfaceNode => Boolean(node)); + const totalVisibleDetailCount = input.rows.reduce((sum, row) => { + const childSurfaceRows = (config[row.key] ?? []) + .map((key) => rowByKey.get(key)) + .filter((candidate): candidate is SurfaceFinancialRow => + Boolean(candidate), + ); + const detailRows = dedupeDetailRowsAgainstChildSurfaces( + [...(input.statementDetails?.[row.key] ?? [])].sort(sortDetailRows), + childSurfaceRows, + ); + + return sum + detailRows.length; + }, 0); + const unmappedDetailCount = + input.statementDetails?.[UNMAPPED_DETAIL_GROUP_KEY]?.length ?? 0; + if (input.categories.length === 0) { const sections: StatementTreeSection[] = rootNodes.length > 0 @@ -415,11 +541,7 @@ export function buildStatementTree(input: { 0, ), totalNodeCount: - input.rows.length + - Object.values(input.statementDetails ?? {}).reduce( - (sum, rows) => sum + rows.length, - 0, - ), + input.rows.length + totalVisibleDetailCount + unmappedDetailCount, }; } @@ -472,18 +594,18 @@ export function buildStatementTree(input: { 0, ), totalNodeCount: - input.rows.length + - Object.values(input.statementDetails ?? {}).reduce( - (sum, rows) => sum + rows.length, - 0, - ), + input.rows.length + totalVisibleDetailCount + unmappedDetailCount, }; } export function resolveStatementSelection(input: { surfaceKind: Extract< FinancialSurfaceKind, - "income_statement" | "balance_sheet" | "cash_flow_statement" + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "equity_statement" + | "disclosures" >; rows: SurfaceFinancialRow[]; statementDetails: SurfaceDetailMap | null; @@ -510,14 +632,16 @@ export function resolveStatementSelection(input: { Boolean(candidate), ) .sort(sortSurfaceRows); + const detailRows = dedupeDetailRowsAgainstChildSurfaces( + [...(input.statementDetails?.[row.key] ?? [])].sort(sortDetailRows), + childSurfaceRows, + ); return { kind: "surface", row, childSurfaceRows, - detailRows: [...(input.statementDetails?.[row.key] ?? [])].sort( - sortDetailRows, - ), + detailRows, }; } diff --git a/lib/graphing/catalog.ts b/lib/graphing/catalog.ts index 66ad4ea..412d4a0 100644 --- a/lib/graphing/catalog.ts +++ b/lib/graphing/catalog.ts @@ -38,10 +38,10 @@ export type GraphingUrlState = { }; export const DEFAULT_GRAPH_TICKERS = ['MSFT', 'AAPL', 'NVDA'] as const; -export const DEFAULT_GRAPH_SURFACE: GraphableFinancialSurfaceKind = 'income_statement'; -export const DEFAULT_GRAPH_CADENCE: FinancialCadence = 'annual'; -export const DEFAULT_GRAPH_CHART: GraphChartKind = 'line'; -export const DEFAULT_GRAPH_SCALE: NumberScaleUnit = 'millions'; +const DEFAULT_GRAPH_SURFACE: GraphableFinancialSurfaceKind = 'income_statement'; +const DEFAULT_GRAPH_CADENCE: FinancialCadence = 'annual'; +const DEFAULT_GRAPH_CHART: GraphChartKind = 'line'; +const DEFAULT_GRAPH_SCALE: NumberScaleUnit = 'millions'; export const GRAPH_SURFACE_LABELS: Record = { income_statement: 'Income Statement', @@ -84,7 +84,7 @@ function buildStatementMetrics( })) satisfies GraphMetricDefinition[]; } -export const GRAPH_METRIC_CATALOG: Record = { +const GRAPH_METRIC_CATALOG: Record = { income_statement: buildStatementMetrics('income_statement', INCOME_STATEMENT_METRIC_DEFINITIONS), balance_sheet: buildStatementMetrics('balance_sheet', BALANCE_SHEET_METRIC_DEFINITIONS), cash_flow_statement: buildStatementMetrics('cash_flow_statement', CASH_FLOW_STATEMENT_METRIC_DEFINITIONS), @@ -99,7 +99,7 @@ export const GRAPH_METRIC_CATALOG: Record = { +const DEFAULT_GRAPH_METRIC_BY_SURFACE: Record = { income_statement: 'revenue', balance_sheet: 'total_assets', cash_flow_statement: 'free_cash_flow', @@ -123,19 +123,19 @@ export function normalizeGraphTickers(value: string | null | undefined) { return [...unique]; } -export function isGraphSurfaceKind(value: string | null | undefined): value is GraphableFinancialSurfaceKind { +function isGraphSurfaceKind(value: string | null | undefined): value is GraphableFinancialSurfaceKind { return GRAPHABLE_FINANCIAL_SURFACES.includes(value as GraphableFinancialSurfaceKind); } -export function isGraphCadence(value: string | null | undefined): value is FinancialCadence { +function isGraphCadence(value: string | null | undefined): value is FinancialCadence { return value === 'annual' || value === 'quarterly' || value === 'ltm'; } -export function isGraphChartKind(value: string | null | undefined): value is GraphChartKind { +function isGraphChartKind(value: string | null | undefined): value is GraphChartKind { return value === 'line' || value === 'bar'; } -export function isNumberScaleUnit(value: string | null | undefined): value is NumberScaleUnit { +function isNumberScaleUnit(value: string | null | undefined): value is NumberScaleUnit { return value === 'thousands' || value === 'millions' || value === 'billions'; } @@ -171,7 +171,7 @@ export function getGraphMetricDefinition( return metricsForSurfaceAndCadence(surface, cadence).find((candidate) => candidate.key === metric) ?? null; } -export function defaultGraphingState(): GraphingUrlState { +function defaultGraphingState(): GraphingUrlState { return { tickers: [...DEFAULT_GRAPH_TICKERS], surface: DEFAULT_GRAPH_SURFACE, @@ -219,7 +219,7 @@ export function serializeGraphingParams(state: GraphingUrlState) { return params.toString(); } -export function withPrimaryGraphTicker(ticker: string | null | undefined) { +function withPrimaryGraphTicker(ticker: string | null | undefined) { const normalized = ticker?.trim().toUpperCase() ?? ''; if (!normalized) { return [...DEFAULT_GRAPH_TICKERS]; diff --git a/lib/graphing/series.test.ts b/lib/graphing/series.test.ts index 1e24d09..8ad5ed2 100644 --- a/lib/graphing/series.test.ts +++ b/lib/graphing/series.test.ts @@ -113,6 +113,10 @@ function createFinancials(input: { kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: [] }, dimensionBreakdown: null diff --git a/lib/graphing/series.ts b/lib/graphing/series.ts index 50494f4..8971ba3 100644 --- a/lib/graphing/series.ts +++ b/lib/graphing/series.ts @@ -55,7 +55,7 @@ export type GraphingChartDatum = Record & { dateMs: number; }; -export type GraphingComparisonData = { +type GraphingComparisonData = { companies: GraphingCompanySeries[]; chartData: GraphingChartDatum[]; latestRows: GraphingLatestValueRow[]; diff --git a/lib/query/options.ts b/lib/query/options.ts index cf0cd49..dfef94b 100644 --- a/lib/query/options.ts +++ b/lib/query/options.ts @@ -171,7 +171,7 @@ export function researchLibraryQueryOptions(input: { }); } -export function researchMemoQueryOptions(ticker: string) { +function researchMemoQueryOptions(ticker: string) { const normalizedTicker = ticker.trim().toUpperCase(); return queryOptions({ @@ -181,7 +181,7 @@ export function researchMemoQueryOptions(ticker: string) { }); } -export function researchPacketQueryOptions(ticker: string) { +function researchPacketQueryOptions(ticker: string) { const normalizedTicker = ticker.trim().toUpperCase(); return queryOptions({ @@ -199,7 +199,7 @@ export function watchlistQueryOptions() { }); } -export function researchJournalQueryOptions(ticker: string) { +function researchJournalQueryOptions(ticker: string) { const normalizedTicker = ticker.trim().toUpperCase(); return queryOptions({ diff --git a/lib/server/ai.ts b/lib/server/ai.ts index 9297f7b..6dff168 100644 --- a/lib/server/ai.ts +++ b/lib/server/ai.ts @@ -147,7 +147,7 @@ export function getAiConfig(options?: GetAiConfigOptions) { return getReportAiConfig(options); } -export function getReportAiConfig(options?: GetAiConfigOptions) { +function getReportAiConfig(options?: GetAiConfigOptions) { const env = options?.env ?? process.env; warnIgnoredZhipuBaseUrl(env, options?.warn ?? console.warn); @@ -167,7 +167,7 @@ export function getExtractionAiConfig(options?: GetAiConfigOptions) { }; } -export function getEmbeddingAiConfig(options?: GetAiConfigOptions) { +function getEmbeddingAiConfig(options?: GetAiConfigOptions) { const env = options?.env ?? process.env; warnIgnoredZhipuBaseUrl(env, options?.warn ?? console.warn); diff --git a/lib/server/api/app.ts b/lib/server/api/app.ts index 77cbe9e..646b84c 100644 --- a/lib/server/api/app.ts +++ b/lib/server/api/app.ts @@ -14,7 +14,8 @@ import type { ResearchMemoConviction, ResearchMemoRating, ResearchMemoSection, - TaskStatus + TaskStatus, + TickerAutomationSource } from '@/lib/types'; import { auth } from '@/lib/auth'; import { requireAuthenticatedSession } from '@/lib/server/auth-session'; @@ -71,6 +72,8 @@ import { upsertWatchlistItemRecord } from '@/lib/server/repos/watchlist'; import { answerSearchQuery, searchKnowledgeBase } from '@/lib/server/search'; +import { shouldQueueTickerAutomation } from '@/lib/server/issuer-overlays'; +import { ensureIssuerOverlayRow } from '@/lib/server/repos/issuer-overlays'; import { enqueueTask, findOrEnqueueTask, @@ -91,6 +94,7 @@ const FINANCIAL_STATEMENT_KINDS: FinancialStatementKind[] = [ 'income', 'balance', 'cash_flow', + 'disclosure', 'equity', 'comprehensive_income' ]; @@ -99,6 +103,8 @@ const FINANCIAL_SURFACES: FinancialSurfaceKind[] = [ 'income_statement', 'balance_sheet', 'cash_flow_statement', + 'equity_statement', + 'disclosures', 'ratios', 'segments_kpis', 'adjusted', @@ -112,6 +118,7 @@ const RESEARCH_ARTIFACT_KINDS: ResearchArtifactKind[] = ['filing', 'ai_report', const RESEARCH_ARTIFACT_SOURCES: ResearchArtifactSource[] = ['system', 'user']; const RESEARCH_MEMO_RATINGS: ResearchMemoRating[] = ['strong_buy', 'buy', 'hold', 'sell']; const RESEARCH_MEMO_CONVICTIONS: ResearchMemoConviction[] = ['low', 'medium', 'high']; +const TICKER_AUTOMATION_SOURCES: TickerAutomationSource[] = ['analysis', 'financials', 'search', 'graphing', 'research']; const RESEARCH_MEMO_SECTIONS: ResearchMemoSection[] = [ 'thesis', 'variant_view', @@ -212,6 +219,10 @@ function surfaceFromLegacyStatement(statement: FinancialStatementKind): Financia return 'balance_sheet'; case 'cash_flow': return 'cash_flow_statement'; + case 'disclosure': + return 'disclosures'; + case 'equity': + return 'equity_statement'; default: return 'income_statement'; } @@ -296,6 +307,12 @@ function asResearchMemoSection(value: unknown) { : undefined; } +function asTickerAutomationSource(value: unknown) { + return TICKER_AUTOMATION_SOURCES.includes(value as TickerAutomationSource) + ? value as TickerAutomationSource + : undefined; +} + function formatLabel(value: string) { return value .split('_') @@ -361,6 +378,42 @@ async function queueAutoFilingSync( } } +async function ensureTickerAutomationTask(input: { + userId: string; + ticker: string; + source: TickerAutomationSource; +}) { + void input.source; + const ticker = input.ticker.trim().toUpperCase(); + await ensureIssuerOverlayRow(ticker); + + if (!(await shouldQueueTickerAutomation(ticker))) { + return { + queued: false, + task: null + }; + } + + const watchlistItem = await getWatchlistItemByTicker(input.userId, ticker); + const task = await findOrEnqueueTask({ + userId: input.userId, + taskType: 'sync_filings', + payload: buildSyncFilingsPayload({ + ticker, + limit: defaultFinancialSyncLimit(), + category: watchlistItem?.category, + tags: watchlistItem?.tags + }), + priority: 89, + resourceKey: `sync_filings:${ticker}` + }); + + return { + queued: true, + task + }; +} + const authHandler = ({ request }: { request: Request }) => auth.handler(request); async function checkWorkflowBackend() { @@ -821,6 +874,45 @@ export const app = new Elysia({ prefix: '/api' }) return Response.json({ insight }); }) + .post('/tickers/ensure', async ({ body }) => { + const { session, response } = await requireAuthenticatedSession(); + if (response) { + return response; + } + + const payload = asRecord(body); + const ticker = typeof payload.ticker === 'string' ? payload.ticker.trim().toUpperCase() : ''; + const source = asTickerAutomationSource(payload.source); + + if (!ticker) { + return jsonError('ticker is required'); + } + + if (!source) { + return jsonError('source is required'); + } + + try { + return Response.json(await ensureTickerAutomationTask({ + userId: session.user.id, + ticker, + source + })); + } catch (error) { + return jsonError(asErrorMessage(error, `Failed to ensure ticker automation for ${ticker}`)); + } + }, { + body: t.Object({ + ticker: t.String({ minLength: 1 }), + source: t.Union([ + t.Literal('analysis'), + t.Literal('financials'), + t.Literal('search'), + t.Literal('graphing'), + t.Literal('research') + ]) + }) + }) .get('/research/workspace', async ({ query }) => { const { session, response } = await requireAuthenticatedSession(); if (response) { @@ -1492,6 +1584,8 @@ export const app = new Elysia({ prefix: '/api' }) t.Literal('income_statement'), t.Literal('balance_sheet'), t.Literal('cash_flow_statement'), + t.Literal('equity_statement'), + t.Literal('disclosures'), t.Literal('ratios'), t.Literal('segments_kpis'), t.Literal('adjusted'), @@ -1506,6 +1600,7 @@ export const app = new Elysia({ prefix: '/api' }) t.Literal('income'), t.Literal('balance'), t.Literal('cash_flow'), + t.Literal('disclosure'), t.Literal('equity'), t.Literal('comprehensive_income') ])), diff --git a/lib/server/api/task-workflow-hybrid.e2e.test.ts b/lib/server/api/task-workflow-hybrid.e2e.test.ts index d5e080e..ffc8781 100644 --- a/lib/server/api/task-workflow-hybrid.e2e.test.ts +++ b/lib/server/api/task-workflow-hybrid.e2e.test.ts @@ -168,6 +168,8 @@ function clearProjectionTables(client: { exec: (query: string) => void }) { client.exec('DELETE FROM portfolio_insight;'); client.exec('DELETE FROM company_overview_cache;'); client.exec('DELETE FROM filing;'); + client.exec('DELETE FROM issuer_overlay;'); + client.exec('DELETE FROM issuer_overlay_revision;'); } function seedFilingRecord(client: Database, input: { @@ -568,6 +570,27 @@ if (process.env.RUN_TASK_WORKFLOW_E2E === '1') { expect(tasks).toHaveLength(1); }); + it('queues ticker automation only once per ticker via the explicit ensure endpoint', async () => { + const first = await jsonRequest('POST', '/api/tickers/ensure', { + ticker: 'NVDA', + source: 'search' + }); + const second = await jsonRequest('POST', '/api/tickers/ensure', { + ticker: 'nvda', + source: 'analysis' + }); + + expect(first.response.status).toBe(200); + expect(second.response.status).toBe(200); + + const firstBody = first.json as { queued: boolean; task: { id: string } | null }; + const secondBody = second.json as { queued: boolean; task: { id: string } | null }; + + expect(firstBody.queued).toBe(true); + expect(secondBody.queued).toBe(true); + expect(secondBody.task?.id).toBe(firstBody.task?.id); + }); + it('lets different tickers queue independent filing sync tasks', async () => { const nvda = await jsonRequest('POST', '/api/filings/sync', { ticker: 'NVDA', limit: 20 }); const msft = await jsonRequest('POST', '/api/filings/sync', { ticker: 'MSFT', limit: 20 }); diff --git a/lib/server/auth-session.ts b/lib/server/auth-session.ts index 4321be3..cfa6de7 100644 --- a/lib/server/auth-session.ts +++ b/lib/server/auth-session.ts @@ -2,7 +2,7 @@ import { headers } from 'next/headers'; import { auth } from '@/lib/auth'; import { asErrorMessage, jsonError } from '@/lib/server/http'; -export type AuthenticatedSession = NonNullable< +type AuthenticatedSession = NonNullable< Awaited> >; @@ -17,7 +17,7 @@ type RequiredSessionResult = ( } ); -export async function getAuthenticatedSession() { +async function getAuthenticatedSession() { const session = await auth.api.getSession({ headers: await headers() }); diff --git a/lib/server/company-analysis.ts b/lib/server/company-analysis.ts index 95f4a95..7a06a93 100644 --- a/lib/server/company-analysis.ts +++ b/lib/server/company-analysis.ts @@ -29,7 +29,7 @@ import { const FINANCIAL_FORMS = new Set(['10-K', '10-Q']); const COMPANY_OVERVIEW_CACHE_TTL_MS = 1000 * 60 * 15; -export type CompanyAnalysisLocalInputs = { +type CompanyAnalysisLocalInputs = { filings: Filing[]; holding: Holding | null; watchlistItem: WatchlistItem | null; @@ -48,7 +48,7 @@ function withFinancialMetricsPolicy(filing: Filing): Filing { }; } -export function buildCompanyAnalysisSourceSignature(input: { +function buildCompanyAnalysisSourceSignature(input: { ticker: string; localInputs: CompanyAnalysisLocalInputs; cacheVersion?: number; @@ -72,7 +72,7 @@ export function buildCompanyAnalysisSourceSignature(input: { return createHash('sha256').update(JSON.stringify(payload)).digest('hex'); } -export function isCompanyOverviewCacheFresh(input: { +function isCompanyOverviewCacheFresh(input: { updatedAt: string; sourceSignature: string; expectedSourceSignature: string; diff --git a/lib/server/db/financial-ingestion-schema.ts b/lib/server/db/financial-ingestion-schema.ts index 0964c4b..397c3cd 100644 --- a/lib/server/db/financial-ingestion-schema.ts +++ b/lib/server/db/financial-ingestion-schema.ts @@ -1,7 +1,7 @@ import type { Database } from 'bun:sqlite'; -export type FinancialSchemaRepairMode = 'auto' | 'check-only' | 'off'; -export type FinancialIngestionHealthMode = 'healthy' | 'repaired' | 'drifted' | 'failed'; +type FinancialSchemaRepairMode = 'auto' | 'check-only' | 'off'; +type FinancialIngestionHealthMode = 'healthy' | 'repaired' | 'drifted' | 'failed'; type CriticalIndexDefinition = { name: string; @@ -16,7 +16,7 @@ type DuplicateRule = { partitionColumns: string[]; }; -export type FinancialIngestionIndexStatus = { +type FinancialIngestionIndexStatus = { name: string; table: string; expectedColumns: string[]; @@ -26,13 +26,13 @@ export type FinancialIngestionIndexStatus = { healthy: boolean; }; -export type FinancialIngestionDuplicateStatus = { +type FinancialIngestionDuplicateStatus = { table: string; duplicateGroups: number; duplicateRows: number; }; -export type FinancialIngestionSchemaReport = { +type FinancialIngestionSchemaReport = { ok: boolean; checkedAt: string; indexes: FinancialIngestionIndexStatus[]; @@ -42,7 +42,7 @@ export type FinancialIngestionSchemaReport = { duplicates: FinancialIngestionDuplicateStatus[]; }; -export type FinancialIngestionSchemaRepairResult = { +type FinancialIngestionSchemaRepairResult = { attempted: boolean; requestedMode: FinancialSchemaRepairMode; missingIndexesBefore: string[]; @@ -56,7 +56,7 @@ export type FinancialIngestionSchemaRepairResult = { reportAfter: FinancialIngestionSchemaReport; }; -export type FinancialIngestionSchemaEnsureResult = { +type FinancialIngestionSchemaEnsureResult = { ok: boolean; mode: FinancialIngestionHealthMode; requestedMode: FinancialSchemaRepairMode; @@ -317,7 +317,7 @@ function createOrRecreateIndex(client: Database, definition: CriticalIndexDefini client.exec(definition.createSql); } -export function repairFinancialIngestionSchema( +function repairFinancialIngestionSchema( client: Database, options: { mode?: FinancialSchemaRepairMode; @@ -496,7 +496,7 @@ export function ensureFinancialIngestionSchemaHealthy( } } -export function isMissingOnConflictConstraintError(error: unknown) { +function isMissingOnConflictConstraintError(error: unknown) { return error instanceof Error && error.message.toLowerCase().includes('on conflict clause does not match any primary key or unique constraint'); } @@ -529,7 +529,7 @@ export async function withFinancialIngestionSchemaRetry(input: { } } -export const __financialIngestionSchemaInternals = { +const __financialIngestionSchemaInternals = { CRITICAL_INDEX_DEFINITIONS, UNIQUE_DUPLICATE_RULES, clearBundleCache, diff --git a/lib/server/db/index.test.ts b/lib/server/db/index.test.ts index b5eeaca..d35acb0 100644 --- a/lib/server/db/index.test.ts +++ b/lib/server/db/index.test.ts @@ -218,6 +218,17 @@ describe("sqlite schema compatibility bootstrap", () => { expect(__dbInternals.hasTable(client, "research_memo")).toBe(true); expect(__dbInternals.hasTable(client, "research_memo_evidence")).toBe(true); expect(__dbInternals.hasTable(client, "company_overview_cache")).toBe(true); + expect(__dbInternals.hasTable(client, "issuer_overlay")).toBe(true); + expect(__dbInternals.hasTable(client, "issuer_overlay_revision")).toBe( + true, + ); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "issuer_overlay_revision_id", + ), + ).toBe(true); __dbInternals.loadSqliteExtensions(client); __dbInternals.ensureSearchVirtualTables(client); diff --git a/lib/server/db/schema.ts b/lib/server/db/schema.ts index 89bd00c..2c59210 100644 --- a/lib/server/db/schema.ts +++ b/lib/server/db/schema.ts @@ -33,6 +33,7 @@ type TaxonomyMetricValidationStatus = | "matched" | "mismatch" | "error"; +type IssuerOverlayStatus = "empty" | "active" | "error"; type CoverageStatus = "backlog" | "active" | "watch" | "archive"; type CoveragePriority = "low" | "medium" | "high"; type ResearchJournalEntryType = "note" | "filing_note" | "status_change"; @@ -65,6 +66,8 @@ type FinancialSurfaceKind = | "income_statement" | "balance_sheet" | "cash_flow_statement" + | "equity_statement" + | "disclosures" | "ratios" | "segments_kpis" | "adjusted" @@ -103,6 +106,44 @@ type FinancialStatementKind = | "equity" | "comprehensive_income"; +export type IssuerOverlayDefinition = { + version: "fiscal-v1"; + ticker: string; + pack: string | null; + mappings: Array<{ + surface_key: string; + statement: FinancialStatementKind; + allowed_source_concepts: string[]; + allowed_authoritative_concepts: string[]; + }>; +}; + +export type IssuerOverlayStats = { + pack: string | null; + sampledSnapshotCount: number; + sampledSnapshotIds: number[]; + acceptedMappingCount: number; + rejectedMappingCount: number; + publishedRevisionNumber: number | null; +}; + +export type IssuerOverlayDiagnostics = { + pack: string | null; + sampledSnapshotIds: number[]; + acceptedMappings: Array<{ + qname: string; + surface_key: string; + statement: FinancialStatementKind; + reason: "authoritative_match" | "local_name_match"; + source_snapshot_ids: number[]; + }>; + rejectedMappings: Array<{ + qname: string; + reason: string; + source_snapshot_ids: number[]; + }>; +}; + type FilingStatementPeriod = { id: string; filingId: number; @@ -208,7 +249,7 @@ type TaxonomySurfaceSnapshotRow = { formulaKey: string | null; hasDimensions: boolean; resolvedSourceRowKeys: Record; - statement?: "income" | "balance" | "cash_flow"; + statement?: "income" | "balance" | "cash_flow" | "equity" | "disclosure"; detailCount?: number; }; @@ -270,6 +311,10 @@ type TaxonomyNormalizationSummary = { kpiRowCount: number; unmappedRowCount: number; materialUnmappedRowCount: number; + residualPrimaryCount: number; + residualDisclosureCount: number; + unsupportedConceptCount: number; + issuerOverlayMatchCount: number; warnings: string[]; }; @@ -640,6 +685,7 @@ export const filingTaxonomySnapshot = sqliteTable( normalization_summary: text("normalization_summary", { mode: "json", }).$type(), + issuer_overlay_revision_id: integer("issuer_overlay_revision_id"), facts_count: integer("facts_count").notNull().default(0), concepts_count: integer("concepts_count").notNull().default(0), dimensions_count: integer("dimensions_count").notNull().default(0), @@ -659,6 +705,65 @@ export const filingTaxonomySnapshot = sqliteTable( }), ); +export const issuerOverlayRevision = sqliteTable( + "issuer_overlay_revision", + { + id: integer("id").primaryKey({ autoIncrement: true }), + ticker: text("ticker").notNull(), + revision_number: integer("revision_number").notNull(), + definition_hash: text("definition_hash").notNull(), + definition_json: text("definition_json", { + mode: "json", + }).$type(), + diagnostics_json: text("diagnostics_json", { + mode: "json", + }).$type(), + source_snapshot_ids: text("source_snapshot_ids", { + mode: "json", + }).$type(), + created_at: text("created_at").notNull(), + }, + (table) => ({ + issuerOverlayRevisionTickerRevisionUnique: uniqueIndex( + "issuer_overlay_revision_ticker_revision_uidx", + ).on(table.ticker, table.revision_number), + issuerOverlayRevisionTickerHashUnique: uniqueIndex( + "issuer_overlay_revision_ticker_hash_uidx", + ).on(table.ticker, table.definition_hash), + issuerOverlayRevisionTickerCreatedIndex: index( + "issuer_overlay_revision_ticker_created_idx", + ).on(table.ticker, table.created_at), + }), +); + +export const issuerOverlay = sqliteTable( + "issuer_overlay", + { + ticker: text("ticker").primaryKey(), + status: text("status") + .$type() + .notNull() + .default("empty"), + active_revision_id: integer("active_revision_id").references( + () => issuerOverlayRevision.id, + { onDelete: "set null" }, + ), + last_built_at: text("last_built_at"), + last_error: text("last_error"), + stats_json: text("stats_json", { + mode: "json", + }).$type(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + issuerOverlayStatusIndex: index("issuer_overlay_status_idx").on( + table.status, + table.updated_at, + ), + }), +); + export const filingTaxonomyContext = sqliteTable( "filing_taxonomy_context", { @@ -1315,6 +1420,8 @@ export const appSchema = { filing, filingStatementSnapshot, filingTaxonomySnapshot, + issuerOverlay, + issuerOverlayRevision, filingTaxonomyAsset, filingTaxonomyConcept, filingTaxonomyFact, diff --git a/lib/server/db/sqlite-schema-compat.ts b/lib/server/db/sqlite-schema-compat.ts index 5a5f75f..0755546 100644 --- a/lib/server/db/sqlite-schema-compat.ts +++ b/lib/server/db/sqlite-schema-compat.ts @@ -35,12 +35,12 @@ export function hasColumn( return rows.some((row) => row.name === columnName); } -export function applySqlFile(client: Database, fileName: string) { +function applySqlFile(client: Database, fileName: string) { const sql = readFileSync(join(process.cwd(), "drizzle", fileName), "utf8"); client.exec(sql); } -export function applyBaseSchemaCompat(client: Database) { +function applyBaseSchemaCompat(client: Database) { const sql = readFileSync( join(process.cwd(), "drizzle", "0000_cold_silver_centurion.sql"), "utf8", @@ -340,6 +340,7 @@ const TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS = [ "kpi_rows", "computed_definitions", "normalization_summary", + "issuer_overlay_revision_id", ] as const; function ensureTaxonomySnapshotCompat(client: Database) { @@ -388,6 +389,10 @@ function ensureTaxonomySnapshotCompat(client: Database) { name: "normalization_summary", sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;", }, + { + name: "issuer_overlay_revision_id", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `issuer_overlay_revision_id` integer REFERENCES `issuer_overlay_revision`(`id`) ON UPDATE no action ON DELETE set null;", + }, ]); for (const columnName of TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS) { @@ -527,6 +532,52 @@ function ensureTaxonomyCompat(client: Database) { ensureTaxonomyFactCompat(client); } +function ensureIssuerOverlaySchema(client: Database) { + if (!hasTable(client, "issuer_overlay_revision")) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`issuer_overlay_revision\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`ticker\` text NOT NULL, + \`revision_number\` integer NOT NULL, + \`definition_hash\` text NOT NULL, + \`definition_json\` text NOT NULL, + \`diagnostics_json\` text, + \`source_snapshot_ids\` text NOT NULL DEFAULT '[]', + \`created_at\` text NOT NULL + ); + `); + } + + if (!hasTable(client, "issuer_overlay")) { + client.exec(` + CREATE TABLE IF NOT EXISTS \`issuer_overlay\` ( + \`ticker\` text PRIMARY KEY NOT NULL, + \`status\` text NOT NULL DEFAULT 'empty', + \`active_revision_id\` integer, + \`last_built_at\` text, + \`last_error\` text, + \`stats_json\` text, + \`created_at\` text NOT NULL, + \`updated_at\` text NOT NULL, + FOREIGN KEY (\`active_revision_id\`) REFERENCES \`issuer_overlay_revision\`(\`id\`) ON UPDATE no action ON DELETE set null + ); + `); + } + + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `issuer_overlay_revision_ticker_revision_uidx` ON `issuer_overlay_revision` (`ticker`, `revision_number`);", + ); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `issuer_overlay_revision_ticker_hash_uidx` ON `issuer_overlay_revision` (`ticker`, `definition_hash`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `issuer_overlay_revision_ticker_created_idx` ON `issuer_overlay_revision` (`ticker`, `created_at`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `issuer_overlay_status_idx` ON `issuer_overlay` (`status`, `updated_at`);", + ); +} + export function ensureLocalSqliteSchema(client: Database) { const missingBaseSchema = [ "filing", @@ -684,6 +735,7 @@ WHERE resource_key IS NOT NULL AND status IN ('queued', 'running');`); if (!hasTable(client, "filing_taxonomy_snapshot")) { applySqlFile(client, "0005_financial_taxonomy_v3.sql"); } + ensureIssuerOverlaySchema(client); ensureTaxonomyCompat(client); if (!hasTable(client, "company_financial_bundle")) { @@ -725,7 +777,7 @@ WHERE resource_key IS NOT NULL AND status IN ('queued', 'running');`); ensureResearchWorkspaceSchema(client); } -export const __sqliteSchemaCompatInternals = { +const __sqliteSchemaCompatInternals = { applyBaseSchemaCompat, applySqlFile, hasColumn, diff --git a/lib/server/financial-statements.ts b/lib/server/financial-statements.ts index ceb1446..b293268 100644 --- a/lib/server/financial-statements.ts +++ b/lib/server/financial-statements.ts @@ -23,7 +23,7 @@ type GetCompanyFinancialStatementsInput = { queuedSync: boolean; }; -export async function getCompanyFinancialStatements( +async function getCompanyFinancialStatements( input: GetCompanyFinancialStatementsInput ): Promise { return await getCompanyFinancials({ @@ -41,7 +41,7 @@ export async function getCompanyFinancialStatements( }); } -export { defaultFinancialSyncLimit }; +; export const __financialStatementsInternals = { defaultFinancialSyncLimit diff --git a/lib/server/financial-taxonomy.ts b/lib/server/financial-taxonomy.ts index 385d1d1..3daffa4 100644 --- a/lib/server/financial-taxonomy.ts +++ b/lib/server/financial-taxonomy.ts @@ -151,11 +151,28 @@ function latestMetrics(snapshots: FilingTaxonomySnapshotRecord[]) { } function defaultDisplayModes(surfaceKind: FinancialSurfaceKind): FinancialDisplayMode[] { + if (surfaceKind === 'equity_statement') { + return ['faithful']; + } + return isStatementSurface(surfaceKind) ? ['standardized', 'faithful'] : ['standardized']; } +function defaultDisplayMode( + surfaceKind: FinancialSurfaceKind, + displayModes = defaultDisplayModes(surfaceKind) +): FinancialDisplayMode { + if (surfaceKind === 'equity_statement') { + return 'faithful'; + } + + return displayModes.includes('standardized') + ? 'standardized' + : displayModes[0] ?? 'standardized'; +} + function rekeyRowsByFilingId; resolvedSourceRowKeys?: Record; @@ -311,6 +328,10 @@ function emptyNormalizationMetadata(): NormalizationMetadata { kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: [] }; } @@ -348,6 +369,22 @@ function buildNormalizationMetadata( (sum, snapshot) => sum + (snapshot.normalization_summary?.materialUnmappedRowCount ?? 0), 0 ), + residualPrimaryCount: snapshots.reduce( + (sum, snapshot) => sum + (snapshot.normalization_summary?.residualPrimaryCount ?? 0), + 0 + ), + residualDisclosureCount: snapshots.reduce( + (sum, snapshot) => sum + (snapshot.normalization_summary?.residualDisclosureCount ?? 0), + 0 + ), + unsupportedConceptCount: snapshots.reduce( + (sum, snapshot) => sum + (snapshot.normalization_summary?.unsupportedConceptCount ?? 0), + 0 + ), + issuerOverlayMatchCount: snapshots.reduce( + (sum, snapshot) => sum + (snapshot.normalization_summary?.issuerOverlayMatchCount ?? 0), + 0 + ), warnings: [...new Set(snapshots.flatMap((snapshot) => snapshot.normalization_summary?.warnings ?? []))] .sort((left, right) => left.localeCompare(right)) }; @@ -568,7 +605,7 @@ function buildLtmDetailRows(input: { detailRows: SurfaceDetailMap; quarterlyPeriods: FinancialStatementPeriod[]; ltmPeriods: FinancialStatementPeriod[]; - statement: Extract; + statement: Extract; }) { const sortedQuarterlyPeriods = [...input.quarterlyPeriods].sort(periodSorter); @@ -586,7 +623,7 @@ function buildLtmDetailRows(input: { const slice = sortedQuarterlyPeriods.slice(anchorIndex - 3, anchorIndex + 1); const sourceValues = slice.map((period) => row.values[period.id] ?? null); - values[ltmPeriod.id] = input.statement === 'balance' + values[ltmPeriod.id] = input.statement === 'balance' || input.statement === 'equity' ? sourceValues[sourceValues.length - 1] ?? null : sourceValues.some((value) => value === null) ? null @@ -606,7 +643,7 @@ function buildLtmDetailRows(input: { } function buildQuarterlyStatementSurfaceRows(input: { - statement: Extract; + statement: Extract; sourcePeriods: FinancialStatementPeriod[]; selectedPeriodIds: Set; faithfulRows: TaxonomyStatementRow[]; @@ -682,6 +719,8 @@ function buildEmptyResponse(input: { nextCursor: string | null; coverageFacts: number; }) { + const displayModes = defaultDisplayModes(input.surfaceKind); + return { company: { ticker: input.ticker, @@ -690,8 +729,8 @@ function buildEmptyResponse(input: { }, surfaceKind: input.surfaceKind, cadence: input.cadence, - displayModes: defaultDisplayModes(input.surfaceKind), - defaultDisplayMode: 'standardized', + displayModes, + defaultDisplayMode: defaultDisplayMode(input.surfaceKind, displayModes), periods: [], statementRows: isStatementSurface(input.surfaceKind) ? { faithful: [], standardized: [] } @@ -728,7 +767,10 @@ function buildEmptyResponse(input: { } async function buildStatementSurfaceBundle(input: { - surfaceKind: Extract; + surfaceKind: Extract< + FinancialSurfaceKind, + 'income_statement' | 'balance_sheet' | 'cash_flow_statement' | 'equity_statement' | 'disclosures' + >; cadence: FinancialCadence; sourcePeriods: FinancialStatementPeriod[]; targetPeriods: FinancialStatementPeriod[]; @@ -752,7 +794,14 @@ async function buildStatementSurfaceBundle(input: { } const statement = surfaceToStatementKind(input.surfaceKind); - if (!statement || (statement !== 'income' && statement !== 'balance' && statement !== 'cash_flow')) { + if ( + !statement || + (statement !== 'income' && + statement !== 'balance' && + statement !== 'cash_flow' && + statement !== 'equity' && + statement !== 'disclosure') + ) { return { rows: [], detailRows: {}, @@ -810,6 +859,16 @@ async function buildStatementSurfaceBundle(input: { return payload; } +async function buildDisclosureSurfaceBundle(input: Omit< + Parameters[0], + 'surfaceKind' +>) { + return await buildStatementSurfaceBundle({ + ...input, + surfaceKind: 'disclosures' + }); +} + async function buildRatioSurfaceBundle(input: { ticker: string; cadence: FinancialCadence; @@ -1039,18 +1098,36 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr const factsForStatement = allFacts.facts.filter((fact) => fact.statement === statement); const factsForStandardization = allFacts.facts; - const standardizedPayload = await buildStatementSurfaceBundle({ - surfaceKind: input.surfaceKind as Extract, - cadence: input.cadence, - sourcePeriods: selection.periods, - targetPeriods: periods, - selectedPeriodIds: selection.selectedPeriodIds, - faithfulRows: baseFaithfulRows, - facts: factsForStandardization, - snapshots: selection.snapshots - }); + const standardizedPayload = input.surfaceKind === 'disclosures' + ? await buildDisclosureSurfaceBundle({ + cadence: input.cadence, + sourcePeriods: selection.periods, + targetPeriods: periods, + selectedPeriodIds: selection.selectedPeriodIds, + faithfulRows: baseFaithfulRows, + facts: factsForStandardization, + snapshots: selection.snapshots + }) + : await buildStatementSurfaceBundle({ + surfaceKind: input.surfaceKind as Extract< + FinancialSurfaceKind, + 'income_statement' | 'balance_sheet' | 'cash_flow_statement' | 'equity_statement' + >, + cadence: input.cadence, + sourcePeriods: selection.periods, + targetPeriods: periods, + selectedPeriodIds: selection.selectedPeriodIds, + faithfulRows: baseFaithfulRows, + facts: factsForStandardization, + snapshots: selection.snapshots + }); const standardizedRows = standardizedPayload.rows; + const statementDisplayModes: FinancialDisplayMode[] = input.surfaceKind === 'equity_statement' + ? (standardizedRows.length > 0 + ? ['faithful', 'standardized'] + : ['faithful']) + : defaultDisplayModes(input.surfaceKind); const rawFacts = input.includeFacts ? await listTaxonomyFactsByTicker({ @@ -1075,8 +1152,8 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr }, surfaceKind: input.surfaceKind, cadence: input.cadence, - displayModes: defaultDisplayModes(input.surfaceKind), - defaultDisplayMode: 'standardized', + displayModes: statementDisplayModes, + defaultDisplayMode: defaultDisplayMode(input.surfaceKind, statementDisplayModes), periods, statementRows: { faithful: faithfulRows, @@ -1187,7 +1264,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr surfaceKind: input.surfaceKind, cadence: input.cadence, displayModes: defaultDisplayModes(input.surfaceKind), - defaultDisplayMode: 'standardized', + defaultDisplayMode: defaultDisplayMode(input.surfaceKind), periods: basePeriods, statementRows: null, statementDetails: null, @@ -1253,7 +1330,7 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr surfaceKind: input.surfaceKind, cadence: input.cadence, displayModes: defaultDisplayModes(input.surfaceKind), - defaultDisplayMode: 'standardized', + defaultDisplayMode: defaultDisplayMode(input.surfaceKind), periods: basePeriods, statementRows: null, statementDetails: null, @@ -1287,11 +1364,11 @@ export async function getCompanyFinancials(input: GetCompanyFinancialsInput): Pr }; } -export async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialsInput) { +async function getCompanyFinancialTaxonomy(input: GetCompanyFinancialsInput) { return await getCompanyFinancials(input); } -export const __financialTaxonomyInternals = { +const __financialTaxonomyInternals = { buildRows, buildDimensionBreakdown, buildNormalizationMetadata, diff --git a/lib/server/financials/bundles.ts b/lib/server/financials/bundles.ts index c98c2be..5f15c61 100644 --- a/lib/server/financials/bundles.ts +++ b/lib/server/financials/bundles.ts @@ -9,7 +9,7 @@ import { } from '@/lib/server/repos/company-financial-bundles'; import type { FilingTaxonomySnapshotRecord } from '@/lib/server/repos/filing-taxonomy'; -export function computeSourceSignature(snapshots: FilingTaxonomySnapshotRecord[]) { +function computeSourceSignature(snapshots: FilingTaxonomySnapshotRecord[]) { return snapshots .map((snapshot) => `${snapshot.id}:${snapshot.updated_at}`) .sort((left, right) => left.localeCompare(right)) diff --git a/lib/server/financials/cadence.ts b/lib/server/financials/cadence.ts index bbf9f08..3174fd2 100644 --- a/lib/server/financials/cadence.ts +++ b/lib/server/financials/cadence.ts @@ -37,7 +37,7 @@ export function periodSorter(left: FinancialStatementPeriod, right: FinancialSta return left.id.localeCompare(right.id); } -export function isInstantPeriod(period: FinancialStatementPeriod) { +function isInstantPeriod(period: FinancialStatementPeriod) { return period.periodStart === null; } @@ -218,7 +218,7 @@ function selectPrimaryPeriodFromSnapshot( candidates.map((period) => [period.id, coverageScore(coverageRows, period.id)]) ); - if (statement === 'balance') { + if (statement === 'balance' || statement === 'equity') { return [...candidates].sort((left, right) => compareBalancePeriods(left, right, rowCoverage))[0] ?? null; } @@ -238,6 +238,10 @@ export function surfaceToStatementKind(surfaceKind: FinancialSurfaceKind): Finan return 'balance'; case 'cash_flow_statement': return 'cash_flow'; + case 'equity_statement': + return 'equity'; + case 'disclosures': + return 'disclosure'; default: return null; } @@ -417,7 +421,7 @@ export function buildLtmFaithfulRows( const sourceValues = slice.map((period) => sourceRow.values[period.id] ?? null); const sourceUnits = slice.map((period) => sourceRow.units[period.id] ?? null).filter((unit): unit is string => unit !== null); - row.values[ltmPeriod.id] = statement === 'balance' + row.values[ltmPeriod.id] = statement === 'balance' || statement === 'equity' ? sourceValues[sourceValues.length - 1] ?? null : aggregateValues(sourceValues); row.units[ltmPeriod.id] = sourceUnits[sourceUnits.length - 1] ?? null; diff --git a/lib/server/financials/kpi-registry.ts b/lib/server/financials/kpi-registry.ts index 3f92c17..825cf30 100644 --- a/lib/server/financials/kpi-registry.ts +++ b/lib/server/financials/kpi-registry.ts @@ -3,7 +3,7 @@ import type { FinancialUnit } from '@/lib/types'; -export type IndustryTemplate = +type IndustryTemplate = | 'internet_platforms' | 'software_saas' | 'semiconductors_industrial_auto'; @@ -80,7 +80,7 @@ const KPI_REGISTRY: RegistryBundle = { tickerDefinitions: {} }; -export function getTickerIndustryTemplate(ticker: string): IndustryTemplate | null { +function getTickerIndustryTemplate(ticker: string): IndustryTemplate | null { return KPI_REGISTRY.tickerTemplates[ticker.trim().toUpperCase()] ?? null; } diff --git a/lib/server/financials/standardize.ts b/lib/server/financials/standardize.ts index d9e1340..f32dd53 100644 --- a/lib/server/financials/standardize.ts +++ b/lib/server/financials/standardize.ts @@ -101,7 +101,7 @@ export function buildDimensionBreakdown( return map.size > 0 ? Object.fromEntries(map.entries()) : null; } -export function cloneStandardizedRows(rows: StandardizedFinancialRow[]) { +function cloneStandardizedRows(rows: StandardizedFinancialRow[]) { return rows.map((row) => ({ ...row, values: { ...row.values }, @@ -116,7 +116,7 @@ export function buildLtmStandardizedRows( quarterlyRows: StandardizedFinancialRow[], quarterlyPeriods: FinancialStatementPeriod[], ltmPeriods: FinancialStatementPeriod[], - statement: Extract + statement: Extract ) { const sortedQuarterlyPeriods = [...quarterlyPeriods].sort((left, right) => { return Date.parse(left.periodEnd ?? left.filingDate) - Date.parse(right.periodEnd ?? right.filingDate); @@ -141,7 +141,7 @@ export function buildLtmStandardizedRows( const slice = sortedQuarterlyPeriods.slice(anchorIndex - 3, anchorIndex + 1); const sourceValues = slice.map((period) => source.values[period.id] ?? null); - row.values[ltmPeriod.id] = statement === 'balance' + row.values[ltmPeriod.id] = statement === 'balance' || statement === 'equity' ? sourceValues[sourceValues.length - 1] ?? null : sumValues(sourceValues); row.resolvedSourceRowKeys[ltmPeriod.id] = source.formulaKey ? null : source.resolvedSourceRowKeys[slice[slice.length - 1]?.id ?? ''] ?? null; diff --git a/lib/server/financials/trend-series.ts b/lib/server/financials/trend-series.ts index a67390e..7090d25 100644 --- a/lib/server/financials/trend-series.ts +++ b/lib/server/financials/trend-series.ts @@ -64,6 +64,12 @@ export function buildTrendSeries(input: { return (input.statementRows ?? []) .filter((row) => row.key === 'operating_cash_flow' || row.key === 'free_cash_flow' || row.key === 'capital_expenditures') .map(toTrendSeriesRow); + case 'equity_statement': + return (input.statementRows ?? []) + .filter((row) => row.key === 'total_equity' || row.key === 'retained_earnings' || row.key === 'common_stock_and_apic') + .map(toTrendSeriesRow); + case 'disclosures': + return []; case 'ratios': return (input.ratioRows ?? []) .filter((row) => row.category === 'margins') diff --git a/lib/server/issuer-overlays.ts b/lib/server/issuer-overlays.ts new file mode 100644 index 0000000..a5613ea --- /dev/null +++ b/lib/server/issuer-overlays.ts @@ -0,0 +1,575 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { + IssuerOverlayDefinition, + IssuerOverlayDiagnostics, + IssuerOverlayStats, +} from "@/lib/server/db/schema"; +import type { + FilingTaxonomyConceptRecord, + FilingTaxonomySnapshotRecord, +} from "@/lib/server/repos/filing-taxonomy"; +import { + listFilingTaxonomyConceptsBySnapshotIds, + listFilingTaxonomySnapshotsByTicker, +} from "@/lib/server/repos/filing-taxonomy"; +import { + ensureIssuerOverlayRow, + getIssuerOverlay, + markIssuerOverlayBuildState, + normalizeIssuerOverlayDefinition, + publishIssuerOverlayRevision, +} from "@/lib/server/repos/issuer-overlays"; + +type SurfaceFile = { + surfaces?: Array<{ + surface_key: string; + statement: string; + allowed_source_concepts?: string[]; + allowed_authoritative_concepts?: string[]; + }>; +}; + +type SurfaceCatalogEntry = { + surface_key: string; + statement: + | "income" + | "balance" + | "cash_flow" + | "disclosure" + | "equity" + | "comprehensive_income"; + sourceConcepts: Set; + authoritativeConcepts: Set; +}; + +type AcceptedCandidate = { + qname: string; + surface_key: string; + statement: SurfaceCatalogEntry["statement"]; + reason: "authoritative_match" | "local_name_match"; + source_snapshot_ids: number[]; +}; + +type RejectedCandidate = { + qname: string; + reason: string; + source_snapshot_ids: number[]; +}; + +function normalizeTicker(ticker: string) { + return ticker.trim().toUpperCase(); +} + +function normalizeLocalName(value: string | null | undefined) { + const normalized = value?.trim() ?? ""; + if (!normalized) { + return null; + } + + const localName = normalized.includes(":") + ? normalized.split(":").pop() ?? normalized + : normalized.includes("#") + ? normalized.split("#").pop() ?? normalized + : normalized; + return localName.trim().toLowerCase() || null; +} + +function taxonomyFilePath(fileName: string) { + return join(process.cwd(), "rust", "taxonomy", "fiscal", "v1", fileName); +} + +function loadOptionalSurfaceFile(fileName: string) { + const path = taxonomyFilePath(fileName); + if (!existsSync(path)) { + return null; + } + + return JSON.parse(readFileSync(path, "utf8")) as SurfaceFile; +} + +function applySurfaceFile( + catalog: Map, + file: SurfaceFile | null, +) { + for (const surface of file?.surfaces ?? []) { + if ( + surface.statement !== "income" && + surface.statement !== "balance" && + surface.statement !== "cash_flow" && + surface.statement !== "disclosure" && + surface.statement !== "equity" && + surface.statement !== "comprehensive_income" + ) { + continue; + } + + const existing = catalog.get(surface.surface_key) ?? { + surface_key: surface.surface_key, + statement: surface.statement, + sourceConcepts: new Set(), + authoritativeConcepts: new Set(), + }; + for (const concept of surface.allowed_source_concepts ?? []) { + existing.sourceConcepts.add(concept); + } + for (const concept of surface.allowed_authoritative_concepts ?? []) { + existing.authoritativeConcepts.add(concept); + } + catalog.set(surface.surface_key, existing); + } +} + +function loadSurfaceCatalog(pack: string | null) { + const normalizedPack = pack?.trim() || "core"; + const catalog = new Map(); + + applySurfaceFile(catalog, loadOptionalSurfaceFile(`${normalizedPack}.surface.json`)); + applySurfaceFile( + catalog, + loadOptionalSurfaceFile(`${normalizedPack}.disclosure.surface.json`), + ); + + if (normalizedPack !== "core") { + applySurfaceFile(catalog, loadOptionalSurfaceFile("core.surface.json")); + applySurfaceFile( + catalog, + loadOptionalSurfaceFile("core.disclosure.surface.json"), + ); + } + + return catalog; +} + +function buildSurfaceIndexes(catalog: Map) { + const authoritative = new Map>(); + const localNames = new Map>(); + + for (const entry of catalog.values()) { + for (const concept of entry.authoritativeConcepts) { + const localName = normalizeLocalName(concept); + if (!localName) { + continue; + } + const surfaces = authoritative.get(localName) ?? new Set(); + surfaces.add(entry.surface_key); + authoritative.set(localName, surfaces); + } + + for (const concept of [...entry.sourceConcepts, ...entry.authoritativeConcepts]) { + const localName = normalizeLocalName(concept); + if (!localName) { + continue; + } + const surfaces = localNames.get(localName) ?? new Set(); + surfaces.add(entry.surface_key); + localNames.set(localName, surfaces); + } + } + + return { authoritative, localNames }; +} + +function selectWorkingPack(snapshots: FilingTaxonomySnapshotRecord[]) { + const counts = new Map(); + for (const snapshot of snapshots) { + const pack = snapshot.fiscal_pack?.trim(); + if (!pack) { + continue; + } + counts.set(pack, (counts.get(pack) ?? 0) + 1); + } + + return ( + [...counts.entries()].sort( + (left, right) => right[1] - left[1] || left[0].localeCompare(right[0]), + )[0]?.[0] ?? + snapshots[0]?.fiscal_pack ?? + "core" + ); +} + +function sourceSnapshotIds(records: FilingTaxonomyConceptRecord[]) { + return [...new Set(records.map((record) => record.snapshot_id))].sort( + (left, right) => left - right, + ); +} + +function chooseCandidate( + records: FilingTaxonomyConceptRecord[], + sampleSnapshots: FilingTaxonomySnapshotRecord[], + catalog: Map, +) { + const statementKinds = [ + ...new Set( + records + .map((record) => record.statement_kind) + .filter((value): value is NonNullable => value !== null), + ), + ]; + if (statementKinds.length !== 1) { + return { + accepted: null, + rejected: { + qname: records[0]?.qname ?? "unknown", + reason: "inconsistent_statement_kind", + source_snapshot_ids: sourceSnapshotIds(records), + } satisfies RejectedCandidate, + }; + } + + const distinctSnapshotIds = sourceSnapshotIds(records); + if (distinctSnapshotIds.length < 2) { + return { + accepted: null, + rejected: { + qname: records[0]?.qname ?? "unknown", + reason: "insufficient_recurrence", + source_snapshot_ids: distinctSnapshotIds, + } satisfies RejectedCandidate, + }; + } + + const catalogIndexes = buildSurfaceIndexes(catalog); + const statementKind = statementKinds[0]; + const authoritativeNames = [ + ...new Set( + records + .map((record) => normalizeLocalName(record.authoritative_concept_key)) + .filter((value): value is string => value !== null), + ), + ]; + + let matchedSurfaceKey: string | null = null; + let reason: AcceptedCandidate["reason"] | null = null; + + if (authoritativeNames.length === 1) { + const candidates = [ + ...(catalogIndexes.authoritative.get(authoritativeNames[0]) ?? new Set()), + ]; + const matchingCandidates = candidates.filter((surfaceKey) => { + return catalog.get(surfaceKey)?.statement === statementKind; + }); + if (matchingCandidates.length === 1) { + matchedSurfaceKey = matchingCandidates[0] ?? null; + reason = "authoritative_match"; + } + } + + if (!matchedSurfaceKey) { + const localNames = [ + ...new Set( + records + .map((record) => normalizeLocalName(record.local_name)) + .filter((value): value is string => value !== null), + ), + ]; + + if (localNames.length === 1) { + const candidates = [ + ...(catalogIndexes.localNames.get(localNames[0]) ?? new Set()), + ]; + const matchingCandidates = candidates.filter((surfaceKey) => { + return catalog.get(surfaceKey)?.statement === statementKind; + }); + if (matchingCandidates.length === 1) { + matchedSurfaceKey = matchingCandidates[0] ?? null; + reason = "local_name_match"; + } + } + } + + if (!matchedSurfaceKey || !reason) { + return { + accepted: null, + rejected: { + qname: records[0]?.qname ?? "unknown", + reason: "no_unique_surface_match", + source_snapshot_ids: distinctSnapshotIds, + } satisfies RejectedCandidate, + }; + } + + const surface = catalog.get(matchedSurfaceKey); + if (!surface) { + return { + accepted: null, + rejected: { + qname: records[0]?.qname ?? "unknown", + reason: "unknown_surface_key", + source_snapshot_ids: distinctSnapshotIds, + } satisfies RejectedCandidate, + }; + } + + const latestTenKSnapshotId = sampleSnapshots.find( + (snapshot) => snapshot.filing_type === "10-K", + )?.id; + if ( + distinctSnapshotIds.length < 2 && + !( + latestTenKSnapshotId !== undefined && + distinctSnapshotIds.includes(latestTenKSnapshotId) + ) + ) { + return { + accepted: null, + rejected: { + qname: records[0]?.qname ?? "unknown", + reason: "promotion_threshold_not_met", + source_snapshot_ids: distinctSnapshotIds, + } satisfies RejectedCandidate, + }; + } + + return { + accepted: { + qname: records[0]?.qname ?? "unknown", + surface_key: surface.surface_key, + statement: surface.statement, + reason, + source_snapshot_ids: distinctSnapshotIds, + } satisfies AcceptedCandidate, + rejected: null, + }; +} + +function mergeOverlayDefinition(input: { + ticker: string; + pack: string | null; + current: IssuerOverlayDefinition | null; + accepted: AcceptedCandidate[]; +}) { + const mappings = new Map< + string, + { + surface_key: string; + statement: IssuerOverlayDefinition["mappings"][number]["statement"]; + allowed_source_concepts: Set; + allowed_authoritative_concepts: Set; + } + >(); + + for (const mapping of input.current?.mappings ?? []) { + const key = `${mapping.statement}:${mapping.surface_key}`; + mappings.set(key, { + surface_key: mapping.surface_key, + statement: mapping.statement, + allowed_source_concepts: new Set(mapping.allowed_source_concepts), + allowed_authoritative_concepts: new Set( + mapping.allowed_authoritative_concepts, + ), + }); + } + + for (const candidate of input.accepted) { + const key = `${candidate.statement}:${candidate.surface_key}`; + const existing = mappings.get(key) ?? { + surface_key: candidate.surface_key, + statement: candidate.statement, + allowed_source_concepts: new Set(), + allowed_authoritative_concepts: new Set(), + }; + existing.allowed_source_concepts.add(candidate.qname); + mappings.set(key, existing); + } + + return normalizeIssuerOverlayDefinition({ + version: "fiscal-v1", + ticker: input.ticker, + pack: input.current?.pack ?? input.pack, + mappings: [...mappings.values()].map((mapping) => ({ + surface_key: mapping.surface_key, + statement: mapping.statement, + allowed_source_concepts: [...mapping.allowed_source_concepts], + allowed_authoritative_concepts: [...mapping.allowed_authoritative_concepts], + })), + }); +} + +export async function shouldQueueTickerAutomation(ticker: string) { + const normalizedTicker = normalizeTicker(ticker); + if (!normalizedTicker) { + return false; + } + + const [overlay, snapshotResult] = await Promise.all([ + getIssuerOverlay(normalizedTicker), + listFilingTaxonomySnapshotsByTicker({ + ticker: normalizedTicker, + window: "all", + filingTypes: ["10-K", "10-Q"], + limit: 5, + }), + ]); + + const latestReadySnapshot = + snapshotResult.snapshots.find((snapshot) => snapshot.parse_status === "ready") ?? + null; + + if (!overlay || !latestReadySnapshot) { + return true; + } + + if (overlay.status === "error") { + return true; + } + + if (overlay.status === "empty") { + return overlay.last_built_at === null; + } + + return ( + overlay.active_revision_id !== latestReadySnapshot.issuer_overlay_revision_id + ); +} + +export async function generateIssuerOverlayForTicker(ticker: string) { + const normalizedTicker = normalizeTicker(ticker); + const currentOverlay = await getIssuerOverlay(normalizedTicker); + const threeYearsAgo = new Date(); + threeYearsAgo.setUTCFullYear(threeYearsAgo.getUTCFullYear() - 3); + + const snapshotResult = await listFilingTaxonomySnapshotsByTicker({ + ticker: normalizedTicker, + window: "all", + filingTypes: ["10-K", "10-Q"], + limit: 24, + }); + const sampleSnapshots = snapshotResult.snapshots + .filter( + (snapshot) => + snapshot.parse_status === "ready" && + Date.parse(snapshot.filing_date) >= threeYearsAgo.getTime(), + ) + .slice(0, 12); + + await ensureIssuerOverlayRow(normalizedTicker); + + if (sampleSnapshots.length === 0) { + const stats: IssuerOverlayStats = { + pack: null, + sampledSnapshotCount: 0, + sampledSnapshotIds: [], + acceptedMappingCount: 0, + rejectedMappingCount: 0, + publishedRevisionNumber: null, + }; + await markIssuerOverlayBuildState({ + ticker: normalizedTicker, + status: currentOverlay?.active_revision_id ? "active" : "empty", + stats, + activeRevisionId: currentOverlay?.active_revision_id ?? null, + }); + return { + published: false, + activeRevisionId: currentOverlay?.active_revision_id ?? null, + sampledSnapshotIds: [], + }; + } + + const pack = selectWorkingPack(sampleSnapshots); + const catalog = loadSurfaceCatalog(pack); + const conceptRows = ( + await listFilingTaxonomyConceptsBySnapshotIds( + sampleSnapshots.map((snapshot) => snapshot.id), + ) + ).filter((record) => { + return ( + record.is_extension && + record.residual_flag && + !record.is_abstract && + record.statement_kind !== null + ); + }); + + const grouped = new Map(); + for (const record of conceptRows) { + const existing = grouped.get(record.qname) ?? []; + existing.push(record); + grouped.set(record.qname, existing); + } + + const acceptedMappings: AcceptedCandidate[] = []; + const rejectedMappings: RejectedCandidate[] = []; + + for (const records of grouped.values()) { + const selection = chooseCandidate(records, sampleSnapshots, catalog); + if (selection.accepted) { + acceptedMappings.push(selection.accepted); + } else if (selection.rejected) { + rejectedMappings.push(selection.rejected); + } + } + + const nextDefinition = + acceptedMappings.length > 0 || currentOverlay?.active_revision?.definition_json + ? mergeOverlayDefinition({ + ticker: normalizedTicker, + pack, + current: currentOverlay?.active_revision?.definition_json ?? null, + accepted: acceptedMappings, + }) + : null; + + const diagnostics: IssuerOverlayDiagnostics = { + pack, + sampledSnapshotIds: sampleSnapshots.map((snapshot) => snapshot.id), + acceptedMappings, + rejectedMappings, + }; + const stats: IssuerOverlayStats = { + pack, + sampledSnapshotCount: sampleSnapshots.length, + sampledSnapshotIds: sampleSnapshots.map((snapshot) => snapshot.id), + acceptedMappingCount: acceptedMappings.length, + rejectedMappingCount: rejectedMappings.length, + publishedRevisionNumber: + currentOverlay?.active_revision?.revision_number ?? null, + }; + + if (!nextDefinition || nextDefinition.mappings.length === 0) { + const overlay = await markIssuerOverlayBuildState({ + ticker: normalizedTicker, + status: currentOverlay?.active_revision_id ? "active" : "empty", + stats, + activeRevisionId: currentOverlay?.active_revision_id ?? null, + }); + return { + published: false, + activeRevisionId: overlay?.active_revision_id ?? null, + sampledSnapshotIds: diagnostics.sampledSnapshotIds, + }; + } + + const published = await publishIssuerOverlayRevision({ + ticker: normalizedTicker, + definition: nextDefinition, + diagnostics, + stats, + }); + stats.publishedRevisionNumber = published.revision.revision_number; + await markIssuerOverlayBuildState({ + ticker: normalizedTicker, + status: "active", + stats, + activeRevisionId: published.revision.id, + }); + + return { + published: published.published, + activeRevisionId: published.revision.id, + sampledSnapshotIds: diagnostics.sampledSnapshotIds, + }; +} + +export async function recordIssuerOverlayBuildFailure( + ticker: string, + error: unknown, +) { + await markIssuerOverlayBuildState({ + ticker, + status: "error", + lastError: error instanceof Error ? error.message : String(error), + }); +} diff --git a/lib/server/prices.ts b/lib/server/prices.ts index 143fa99..2ad1919 100644 --- a/lib/server/prices.ts +++ b/lib/server/prices.ts @@ -3,12 +3,12 @@ const QUOTE_CACHE_TTL_MS = 1000 * 60; const PRICE_HISTORY_CACHE_TTL_MS = 1000 * 60 * 15; const FAILURE_CACHE_TTL_MS = 1000 * 30; -export type QuoteResult = { +type QuoteResult = { value: number | null; stale: boolean; }; -export type PriceHistoryResult = { +type PriceHistoryResult = { value: Array<{ date: string; close: number }> | null; stale: boolean; }; @@ -95,7 +95,7 @@ export async function getQuote(ticker: string): Promise { } } -export async function getQuoteOrNull(ticker: string): Promise { +async function getQuoteOrNull(ticker: string): Promise { const result = await getQuote(ticker); return result.value; } diff --git a/lib/server/recent-developments.ts b/lib/server/recent-developments.ts index a72cc60..408121a 100644 --- a/lib/server/recent-developments.ts +++ b/lib/server/recent-developments.ts @@ -1,12 +1,12 @@ import { format } from 'date-fns'; import type { Filing, RecentDevelopmentItem, RecentDevelopments } from '@/lib/types'; -export type RecentDevelopmentSourceContext = { +type RecentDevelopmentSourceContext = { filings: Filing[]; now?: Date; }; -export type RecentDevelopmentSource = { +type RecentDevelopmentSource = { name: string; fetch: (ticker: string, context: RecentDevelopmentSourceContext) => Promise; }; @@ -104,8 +104,8 @@ export const secFilingsDevelopmentSource: RecentDevelopmentSource = { } }; -export const yahooDevelopmentSource: RecentDevelopmentSource | null = null; -export const investorRelationsRssSource: RecentDevelopmentSource | null = null; +const yahooDevelopmentSource: RecentDevelopmentSource | null = null; +const investorRelationsRssSource: RecentDevelopmentSource | null = null; export async function getRecentDevelopments( ticker: string, diff --git a/lib/server/repos/company-financial-bundles.ts b/lib/server/repos/company-financial-bundles.ts index 4c0e6da..38c575b 100644 --- a/lib/server/repos/company-financial-bundles.ts +++ b/lib/server/repos/company-financial-bundles.ts @@ -9,7 +9,7 @@ import { companyFinancialBundle } from '@/lib/server/db/schema'; export const CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION = 15; -export type CompanyFinancialBundleRecord = { +type CompanyFinancialBundleRecord = { id: number; ticker: string; surface_kind: FinancialSurfaceKind; @@ -107,6 +107,6 @@ export async function deleteCompanyFinancialBundlesForTicker(ticker: string) { .where(eq(companyFinancialBundle.ticker, ticker.trim().toUpperCase())); } -export const __companyFinancialBundlesInternals = { +const __companyFinancialBundlesInternals = { BUNDLE_VERSION: CURRENT_COMPANY_FINANCIAL_BUNDLE_VERSION }; diff --git a/lib/server/repos/company-overview-cache.ts b/lib/server/repos/company-overview-cache.ts index 3707d7d..786e6bb 100644 --- a/lib/server/repos/company-overview-cache.ts +++ b/lib/server/repos/company-overview-cache.ts @@ -6,7 +6,7 @@ import { companyOverviewCache, schema } from '@/lib/server/db/schema'; export const CURRENT_COMPANY_OVERVIEW_CACHE_VERSION = 1; -export type CompanyOverviewCacheRecord = { +type CompanyOverviewCacheRecord = { id: number; user_id: string; ticker: string; @@ -86,7 +86,7 @@ export async function upsertCompanyOverviewCache(input: { return toRecord(saved); } -export async function deleteCompanyOverviewCache(input: { userId: string; ticker: string }) { +async function deleteCompanyOverviewCache(input: { userId: string; ticker: string }) { const normalizedTicker = input.ticker.trim().toUpperCase(); return await getDb() @@ -97,6 +97,6 @@ export async function deleteCompanyOverviewCache(input: { userId: string; ticker )); } -export const __companyOverviewCacheInternals = { +const __companyOverviewCacheInternals = { CACHE_VERSION: CURRENT_COMPANY_OVERVIEW_CACHE_VERSION }; diff --git a/lib/server/repos/filing-statements.ts b/lib/server/repos/filing-statements.ts index 115be68..45a876e 100644 --- a/lib/server/repos/filing-statements.ts +++ b/lib/server/repos/filing-statements.ts @@ -80,7 +80,7 @@ export type DimensionStatementBundle = { statements: Record; }; -export type FilingStatementSnapshotRecord = { +type FilingStatementSnapshotRecord = { id: number; filing_id: number; ticker: string; @@ -97,7 +97,7 @@ export type FilingStatementSnapshotRecord = { updated_at: string; }; -export type UpsertFilingStatementSnapshotInput = { +type UpsertFilingStatementSnapshotInput = { filing_id: number; ticker: string; filing_date: string; @@ -191,7 +191,7 @@ export async function upsertFilingStatementSnapshot( return toSnapshotRecord(saved); } -export async function listFilingStatementSnapshotsByTicker(input: { +async function listFilingStatementSnapshotsByTicker(input: { ticker: string; window: "10y" | "all"; limit?: number; @@ -235,7 +235,7 @@ export async function listFilingStatementSnapshotsByTicker(input: { }; } -export async function countFilingStatementSnapshotStatuses(ticker: string) { +async function countFilingStatementSnapshotStatuses(ticker: string) { const rows = await db .select({ status: filingStatementSnapshot.parse_status, diff --git a/lib/server/repos/filing-taxonomy.test.ts b/lib/server/repos/filing-taxonomy.test.ts index 041cb61..ea78690 100644 --- a/lib/server/repos/filing-taxonomy.test.ts +++ b/lib/server/repos/filing-taxonomy.test.ts @@ -149,6 +149,10 @@ describe("filing taxonomy snapshot normalization", () => { kpi_row_count: 1, unmapped_row_count: 0, material_unmapped_row_count: 0, + residual_primary_count: 0, + residual_disclosure_count: 0, + unsupported_concept_count: 0, + issuer_overlay_match_count: 0, warnings: ["legacy_warning"], }, facts_count: 3, @@ -223,6 +227,10 @@ describe("filing taxonomy snapshot normalization", () => { kpiRowCount: 1, unmappedRowCount: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: ["legacy_warning"], }); }); @@ -338,6 +346,10 @@ describe("filing taxonomy snapshot normalization", () => { kpiRowCount: 0, unmapped_row_count: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: [], }, }); @@ -355,6 +367,10 @@ describe("filing taxonomy snapshot normalization", () => { kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, warnings: [], }); expect(normalized.computed_definitions).toEqual([ diff --git a/lib/server/repos/filing-taxonomy.ts b/lib/server/repos/filing-taxonomy.ts index a4149eb..0f66984 100644 --- a/lib/server/repos/filing-taxonomy.ts +++ b/lib/server/repos/filing-taxonomy.ts @@ -73,6 +73,7 @@ export type FilingTaxonomySnapshotRecord = { derived_metrics: Filing["metrics"]; validation_result: MetricValidationResult | null; normalization_summary: NormalizationSummary | null; + issuer_overlay_revision_id: number | null; facts_count: number; concepts_count: number; dimensions_count: number; @@ -80,7 +81,7 @@ export type FilingTaxonomySnapshotRecord = { updated_at: string; }; -export type FilingTaxonomyContextRecord = { +type FilingTaxonomyContextRecord = { id: number; snapshot_id: number; context_id: string; @@ -94,7 +95,7 @@ export type FilingTaxonomyContextRecord = { created_at: string; }; -export type FilingTaxonomyAssetRecord = { +type FilingTaxonomyAssetRecord = { id: number; snapshot_id: number; asset_type: FilingTaxonomyAssetType; @@ -133,7 +134,7 @@ export type FilingTaxonomyConceptRecord = { created_at: string; }; -export type FilingTaxonomyFactRecord = { +type FilingTaxonomyFactRecord = { id: number; snapshot_id: number; concept_key: string; @@ -164,7 +165,7 @@ export type FilingTaxonomyFactRecord = { created_at: string; }; -export type FilingTaxonomyMetricValidationRecord = { +type FilingTaxonomyMetricValidationRecord = { id: number; snapshot_id: number; metric_key: keyof NonNullable; @@ -182,7 +183,7 @@ export type FilingTaxonomyMetricValidationRecord = { updated_at: string; }; -export type UpsertFilingTaxonomySnapshotInput = { +type UpsertFilingTaxonomySnapshotInput = { filing_id: number; ticker: string; filing_date: string; @@ -204,6 +205,7 @@ export type UpsertFilingTaxonomySnapshotInput = { derived_metrics: Filing["metrics"]; validation_result: MetricValidationResult | null; normalization_summary: NormalizationSummary | null; + issuer_overlay_revision_id?: number | null; facts_count: number; concepts_count: number; dimensions_count: number; @@ -294,6 +296,7 @@ const FINANCIAL_STATEMENT_KINDS = [ "income", "balance", "cash_flow", + "disclosure", "equity", "comprehensive_income", ] as const satisfies FinancialStatementKind[]; @@ -351,6 +354,7 @@ function asStatementKind(value: unknown): FinancialStatementKind | null { return value === "income" || value === "balance" || value === "cash_flow" || + value === "disclosure" || value === "equity" || value === "comprehensive_income" ? value @@ -576,7 +580,9 @@ function normalizeSurfaceRows( if ( normalizedStatement === "income" || normalizedStatement === "balance" || - normalizedStatement === "cash_flow" + normalizedStatement === "cash_flow" || + normalizedStatement === "equity" || + normalizedStatement === "disclosure" ) { normalizedRow.statement = normalizedStatement; } @@ -856,6 +862,17 @@ function normalizeNormalizationSummary(value: unknown) { asNumber( row.materialUnmappedRowCount ?? row.material_unmapped_row_count, ) ?? 0, + residualPrimaryCount: + asNumber(row.residualPrimaryCount ?? row.residual_primary_count) ?? 0, + residualDisclosureCount: + asNumber(row.residualDisclosureCount ?? row.residual_disclosure_count) ?? + 0, + unsupportedConceptCount: + asNumber(row.unsupportedConceptCount ?? row.unsupported_concept_count) ?? + 0, + issuerOverlayMatchCount: + asNumber(row.issuerOverlayMatchCount ?? row.issuer_overlay_match_count) ?? + 0, warnings: normalizeStringArray(row.warnings), } satisfies NormalizationSummary; } @@ -962,6 +979,7 @@ function toSnapshotRecord( derived_metrics: row.derived_metrics ?? null, validation_result: row.validation_result ?? null, normalization_summary: normalized.normalization_summary, + issuer_overlay_revision_id: row.issuer_overlay_revision_id ?? null, facts_count: row.facts_count, concepts_count: row.concepts_count, dimensions_count: row.dimensions_count, @@ -1107,7 +1125,7 @@ export async function getFilingTaxonomySnapshotByFilingId(filingId: number) { return row ? toSnapshotRecord(row) : null; } -export async function listFilingTaxonomyAssets(snapshotId: number) { +async function listFilingTaxonomyAssets(snapshotId: number) { const rows = await db .select() .from(filingTaxonomyAsset) @@ -1117,7 +1135,7 @@ export async function listFilingTaxonomyAssets(snapshotId: number) { return rows.map(toAssetRecord); } -export async function listFilingTaxonomyContexts(snapshotId: number) { +async function listFilingTaxonomyContexts(snapshotId: number) { const rows = await db .select() .from(filingTaxonomyContext) @@ -1127,7 +1145,7 @@ export async function listFilingTaxonomyContexts(snapshotId: number) { return rows.map(toContextRecord); } -export async function listFilingTaxonomyConcepts(snapshotId: number) { +async function listFilingTaxonomyConcepts(snapshotId: number) { const rows = await db .select() .from(filingTaxonomyConcept) @@ -1137,7 +1155,7 @@ export async function listFilingTaxonomyConcepts(snapshotId: number) { return rows.map(toConceptRecord); } -export async function listFilingTaxonomyFacts(snapshotId: number) { +async function listFilingTaxonomyFacts(snapshotId: number) { const rows = await db .select() .from(filingTaxonomyFact) @@ -1147,7 +1165,7 @@ export async function listFilingTaxonomyFacts(snapshotId: number) { return rows.map(toFactRecord); } -export async function listFilingTaxonomyMetricValidations(snapshotId: number) { +async function listFilingTaxonomyMetricValidations(snapshotId: number) { const rows = await db .select() .from(filingTaxonomyMetricValidation) @@ -1188,6 +1206,7 @@ export async function upsertFilingTaxonomySnapshot( derived_metrics: input.derived_metrics, validation_result: input.validation_result, normalization_summary: normalized.normalization_summary, + issuer_overlay_revision_id: input.issuer_overlay_revision_id ?? null, facts_count: input.facts_count, concepts_count: input.concepts_count, dimensions_count: input.dimensions_count, @@ -1217,6 +1236,7 @@ export async function upsertFilingTaxonomySnapshot( derived_metrics: input.derived_metrics, validation_result: input.validation_result, normalization_summary: normalized.normalization_summary, + issuer_overlay_revision_id: input.issuer_overlay_revision_id ?? null, facts_count: input.facts_count, concepts_count: input.concepts_count, dimensions_count: input.dimensions_count, @@ -1579,7 +1599,7 @@ export async function listTaxonomyFactsByTicker(input: { }; } -export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) { +async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) { if (snapshotIds.length === 0) { return []; } @@ -1593,6 +1613,22 @@ export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) { return rows.map(toAssetRecord); } +export async function listFilingTaxonomyConceptsBySnapshotIds( + snapshotIds: number[], +) { + if (snapshotIds.length === 0) { + return []; + } + + const rows = await db + .select() + .from(filingTaxonomyConcept) + .where(inArray(filingTaxonomyConcept.snapshot_id, snapshotIds)) + .orderBy(desc(filingTaxonomyConcept.id)); + + return rows.map(toConceptRecord); +} + export const __filingTaxonomyInternals = { normalizeFilingTaxonomySnapshotPayload, toSnapshotRecord, diff --git a/lib/server/repos/issuer-overlays.test.ts b/lib/server/repos/issuer-overlays.test.ts new file mode 100644 index 0000000..083eb21 --- /dev/null +++ b/lib/server/repos/issuer-overlays.test.ts @@ -0,0 +1,146 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; +import { __dbInternals } from "@/lib/server/db"; +import type { + IssuerOverlayDefinition, + IssuerOverlayDiagnostics, + IssuerOverlayStats, +} from "@/lib/server/db/schema"; + +let tempDir: string | null = null; +let sqliteClient: Database | null = null; +let overlayRepo: typeof import("./issuer-overlays") | null = null; + +function resetDbSingletons() { + const globalState = globalThis as typeof globalThis & { + __fiscalSqliteClient?: Database; + __fiscalDrizzleDb?: unknown; + __financialIngestionSchemaStatus?: unknown; + }; + + globalState.__fiscalSqliteClient?.close(); + globalState.__fiscalSqliteClient = undefined; + globalState.__fiscalDrizzleDb = undefined; + globalState.__financialIngestionSchemaStatus = undefined; +} + +function sampleDefinition(): IssuerOverlayDefinition { + return { + version: "fiscal-v1", + ticker: "AAPL", + pack: "core", + mappings: [ + { + surface_key: "revenue", + statement: "income", + allowed_source_concepts: ["aapl:ServicesRevenue", "aapl:ProductRevenue"], + allowed_authoritative_concepts: [], + }, + ], + }; +} + +function sampleDiagnostics(): IssuerOverlayDiagnostics { + return { + pack: "core", + sampledSnapshotIds: [11, 12], + acceptedMappings: [ + { + qname: "aapl:ServicesRevenue", + surface_key: "revenue", + statement: "income", + reason: "local_name_match", + source_snapshot_ids: [11, 12], + }, + ], + rejectedMappings: [], + }; +} + +function sampleStats(): IssuerOverlayStats { + return { + pack: "core", + sampledSnapshotCount: 2, + sampledSnapshotIds: [11, 12], + acceptedMappingCount: 1, + rejectedMappingCount: 0, + publishedRevisionNumber: null, + }; +} + +describe("issuer overlay repo", () => { + beforeAll(async () => { + tempDir = mkdtempSync(join(tmpdir(), "fiscal-issuer-overlay-")); + const env = process.env as Record; + env.DATABASE_URL = `file:${join(tempDir, "repo.sqlite")}`; + env.NODE_ENV = "test"; + + resetDbSingletons(); + + sqliteClient = new Database(join(tempDir, "repo.sqlite"), { create: true }); + sqliteClient.exec("PRAGMA foreign_keys = ON;"); + __dbInternals.ensureLocalSqliteSchema(sqliteClient); + + const globalState = globalThis as typeof globalThis & { + __fiscalSqliteClient?: Database; + __fiscalDrizzleDb?: unknown; + }; + globalState.__fiscalSqliteClient = sqliteClient; + globalState.__fiscalDrizzleDb = undefined; + + overlayRepo = await import("./issuer-overlays"); + }); + + afterAll(() => { + sqliteClient?.close(); + resetDbSingletons(); + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + beforeEach(() => { + sqliteClient?.exec("DELETE FROM issuer_overlay;"); + sqliteClient?.exec("DELETE FROM issuer_overlay_revision;"); + }); + + it("creates an empty overlay row on ensure", async () => { + if (!overlayRepo) { + throw new Error("overlay repo not initialized"); + } + + const overlay = await overlayRepo.ensureIssuerOverlayRow("aapl"); + expect(overlay?.ticker).toBe("AAPL"); + expect(overlay?.status).toBe("empty"); + expect(overlay?.active_revision).toBeNull(); + }); + + it("publishes and deduplicates overlay revisions by content hash", async () => { + if (!overlayRepo) { + throw new Error("overlay repo not initialized"); + } + + const first = await overlayRepo.publishIssuerOverlayRevision({ + ticker: "AAPL", + definition: sampleDefinition(), + diagnostics: sampleDiagnostics(), + stats: sampleStats(), + }); + const second = await overlayRepo.publishIssuerOverlayRevision({ + ticker: "AAPL", + definition: sampleDefinition(), + diagnostics: sampleDiagnostics(), + stats: sampleStats(), + }); + const overlay = await overlayRepo.getIssuerOverlay("AAPL"); + const revisions = await overlayRepo.listIssuerOverlayRevisions("AAPL"); + + expect(first.published).toBe(true); + expect(second.published).toBe(false); + expect(overlay?.active_revision?.id).toBe(first.revision.id); + expect(revisions).toHaveLength(1); + }); +}); diff --git a/lib/server/repos/issuer-overlays.ts b/lib/server/repos/issuer-overlays.ts new file mode 100644 index 0000000..50acbeb --- /dev/null +++ b/lib/server/repos/issuer-overlays.ts @@ -0,0 +1,331 @@ +import { createHash } from "node:crypto"; +import { and, desc, eq, max } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/bun-sqlite"; +import { getSqliteClient } from "@/lib/server/db"; +import { + issuerOverlay, + issuerOverlayRevision, + schema, + type IssuerOverlayDefinition, + type IssuerOverlayDiagnostics, + type IssuerOverlayStats, +} from "@/lib/server/db/schema"; + +export type IssuerOverlayRevisionRecord = { + id: number; + ticker: string; + revision_number: number; + definition_hash: string; + definition_json: IssuerOverlayDefinition; + diagnostics_json: IssuerOverlayDiagnostics | null; + source_snapshot_ids: number[]; + created_at: string; +}; + +export type IssuerOverlayRecord = { + ticker: string; + status: "empty" | "active" | "error"; + active_revision_id: number | null; + last_built_at: string | null; + last_error: string | null; + stats_json: IssuerOverlayStats | null; + created_at: string; + updated_at: string; + active_revision: IssuerOverlayRevisionRecord | null; +}; + +function getDb() { + return drizzle(getSqliteClient(), { schema }); +} + +function normalizeTicker(ticker: string) { + return ticker.trim().toUpperCase(); +} + +function uniqueSorted(values: string[]) { + return [...new Set(values.filter((value) => value.trim().length > 0))].sort( + (left, right) => left.localeCompare(right), + ); +} + +export function normalizeIssuerOverlayDefinition( + input: IssuerOverlayDefinition, +): IssuerOverlayDefinition { + return { + version: "fiscal-v1", + ticker: normalizeTicker(input.ticker), + pack: input.pack?.trim() ? input.pack.trim() : null, + mappings: [...input.mappings] + .map((mapping) => ({ + surface_key: mapping.surface_key, + statement: mapping.statement, + allowed_source_concepts: uniqueSorted(mapping.allowed_source_concepts), + allowed_authoritative_concepts: uniqueSorted( + mapping.allowed_authoritative_concepts, + ), + })) + .filter( + (mapping) => + mapping.allowed_source_concepts.length > 0 || + mapping.allowed_authoritative_concepts.length > 0, + ) + .sort((left, right) => { + return ( + left.statement.localeCompare(right.statement) || + left.surface_key.localeCompare(right.surface_key) + ); + }), + }; +} + +function definitionHash(definition: IssuerOverlayDefinition) { + return createHash("sha256") + .update(JSON.stringify(normalizeIssuerOverlayDefinition(definition))) + .digest("hex"); +} + +function toRevisionRecord( + row: typeof issuerOverlayRevision.$inferSelect, +): IssuerOverlayRevisionRecord { + const definition = row.definition_json; + if (!definition) { + throw new Error( + `Issuer overlay revision ${row.id} is missing definition_json`, + ); + } + + return { + id: row.id, + ticker: row.ticker, + revision_number: row.revision_number, + definition_hash: row.definition_hash, + definition_json: normalizeIssuerOverlayDefinition(definition), + diagnostics_json: row.diagnostics_json ?? null, + source_snapshot_ids: row.source_snapshot_ids ?? [], + created_at: row.created_at, + }; +} + +async function getRevisionById(id: number | null) { + if (!id) { + return null; + } + + const [row] = await getDb() + .select() + .from(issuerOverlayRevision) + .where(eq(issuerOverlayRevision.id, id)) + .limit(1); + + return row ? toRevisionRecord(row) : null; +} + +export async function getIssuerOverlay(ticker: string) { + const normalizedTicker = normalizeTicker(ticker); + if (!normalizedTicker) { + return null; + } + + const [row] = await getDb() + .select() + .from(issuerOverlay) + .where(eq(issuerOverlay.ticker, normalizedTicker)) + .limit(1); + + if (!row) { + return null; + } + + return { + ticker: row.ticker, + status: row.status, + active_revision_id: row.active_revision_id ?? null, + last_built_at: row.last_built_at, + last_error: row.last_error, + stats_json: row.stats_json ?? null, + created_at: row.created_at, + updated_at: row.updated_at, + active_revision: await getRevisionById(row.active_revision_id ?? null), + } satisfies IssuerOverlayRecord; +} + +export async function ensureIssuerOverlayRow(ticker: string) { + const normalizedTicker = normalizeTicker(ticker); + if (!normalizedTicker) { + return null; + } + + const now = new Date().toISOString(); + await getDb() + .insert(issuerOverlay) + .values({ + ticker: normalizedTicker, + status: "empty", + active_revision_id: null, + last_built_at: null, + last_error: null, + stats_json: null, + created_at: now, + updated_at: now, + }) + .onConflictDoNothing(); + + return await getIssuerOverlay(normalizedTicker); +} + +export async function markIssuerOverlayBuildState(input: { + ticker: string; + status: "empty" | "active" | "error"; + lastError?: string | null; + stats?: IssuerOverlayStats | null; + activeRevisionId?: number | null; +}) { + const normalizedTicker = normalizeTicker(input.ticker); + const now = new Date().toISOString(); + await ensureIssuerOverlayRow(normalizedTicker); + + await getDb() + .update(issuerOverlay) + .set({ + status: input.status, + last_error: input.lastError ?? null, + last_built_at: now, + stats_json: input.stats ?? null, + active_revision_id: + input.activeRevisionId === undefined ? undefined : input.activeRevisionId, + updated_at: now, + }) + .where(eq(issuerOverlay.ticker, normalizedTicker)); + + return await getIssuerOverlay(normalizedTicker); +} + +export async function getActiveIssuerOverlayDefinition(ticker: string) { + const overlay = await getIssuerOverlay(ticker); + return overlay?.active_revision?.definition_json ?? null; +} + +export async function publishIssuerOverlayRevision(input: { + ticker: string; + definition: IssuerOverlayDefinition; + diagnostics: IssuerOverlayDiagnostics; + stats: IssuerOverlayStats; +}) { + const normalizedTicker = normalizeTicker(input.ticker); + const normalizedDefinition = normalizeIssuerOverlayDefinition({ + ...input.definition, + ticker: normalizedTicker, + }); + const hash = definitionHash(normalizedDefinition); + const now = new Date().toISOString(); + + return await getDb().transaction(async (tx) => { + const [currentOverlay] = await tx + .select() + .from(issuerOverlay) + .where(eq(issuerOverlay.ticker, normalizedTicker)) + .limit(1); + + if (!currentOverlay) { + await tx.insert(issuerOverlay).values({ + ticker: normalizedTicker, + status: "empty", + active_revision_id: null, + last_built_at: now, + last_error: null, + stats_json: null, + created_at: now, + updated_at: now, + }); + } + + const [existingRevision] = await tx + .select() + .from(issuerOverlayRevision) + .where( + and( + eq(issuerOverlayRevision.ticker, normalizedTicker), + eq(issuerOverlayRevision.definition_hash, hash), + ), + ) + .limit(1); + + const currentActiveRevisionId = currentOverlay?.active_revision_id ?? null; + if (existingRevision) { + await tx + .update(issuerOverlay) + .set({ + status: + normalizedDefinition.mappings.length > 0 ? "active" : "empty", + active_revision_id: + normalizedDefinition.mappings.length > 0 + ? existingRevision.id + : currentActiveRevisionId, + last_built_at: now, + last_error: null, + stats_json: input.stats, + updated_at: now, + }) + .where(eq(issuerOverlay.ticker, normalizedTicker)); + + return { + published: + normalizedDefinition.mappings.length > 0 && + currentActiveRevisionId !== existingRevision.id, + revision: toRevisionRecord(existingRevision), + }; + } + + const [currentRevisionNumberRow] = await tx + .select({ + value: max(issuerOverlayRevision.revision_number), + }) + .from(issuerOverlayRevision) + .where(eq(issuerOverlayRevision.ticker, normalizedTicker)); + const nextRevisionNumber = (currentRevisionNumberRow?.value ?? 0) + 1; + + const [savedRevision] = await tx + .insert(issuerOverlayRevision) + .values({ + ticker: normalizedTicker, + revision_number: nextRevisionNumber, + definition_hash: hash, + definition_json: normalizedDefinition, + diagnostics_json: input.diagnostics, + source_snapshot_ids: input.diagnostics.sampledSnapshotIds, + created_at: now, + }) + .returning(); + + await tx + .update(issuerOverlay) + .set({ + status: normalizedDefinition.mappings.length > 0 ? "active" : "empty", + active_revision_id: + normalizedDefinition.mappings.length > 0 + ? savedRevision.id + : currentActiveRevisionId, + last_built_at: now, + last_error: null, + stats_json: input.stats, + updated_at: now, + }) + .where(eq(issuerOverlay.ticker, normalizedTicker)); + + return { + published: normalizedDefinition.mappings.length > 0, + revision: toRevisionRecord(savedRevision), + }; + }); +} + +export async function listIssuerOverlayRevisions(ticker: string) { + const normalizedTicker = normalizeTicker(ticker); + const rows = await getDb() + .select() + .from(issuerOverlayRevision) + .where(eq(issuerOverlayRevision.ticker, normalizedTicker)) + .orderBy(desc(issuerOverlayRevision.revision_number)); + + return rows.map(toRevisionRecord); +} diff --git a/lib/server/repos/research-library.ts b/lib/server/repos/research-library.ts index a6f8ea0..a44e25a 100644 --- a/lib/server/repos/research-library.ts +++ b/lib/server/repos/research-library.ts @@ -457,7 +457,7 @@ export async function createResearchArtifactRecord(input: { return toResearchArtifact(created); } -export async function upsertSystemResearchArtifact(input: { +async function upsertSystemResearchArtifact(input: { userId: string; organizationId?: string | null; ticker: string; @@ -837,7 +837,7 @@ export async function deleteResearchMemoEvidenceLink(userId: string, memoId: num return rows.length > 0; } -export async function listResearchMemoEvidenceLinks(userId: string, ticker: string): Promise { +async function listResearchMemoEvidenceLinks(userId: string, ticker: string): Promise { const memo = await getResearchMemoByTicker(userId, ticker); if (!memo) { return []; @@ -1116,7 +1116,7 @@ export async function getResearchArtifactFileResponse(userId: string, id: number }); } -export function rebuildResearchArtifactIndex() { +function rebuildResearchArtifactIndex() { rebuildArtifactSearchIndex(); } diff --git a/lib/server/repos/tasks.ts b/lib/server/repos/tasks.ts index 438c220..c30491e 100644 --- a/lib/server/repos/tasks.ts +++ b/lib/server/repos/tasks.ts @@ -153,7 +153,7 @@ export async function createTaskRunRecord(input: CreateTaskInput) { }); } -export type AtomicCreateResult = +type AtomicCreateResult = | { task: Task; created: true } | { task: null; created: false }; diff --git a/lib/server/sec-company-profile.ts b/lib/server/sec-company-profile.ts index eaee1ba..990bb52 100644 --- a/lib/server/sec-company-profile.ts +++ b/lib/server/sec-company-profile.ts @@ -41,7 +41,7 @@ type ExchangeDirectoryRecord = { exchange: string | null; }; -export type SecCompanyProfileResult = { +type SecCompanyProfileResult = { ticker: string; cik: string | null; companyName: string | null; diff --git a/lib/server/sec.ts b/lib/server/sec.ts index 5431b3c..d48dd79 100644 --- a/lib/server/sec.ts +++ b/lib/server/sec.ts @@ -72,7 +72,7 @@ type FetchPrimaryFilingTextOptions = { maxChars?: number; }; -export type FilingDocumentText = { +type FilingDocumentText = { source: "primary_document"; url: string; text: string; @@ -573,7 +573,7 @@ export async function fetchRecentFilings( return filings; } -export async function fetchLatestFilingMetrics(cik: string) { +async function fetchLatestFilingMetrics(cik: string) { const normalized = cik.padStart(10, "0"); const payload = await fetchJson( `https://data.sec.gov/api/xbrl/companyfacts/CIK${normalized}.json`, diff --git a/lib/server/task-errors.ts b/lib/server/task-errors.ts index 060061f..e5e634c 100644 --- a/lib/server/task-errors.ts +++ b/lib/server/task-errors.ts @@ -8,7 +8,7 @@ type TaskErrorSource = { payload?: Record | null; }; -export type TaskFailureMessage = { +type TaskFailureMessage = { summary: string; detail: string; }; diff --git a/lib/server/task-processors.outcomes.test.ts b/lib/server/task-processors.outcomes.test.ts index 83b2358..0d9b430 100644 --- a/lib/server/task-processors.outcomes.test.ts +++ b/lib/server/task-processors.outcomes.test.ts @@ -158,6 +158,9 @@ const mockUpsertFilingsRecords = mock(async () => ({ const mockDeleteCompanyFinancialBundlesForTicker = mock(async () => {}); const mockGetFilingTaxonomySnapshotByFilingId = mock(async () => null); +const mockNormalizeFilingTaxonomySnapshotPayload = mock((input: unknown) => { + return input as Record; +}); const mockUpsertFilingTaxonomySnapshot = mock(async () => {}); const mockValidateMetricsWithPdfLlm = mock(async () => ({ validation_result: { @@ -252,6 +255,14 @@ const mockFetchRecentFilings = mock(async () => [ const mockEnqueueTask = mock(async () => ({ id: "search-task-1", })); +const mockGenerateIssuerOverlayForTicker = mock(async () => ({ + published: false, + activeRevisionId: null, + sampledSnapshotIds: [1], +})); +const mockRecordIssuerOverlayBuildFailure = mock(async () => {}); +const mockGetActiveIssuerOverlayDefinition = mock(async () => null); +const mockGetIssuerOverlay = mock(async () => null); const mockHydrateFilingTaxonomySnapshot = mock( async (input: { filingId: number }) => ({ filing_id: input.filingId, @@ -338,6 +349,8 @@ mock.module("@/lib/server/repos/company-financial-bundles", () => ({ })); mock.module("@/lib/server/repos/filing-taxonomy", () => ({ getFilingTaxonomySnapshotByFilingId: mockGetFilingTaxonomySnapshotByFilingId, + normalizeFilingTaxonomySnapshotPayload: + mockNormalizeFilingTaxonomySnapshotPayload, upsertFilingTaxonomySnapshot: mockUpsertFilingTaxonomySnapshot, })); mock.module("@/lib/server/repos/holdings", () => ({ @@ -358,6 +371,14 @@ mock.module("@/lib/server/sec", () => ({ mock.module("@/lib/server/tasks", () => ({ enqueueTask: mockEnqueueTask, })); +mock.module("@/lib/server/issuer-overlays", () => ({ + generateIssuerOverlayForTicker: mockGenerateIssuerOverlayForTicker, + recordIssuerOverlayBuildFailure: mockRecordIssuerOverlayBuildFailure, +})); +mock.module("@/lib/server/repos/issuer-overlays", () => ({ + getActiveIssuerOverlayDefinition: mockGetActiveIssuerOverlayDefinition, + getIssuerOverlay: mockGetIssuerOverlay, +})); mock.module("@/lib/server/taxonomy/engine", () => ({ hydrateFilingTaxonomySnapshot: mockHydrateFilingTaxonomySnapshot, })); @@ -413,6 +434,11 @@ describe("task processor outcomes", () => { mockUpdateTaskStage.mockClear(); mockEnqueueTask.mockClear(); mockValidateMetricsWithPdfLlm.mockClear(); + mockGenerateIssuerOverlayForTicker.mockClear(); + mockRecordIssuerOverlayBuildFailure.mockClear(); + mockGetActiveIssuerOverlayDefinition.mockClear(); + mockGetIssuerOverlay.mockClear(); + mockNormalizeFilingTaxonomySnapshotPayload.mockClear(); }); it("returns sync filing completion detail and progress context", async () => { diff --git a/lib/server/task-processors.ts b/lib/server/task-processors.ts index 7fec23b..ca2465b 100644 --- a/lib/server/task-processors.ts +++ b/lib/server/task-processors.ts @@ -32,6 +32,14 @@ import { import { createPortfolioInsight } from "@/lib/server/repos/insights"; import { updateTaskStage } from "@/lib/server/repos/tasks"; import { fetchPrimaryFilingText, fetchRecentFilings } from "@/lib/server/sec"; +import { + generateIssuerOverlayForTicker, + recordIssuerOverlayBuildFailure, +} from "@/lib/server/issuer-overlays"; +import { + getActiveIssuerOverlayDefinition, + getIssuerOverlay, +} from "@/lib/server/repos/issuer-overlays"; import { enqueueTask } from "@/lib/server/tasks"; import { hydrateFilingTaxonomySnapshot } from "@/lib/server/taxonomy/engine"; import { validateMetricsWithPdfLlm } from "@/lib/server/taxonomy/pdf-validation"; @@ -143,6 +151,12 @@ export type TaskExecutionOutcome = { completionContext?: TaskStageContext | null; }; +type TaxonomyHydrationRunResult = { + hydrated: number; + failed: number; + touchedFilingIds: Set; +}; + function buildTaskOutcome( result: unknown, completionDetail: string, @@ -155,6 +169,259 @@ function buildTaskOutcome( }; } +function shouldRefreshTaxonomySnapshot(input: { + existingSnapshot: Awaited< + ReturnType + > | null; + filingUpdatedAt: string; + overlayRevisionId: number | null; +}) { + if (!input.existingSnapshot) { + return true; + } + + if ( + Date.parse(input.existingSnapshot.updated_at) < + Date.parse(input.filingUpdatedAt) + ) { + return true; + } + + return ( + (input.existingSnapshot.issuer_overlay_revision_id ?? null) !== + input.overlayRevisionId + ); +} + +async function hydrateTaxonomySnapshotsForCandidates(input: { + task: Task; + ticker: string; + filingsCount: number; + saveResult: { inserted: number; updated: number }; + candidates: Array; + overlayRevisionId: number | null; +}) { + const overlayDefinition = + input.overlayRevisionId === null + ? null + : await getActiveIssuerOverlayDefinition(input.ticker); + let hydrated = 0; + let failed = 0; + const touchedFilingIds = new Set(); + + for (let index = 0; index < input.candidates.length; index += 1) { + const filing = input.candidates[index]; + const existingSnapshot = await getFilingTaxonomySnapshotByFilingId( + filing.id, + ); + if ( + !shouldRefreshTaxonomySnapshot({ + existingSnapshot, + filingUpdatedAt: filing.updated_at, + overlayRevisionId: input.overlayRevisionId, + }) + ) { + continue; + } + + touchedFilingIds.add(filing.id); + const stageContext = (stage: TaskStage) => + buildProgressContext({ + current: index + 1, + total: input.candidates.length, + unit: "filings", + counters: { + fetched: input.filingsCount, + inserted: input.saveResult.inserted, + updated: input.saveResult.updated, + hydrated, + failed, + }, + subject: { + ticker: input.ticker, + accessionNumber: filing.accession_number, + label: stage, + }, + }); + + try { + await setProjectionStage( + input.task, + "sync.extract_taxonomy", + `Extracting XBRL taxonomy for ${filing.accession_number}`, + stageContext("sync.extract_taxonomy"), + ); + const snapshot = await hydrateFilingTaxonomySnapshot({ + filingId: filing.id, + ticker: filing.ticker, + cik: filing.cik, + accessionNumber: filing.accession_number, + filingDate: filing.filing_date, + filingType: filing.filing_type, + filingUrl: filing.filing_url, + primaryDocument: filing.primary_document ?? null, + issuerOverlay: overlayDefinition, + }); + let pdfValidation = { + validation_result: snapshot.validation_result, + metric_validations: snapshot.metric_validations, + }; + + try { + pdfValidation = await validateMetricsWithPdfLlm({ + metrics: snapshot.derived_metrics, + assets: snapshot.assets, + }); + } catch (error) { + const message = + error instanceof Error + ? error.message + : "PDF metric validation failed"; + pdfValidation = { + validation_result: { + status: "error", + checks: [], + validatedAt: new Date().toISOString(), + }, + metric_validations: snapshot.metric_validations.map((check) => ({ + ...check, + error: check.error ?? message, + })), + }; + } + + const normalizedSnapshot = { + ...snapshot, + validation_result: pdfValidation.validation_result, + metric_validations: pdfValidation.metric_validations, + issuer_overlay_revision_id: input.overlayRevisionId, + ...normalizeFilingTaxonomySnapshotPayload(snapshot), + }; + + await setProjectionStage( + input.task, + "sync.normalize_taxonomy", + `Materializing statements for ${filing.accession_number}`, + stageContext("sync.normalize_taxonomy"), + ); + await setProjectionStage( + input.task, + "sync.derive_metrics", + `Deriving taxonomy metrics for ${filing.accession_number}`, + stageContext("sync.derive_metrics"), + ); + await setProjectionStage( + input.task, + "sync.validate_pdf_metrics", + `Validating metrics via PDF + LLM for ${filing.accession_number}`, + stageContext("sync.validate_pdf_metrics"), + ); + await setProjectionStage( + input.task, + "sync.persist_taxonomy", + `Persisting taxonomy snapshot for ${filing.accession_number}`, + stageContext("sync.persist_taxonomy"), + ); + + await upsertFilingTaxonomySnapshot(normalizedSnapshot); + await updateFilingMetricsById( + filing.id, + normalizedSnapshot.derived_metrics, + ); + await deleteCompanyFinancialBundlesForTicker(filing.ticker); + hydrated += 1; + } catch (error) { + const now = new Date().toISOString(); + await upsertFilingTaxonomySnapshot({ + filing_id: filing.id, + ticker: filing.ticker, + filing_date: filing.filing_date, + filing_type: filing.filing_type, + parse_status: "failed", + parse_error: + error instanceof Error ? error.message : "Taxonomy hydration failed", + source: "legacy_html_fallback", + parser_engine: "fiscal-xbrl", + parser_version: "unknown", + taxonomy_regime: "unknown", + fiscal_pack: "core", + periods: [], + faithful_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + disclosure: [], + }, + statement_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + disclosure: [], + }, + surface_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + disclosure: [], + }, + detail_rows: { + income: {}, + balance: {}, + cash_flow: {}, + equity: {}, + comprehensive_income: {}, + disclosure: {}, + }, + kpi_rows: [], + computed_definitions: [], + contexts: [], + derived_metrics: filing.metrics ?? null, + validation_result: { + status: "error", + checks: [], + validatedAt: now, + }, + normalization_summary: { + surfaceRowCount: 0, + detailRowCount: 0, + kpiRowCount: 0, + unmappedRowCount: 0, + materialUnmappedRowCount: 0, + residualPrimaryCount: 0, + residualDisclosureCount: 0, + unsupportedConceptCount: 0, + issuerOverlayMatchCount: 0, + warnings: [], + }, + issuer_overlay_revision_id: input.overlayRevisionId, + facts_count: 0, + concepts_count: 0, + dimensions_count: 0, + assets: [], + concepts: [], + facts: [], + metric_validations: [], + }); + await deleteCompanyFinancialBundlesForTicker(filing.ticker); + failed += 1; + } + + await Bun.sleep(STATEMENT_HYDRATION_DELAY_MS); + } + + return { + hydrated, + failed, + touchedFilingIds, + } satisfies TaxonomyHydrationRunResult; +} + async function setProjectionStage( task: Task, stage: TaskStage, @@ -756,200 +1023,77 @@ async function processSyncFilings(task: Task) { subject: tickerSubject, }), ); - for (let index = 0; index < hydrateCandidates.length; index += 1) { - const filing = hydrateCandidates[index]; - const existingSnapshot = await getFilingTaxonomySnapshotByFilingId( - filing.id, - ); - const shouldRefresh = - !existingSnapshot || - Date.parse(existingSnapshot.updated_at) < Date.parse(filing.updated_at); + const currentOverlay = await getIssuerOverlay(ticker); + const firstPass = await hydrateTaxonomySnapshotsForCandidates({ + task, + ticker, + filingsCount: filings.length, + saveResult, + candidates: hydrateCandidates, + overlayRevisionId: currentOverlay?.active_revision_id ?? null, + }); + taxonomySnapshotsHydrated += firstPass.hydrated; + taxonomySnapshotsFailed += firstPass.failed; - if (!shouldRefresh) { - continue; - } - - const stageContext = (stage: TaskStage) => - buildProgressContext({ - current: index + 1, - total: hydrateCandidates.length, - unit: "filings", + let overlayPublished = false; + let activeOverlayRevisionId = currentOverlay?.active_revision_id ?? null; + try { + await setProjectionStage( + task, + "sync.persist_taxonomy", + `Building issuer overlay for ${ticker}`, + { counters: { - fetched: filings.length, - inserted: saveResult.inserted, - updated: saveResult.updated, + sampledFilings: Math.min(hydrateCandidates.length, 12), hydrated: taxonomySnapshotsHydrated, failed: taxonomySnapshotsFailed, }, - subject: { - ticker, - accessionNumber: filing.accession_number, - label: stage, - }, - }); + subject: tickerSubject, + }, + ); + const overlayResult = await generateIssuerOverlayForTicker(ticker); + overlayPublished = overlayResult.published; + activeOverlayRevisionId = overlayResult.activeRevisionId; + } catch (error) { + await recordIssuerOverlayBuildFailure(ticker, error); + console.error(`[issuer-overlay] failed for ${ticker}:`, error); + } - try { - await setProjectionStage( - task, - "sync.extract_taxonomy", - `Extracting XBRL taxonomy for ${filing.accession_number}`, - stageContext("sync.extract_taxonomy"), - ); - const snapshot = await hydrateFilingTaxonomySnapshot({ - filingId: filing.id, - ticker: filing.ticker, - cik: filing.cik, - accessionNumber: filing.accession_number, - filingDate: filing.filing_date, - filingType: filing.filing_type, - filingUrl: filing.filing_url, - primaryDocument: filing.primary_document ?? null, - }); - let pdfValidation = { - validation_result: snapshot.validation_result, - metric_validations: snapshot.metric_validations, - }; + if ( + overlayPublished && + activeOverlayRevisionId !== null && + activeOverlayRevisionId !== currentOverlay?.active_revision_id + ) { + const prioritizedCandidates = [ + ...hydrateCandidates.filter((filing) => + firstPass.touchedFilingIds.has(filing.id), + ), + ...hydrateCandidates.filter( + (filing) => !firstPass.touchedFilingIds.has(filing.id), + ), + ]; + const uniqueCandidates = prioritizedCandidates.filter( + (filing, index, rows) => { + return ( + rows.findIndex((candidate) => candidate.id === filing.id) === index + ); + }, + ); + const rehydrateCandidates = uniqueCandidates.slice( + 0, + Math.max(firstPass.touchedFilingIds.size, 8), + ); - try { - pdfValidation = await validateMetricsWithPdfLlm({ - metrics: snapshot.derived_metrics, - assets: snapshot.assets, - }); - } catch (error) { - const message = - error instanceof Error - ? error.message - : "PDF metric validation failed"; - pdfValidation = { - validation_result: { - status: "error", - checks: [], - validatedAt: new Date().toISOString(), - }, - metric_validations: snapshot.metric_validations.map((check) => ({ - ...check, - error: check.error ?? message, - })), - }; - } - - const normalizedSnapshot = { - ...snapshot, - validation_result: pdfValidation.validation_result, - metric_validations: pdfValidation.metric_validations, - ...normalizeFilingTaxonomySnapshotPayload(snapshot), - }; - - await setProjectionStage( - task, - "sync.normalize_taxonomy", - `Materializing statements for ${filing.accession_number}`, - stageContext("sync.normalize_taxonomy"), - ); - await setProjectionStage( - task, - "sync.derive_metrics", - `Deriving taxonomy metrics for ${filing.accession_number}`, - stageContext("sync.derive_metrics"), - ); - await setProjectionStage( - task, - "sync.validate_pdf_metrics", - `Validating metrics via PDF + LLM for ${filing.accession_number}`, - stageContext("sync.validate_pdf_metrics"), - ); - await setProjectionStage( - task, - "sync.persist_taxonomy", - `Persisting taxonomy snapshot for ${filing.accession_number}`, - stageContext("sync.persist_taxonomy"), - ); - - await upsertFilingTaxonomySnapshot(normalizedSnapshot); - await updateFilingMetricsById( - filing.id, - normalizedSnapshot.derived_metrics, - ); - await deleteCompanyFinancialBundlesForTicker(filing.ticker); - taxonomySnapshotsHydrated += 1; - } catch (error) { - const now = new Date().toISOString(); - await upsertFilingTaxonomySnapshot({ - filing_id: filing.id, - ticker: filing.ticker, - filing_date: filing.filing_date, - filing_type: filing.filing_type, - parse_status: "failed", - parse_error: - error instanceof Error ? error.message : "Taxonomy hydration failed", - source: "legacy_html_fallback", - parser_engine: "fiscal-xbrl", - parser_version: "unknown", - taxonomy_regime: "unknown", - fiscal_pack: "core", - periods: [], - faithful_rows: { - income: [], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [], - disclosure: [], - }, - statement_rows: { - income: [], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [], - disclosure: [], - }, - surface_rows: { - income: [], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [], - disclosure: [], - }, - detail_rows: { - income: {}, - balance: {}, - cash_flow: {}, - equity: {}, - comprehensive_income: {}, - disclosure: {}, - }, - kpi_rows: [], - computed_definitions: [], - contexts: [], - derived_metrics: filing.metrics ?? null, - validation_result: { - status: "error", - checks: [], - validatedAt: now, - }, - normalization_summary: { - surfaceRowCount: 0, - detailRowCount: 0, - kpiRowCount: 0, - unmappedRowCount: 0, - materialUnmappedRowCount: 0, - warnings: [], - }, - facts_count: 0, - concepts_count: 0, - dimensions_count: 0, - assets: [], - concepts: [], - facts: [], - metric_validations: [], - }); - await deleteCompanyFinancialBundlesForTicker(filing.ticker); - taxonomySnapshotsFailed += 1; - } - - await Bun.sleep(STATEMENT_HYDRATION_DELAY_MS); + const secondPass = await hydrateTaxonomySnapshotsForCandidates({ + task, + ticker, + filingsCount: filings.length, + saveResult, + candidates: rehydrateCandidates, + overlayRevisionId: activeOverlayRevisionId, + }); + taxonomySnapshotsHydrated += secondPass.hydrated; + taxonomySnapshotsFailed += secondPass.failed; } try { @@ -977,6 +1121,8 @@ async function processSyncFilings(task: Task) { updated: saveResult.updated, taxonomySnapshotsHydrated, taxonomySnapshotsFailed, + overlayPublished, + activeOverlayRevisionId, searchTaskId, }; diff --git a/lib/server/taxonomy/engine.test.ts b/lib/server/taxonomy/engine.test.ts index a061269..2729848 100644 --- a/lib/server/taxonomy/engine.test.ts +++ b/lib/server/taxonomy/engine.test.ts @@ -55,6 +55,10 @@ function createHydrationResult(): TaxonomyHydrationResult { kpi_row_count: 0, unmapped_row_count: 0, material_unmapped_row_count: 0, + residual_primary_count: 0, + residual_disclosure_count: 0, + unsupported_concept_count: 0, + issuer_overlay_match_count: 0, warnings: ["rust_warning"], }, xbrl_validation: { diff --git a/lib/server/taxonomy/parser-client.test.ts b/lib/server/taxonomy/parser-client.test.ts index 4dda9e4..650f118 100644 --- a/lib/server/taxonomy/parser-client.test.ts +++ b/lib/server/taxonomy/parser-client.test.ts @@ -4,6 +4,7 @@ import type { TaxonomyHydrationInput, TaxonomyHydrationResult, } from "@/lib/server/taxonomy/types"; +import type { IssuerOverlayDefinition } from "@/lib/server/db/schema"; import { __parserClientInternals } from "@/lib/server/taxonomy/parser-client"; function streamFromText(text: string) { @@ -81,6 +82,10 @@ function sampleHydrationResult(): TaxonomyHydrationResult { kpi_row_count: 0, unmapped_row_count: 0, material_unmapped_row_count: 0, + residual_primary_count: 0, + residual_disclosure_count: 0, + unsupported_concept_count: 0, + issuer_overlay_match_count: 0, warnings: [], }, xbrl_validation: { @@ -103,6 +108,22 @@ function sampleInput(): TaxonomyHydrationInput { }; } +function sampleOverlay(): IssuerOverlayDefinition { + return { + version: "fiscal-v1", + ticker: "AAPL", + pack: "core", + mappings: [ + { + surface_key: "revenue", + statement: "income", + allowed_source_concepts: ["aapl:ServicesRevenue"], + allowed_authoritative_concepts: [], + }, + ], + }; +} + const passThroughTimeout = ((handler: TimerHandler, timeout?: number) => globalThis.setTimeout( handler, @@ -156,6 +177,47 @@ describe("parser client", () => { expect(result.parser_engine).toBe("fiscal-xbrl"); expect(stdinWrite).toHaveBeenCalledTimes(1); expect(stdinEnd).toHaveBeenCalledTimes(1); + const [firstCall] = stdinWrite.mock.calls as unknown[][]; + const firstWrite = firstCall?.[0]; + expect(firstWrite).toBeDefined(); + const payload = JSON.parse( + new TextDecoder().decode(firstWrite as unknown as Uint8Array), + ) as { issuerOverlay?: IssuerOverlayDefinition | null }; + expect(payload.issuerOverlay).toBeNull(); + }); + + it("includes issuer overlay payload when provided", async () => { + const stdinWrite = mock(() => {}); + + await __parserClientInternals.hydrateFromSidecarImpl( + { + ...sampleInput(), + issuerOverlay: sampleOverlay(), + }, + { + existsSync: () => true, + spawn: mock(() => ({ + stdin: { + write: stdinWrite, + end: () => {}, + }, + stdout: streamFromText(JSON.stringify(sampleHydrationResult())), + stderr: streamFromText(""), + exited: Promise.resolve(0), + kill: mock(() => {}), + })) as never, + setTimeout: passThroughTimeout, + clearTimeout, + }, + ); + + const [firstCall] = stdinWrite.mock.calls as unknown[][]; + const firstWrite = firstCall?.[0]; + expect(firstWrite).toBeDefined(); + const payload = JSON.parse( + new TextDecoder().decode(firstWrite as unknown as Uint8Array), + ) as { issuerOverlay?: IssuerOverlayDefinition | null }; + expect(payload.issuerOverlay).toEqual(sampleOverlay()); }); it("throws when the sidecar exits non-zero", async () => { diff --git a/lib/server/taxonomy/parser-client.ts b/lib/server/taxonomy/parser-client.ts index 916b60d..9b59760 100644 --- a/lib/server/taxonomy/parser-client.ts +++ b/lib/server/taxonomy/parser-client.ts @@ -32,7 +32,7 @@ function candidateBinaryPaths() { ); } -export function resolveFiscalXbrlBinary() { +function resolveFiscalXbrlBinary() { return resolveFiscalXbrlBinaryWithDeps({ existsSync, }); @@ -93,6 +93,7 @@ async function hydrateFromSidecarImpl( filingType: input.filingType, filingUrl: input.filingUrl, primaryDocument: input.primaryDocument, + issuerOverlay: input.issuerOverlay ?? null, cacheDir: process.env.FISCAL_XBRL_CACHE_DIR ?? join(process.cwd(), ".cache", "xbrl"), diff --git a/lib/server/taxonomy/types.ts b/lib/server/taxonomy/types.ts index 46f45bb..82ed1ce 100644 --- a/lib/server/taxonomy/types.ts +++ b/lib/server/taxonomy/types.ts @@ -2,6 +2,7 @@ import type { Filing, FinancialStatementKind, MetricValidationResult, + TickerAutomationSource, } from "@/lib/types"; import type { ComputedDefinition } from "@/lib/generated"; import type { @@ -10,6 +11,7 @@ import type { FilingTaxonomyPeriod, FilingTaxonomySource, } from "@/lib/server/repos/filing-taxonomy"; +import type { IssuerOverlayDefinition } from "@/lib/server/db/schema"; export type TaxonomyAsset = { asset_type: FilingTaxonomyAssetType; @@ -20,9 +22,9 @@ export type TaxonomyAsset = { is_selected: boolean; }; -export type TaxonomyNamespaceMap = Record; +type TaxonomyNamespaceMap = Record; -export type TaxonomyContext = { +type TaxonomyContext = { id: string; entityIdentifier: string | null; entityScheme: string | null; @@ -40,7 +42,7 @@ export type TaxonomyContext = { } | null; }; -export type TaxonomyUnit = { +type TaxonomyUnit = { id: string; measure: string | null; }; @@ -65,7 +67,7 @@ export type TaxonomyFact = { sourceFile: string | null; }; -export type TaxonomyPresentationConcept = { +type TaxonomyPresentationConcept = { conceptKey: string; qname: string; roleUri: string; @@ -157,7 +159,7 @@ export type TaxonomyHydrationSurfaceRow = { formula_key: string | null; has_dimensions: boolean; resolved_source_row_keys: Record; - statement?: "income" | "balance" | "cash_flow"; + statement?: "income" | "balance" | "cash_flow" | "equity" | "disclosure"; detail_count?: number; resolution_method?: | "direct" @@ -206,6 +208,10 @@ export type TaxonomyHydrationNormalizationSummary = { kpi_row_count: number; unmapped_row_count: number; material_unmapped_row_count: number; + residual_primary_count: number; + residual_disclosure_count: number; + unsupported_concept_count: number; + issuer_overlay_match_count: number; warnings: string[]; }; @@ -223,6 +229,12 @@ export type TaxonomyHydrationInput = { filingType: "10-K" | "10-Q"; filingUrl: string | null; primaryDocument: string | null; + issuerOverlay?: IssuerOverlayDefinition | null; +}; + +export type TickerAutomationRequest = { + ticker: string; + source: TickerAutomationSource; }; export type TaxonomyHydrationResult = { diff --git a/lib/task-workflow.ts b/lib/task-workflow.ts index 87c2b35..13f051d 100644 --- a/lib/task-workflow.ts +++ b/lib/task-workflow.ts @@ -5,7 +5,7 @@ import type { TaskType } from '@/lib/types'; -export type StageTimelineItem = { +type StageTimelineItem = { stage: TaskStage; label: string; state: 'completed' | 'active' | 'pending' | 'failed'; @@ -115,7 +115,7 @@ export function stageLabel(stage: TaskStage) { return STAGE_LABELS[stage] ?? stage; } -export function taskStageOrder(taskType: TaskType) { +function taskStageOrder(taskType: TaskType) { return TASK_STAGE_ORDER[taskType] ?? ['queued', 'running', 'completed']; } diff --git a/lib/types.ts b/lib/types.ts index 363e51a..f47720c 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -126,6 +126,12 @@ export type TaskType = | "analyze_filing" | "portfolio_insights" | "index_search"; +export type TickerAutomationSource = + | "analysis" + | "financials" + | "search" + | "graphing" + | "research"; export type TaskStage = | "queued" | "running" @@ -428,13 +434,15 @@ export type FinancialStatementKind = | "disclosure" | "equity" | "comprehensive_income"; -export type FinancialHistoryWindow = "10y" | "all"; +type FinancialHistoryWindow = "10y" | "all"; export type FinancialCadence = "annual" | "quarterly" | "ltm"; export type FinancialDisplayMode = "faithful" | "standardized"; export type FinancialSurfaceKind = | "income_statement" | "balance_sheet" | "cash_flow_statement" + | "equity_statement" + | "disclosures" | "ratios" | "segments_kpis" | "adjusted" @@ -482,7 +490,7 @@ export type TaxonomyStatementRow = { sourceFactIds: number[]; }; -export type FinancialStatementSurfaceKind = FinancialDisplayMode; +type FinancialStatementSurfaceKind = FinancialDisplayMode; export type DerivedFinancialRow = { key: string; @@ -501,11 +509,11 @@ export type DerivedFinancialRow = { }; export type StandardizedFinancialRow = DerivedFinancialRow; -export type StandardizedStatementRow = StandardizedFinancialRow; +type StandardizedStatementRow = StandardizedFinancialRow; export type SurfaceFinancialRow = StandardizedFinancialRow & { statement?: Extract< FinancialStatementKind, - "income" | "balance" | "cash_flow" + "income" | "balance" | "cash_flow" | "equity" | "disclosure" >; detailCount?: number; resolutionMethod?: @@ -541,6 +549,10 @@ export type NormalizationSummary = { kpiRowCount: number; unmappedRowCount: number; materialUnmappedRowCount: number; + residualPrimaryCount: number; + residualDisclosureCount: number; + unsupportedConceptCount: number; + issuerOverlayMatchCount: number; warnings: string[]; }; @@ -554,6 +566,10 @@ export type NormalizationMetadata = { kpiRowCount: number; unmappedRowCount: number; materialUnmappedRowCount: number; + residualPrimaryCount: number; + residualDisclosureCount: number; + unsupportedConceptCount: number; + issuerOverlayMatchCount: number; warnings: string[]; }; @@ -620,7 +636,7 @@ export type MetricValidationResult = { validatedAt: string | null; }; -export type FilingFaithfulStatementRow = { +type FilingFaithfulStatementRow = { key: string; label: string; concept: string | null; diff --git a/package.json b/package.json index a2694d4..371322e 100644 --- a/package.json +++ b/package.json @@ -33,14 +33,14 @@ "test:e2e:headed": "bun run scripts/e2e-run.ts --headed", "test:e2e:install": "playwright install chromium", "test:e2e:ui": "bun run scripts/e2e-run.ts --ui", - "test:e2e:workflow": "RUN_TASK_WORKFLOW_E2E=1 bun test lib/server/api/task-workflow-hybrid.e2e.test.ts" + "test:e2e:workflow": "RUN_TASK_WORKFLOW_E2E=1 bun test lib/server/api/task-workflow-hybrid.e2e.test.ts", + "knip": "knip" }, "dependencies": { "@elysiajs/eden": "^1.4.8", "@libsql/client": "^0.17.0", "@tailwindcss/postcss": "^4.2.1", "@tanstack/react-query": "^5.90.21", - "@tanstack/react-virtual": "^3.13.23", "@workflow/world-postgres": "^4.1.0-beta.42", "ai": "^6.0.116", "better-auth": "^1.5.4", @@ -57,18 +57,17 @@ "recharts": "^3.8.0", "sonner": "^2.0.7", "sqlite-vec": "^0.1.7-alpha.2", - "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2", "workflow": "^4.1.0-beta.63", "zhipu-ai-provider": "^0.2.2" }, "devDependencies": { "@playwright/test": "^1.58.2", - "@types/node": "^25.3.5", + "@types/node": "^25.5.0", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", - "autoprefixer": "^10.4.27", "bun-types": "^1.3.10", "drizzle-kit": "^0.31.9", + "knip": "^5.88.1", "postcss": "^8.5.8", "prettier": "^3.6.2", "tailwindcss": "^4.2.1", diff --git a/rust/fiscal-xbrl-core/src/lib.rs b/rust/fiscal-xbrl-core/src/lib.rs index 3d2f8dc..29e5bc6 100644 --- a/rust/fiscal-xbrl-core/src/lib.rs +++ b/rust/fiscal-xbrl-core/src/lib.rs @@ -6,6 +6,7 @@ use regex::Regex; use reqwest::blocking::Client; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs; use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -17,7 +18,7 @@ mod surface_mapper; mod taxonomy_loader; mod universal_income; -use taxonomy_loader::{ComputationSpec, ComputedDefinition}; +use taxonomy_loader::{ComputationSpec, ComputedDefinition, RuntimeIssuerOverlay, SurfacePackFile}; pub const PARSER_ENGINE: &str = "fiscal-xbrl"; pub const PARSER_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -26,6 +27,316 @@ const DEFAULT_SEC_RATE_LIMIT_MS: u64 = 100; const HTTP_TIMEOUT_SECS: u64 = 30; static RATE_LIMITER: Lazy> = Lazy::new(|| Mutex::new(Instant::now())); +static PRIMARY_STATEMENT_CONCEPT_INDEX: Lazy = + Lazy::new(load_primary_statement_concept_index); + +#[derive(Debug, Default, Clone)] +struct SurfaceConceptIndex { + primary_by_qname: HashMap>, + primary_by_local_name: HashMap>, + disclosure_by_qname: HashMap>, + disclosure_by_local_name: HashMap>, +} + +#[derive(Debug, Default, Clone)] +struct SurfaceConceptMembership { + primary_statements: HashSet, + disclosure_surfaces: HashSet, +} + +impl SurfaceConceptIndex { + fn insert(&mut self, concept: &str, statement: &str, surface_key: &str) { + let Some(normalized_concept) = normalize_taxonomy_identity(concept) else { + return; + }; + let Some(local_name) = normalize_taxonomy_local_name(concept) else { + return; + }; + + if statement == "disclosure" { + self.disclosure_by_qname + .entry(normalized_concept) + .or_default() + .insert(surface_key.to_string()); + self.disclosure_by_local_name + .entry(local_name) + .or_default() + .insert(surface_key.to_string()); + return; + } + + if !is_primary_statement_kind(statement) { + return; + } + + self.primary_by_qname + .entry(normalized_concept) + .or_default() + .insert(statement.to_string()); + self.primary_by_local_name + .entry(local_name) + .or_default() + .insert(statement.to_string()); + } + + fn membership_for(&self, qname: &str, local_name: &str) -> SurfaceConceptMembership { + let mut membership = SurfaceConceptMembership::default(); + + if let Some(statement) = statement_overlap_override(local_name) { + membership.primary_statements.insert(statement.to_string()); + } + + for value in self + .resolve_from_map(&self.primary_by_qname, qname) + .into_iter() + .flatten() + { + membership.primary_statements.insert(value); + } + for value in self + .resolve_from_map(&self.primary_by_local_name, local_name) + .into_iter() + .flatten() + { + membership.primary_statements.insert(value); + } + for value in self + .resolve_from_map(&self.disclosure_by_qname, qname) + .into_iter() + .flatten() + { + membership.disclosure_surfaces.insert(value); + } + for value in self + .resolve_from_map(&self.disclosure_by_local_name, local_name) + .into_iter() + .flatten() + { + membership.disclosure_surfaces.insert(value); + } + + membership + } + + fn resolve_primary_statement(&self, qname: &str, local_name: &str) -> Option { + if let Some(statement) = statement_overlap_override(local_name) { + return Some(statement.to_string()); + } + + self.resolve_unique_from_map(&self.primary_by_qname, qname) + .or_else(|| self.resolve_unique_from_map(&self.primary_by_local_name, local_name)) + } + + fn resolve_unique_from_map( + &self, + map: &HashMap>, + value: &str, + ) -> Option { + let normalized = normalize_taxonomy_identity(value)?; + let statements = map.get(&normalized)?; + if statements.len() != 1 { + return None; + } + + statements.iter().next().cloned() + } + + fn resolve_from_map( + &self, + map: &HashMap>, + value: &str, + ) -> Option> { + let normalized = normalize_taxonomy_identity(value)?; + map.get(&normalized).cloned() + } +} + +#[derive(Debug, Default)] +struct PrimaryStatementConceptIndex { + by_qname: HashMap>, + by_local_name: HashMap>, +} + +impl PrimaryStatementConceptIndex { + fn insert(&mut self, concept: &str, statement: &str) { + if !is_primary_statement_kind(statement) { + return; + } + + let Some(normalized_concept) = normalize_taxonomy_identity(concept) else { + return; + }; + self.by_qname + .entry(normalized_concept) + .or_default() + .insert(statement.to_string()); + + let Some(local_name) = normalize_taxonomy_local_name(concept) else { + return; + }; + self.by_local_name + .entry(local_name) + .or_default() + .insert(statement.to_string()); + } + + fn resolve(&self, qname: &str, local_name: &str) -> Option { + if let Some(statement) = statement_overlap_override(local_name) { + return Some(statement.to_string()); + } + + self.resolve_from_map(&self.by_qname, qname) + .or_else(|| self.resolve_from_map(&self.by_local_name, local_name)) + } + + fn resolve_from_map( + &self, + map: &HashMap>, + value: &str, + ) -> Option { + let normalized = normalize_taxonomy_identity(value)?; + let statements = map.get(&normalized)?; + if statements.len() != 1 { + return None; + } + + statements.iter().next().cloned() + } +} + +fn is_primary_statement_kind(statement: &str) -> bool { + matches!( + statement, + "income" | "balance" | "cash_flow" | "equity" | "comprehensive_income" + ) +} + +fn normalize_taxonomy_identity(value: &str) -> Option { + let trimmed = value.trim().to_ascii_lowercase(); + if trimmed.is_empty() { + None + } else { + Some(trimmed) + } +} + +fn normalize_taxonomy_local_name(value: &str) -> Option { + let normalized = normalize_taxonomy_identity(value)?; + let local_name = normalized + .rsplit_once(':') + .map(|(_, local_name)| local_name) + .unwrap_or(normalized.as_str()) + .trim() + .to_string(); + + if local_name.is_empty() { + None + } else { + Some(local_name) + } +} + +fn statement_overlap_override(local_name: &str) -> Option<&'static str> { + match normalize_taxonomy_local_name(local_name).as_deref() { + Some("stockholdersequity") => Some("equity"), + Some("retainedearningsaccumulateddeficit") => Some("equity"), + Some("commonstocksincludingadditionalpaidincapital") => Some("equity"), + Some("accumulatedothercomprehensiveincomelossnetoftax") => Some("equity"), + Some("stockholdersequityother") => Some("equity"), + Some("liabilitiesandstockholdersequity") => Some("balance"), + _ => None, + } +} + +fn load_primary_statement_concept_index() -> PrimaryStatementConceptIndex { + let mut index = PrimaryStatementConceptIndex::default(); + + let taxonomy_dir = match taxonomy_loader::resolve_taxonomy_dir() { + Ok(path) => path, + Err(error) => { + eprintln!("[fiscal-xbrl] primary statement taxonomy index unavailable: {error}"); + return index; + } + }; + + let fiscal_dir = taxonomy_dir.join("fiscal").join("v1"); + let entries = match fs::read_dir(&fiscal_dir) { + Ok(entries) => entries, + Err(error) => { + eprintln!( + "[fiscal-xbrl] unable to read taxonomy surface directory {}: {error}", + fiscal_dir.display() + ); + return index; + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + let file_name = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or(""); + if !file_name.ends_with(".surface.json") || file_name == "universal_income.surface.json" { + continue; + } + + let raw = match fs::read_to_string(&path) { + Ok(raw) => raw, + Err(error) => { + eprintln!( + "[fiscal-xbrl] unable to read surface taxonomy {}: {error}", + path.display() + ); + continue; + } + }; + let pack = match serde_json::from_str::(&raw) { + Ok(pack) => pack, + Err(error) => { + eprintln!( + "[fiscal-xbrl] unable to parse surface taxonomy {}: {error}", + path.display() + ); + continue; + } + }; + + for surface in pack + .surfaces + .iter() + .filter(|surface| is_primary_statement_kind(&surface.statement)) + { + for concept in surface + .allowed_source_concepts + .iter() + .chain(surface.allowed_authoritative_concepts.iter()) + { + index.insert(concept, &surface.statement); + } + } + } + + index +} + +fn build_surface_concept_index(surface_pack: &SurfacePackFile) -> SurfaceConceptIndex { + let mut index = SurfaceConceptIndex::default(); + + for surface in &surface_pack.surfaces { + for concept in surface + .allowed_source_concepts + .iter() + .chain(surface.allowed_authoritative_concepts.iter()) + .chain(surface.issuer_overlay_source_concepts.iter()) + .chain(surface.issuer_overlay_authoritative_concepts.iter()) + { + index.insert(concept, &surface.statement, &surface.surface_key); + } + } + + index +} fn sec_rate_limit_delay() -> u64 { std::env::var("SEC_RATE_LIMIT_MS") @@ -89,6 +400,8 @@ pub struct HydrateFilingRequest { pub filing_type: String, pub filing_url: Option, pub primary_document: Option, + #[serde(default)] + pub issuer_overlay: Option, pub cache_dir: Option, } @@ -420,6 +733,10 @@ pub struct NormalizationSummaryOutput { pub kpi_row_count: usize, pub unmapped_row_count: usize, pub material_unmapped_row_count: usize, + pub residual_primary_count: usize, + pub residual_disclosure_count: usize, + pub unsupported_concept_count: usize, + pub issuer_overlay_match_count: usize, pub warnings: Vec, } @@ -520,6 +837,10 @@ pub fn hydrate_filing(input: HydrateFilingRequest) -> Result Result Result, facts: Vec, + unsupported_concept_count: usize, } fn materialize_taxonomy_statements( @@ -1475,6 +1822,7 @@ fn materialize_taxonomy_statements( facts: &[ParsedFact], presentation: &[PresentationNode], label_by_concept: &HashMap, + surface_index: Option<&SurfaceConceptIndex>, ) -> MaterializedStatements { let compact_accession = accession_number.replace('-', ""); let mut period_by_signature = HashMap::::new(); @@ -1551,6 +1899,7 @@ fn materialize_taxonomy_statements( let mut grouped_by_statement = empty_parsed_fact_map(); let mut enriched_facts = Vec::new(); + let mut unsupported_concepts = HashSet::::new(); for (index, fact) in facts.iter().enumerate() { let nodes = presentation_by_concept @@ -1558,9 +1907,28 @@ fn materialize_taxonomy_statements( .cloned() .unwrap_or_default(); let best_node = nodes.first().copied(); - let statement_kind = best_node - .and_then(|node| classify_statement_role(&node.role_uri)) - .or_else(|| concept_statement_fallback(&fact.local_name)); + let role_statement_kind = + best_node.and_then(|node| classify_statement_role(&node.role_uri)); + let disclosure_role_family = + best_node.and_then(|node| classify_disclosure_role(&node.role_uri)); + let statement_kind = if let Some(statement_kind) = role_statement_kind { + Some(statement_kind) + } else if disclosure_role_family.is_some() { + Some("disclosure".to_string()) + } else if let Some(index) = surface_index { + let membership = index.membership_for(&fact.qname, &fact.local_name); + if !membership.disclosure_surfaces.is_empty() { + Some("disclosure".to_string()) + } else { + index.resolve_primary_statement(&fact.qname, &fact.local_name) + } + } else { + concept_statement_fallback(&fact.qname, &fact.local_name) + }; + + if statement_kind.is_none() { + unsupported_concepts.insert(fact.concept_key.clone()); + } let fact_output = FactOutput { concept_key: fact.concept_key.clone(), @@ -1762,6 +2130,7 @@ fn materialize_taxonomy_statements( statement_rows, concepts, facts: enriched_facts, + unsupported_concept_count: unsupported_concepts.len(), } } @@ -1795,11 +2164,12 @@ fn empty_detail_row_map() -> DetailRowStatementMap { .collect() } -fn statement_keys() -> [&'static str; 5] { +fn statement_keys() -> [&'static str; 6] { [ "income", "balance", "cash_flow", + "disclosure", "equity", "comprehensive_income", ] @@ -1810,6 +2180,7 @@ fn statement_key_ref(value: &str) -> Option<&'static str> { "income" => Some("income"), "balance" => Some("balance"), "cash_flow" => Some("cash_flow"), + "disclosure" => Some("disclosure"), "equity" => Some("equity"), "comprehensive_income" => Some("comprehensive_income"), _ => None, @@ -1913,52 +2284,62 @@ fn classify_statement_role(role_uri: &str) -> Option { None } -fn concept_statement_fallback(local_name: &str) -> Option { - let normalized = local_name.to_ascii_lowercase(); - if Regex::new(r#"equity|retainedearnings|additionalpaidincapital"#) - .unwrap() - .is_match(&normalized) +pub fn classify_disclosure_role(role_uri: &str) -> Option { + let normalized = role_uri.to_ascii_lowercase(); + + if normalized.contains("tax") { + return Some("income_tax_disclosure".to_string()); + } + if normalized.contains("debt") + || normalized.contains("borrow") + || normalized.contains("notespayable") { - return Some("equity".to_string()); + return Some("debt_disclosure".to_string()); } - if normalized.contains("comprehensiveincome") { - return Some("comprehensive_income".to_string()); + if normalized.contains("lease") { + return Some("lease_disclosure".to_string()); } - if Regex::new( - r#"deferredpolicyacquisitioncosts(andvalueofbusinessacquired)?$|supplementaryinsuranceinformationdeferredpolicyacquisitioncosts$|deferredacquisitioncosts$"#, - ) - .unwrap() - .is_match(&normalized) + if normalized.contains("derivative") || normalized.contains("hedg") { + return Some("derivative_instruments_disclosure".to_string()); + } + if normalized.contains("intangible") || normalized.contains("goodwill") { + return Some("intangible_assets_disclosure".to_string()); + } + if normalized.contains("revenue") + || normalized.contains("performanceobligation") + || normalized.contains("contractwithcustomer") { - return Some("balance".to_string()); + return Some("revenue_disclosure".to_string()); } - if Regex::new( - r#"netcashprovidedbyusedin.*activities|increasedecreasein|paymentstoacquire|paymentsforcapitalimprovements$|paymentsfordepositsonrealestateacquisitions$|paymentsforrepurchase|paymentsofdividends|dividendscommonstockcash$|proceedsfrom|repaymentsofdebt|sharebasedcompensation$|allocatedsharebasedcompensationexpense$|depreciationdepletionandamortization$|depreciationamortizationandaccretionnet$|depreciationandamortization$|depreciationamortizationandother$|otheradjustmentstoreconcilenetincomelosstocashprovidedbyusedinoperatingactivities"#, - ) - .unwrap() - .is_match(&normalized) + if normalized.contains("restrictedcash") { + return Some("cash_flow_disclosure".to_string()); + } + if normalized.contains("businesscombination") + || normalized.contains("acquisition") + || normalized.contains("purchaseprice") { - return Some("cash_flow".to_string()); + return Some("business_combinations_disclosure".to_string()); } - if Regex::new( - r#"asset|liabilit|debt|financingreceivable|loansreceivable|deposits|allowanceforcreditloss|futurepolicybenefits|policyholderaccountbalances|unearnedpremiums|realestateinvestmentproperty|grossatcarryingvalue|investmentproperty"#, - ) - .unwrap() - .is_match(&normalized) - { - return Some("balance".to_string()); + if normalized.contains("sharebased") || normalized.contains("stockcompensation") { + return Some("share_based_compensation_disclosure".to_string()); } - if Regex::new( - r#"revenue|income|profit|expense|costof|leaseincome|rental|premiums|claims|underwriting|policyacquisition|interestincome|interestexpense|noninterest|leasedandrentedproperty"#, - ) - .unwrap() - .is_match(&normalized) - { - return Some("income".to_string()); + if normalized.contains("equitymethod") || normalized.contains("equitysecurity") { + return Some("equity_investments_disclosure".to_string()); } + if normalized.contains("deferredtax") { + return Some("deferred_tax_balance_disclosure".to_string()); + } + if normalized.contains("othercomprehensiveincome") || normalized.contains("oci") { + return Some("other_comprehensive_income_disclosure".to_string()); + } + None } +fn concept_statement_fallback(qname: &str, local_name: &str) -> Option { + PRIMARY_STATEMENT_CONCEPT_INDEX.resolve(qname, local_name) +} + fn is_standard_namespace(namespace_uri: &str) -> bool { let lower = namespace_uri.to_ascii_lowercase(); lower.contains("us-gaap") @@ -2110,6 +2491,8 @@ mod tests { &statement_rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("core pack should load and map"); @@ -2218,60 +2601,124 @@ mod tests { fn classifies_pack_specific_concepts_without_presentation_roles() { assert_eq!( concept_statement_fallback( + "us-gaap:FinancingReceivableExcludingAccruedInterestAfterAllowanceForCreditLoss", "FinancingReceivableExcludingAccruedInterestAfterAllowanceForCreditLoss" ) .as_deref(), Some("balance") ); assert_eq!( - concept_statement_fallback("Deposits").as_deref(), + concept_statement_fallback("us-gaap:Deposits", "Deposits").as_deref(), Some("balance") ); assert_eq!( - concept_statement_fallback("RealEstateInvestmentPropertyNet").as_deref(), + concept_statement_fallback( + "us-gaap:RealEstateInvestmentPropertyNet", + "RealEstateInvestmentPropertyNet" + ) + .as_deref(), Some("balance") ); assert_eq!( - concept_statement_fallback("DeferredPolicyAcquisitionCosts").as_deref(), + concept_statement_fallback( + "us-gaap:DeferredPolicyAcquisitionCosts", + "DeferredPolicyAcquisitionCosts" + ) + .as_deref(), Some("balance") ); assert_eq!( - concept_statement_fallback("DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired") + concept_statement_fallback( + "us-gaap:DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired", + "DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired" + ) + .as_deref(), + Some("balance") + ); + assert_eq!( + concept_statement_fallback( + "us-gaap:IncreaseDecreaseInAccountsReceivable", + "IncreaseDecreaseInAccountsReceivable" + ) + .as_deref(), + Some("cash_flow") + ); + assert_eq!( + concept_statement_fallback("us-gaap:PaymentsOfDividends", "PaymentsOfDividends") .as_deref(), - Some("balance") - ); - assert_eq!( - concept_statement_fallback("IncreaseDecreaseInAccountsReceivable").as_deref(), Some("cash_flow") ); assert_eq!( - concept_statement_fallback("PaymentsOfDividends").as_deref(), + concept_statement_fallback("us-gaap:RepaymentsOfDebt", "RepaymentsOfDebt").as_deref(), Some("cash_flow") ); assert_eq!( - concept_statement_fallback("RepaymentsOfDebt").as_deref(), + concept_statement_fallback("us-gaap:ShareBasedCompensation", "ShareBasedCompensation") + .as_deref(), + None + ); + assert_eq!( + concept_statement_fallback( + "us-gaap:PaymentsForCapitalImprovements", + "PaymentsForCapitalImprovements" + ) + .as_deref(), Some("cash_flow") ); assert_eq!( - concept_statement_fallback("ShareBasedCompensation").as_deref(), + concept_statement_fallback( + "us-gaap:PaymentsForDepositsOnRealEstateAcquisitions", + "PaymentsForDepositsOnRealEstateAcquisitions" + ) + .as_deref(), Some("cash_flow") ); assert_eq!( - concept_statement_fallback("PaymentsForCapitalImprovements").as_deref(), - Some("cash_flow") - ); - assert_eq!( - concept_statement_fallback("PaymentsForDepositsOnRealEstateAcquisitions").as_deref(), - Some("cash_flow") - ); - assert_eq!( - concept_statement_fallback("LeaseIncome").as_deref(), + concept_statement_fallback("us-gaap:LeaseIncome", "LeaseIncome").as_deref(), Some("income") ); assert_eq!( - concept_statement_fallback("DirectCostsOfLeasedAndRentedPropertyOrEquipment") - .as_deref(), + concept_statement_fallback( + "us-gaap:DirectCostsOfLeasedAndRentedPropertyOrEquipment", + "DirectCostsOfLeasedAndRentedPropertyOrEquipment" + ) + .as_deref(), Some("income") ); } + + #[test] + fn keeps_disclosure_only_concepts_out_of_primary_statement_fallback() { + assert_eq!( + concept_statement_fallback( + "us-gaap:RevenueRemainingPerformanceObligation", + "RevenueRemainingPerformanceObligation" + ), + None + ); + assert_eq!( + concept_statement_fallback( + "us-gaap:EquitySecuritiesFVNINoncurrent", + "EquitySecuritiesFVNINoncurrent" + ), + None + ); + } + + #[test] + fn applies_explicit_balance_equity_overlap_overrides() { + assert_eq!( + concept_statement_fallback("us-gaap:StockholdersEquity", "StockholdersEquity") + .as_deref(), + Some("equity") + ); + assert_eq!( + concept_statement_fallback( + "us-gaap:LiabilitiesAndStockholdersEquity", + "LiabilitiesAndStockholdersEquity" + ) + .as_deref(), + Some("balance") + ); + } } diff --git a/rust/fiscal-xbrl-core/src/surface_mapper.rs b/rust/fiscal-xbrl-core/src/surface_mapper.rs index e1dbda5..bf2ca2f 100644 --- a/rust/fiscal-xbrl-core/src/surface_mapper.rs +++ b/rust/fiscal-xbrl-core/src/surface_mapper.rs @@ -3,8 +3,9 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::pack_selector::FiscalPack; use crate::taxonomy_loader::{ - load_crosswalk, load_income_bridge, load_surface_pack, CrosswalkFile, IncomeBridgeFile, - IncomeBridgeRow, SurfaceDefinition, SurfaceFormula, SurfaceFormulaOp, SurfaceSignTransform, + load_crosswalk, load_income_bridge, load_surface_pack_with_overlays, CrosswalkFile, + IncomeBridgeFile, IncomeBridgeRow, RuntimeIssuerOverlay, SurfaceDefinition, SurfaceFormula, + SurfaceFormulaOp, SurfaceOrigin, SurfaceSignTransform, }; use crate::{ ConceptOutput, DetailRowOutput, DetailRowStatementMap, FactOutput, NormalizationSummaryOutput, @@ -63,6 +64,7 @@ struct MatchedStatementRow<'a> { mapping_method: MappingMethod, match_role: MatchRole, rank: i64, + issuer_overlay_match: bool, } #[derive(Debug, Default, Clone)] @@ -110,9 +112,11 @@ pub fn build_compact_surface_model( statement_rows: &StatementRowMap, taxonomy_regime: &str, fiscal_pack: FiscalPack, + ticker: Option<&str>, + runtime_overlay: Option<&RuntimeIssuerOverlay>, warnings: Vec, ) -> Result { - let pack = load_surface_pack(fiscal_pack)?; + let pack = load_surface_pack_with_overlays(fiscal_pack, ticker, runtime_overlay)?; let crosswalk = load_crosswalk(taxonomy_regime)?; let income_bridge = load_income_bridge(fiscal_pack).ok(); let mut surface_rows = empty_surface_row_map(); @@ -122,6 +126,9 @@ pub fn build_compact_surface_model( let mut detail_row_count = 0usize; let mut unmapped_row_count = 0usize; let mut material_unmapped_row_count = 0usize; + let mut residual_primary_count = 0usize; + let mut residual_disclosure_count = 0usize; + let mut issuer_overlay_match_keys = HashSet::::new(); for statement in statement_keys() { let rows = statement_rows.get(statement).cloned().unwrap_or_default(); @@ -130,6 +137,11 @@ pub fn build_compact_surface_model( .iter() .filter(|definition| definition.statement == statement) .collect::>(); + if statement_definitions.is_empty() { + surface_rows.insert(statement.to_string(), Vec::new()); + detail_rows.insert(statement.to_string(), BTreeMap::new()); + continue; + } statement_definitions.sort_by(|left, right| { left.order .cmp(&right.order) @@ -203,6 +215,9 @@ pub fn build_compact_surface_model( residual_flag: false, }, ); + if matched.issuer_overlay_match { + issuer_overlay_match_keys.insert(matched.row.concept_key.clone()); + } } let details = detail_matches @@ -221,6 +236,9 @@ pub fn build_compact_surface_model( residual_flag: false, }, ); + if matched.issuer_overlay_match { + issuer_overlay_match_keys.insert(matched.row.concept_key.clone()); + } build_detail_row( matched.row, &definition.surface_key, @@ -310,6 +328,11 @@ pub fn build_compact_surface_model( if !residual_rows.is_empty() { unmapped_row_count += residual_rows.len(); + if statement == "disclosure" { + residual_disclosure_count += residual_rows.len(); + } else { + residual_primary_count += residual_rows.len(); + } material_unmapped_row_count += residual_rows .iter() .filter(|row| max_abs_value(&row.values) >= threshold) @@ -331,6 +354,10 @@ pub fn build_compact_surface_model( kpi_row_count: 0, unmapped_row_count, material_unmapped_row_count, + residual_primary_count, + residual_disclosure_count, + unsupported_concept_count: 0, + issuer_overlay_match_count: issuer_overlay_match_keys.len(), warnings, }, concept_mappings, @@ -734,21 +761,38 @@ fn match_statement_row<'a>( }) || authoritative_mapping .map(|mapping| mapping.surface_key == definition.surface_key) .unwrap_or(false); + let matches_overlay_authoritative = + authoritative_concept_key.as_ref().map_or(false, |concept| { + definition + .issuer_overlay_authoritative_concepts + .iter() + .any(|candidate| candidate_matches(candidate, concept)) + }); - if matches_authoritative { + if matches_authoritative || matches_overlay_authoritative { return Some(MatchedStatementRow { row, authoritative_concept_key, mapping_method: MappingMethod::AuthoritativeDirect, match_role: MatchRole::Surface, rank: 0, + issuer_overlay_match: matches_overlay_authoritative + || definition.origin == SurfaceOrigin::IssuerOverlay, }); } let matches_source = definition.allowed_source_concepts.iter().any(|candidate| { candidate_matches(candidate, &row.qname) || candidate_matches(candidate, &row.local_name) }); - if matches_source { + let matches_overlay_source = + definition + .issuer_overlay_source_concepts + .iter() + .any(|candidate| { + candidate_matches(candidate, &row.qname) + || candidate_matches(candidate, &row.local_name) + }); + if matches_source || matches_overlay_source { return Some(MatchedStatementRow { row, authoritative_concept_key, @@ -759,6 +803,8 @@ fn match_statement_row<'a>( MatchRole::Surface }, rank: 1, + issuer_overlay_match: matches_overlay_source + || definition.origin == SurfaceOrigin::IssuerOverlay, }); } @@ -832,6 +878,7 @@ fn match_income_bridge_detail_row<'a>( mapping_method: MappingMethod::AggregateChildren, match_role: MatchRole::Detail, rank: 2, + issuer_overlay_match: false, }) } @@ -1033,11 +1080,12 @@ fn candidate_matches(candidate: &str, actual: &str) -> bool { .unwrap_or(false) } -fn statement_keys() -> [&'static str; 5] { +fn statement_keys() -> [&'static str; 6] { [ "income", "balance", "cash_flow", + "disclosure", "equity", "comprehensive_income", ] @@ -1122,6 +1170,7 @@ mod tests { ("income".to_string(), Vec::new()), ("balance".to_string(), Vec::new()), ("cash_flow".to_string(), Vec::new()), + ("disclosure".to_string(), Vec::new()), ("equity".to_string(), Vec::new()), ("comprehensive_income".to_string(), Vec::new()), ]) @@ -1151,6 +1200,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1178,6 +1229,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1220,6 +1273,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1290,9 +1345,16 @@ mod tests { }, ]; - let model = - build_compact_surface_model(&periods, &rows, "us-gaap", FiscalPack::Core, vec![]) - .expect("compact model should build"); + let model = build_compact_surface_model( + &periods, + &rows, + "us-gaap", + FiscalPack::Core, + None, + None, + vec![], + ) + .expect("compact model should build"); let receivables = model .surface_rows @@ -1365,6 +1427,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1430,6 +1494,8 @@ mod tests { &rows, "us-gaap", FiscalPack::BankLender, + None, + None, vec![], ) .expect("compact model should build"); @@ -1473,6 +1539,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Insurance, + None, + None, vec![], ) .expect("compact model should build"); @@ -1557,6 +1625,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1640,6 +1710,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Core, + None, + None, vec![], ) .expect("compact model should build"); @@ -1686,6 +1758,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Insurance, + None, + None, vec![], ) .expect("compact model should build"); @@ -1742,6 +1816,8 @@ mod tests { &rows, "us-gaap", FiscalPack::ReitRealEstate, + None, + None, vec![], ) .expect("compact model should build"); @@ -1786,6 +1862,8 @@ mod tests { &rows, "us-gaap", FiscalPack::Software, + None, + None, vec![], ) .expect("compact model should build"); @@ -1825,6 +1903,8 @@ mod tests { &rows, "us-gaap", FiscalPack::EntertainmentBroadcasters, + None, + None, vec![], ) .expect("compact model should build"); @@ -1866,6 +1946,8 @@ mod tests { &rows, "us-gaap", FiscalPack::PlanDefinedBenefit, + None, + None, vec![], ) .expect("compact model should build"); diff --git a/rust/fiscal-xbrl-core/src/taxonomy_loader.rs b/rust/fiscal-xbrl-core/src/taxonomy_loader.rs index bf4839b..b7a05ce 100644 --- a/rust/fiscal-xbrl-core/src/taxonomy_loader.rs +++ b/rust/fiscal-xbrl-core/src/taxonomy_loader.rs @@ -11,6 +11,19 @@ fn default_include_in_output() -> bool { true } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SurfaceOrigin { + PackPrimary, + PackDisclosure, + CorePrimary, + CoreDisclosure, + IssuerOverlay, +} + +fn default_surface_origin() -> SurfaceOrigin { + SurfaceOrigin::CorePrimary +} + #[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum SurfaceSignTransform { @@ -25,6 +38,25 @@ pub struct SurfacePackFile { pub surfaces: Vec, } +#[derive(Debug, Deserialize, Clone)] +pub struct RuntimeIssuerOverlay { + pub version: String, + pub ticker: String, + pub pack: Option, + #[serde(default)] + pub mappings: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct RuntimeIssuerOverlayMapping { + pub surface_key: String, + pub statement: String, + #[serde(default)] + pub allowed_source_concepts: Vec, + #[serde(default)] + pub allowed_authoritative_concepts: Vec, +} + #[derive(Debug, Deserialize, Clone)] pub struct SurfaceDefinition { pub surface_key: String, @@ -43,6 +75,12 @@ pub struct SurfaceDefinition { pub include_in_output: bool, #[serde(default)] pub sign_transform: Option, + #[serde(skip, default = "default_surface_origin")] + pub origin: SurfaceOrigin, + #[serde(skip, default)] + pub issuer_overlay_source_concepts: Vec, + #[serde(skip, default)] + pub issuer_overlay_authoritative_concepts: Vec, } #[derive(Debug, Deserialize, Clone)] @@ -244,44 +282,114 @@ pub fn resolve_taxonomy_dir() -> Result { } pub fn load_surface_pack(pack: FiscalPack) -> Result { + load_surface_pack_with_overlays(pack, None, None) +} + +pub fn load_surface_pack_with_overlays( + pack: FiscalPack, + ticker: Option<&str>, + runtime_overlay: Option<&RuntimeIssuerOverlay>, +) -> Result { let taxonomy_dir = resolve_taxonomy_dir()?; - let path = taxonomy_dir - .join("fiscal") - .join("v1") - .join(format!("{}.surface.json", pack.as_str())); - let mut file = load_surface_pack_file(&path)?; + let fiscal_dir = taxonomy_dir.join("fiscal").join("v1"); + let mut file = SurfacePackFile { + version: "fiscal-v1".to_string(), + pack: pack.as_str().to_string(), + surfaces: Vec::new(), + }; + + merge_surface_pack_file( + &mut file, + load_surface_pack_file(&fiscal_dir.join(format!("{}.surface.json", pack.as_str())))?, + if matches!(pack, FiscalPack::Core) { + SurfaceOrigin::CorePrimary + } else { + SurfaceOrigin::PackPrimary + }, + ); + merge_optional_surface_pack_file( + &mut file, + load_optional_surface_pack_file( + &fiscal_dir.join(format!("{}.disclosure.surface.json", pack.as_str())), + )?, + if matches!(pack, FiscalPack::Core) { + SurfaceOrigin::CoreDisclosure + } else { + SurfaceOrigin::PackDisclosure + }, + ); if !matches!(pack, FiscalPack::Core) { - let core_path = taxonomy_dir - .join("fiscal") - .join("v1") - .join("core.surface.json"); - let core_file = load_surface_pack_file(&core_path)?; - let pack_inherited_keys = file - .surfaces - .iter() - .filter(|surface| surface.statement == "balance" || surface.statement == "cash_flow") - .map(|surface| (surface.statement.clone(), surface.surface_key.clone())) - .collect::>(); - - file.surfaces.extend( - core_file - .surfaces - .into_iter() - .filter(|surface| { - surface.statement == "balance" || surface.statement == "cash_flow" - }) - .filter(|surface| { - !pack_inherited_keys - .contains(&(surface.statement.clone(), surface.surface_key.clone())) - }), + merge_surface_pack_file( + &mut file, + load_surface_pack_file(&fiscal_dir.join("core.surface.json"))?, + SurfaceOrigin::CorePrimary, ); } + merge_optional_surface_pack_file( + &mut file, + load_optional_surface_pack_file(&fiscal_dir.join("core.disclosure.surface.json"))?, + SurfaceOrigin::CoreDisclosure, + ); + + if let Some(normalized_ticker) = ticker + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| !value.is_empty()) + { + merge_optional_surface_pack_file( + &mut file, + load_optional_surface_pack_file( + &fiscal_dir + .join("issuers") + .join(format!("{normalized_ticker}.surface.json")), + )?, + SurfaceOrigin::IssuerOverlay, + ); + } + + if let Some(runtime_overlay) = runtime_overlay { + merge_runtime_issuer_overlay(&mut file, runtime_overlay); + } let _ = (&file.version, &file.pack); Ok(file) } +fn merge_runtime_issuer_overlay(target: &mut SurfacePackFile, overlay: &RuntimeIssuerOverlay) { + for mapping in &overlay.mappings { + let Some(existing) = target + .surfaces + .iter_mut() + .find(|candidate| candidate.surface_key == mapping.surface_key) + else { + continue; + }; + + if existing.statement != mapping.statement { + continue; + } + + existing.issuer_overlay_source_concepts.extend( + mapping + .allowed_source_concepts + .iter() + .cloned() + .filter(|concept| !concept.trim().is_empty()), + ); + existing.issuer_overlay_authoritative_concepts.extend( + mapping + .allowed_authoritative_concepts + .iter() + .cloned() + .filter(|concept| !concept.trim().is_empty()), + ); + existing.issuer_overlay_source_concepts.sort(); + existing.issuer_overlay_source_concepts.dedup(); + existing.issuer_overlay_authoritative_concepts.sort(); + existing.issuer_overlay_authoritative_concepts.dedup(); + } +} + fn load_surface_pack_file(path: &PathBuf) -> Result { let raw = fs::read_to_string(path).with_context(|| { format!( @@ -297,6 +405,128 @@ fn load_surface_pack_file(path: &PathBuf) -> Result { }) } +fn load_optional_surface_pack_file(path: &PathBuf) -> Result> { + if !path.exists() { + return Ok(None); + } + + load_surface_pack_file(path).map(Some) +} + +fn merge_optional_surface_pack_file( + target: &mut SurfacePackFile, + file: Option, + origin: SurfaceOrigin, +) { + if let Some(file) = file { + merge_surface_pack_file(target, file, origin); + } +} + +fn merge_surface_pack_file( + target: &mut SurfacePackFile, + mut incoming: SurfacePackFile, + origin: SurfaceOrigin, +) { + for mut surface in incoming.surfaces.drain(..) { + surface.origin = origin; + + if let Some(existing) = target + .surfaces + .iter_mut() + .find(|candidate| candidate.surface_key == surface.surface_key) + { + if surface.origin == SurfaceOrigin::IssuerOverlay { + merge_surface_definition(existing, surface); + } else if matches!( + existing.origin, + SurfaceOrigin::PackPrimary | SurfaceOrigin::PackDisclosure + ) && matches!( + surface.origin, + SurfaceOrigin::CorePrimary | SurfaceOrigin::CoreDisclosure + ) { + continue; + } else { + merge_surface_definition(existing, surface); + } + } else { + target.surfaces.push(surface); + } + } +} + +fn merge_surface_definition(existing: &mut SurfaceDefinition, incoming: SurfaceDefinition) { + if incoming.origin == SurfaceOrigin::IssuerOverlay { + existing.issuer_overlay_source_concepts.extend( + incoming + .allowed_source_concepts + .iter() + .cloned() + .filter(|concept| !concept.trim().is_empty()), + ); + existing.issuer_overlay_authoritative_concepts.extend( + incoming + .allowed_authoritative_concepts + .iter() + .cloned() + .filter(|concept| !concept.trim().is_empty()), + ); + + if existing.statement.is_empty() { + existing.statement = incoming.statement; + } + if existing.label.is_empty() { + existing.label = incoming.label; + } + if existing.category.is_empty() { + existing.category = incoming.category; + } + if existing.unit.is_empty() { + existing.unit = incoming.unit; + } + if existing.formula_fallback.is_none() { + existing.formula_fallback = incoming.formula_fallback; + } + if existing.sign_transform.is_none() { + existing.sign_transform = incoming.sign_transform; + } + existing.include_in_output = existing.include_in_output || incoming.include_in_output; + existing.materiality_policy = if existing.materiality_policy.is_empty() { + incoming.materiality_policy + } else { + existing.materiality_policy.clone() + }; + existing.detail_grouping_policy = if existing.detail_grouping_policy.is_empty() { + incoming.detail_grouping_policy + } else { + existing.detail_grouping_policy.clone() + }; + existing.rollup_policy = if existing.rollup_policy.is_empty() { + incoming.rollup_policy + } else { + existing.rollup_policy.clone() + }; + return; + } + + existing.allowed_source_concepts.extend( + incoming + .allowed_source_concepts + .into_iter() + .filter(|concept| !concept.trim().is_empty()), + ); + existing.allowed_authoritative_concepts.extend( + incoming + .allowed_authoritative_concepts + .into_iter() + .filter(|concept| !concept.trim().is_empty()), + ); + existing.allowed_source_concepts.sort(); + existing.allowed_source_concepts.dedup(); + existing.allowed_authoritative_concepts.sort(); + existing.allowed_authoritative_concepts.dedup(); +} + pub fn load_crosswalk(regime: &str) -> Result> { let file_name = match regime { "us-gaap" => "us-gaap.json", diff --git a/rust/taxonomy/crosswalk/us-gaap.json b/rust/taxonomy/crosswalk/us-gaap.json index ebb454c..077c1b3 100644 --- a/rust/taxonomy/crosswalk/us-gaap.json +++ b/rust/taxonomy/crosswalk/us-gaap.json @@ -535,19 +535,19 @@ "authoritative_concept_key": "us-gaap:FinanceLeaseRightOfUseAssetAmortization" }, "us-gaap:DerivativeAssets": { - "surface_key": "derivative_disclosure", + "surface_key": "derivative_instruments_disclosure", "authoritative_concept_key": "us-gaap:DerivativeAssets" }, "us-gaap:DerivativeLiabilities": { - "surface_key": "derivative_disclosure", + "surface_key": "derivative_instruments_disclosure", "authoritative_concept_key": "us-gaap:DerivativeLiabilities" }, "us-gaap:DerivativeAssetsCurrent": { - "surface_key": "derivative_disclosure", + "surface_key": "derivative_instruments_disclosure", "authoritative_concept_key": "us-gaap:DerivativeAssetsCurrent" }, "us-gaap:DerivativeLiabilitiesCurrent": { - "surface_key": "derivative_disclosure", + "surface_key": "derivative_instruments_disclosure", "authoritative_concept_key": "us-gaap:DerivativeLiabilitiesCurrent" }, "us-gaap:IncreaseDecreaseInContractWithCustomerLiability": { diff --git a/rust/taxonomy/fiscal/v1/core.disclosure.surface.json b/rust/taxonomy/fiscal/v1/core.disclosure.surface.json new file mode 100644 index 0000000..0f1effe --- /dev/null +++ b/rust/taxonomy/fiscal/v1/core.disclosure.surface.json @@ -0,0 +1,389 @@ +{ + "version": "fiscal-v1", + "pack": "core", + "surfaces": [ + { + "surface_key": "income_tax_disclosure", + "statement": "disclosure", + "label": "Income Tax Disclosures", + "category": "tax", + "order": 100, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:CurrentIncomeTaxExpenseBenefit", + "us-gaap:DeferredIncomeTaxExpenseBenefit", + "us-gaap:CurrentFederalTaxExpenseBenefit", + "us-gaap:CurrentForeignTaxExpenseBenefit", + "us-gaap:CurrentStateAndLocalTaxExpenseBenefit", + "us-gaap:DeferredFederalIncomeTaxExpenseBenefit", + "us-gaap:DeferredForeignIncomeTaxExpenseBenefit", + "us-gaap:DeferredStateAndLocalIncomeTaxExpenseBenefit", + "us-gaap:EffectiveIncomeTaxRateContinuingOperations", + "us-gaap:EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate", + "us-gaap:EffectiveIncomeTaxRateReconciliationDeductionsExcessTaxBenefitsStockBasedCompensation", + "us-gaap:EffectiveIncomeTaxRateReconciliationFdiiPercent", + "us-gaap:EffectiveIncomeTaxRateReconciliationForeignIncomeTaxRateDifferential", + "us-gaap:EffectiveIncomeTaxRateReconciliationIntangiblePropertyTransfers", + "us-gaap:EffectiveIncomeTaxRateReconciliationInterestIncomeExpense", + "us-gaap:EffectiveIncomeTaxRateReconciliationOtherAdjustments", + "us-gaap:EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes", + "us-gaap:EffectiveIncomeTaxRateReconciliationTaxCreditsResearch", + "us-gaap:EmployeeServiceShareBasedCompensationTaxBenefitFromCompensationExpense", + "us-gaap:IncomeTaxesPaidNet", + "us-gaap:IncomeTaxExaminationYearUnderExamination", + "us-gaap:IncomeLossFromContinuingOperationsBeforeIncomeTaxesDomestic", + "us-gaap:IncomeLossFromContinuingOperationsBeforeIncomeTaxesForeign", + "us-gaap:UnrecognizedTaxBenefitsIncomeTaxPenaltiesAndInterestAccrued", + "us-gaap:UnrecognizedTaxBenefitsIncomeTaxPenaltiesAndInterestExpense" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "debt_disclosure", + "statement": "disclosure", + "label": "Debt Disclosures", + "category": "debt", + "order": 200, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:DebtInstrumentFaceAmount", + "us-gaap:DebtInstrumentFairValue", + "us-gaap:DebtInstrumentInterestRateEffectivePercentage", + "us-gaap:DebtInstrumentInterestRateStatedPercentage", + "us-gaap:DebtInstrumentUnamortizedDiscountPremiumAndDebtIssuanceCostsNet", + "us-gaap:ExtinguishmentOfDebtAmount", + "us-gaap:LongTermDebtFairValue", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalRemainderOfFiscalYear", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive", + "us-gaap:LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive", + "us-gaap:PaymentsOfDebtRestructuringCosts", + "us-gaap:ProceedsFromDebtMaturingInMoreThanThreeMonths", + "us-gaap:ProceedsFromRepaymentsOfShortTermDebtMaturingInThreeMonthsOrLess", + "us-gaap:ProceedsFromShortTermDebtMaturingInThreeMonthsOrLess", + "us-gaap:RepaymentsOfShortTermDebtMaturingInThreeMonthsOrLess", + "us-gaap:ShortTermDebtWeightedAverageInterestRate" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "investment_securities_disclosure", + "statement": "disclosure", + "label": "Investment Securities Disclosures", + "category": "securities", + "order": 300, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax", + "us-gaap:AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax", + "us-gaap:AvailableForSaleDebtSecuritiesAmortizedCostBasis", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesNextRollingTwelveMonthsAmortizedCostBasis", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesNextRollingTwelveMonthsFairValue", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingAfterYearTenAmortizedCostBasis", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingAfterYearTenFairValue", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingYearSixThroughTenAmortizedCostBasis", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingYearSixThroughTenFairValue", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingYearTwoThroughFiveAmortizedCostBasis", + "us-gaap:AvailableForSaleSecuritiesDebtMaturitiesRollingYearTwoThroughFiveFairValue", + "us-gaap:DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger", + "us-gaap:DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss", + "us-gaap:DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months", + "us-gaap:DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss", + "us-gaap:DebtSecuritiesAvailableForSaleRealizedGain", + "us-gaap:DebtSecuritiesAvailableForSaleRealizedLoss", + "us-gaap:DebtSecuritiesAvailableForSaleUnrealizedLossPosition", + "us-gaap:DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "derivative_instruments_disclosure", + "statement": "disclosure", + "label": "Derivative Instruments Disclosures", + "category": "derivatives", + "order": 400, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:DerivativeAssets", + "us-gaap:DerivativeAssetsCurrent", + "us-gaap:DerivativeAssetsNoncurrent", + "us-gaap:DerivativeLiabilities", + "us-gaap:DerivativeLiabilitiesCurrent", + "us-gaap:DerivativeLiabilitiesNoncurrent", + "us-gaap:DerivativeFairValueOfDerivativeAsset", + "us-gaap:DerivativeFairValueOfDerivativeLiability", + "us-gaap:DerivativeAssetFairValueGrossAssetIncludingNotSubjectToMasterNettingArrangement", + "us-gaap:DerivativeAssetFairValueGrossLiability", + "us-gaap:DerivativeLiabilityFairValueGrossAsset", + "us-gaap:DerivativeLiabilityFairValueGrossLiabilityIncludingNotSubjectToMasterNettingArrangement" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "lease_disclosure", + "statement": "disclosure", + "label": "Lease Obligations Disclosures", + "category": "leases", + "order": 500, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:FinanceLeaseInterestPaymentOnLiability", + "us-gaap:FinanceLeaseLiabilityPaymentsDue", + "us-gaap:FinanceLeaseLiabilityPaymentsRemainderOfFiscalYear", + "us-gaap:FinanceLeaseLiabilityPaymentsDueNextTwelveMonths", + "us-gaap:FinanceLeaseLiabilityPaymentsDueYearTwo", + "us-gaap:FinanceLeaseLiabilityPaymentsDueYearThree", + "us-gaap:FinanceLeaseLiabilityPaymentsDueYearFour", + "us-gaap:FinanceLeaseLiabilityPaymentsDueYearFive", + "us-gaap:FinanceLeaseLiabilityPaymentsDueAfterYearFive", + "us-gaap:FinanceLeaseLiabilityUndiscountedExcessAmount", + "us-gaap:FinanceLeaseRightOfUseAssetAmortization", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDue", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsRemainderOfFiscalYear", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueNextTwelveMonths", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueYearTwo", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueYearThree", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueYearFour", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueYearFive", + "us-gaap:LesseeOperatingLeaseLiabilityPaymentsDueAfterYearFive", + "us-gaap:LesseeOperatingLeaseLiabilityUndiscountedExcessAmount", + "us-gaap:RightOfUseAssetObtainedInExchangeForFinanceLeaseLiability", + "us-gaap:RightOfUseAssetObtainedInExchangeForOperatingLeaseLiability" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "intangible_assets_disclosure", + "statement": "disclosure", + "label": "Intangible Assets Disclosures", + "category": "intangibles", + "order": 600, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:AmortizationOfIntangibleAssets", + "us-gaap:FiniteLivedIntangibleAssetsAccumulatedAmortization", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseRemainderOfFiscalYear", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseYearThree", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseYearFour", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseYearFive", + "us-gaap:FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive", + "us-gaap:FiniteLivedIntangibleAssetsGross", + "us-gaap:FinitelivedIntangibleAssetsAcquired1" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "business_combinations_disclosure", + "statement": "disclosure", + "label": "Business Combinations Disclosures", + "category": "ma", + "order": 700, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:BusinessAcquisitionsProFormaNetIncomeLoss", + "us-gaap:BusinessAcquisitionsProFormaRevenue", + "us-gaap:BusinessCombinationProFormaInformationRevenueOfAcquireeSinceAcquisitionDateActual" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "revenue_disclosure", + "statement": "disclosure", + "label": "Revenue Disclosures", + "category": "revenue", + "order": 800, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:RevenueRemainingPerformanceObligation", + "us-gaap:RevenueRemainingPerformanceObligationPercentage" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "cash_flow_disclosure", + "statement": "disclosure", + "label": "Cash Flow Disclosures", + "category": "cash_flow", + "order": 900, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "equity_investments_disclosure", + "statement": "disclosure", + "label": "Equity Investments Disclosures", + "category": "equity_investments", + "order": 1000, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:EquityMethodInvestmentOwnershipPercentage", + "us-gaap:EquityMethodInvestments", + "us-gaap:EquityMethodInvestmentsFunded", + "us-gaap:EquitySecuritiesFVNINoncurrent", + "us-gaap:EquitySecuritiesFvNiRealizedGainLoss", + "us-gaap:EquitySecuritiesFvNiUnrealizedGainLoss", + "us-gaap:EquitySecuritiesWithoutReadilyDeterminableFairValueAmount" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "share_based_compensation_disclosure", + "statement": "disclosure", + "label": "Share-Based Compensation Disclosures", + "category": "share_based_compensation", + "order": 1100, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsForfeitedInPeriod", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsForfeituresWeightedAverageGrantDateFairValue", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriodWeightedAverageGrantDateFairValue", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsNonvestedNumber", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsNonvestedWeightedAverageGrantDateFairValue", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsVestedInPeriod", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsVestedInPeriodTotalFairValue", + "us-gaap:ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsVestedInPeriodWeightedAverageGrantDateFairValue" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "deferred_tax_balance_disclosure", + "statement": "disclosure", + "label": "Deferred Tax Balance Disclosures", + "category": "deferred_tax_balances", + "order": 1200, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:AccruedIncomeTaxesCurrent", + "us-gaap:AccruedIncomeTaxesNoncurrent", + "us-gaap:DeferredTaxAssetsDeferredIncome", + "us-gaap:DeferredTaxAssetsGross", + "us-gaap:DeferredTaxAssetsLiabilitiesNet", + "us-gaap:DeferredTaxAssetsOther", + "us-gaap:DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost", + "us-gaap:DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsOther", + "us-gaap:DeferredTaxAssetsValuationAllowance", + "us-gaap:DeferredTaxLiabilitiesLeasingArrangements", + "us-gaap:DeferredTaxLiabilitiesOther" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "contract_liability_disclosure", + "statement": "disclosure", + "label": "Contract Liability Disclosures", + "category": "contract_liabilities", + "order": 1300, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:ContractWithCustomerLiabilityIncurred", + "us-gaap:ContractWithCustomerLiabilityRevenueRecognized" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "acquisition_allocation_disclosure", + "statement": "disclosure", + "label": "Acquisition Allocation Disclosures", + "category": "acquisition_allocation", + "order": 1400, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedCashAndEquivalents", + "us-gaap:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedDeferredTaxLiabilities", + "us-gaap:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedIntangibles", + "us-gaap:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedNoncurrentLiabilitiesLongTermDebt", + "us-gaap:BusinessCombinationRecognizedIdentifiableAssetsAcquiredGoodwillAndLiabilitiesAssumedNet" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "other_comprehensive_income_disclosure", + "statement": "disclosure", + "label": "Other Comprehensive Income Disclosures", + "category": "oci", + "order": 1500, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "us-gaap:OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax", + "us-gaap:OtherComprehensiveIncomeLossBeforeReclassificationsTax", + "us-gaap:OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationAndTax", + "us-gaap:OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationAfterTax", + "us-gaap:OtherComprehensiveIncomeLossCashFlowHedgeGainLossReclassificationBeforeTax", + "us-gaap:OtherComprehensiveIncomeLossForeignCurrencyTransactionAndTranslationAdjustmentNetOfTax", + "us-gaap:OtherComprehensiveIncomeLossNetOfTaxPortionAttributableToParent", + "us-gaap:OtherComprehensiveIncomeLossTaxPortionAttributableToParent1" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + } + ] +} diff --git a/rust/taxonomy/fiscal/v1/core.surface.json b/rust/taxonomy/fiscal/v1/core.surface.json index a05b8f9..284c161 100644 --- a/rust/taxonomy/fiscal/v1/core.surface.json +++ b/rust/taxonomy/fiscal/v1/core.surface.json @@ -1538,31 +1538,6 @@ "materiality_policy": "cash_flow_default", "include_in_output": false }, - { - "surface_key": "derivative_disclosure", - "statement": "balance", - "label": "Derivative Instruments Disclosure", - "category": "disclosure", - "order": 181, - "unit": "currency", - "rollup_policy": "aggregate_children", - "allowed_source_concepts": [ - "us-gaap:DerivativeAssets", - "us-gaap:DerivativeAssetsCurrent", - "us-gaap:DerivativeAssetsNoncurrent", - "us-gaap:DerivativeLiabilities", - "us-gaap:DerivativeLiabilitiesCurrent", - "us-gaap:DerivativeLiabilitiesNoncurrent", - "us-gaap:DerivativeFairValueOfDerivativeAsset", - "us-gaap:DerivativeFairValueOfDerivativeLiability", - "us-gaap:DerivativeAssetFairValueGrossAssetIncludingNotSubjectToMasterNettingArrangement", - "us-gaap:DerivativeLiabilityFairValueGrossLiabilityIncludingNotSubjectToMasterNettingArrangement" - ], - "allowed_authoritative_concepts": [], - "formula_fallback": null, - "detail_grouping_policy": "group_all_children", - "materiality_policy": "disclosure" - }, { "surface_key": "changes_other_noncurrent_liabilities", "statement": "cash_flow", @@ -1877,6 +1852,98 @@ "detail_grouping_policy": "top_level_only", "materiality_policy": "cash_flow_default" }, + { + "surface_key": "common_stock_and_apic", + "statement": "equity", + "label": "Common Stock and APIC", + "category": "equity", + "order": 10, + "unit": "currency", + "rollup_policy": "direct_only", + "allowed_source_concepts": [ + "us-gaap:CommonStocksIncludingAdditionalPaidInCapital" + ], + "allowed_authoritative_concepts": [ + "us-gaap:CommonStocksIncludingAdditionalPaidInCapital" + ], + "formula_fallback": null, + "detail_grouping_policy": "top_level_only", + "materiality_policy": "balance_default" + }, + { + "surface_key": "retained_earnings", + "statement": "equity", + "label": "Retained Earnings", + "category": "equity", + "order": 20, + "unit": "currency", + "rollup_policy": "direct_only", + "allowed_source_concepts": [ + "us-gaap:RetainedEarningsAccumulatedDeficit" + ], + "allowed_authoritative_concepts": [ + "us-gaap:RetainedEarningsAccumulatedDeficit" + ], + "formula_fallback": null, + "detail_grouping_policy": "top_level_only", + "materiality_policy": "balance_default" + }, + { + "surface_key": "accumulated_other_comprehensive_income", + "statement": "equity", + "label": "Accumulated Other Comprehensive Income", + "category": "equity", + "order": 30, + "unit": "currency", + "rollup_policy": "direct_only", + "allowed_source_concepts": [ + "us-gaap:AccumulatedOtherComprehensiveIncomeLossNetOfTax" + ], + "allowed_authoritative_concepts": [ + "us-gaap:AccumulatedOtherComprehensiveIncomeLossNetOfTax" + ], + "formula_fallback": null, + "detail_grouping_policy": "top_level_only", + "materiality_policy": "balance_default" + }, + { + "surface_key": "other_equity", + "statement": "equity", + "label": "Other Equity", + "category": "equity", + "order": 40, + "unit": "currency", + "rollup_policy": "direct_only", + "allowed_source_concepts": [ + "us-gaap:StockholdersEquityOther" + ], + "allowed_authoritative_concepts": [ + "us-gaap:StockholdersEquityOther" + ], + "formula_fallback": null, + "detail_grouping_policy": "top_level_only", + "materiality_policy": "balance_default" + }, + { + "surface_key": "total_equity", + "statement": "equity", + "label": "Total Equity", + "category": "equity", + "order": 50, + "unit": "currency", + "rollup_policy": "direct_only", + "allowed_source_concepts": [ + "us-gaap:StockholdersEquity", + "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" + ], + "allowed_authoritative_concepts": [ + "us-gaap:StockholdersEquity", + "us-gaap:StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" + ], + "formula_fallback": null, + "detail_grouping_policy": "top_level_only", + "materiality_policy": "balance_default" + }, { "surface_key": "income_tax_disclosure", "statement": "disclosure", diff --git a/rust/taxonomy/fiscal/v1/issuers/msft.surface.json b/rust/taxonomy/fiscal/v1/issuers/msft.surface.json new file mode 100644 index 0000000..ceb3f0c --- /dev/null +++ b/rust/taxonomy/fiscal/v1/issuers/msft.surface.json @@ -0,0 +1,184 @@ +{ + "version": "fiscal-v1", + "pack": "msft", + "surfaces": [ + { + "surface_key": "contract_liability_disclosure", + "statement": "disclosure", + "label": "Contract Liability Disclosures", + "category": "contract_liabilities", + "order": 1300, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:ContractWithCustomerLiabilityRevenueDeferred", + "msft:ContractWithCustomerLiabilityRevenueRecognizedIncludingAdditions" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "business_combinations_disclosure", + "statement": "disclosure", + "label": "Business Combinations Disclosures", + "category": "ma", + "order": 700, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:AcquisitionsNetOfCashAcquiredAndPurchasesOfIntangibleAndOtherAssets" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "debt_disclosure", + "statement": "disclosure", + "label": "Debt Disclosures", + "category": "debt", + "order": 200, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:DebtInstrumentIssuanceYear", + "msft:DebtInstrumentMaturityYear", + "msft:PremiumOnDebtExchange1", + "msft:LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFour" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "investment_securities_disclosure", + "statement": "disclosure", + "label": "Investment Securities Disclosures", + "category": "securities", + "order": 300, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:ImpairmentOfEquityInvestments", + "msft:ProceedsFromInvestments" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "lease_disclosure", + "statement": "disclosure", + "label": "Lease Obligations Disclosures", + "category": "leases", + "order": 500, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:FinanceLeaseLiabilityPaymentsDueAfterYearFour", + "msft:LesseeOperatingLeaseLiabilityPaymentsDueAfterYearFour" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "intangible_assets_disclosure", + "statement": "disclosure", + "label": "Intangible Assets Disclosures", + "category": "intangibles", + "order": 600, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFour" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "acquisition_allocation_disclosure", + "statement": "disclosure", + "label": "Acquisition Allocation Disclosures", + "category": "acquisition_allocation", + "order": 1400, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedIncomeTaxLiabilitiesNoncurrent", + "msft:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedOtherAssets", + "msft:BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedOtherLiabilities" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "equity_investments_disclosure", + "statement": "disclosure", + "label": "Equity Investments Disclosures", + "category": "equity_investments", + "order": 1000, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:EquityInterestPercentage", + "msft:EquityMethodInvestmentsTotalFundingCommitments" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "deferred_tax_balance_disclosure", + "statement": "disclosure", + "label": "Deferred Tax Balance Disclosures", + "category": "deferred_tax_balances", + "order": 1200, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:DeferredTaxAssetsAmortization", + "msft:DeferredTaxAssetsBookTaxBasisDifferencesInInvestmentsAndDebt", + "msft:DeferredTaxAssetsCapitalizedResearchAndDevelopment", + "msft:DeferredTaxAssetsLeasingLiabilities", + "msft:DeferredTaxAssetsOperatingLossAndTaxCreditCarryForwards", + "msft:DeferredTaxLiabilitiesBookTaxBasisDifferencesInvestmentsAndDebt", + "msft:DeferredTaxLiabilitiesDeferredTaxOnForeignEarnings", + "msft:DeferredTaxLiabilitiesDepreciation" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + }, + { + "surface_key": "income_tax_disclosure", + "statement": "disclosure", + "label": "Income Tax Disclosures", + "category": "tax", + "order": 100, + "unit": "currency", + "rollup_policy": "aggregate_children", + "allowed_source_concepts": [ + "msft:EffectiveIncomeTaxRateReconciliationDeductionsExcessTaxBenefitsStockBasedCompensation", + "msft:EffectiveIncomeTaxRateReconciliationInterestIncomeExpense", + "msft:EffectiveIncomeTaxRateReconciliationIntangiblePropertyTransfers" + ], + "allowed_authoritative_concepts": [], + "formula_fallback": null, + "detail_grouping_policy": "group_all_children", + "materiality_policy": "disclosure" + } + ] +} diff --git a/scripts/dev-env.ts b/scripts/dev-env.ts index 77c3443..9d2aa0b 100644 --- a/scripts/dev-env.ts +++ b/scripts/dev-env.ts @@ -19,7 +19,7 @@ type LocalDevOverrideSummary = { workflowChanged: boolean; }; -export type LocalDevConfig = { +type LocalDevConfig = { bindHost: string; env: EnvMap; overrides: LocalDevOverrideSummary; diff --git a/scripts/report-taxonomy-health.ts b/scripts/report-taxonomy-health.ts index e154605..78378c4 100644 --- a/scripts/report-taxonomy-health.ts +++ b/scripts/report-taxonomy-health.ts @@ -1,16 +1,20 @@ -import { and, desc, eq, gte, lte } from 'drizzle-orm'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { and, desc, eq, gte, inArray, lte } from 'drizzle-orm'; import { db } from '@/lib/server/db'; -import { filingTaxonomySnapshot } from '@/lib/server/db/schema'; +import { filingTaxonomyConcept, filingTaxonomySnapshot } from '@/lib/server/db/schema'; type ScriptOptions = { ticker: string | null; from: string | null; to: string | null; sampleLimit: number; + failOnResiduals: boolean; }; type SnapshotRow = { + id: number; filing_id: number; ticker: string; filing_date: string; @@ -21,17 +25,38 @@ type SnapshotRow = { parser_version: string; fiscal_pack: string | null; normalization_summary: { + issuerOverlayMatchCount?: number; + residualDisclosureCount?: number; + residualPrimaryCount?: number; + unsupportedConceptCount?: number; warnings?: string[]; } | null; + surface_rows: Record> | null; updated_at: string; }; +type ResidualConceptRow = { + snapshot_id: number; + qname: string; + statement_kind: string | null; + role_uri: string | null; +}; + +type SurfacePackFile = { + surfaces: Array<{ + statement: string; + allowed_source_concepts?: string[]; + allowed_authoritative_concepts?: string[]; + }>; +}; + function parseOptions(argv: string[]): ScriptOptions { const options: ScriptOptions = { ticker: null, from: null, to: null, - sampleLimit: 5 + sampleLimit: 5, + failOnResiduals: false }; for (const arg of argv) { @@ -39,10 +64,15 @@ function parseOptions(argv: string[]): ScriptOptions { console.log('Report taxonomy snapshot health from the local database.'); console.log(''); console.log('Usage:'); - console.log(' bun run scripts/report-taxonomy-health.ts [--ticker=SYMBOL] [--from=YYYY-MM-DD] [--to=YYYY-MM-DD] [--sample-limit=N]'); + console.log(' bun run scripts/report-taxonomy-health.ts [--ticker=SYMBOL] [--from=YYYY-MM-DD] [--to=YYYY-MM-DD] [--sample-limit=N] [--fail-on-residuals]'); process.exit(0); } + if (arg === '--fail-on-residuals') { + options.failOnResiduals = true; + continue; + } + if (arg.startsWith('--ticker=')) { const value = arg.slice('--ticker='.length).trim().toUpperCase(); options.ticker = value.length > 0 ? value : null; @@ -121,6 +151,7 @@ async function loadRows(options: ScriptOptions): Promise { const whereClause = conditions.length > 0 ? and(...conditions) : undefined; const baseQuery = db.select({ + id: filingTaxonomySnapshot.id, filing_id: filingTaxonomySnapshot.filing_id, ticker: filingTaxonomySnapshot.ticker, filing_date: filingTaxonomySnapshot.filing_date, @@ -131,6 +162,7 @@ async function loadRows(options: ScriptOptions): Promise { parser_version: filingTaxonomySnapshot.parser_version, fiscal_pack: filingTaxonomySnapshot.fiscal_pack, normalization_summary: filingTaxonomySnapshot.normalization_summary, + surface_rows: filingTaxonomySnapshot.surface_rows, updated_at: filingTaxonomySnapshot.updated_at }).from(filingTaxonomySnapshot).orderBy(desc(filingTaxonomySnapshot.updated_at)); @@ -141,15 +173,77 @@ async function loadRows(options: ScriptOptions): Promise { return await baseQuery; } +async function loadResidualConceptRows(snapshotIds: number[]): Promise { + if (snapshotIds.length === 0) { + return []; + } + + return await db.select({ + snapshot_id: filingTaxonomyConcept.snapshot_id, + qname: filingTaxonomyConcept.qname, + statement_kind: filingTaxonomyConcept.statement_kind, + role_uri: filingTaxonomyConcept.role_uri + }).from(filingTaxonomyConcept).where(and( + inArray(filingTaxonomyConcept.snapshot_id, snapshotIds), + eq(filingTaxonomyConcept.residual_flag, true) + )); +} + +function normalizeConcept(value: string) { + return value.trim().toLowerCase(); +} + +function loadTaxonomyStatementIndex() { + const index = new Map>(); + const taxonomyDir = join(process.cwd(), 'rust', 'taxonomy', 'fiscal', 'v1'); + const fileNames = readdirSync(taxonomyDir) + .filter((fileName) => fileName.endsWith('.surface.json') && fileName !== 'universal_income.surface.json') + .sort((left, right) => left.localeCompare(right)); + + for (const fileName of fileNames) { + const file = JSON.parse(readFileSync(join(taxonomyDir, fileName), 'utf8')) as SurfacePackFile; + for (const surface of file.surfaces ?? []) { + const concepts = [ + ...(surface.allowed_source_concepts ?? []), + ...(surface.allowed_authoritative_concepts ?? []) + ]; + for (const concept of concepts) { + const normalized = normalizeConcept(concept); + const statements = index.get(normalized) ?? new Set(); + statements.add(surface.statement); + index.set(normalized, statements); + } + } + } + + return index; +} + +function statementsForConcept(index: Map>, qname: string) { + return [...(index.get(normalizeConcept(qname)) ?? new Set())] + .sort((left, right) => left.localeCompare(right)); +} + async function main() { const options = parseOptions(process.argv.slice(2)); const rows = await loadRows(options); + const residualRows = await loadResidualConceptRows(rows.map((row) => row.id)); + const taxonomyStatementIndex = loadTaxonomyStatementIndex(); const statusCounts = new Map(); const parserCounts = new Map(); const packCounts = new Map(); const warningCounts = new Map(); const parserVersionCounts = new Map(); + const residualCounts = new Map(); + const residualNoRoleCounts = new Map(); + const residualDisclosureOnlyCounts = new Map(); + const residualEquityMatchCounts = new Map(); + const residualDifferentStatementCounts = new Map(); + const residualAbsentCounts = new Map(); + const disclosureSurfaceCounts = new Map(); + const issuerOverlayMatchCounts = new Map(); + const unsupportedSurfacedConcepts = new Map(); for (const row of rows) { incrementCount(statusCounts, row.parse_status); @@ -160,6 +254,59 @@ async function main() { for (const warning of row.normalization_summary?.warnings ?? []) { incrementCount(warningCounts, warning); } + + for (const disclosureRow of row.surface_rows?.disclosure ?? []) { + incrementCount(disclosureSurfaceCounts, disclosureRow.key); + } + + const issuerOverlayMatches = row.normalization_summary?.issuerOverlayMatchCount ?? 0; + if (issuerOverlayMatches > 0) { + issuerOverlayMatchCounts.set( + row.ticker, + (issuerOverlayMatchCounts.get(row.ticker) ?? 0) + issuerOverlayMatches + ); + } + + const unsupportedConcepts = row.normalization_summary?.unsupportedConceptCount ?? 0; + if (unsupportedConcepts > 0) { + unsupportedSurfacedConcepts.set( + row.ticker, + (unsupportedSurfacedConcepts.get(row.ticker) ?? 0) + unsupportedConcepts + ); + } + } + + for (const row of residualRows) { + incrementCount(residualCounts, row.qname); + if (!row.role_uri || row.role_uri.trim().length === 0) { + incrementCount(residualNoRoleCounts, row.qname); + } + + const matchedStatements = statementsForConcept(taxonomyStatementIndex, row.qname); + const primaryStatements = matchedStatements.filter((statement) => statement !== 'disclosure'); + + if (matchedStatements.length > 0 && primaryStatements.length === 0) { + incrementCount(residualDisclosureOnlyCounts, row.qname); + } + + if (matchedStatements.includes('equity')) { + incrementCount(residualEquityMatchCounts, row.qname); + } + + if ( + row.statement_kind + && primaryStatements.length > 0 + && !primaryStatements.includes(row.statement_kind) + ) { + incrementCount( + residualDifferentStatementCounts, + `${row.qname}::${row.statement_kind}->${primaryStatements.join('|')}` + ); + } + + if (matchedStatements.length === 0) { + incrementCount(residualAbsentCounts, row.qname); + } } const failedRows = rows @@ -183,14 +330,28 @@ async function main() { console.log(`[report-taxonomy-health] legacy_ts=${legacyCount}`); console.log(`[report-taxonomy-health] deferred_to_typescript=${deferredCount}`); console.log(`[report-taxonomy-health] ts_compact_surface_fallback=${fallbackCount}`); + console.log(`[report-taxonomy-health] residual_rows=${residualRows.length}`); printCountMap('parse_status', statusCounts); printCountMap('parser_engine', parserCounts); printCountMap('parser_version', parserVersionCounts); printCountMap('fiscal_pack', packCounts); printCountMap('warnings', warningCounts); + printCountMap('residual_top_concepts', residualCounts); + printCountMap('residual_missing_role_uri', residualNoRoleCounts); + printCountMap('residual_disclosure_only', residualDisclosureOnlyCounts); + printCountMap('residual_matching_equity_taxonomy', residualEquityMatchCounts); + printCountMap('residual_different_primary_statement', residualDifferentStatementCounts); + printCountMap('residual_absent_from_taxonomy', residualAbsentCounts); + printCountMap('disclosure_surface_counts', disclosureSurfaceCounts); + printCountMap('issuer_overlay_match_counts', issuerOverlayMatchCounts); + printCountMap('unsupported_surfaced_concepts', unsupportedSurfacedConcepts); printSamples('failed_samples', failedRows); printSamples('warning_samples', warningRows); + + if (options.failOnResiduals && residualRows.length > 0) { + throw new Error(`strict mode failed: surfaced residual_rows=${residualRows.length}`); + } } void main().catch((error) => { diff --git a/scripts/sqlite-vector-env.ts b/scripts/sqlite-vector-env.ts index c864692..012519e 100644 --- a/scripts/sqlite-vector-env.ts +++ b/scripts/sqlite-vector-env.ts @@ -6,7 +6,7 @@ const HOMEBREW_SQLITE_LIBRARY_PATHS = [ "/usr/local/opt/sqlite/lib/libsqlite3.dylib", ] as const; -export type LocalSqliteVectorConfig = +type LocalSqliteVectorConfig = | { mode: "native"; source: "explicit-env" | "autodetect-homebrew"; diff --git a/vitest.config.mts b/vitest.config.mts index 5c1fe33..1b238a8 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -10,6 +10,7 @@ export default defineConfig({ }, test: { environment: "node", + exclude: [".next/**", "node_modules/**"], setupFiles: ["./test/vitest.setup.ts"], }, });