From a58b07456ea6d0bd5de3bc5f7b5b54ecb5ce77b3 Mon Sep 17 00:00:00 2001 From: francy51 Date: Mon, 16 Mar 2026 15:18:01 -0400 Subject: [PATCH] Integrate crabrl parser into taxonomy hydration --- README.md | 23 + bun.lock | 180 +- .../0014_taxonomy_computed_definitions.sql | 2 + lib/server/db/index.test.ts | 433 +++-- lib/server/db/schema.ts | 1654 +++++++++++------ lib/server/db/sqlite-schema-compat.ts | 415 +++-- lib/server/repos/filing-taxonomy.test.ts | 513 ++--- lib/server/repos/filing-taxonomy.ts | 806 +++++--- lib/server/task-processors.outcomes.test.ts | 653 ++++--- lib/server/task-processors.ts | 1006 ++++++---- lib/server/taxonomy/engine.test.ts | 70 +- lib/server/taxonomy/parser-client.test.ts | 286 +++ lib/server/taxonomy/parser-client.ts | 120 +- lib/server/taxonomy/types.ts | 57 +- package.json | 9 +- rust/fiscal-xbrl-core/src/crabrl_adapter.rs | 231 +++ rust/fiscal-xbrl-core/src/lib.rs | 359 +--- rust/fiscal-xbrl-core/src/surface_mapper.rs | 146 +- rust/fiscal-xbrl-core/src/universal_income.rs | 104 +- scripts/e2e-prepare.test.ts | 41 +- test/bun-test-shim.ts | 33 + test/vitest.setup.ts | 6 + vitest.config.mts | 15 + 23 files changed, 4696 insertions(+), 2466 deletions(-) create mode 100644 drizzle/0014_taxonomy_computed_definitions.sql create mode 100644 lib/server/taxonomy/parser-client.test.ts create mode 100644 rust/fiscal-xbrl-core/src/crabrl_adapter.rs create mode 100644 test/bun-test-shim.ts create mode 100644 test/vitest.setup.ts create mode 100644 vitest.config.mts diff --git a/README.md b/README.md index bafdcb4..eec8ae9 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,20 @@ bun run build bun run start ``` +## Verification + +Run the required repo checks before considering work complete: + +```bash +bun fmt +bun lint +bun typecheck +cargo test -p fiscal-xbrl-core --manifest-path rust/Cargo.toml +bun run validate:taxonomy-packs +``` + +Use `bun run test` for Vitest-based tests when needed. Do not use `bun test` directly. + ## Browser E2E tests Install Playwright's Chromium browser once: @@ -80,6 +94,7 @@ Zhipu always targets the Coding API endpoint (`https://api.z.ai/api/coding/paas/ On container startup, the app applies Drizzle migrations automatically before launching Next.js. The app stores SQLite data in Docker volume `fiscal_sqlite_data` (mounted to `/app/data`) and workflow world data in Postgres volume `workflow_postgres_data`. Container startup runs: + 1. `workflow-postgres-setup` (idempotent Workflow world bootstrap) 2. Programmatic Drizzle migrations for SQLite app tables 3. Next.js server boot @@ -142,6 +157,12 @@ AI_TEMPERATURE=0.2 SEC_USER_AGENT=Fiscal Clone +# Rust XBRL sidecar +FISCAL_XBRL_BIN=rust/target/release/fiscal-xbrl +FISCAL_XBRL_CACHE_DIR=.cache/xbrl +XBRL_ENGINE_TIMEOUT_MS=45000 +FISCAL_TAXONOMY_DIR=rust/taxonomy + # local dev default WORKFLOW_TARGET_WORLD=local @@ -161,6 +182,8 @@ WORKFLOW_LOCAL_QUEUE_CONCURRENCY=100 `ZHIPU_BASE_URL` is deprecated and ignored; runtime always uses `https://api.z.ai/api/coding/paas/v4`. `bun run dev` will still normalize Better Auth origin, same-origin API routing, SQLite path, and Workflow runtime for localhost boot if this file contains deployment values. +For the Rust XBRL sidecar, `FISCAL_XBRL_BIN` should point to the built `fiscal-xbrl` executable, `FISCAL_XBRL_CACHE_DIR` controls local filing cache storage, `XBRL_ENGINE_TIMEOUT_MS` bounds sidecar execution time, and `FISCAL_TAXONOMY_DIR` overrides the taxonomy pack directory when needed. Build the sidecar with `bun run build:sidecar` or `cargo build --manifest-path rust/Cargo.toml --release --bin fiscal-xbrl`. + ## API surface All endpoints below are defined in Elysia at `lib/server/api/app.ts` and exposed via `app/api/[[...slugs]]/route.ts`. diff --git a/bun.lock b/bun.lock index 17b0e95..afc4264 100644 --- a/bun.lock +++ b/bun.lock @@ -37,8 +37,10 @@ "bun-types": "^1.3.10", "drizzle-kit": "^0.31.9", "postcss": "^8.5.8", + "prettier": "^3.6.2", "tailwindcss": "^4.2.1", "typescript": "^5.9.3", + "vitest": "^3.2.4", }, }, }, @@ -401,6 +403,56 @@ "@reduxjs/toolkit": ["@reduxjs/toolkit@2.11.2", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@standard-schema/utils": "^0.3.0", "immer": "^11.0.0", "redux": "^5.0.1", "redux-thunk": "^3.1.0", "reselect": "^5.1.0" }, "peerDependencies": { "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" } }, "sha512-Kd6kAHTA6/nUpp8mySPqj3en3dm0tdMIgbttnQ1xFMVpufoj+ADi8pXLBsd4xzTRHQa7t/Jv8W5UnCuW4kuWMQ=="], + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.59.0", "", { "os": "android", "cpu": "arm" }, "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg=="], + + "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.59.0", "", { "os": "android", "cpu": "arm64" }, "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q=="], + + "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.59.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg=="], + + "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.59.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w=="], + + "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.59.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA=="], + + "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.59.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg=="], + + "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw=="], + + "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA=="], + + "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA=="], + + "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA=="], + + "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg=="], + + "@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q=="], + + "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA=="], + + "@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA=="], + + "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg=="], + + "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg=="], + + "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.59.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w=="], + + "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg=="], + + "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg=="], + + "@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.59.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ=="], + + "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.59.0", "", { "os": "none", "cpu": "arm64" }, "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA=="], + + "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.59.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A=="], + + "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.59.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA=="], + + "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA=="], + + "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA=="], + "@sinclair/typebox": ["@sinclair/typebox@0.34.48", "", {}, "sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA=="], "@sindresorhus/is": ["@sindresorhus/is@5.6.0", "", {}, "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g=="], @@ -559,6 +611,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + "@types/d3-array": ["@types/d3-array@3.2.2", "", {}, "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="], "@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="], @@ -579,6 +633,8 @@ "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/http-cache-semantics": ["@types/http-cache-semantics@4.2.0", "", {}, "sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q=="], @@ -613,6 +669,20 @@ "@vercel/queue": ["@vercel/queue@0.1.1", "", { "dependencies": { "@vercel/oidc": "^3.0.5", "mixpart": "0.0.5" } }, "sha512-ozO0tSBXUYN4gUkK65GbcqgxpC55qaaiY9MzNuXW4cvOSJ5nCkcgO+DQXcfyfL7h+0uIC5HTcP0mPvQ3dW3EhQ=="], + "@vitest/expect": ["@vitest/expect@3.2.4", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/spy": "3.2.4", "@vitest/utils": "3.2.4", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" } }, "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig=="], + + "@vitest/mocker": ["@vitest/mocker@3.2.4", "", { "dependencies": { "@vitest/spy": "3.2.4", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ=="], + + "@vitest/pretty-format": ["@vitest/pretty-format@3.2.4", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA=="], + + "@vitest/runner": ["@vitest/runner@3.2.4", "", { "dependencies": { "@vitest/utils": "3.2.4", "pathe": "^2.0.3", "strip-literal": "^3.0.0" } }, "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ=="], + + "@vitest/snapshot": ["@vitest/snapshot@3.2.4", "", { "dependencies": { "@vitest/pretty-format": "3.2.4", "magic-string": "^0.30.17", "pathe": "^2.0.3" } }, "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ=="], + + "@vitest/spy": ["@vitest/spy@3.2.4", "", { "dependencies": { "tinyspy": "^4.0.3" } }, "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw=="], + + "@vitest/utils": ["@vitest/utils@3.2.4", "", { "dependencies": { "@vitest/pretty-format": "3.2.4", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA=="], + "@workflow/astro": ["@workflow/astro@4.0.0-beta.37", "", { "dependencies": { "@swc/core": "1.15.3", "@workflow/builders": "4.0.1-beta.54", "@workflow/rollup": "4.0.0-beta.20", "@workflow/swc-plugin": "4.1.0-beta.18", "@workflow/vite": "4.0.0-beta.13", "exsolve": "^1.0.8", "pathe": "^2.0.3" } }, "sha512-vK5a5YoDtrDlF7E00CD3vraOpspNRxDTUZ327Gfe0M9hL4BvDSbuOnTxZehsZlqij0rv1PH2YPkXjZOdEW5Hlg=="], "@workflow/builders": ["@workflow/builders@4.0.1-beta.54", "", { "dependencies": { "@swc/core": "1.15.3", "@workflow/core": "4.1.0-beta.63", "@workflow/errors": "4.1.0-beta.17", "@workflow/swc-plugin": "4.1.0-beta.18", "@workflow/utils": "4.1.0-beta.13", "builtin-modules": "5.0.0", "chalk": "5.6.2", "enhanced-resolve": "5.19.0", "esbuild": "^0.27.3", "find-up": "7.0.0", "json5": "2.2.3", "tinyglobby": "0.2.15" } }, "sha512-IxLk/+/Nf7ENqHX2BRQkzD8myrtbmUiuYO/OF5gnwuaM2AavLZOE027hOgGoT6UBlyt8ZhzYi95q65jYdF6ojQ=="], @@ -697,6 +767,8 @@ "array-flatten": ["array-flatten@1.1.1", "", {}, "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], + "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], "async-listen": ["async-listen@3.0.0", "", {}, "sha512-V+SsTpDqkrWTimiotsyl33ePSjA5/KrithwupuvJ6ztsqPvGv6ge4OredFhPffVXiLN/QUWvE0XcqJaYgt6fOg=="], @@ -755,6 +827,8 @@ "c12": ["c12@3.1.0", "", { "dependencies": { "chokidar": "^4.0.3", "confbox": "^0.2.2", "defu": "^6.1.4", "dotenv": "^16.6.1", "exsolve": "^1.0.7", "giget": "^2.0.0", "jiti": "^2.4.2", "ohash": "^2.0.11", "pathe": "^2.0.3", "perfect-debounce": "^1.0.0", "pkg-types": "^2.2.0", "rc9": "^2.1.2" }, "peerDependencies": { "magicast": "^0.3.5" }, "optionalPeers": ["magicast"] }, "sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw=="], + "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], + "cacheable-lookup": ["cacheable-lookup@7.0.0", "", {}, "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w=="], "cacheable-request": ["cacheable-request@10.2.14", "", { "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", "http-cache-semantics": "^4.1.1", "keyv": "^4.5.3", "mimic-response": "^4.0.0", "normalize-url": "^8.0.0", "responselike": "^3.0.0" } }, "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ=="], @@ -773,8 +847,12 @@ "cbor-x": ["cbor-x@1.6.0", "", { "optionalDependencies": { "cbor-extract": "^2.2.0" } }, "sha512-0kareyRwHSkL6ws5VXHEf8uY1liitysCVJjlmhaLG+IXLqhSaOO+t63coaso7yjwEzWZzLy8fJo06gZDVQM9Qg=="], + "chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], + "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], + "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], + "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], @@ -863,6 +941,8 @@ "decompress-response": ["decompress-response@6.0.0", "", { "dependencies": { "mimic-response": "^3.1.0" } }, "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ=="], + "deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="], + "deepmerge-ts": ["deepmerge-ts@7.1.5", "", {}, "sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw=="], "default-browser": ["default-browser@5.5.0", "", { "dependencies": { "bundle-name": "^4.1.0", "default-browser-id": "^5.0.0" } }, "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw=="], @@ -939,6 +1019,8 @@ "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], "es-toolkit": ["es-toolkit@1.44.0", "", {}, "sha512-6penXeZalaV88MM3cGkFZZfOoLGWshWWfdy0tWw/RlVVyhvMaWSBTOvXNeiW3e5FwdS5ePW0LGEu17zT139ktg=="], @@ -967,6 +1049,8 @@ "execa": ["execa@5.1.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" } }, "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg=="], + "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], + "express": ["express@4.22.1", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g=="], "exsolve": ["exsolve@1.0.8", "", {}, "sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA=="], @@ -1139,7 +1223,7 @@ "js-base64": ["js-base64@3.7.8", "", {}, "sha512-hNngCeKxIUQiEUN3GPJOkz4wF/YvdUdbNL9hsBcMQTkKzboD7T/q3OYOuuPZLUE6dBxSGpwhk5mwuDud7JVAow=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], @@ -1203,6 +1287,8 @@ "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], + "lowercase-keys": ["lowercase-keys@3.0.0", "", {}, "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="], "lru.min": ["lru.min@1.1.4", "", {}, "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA=="], @@ -1323,6 +1409,8 @@ "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="], + "pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="], "perfect-debounce": ["perfect-debounce@1.0.0", "", {}, "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA=="], @@ -1369,6 +1457,8 @@ "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + "prettier": ["prettier@3.8.1", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg=="], + "prisma": ["prisma@7.4.2", "", { "dependencies": { "@prisma/config": "7.4.2", "@prisma/dev": "0.20.0", "@prisma/engines": "7.4.2", "@prisma/studio-core": "0.13.1", "mysql2": "3.15.3", "postgres": "3.4.7" }, "peerDependencies": { "better-sqlite3": ">=9.0.0", "typescript": ">=5.4.0" }, "optionalPeers": ["better-sqlite3", "typescript"], "bin": { "prisma": "build/index.js" } }, "sha512-2bP8Ruww3Q95Z2eH4Yqh4KAENRsj/SxbdknIVBfd6DmjPwmpsC4OVFMLOeHt6tM3Amh8ebjvstrUz3V/hOe1dA=="], "promise-limit": ["promise-limit@2.7.0", "", {}, "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw=="], @@ -1431,6 +1521,8 @@ "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], + "rollup": ["rollup@4.59.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.59.0", "@rollup/rollup-android-arm64": "4.59.0", "@rollup/rollup-darwin-arm64": "4.59.0", "@rollup/rollup-darwin-x64": "4.59.0", "@rollup/rollup-freebsd-arm64": "4.59.0", "@rollup/rollup-freebsd-x64": "4.59.0", "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", "@rollup/rollup-linux-arm-musleabihf": "4.59.0", "@rollup/rollup-linux-arm64-gnu": "4.59.0", "@rollup/rollup-linux-arm64-musl": "4.59.0", "@rollup/rollup-linux-loong64-gnu": "4.59.0", "@rollup/rollup-linux-loong64-musl": "4.59.0", "@rollup/rollup-linux-ppc64-gnu": "4.59.0", "@rollup/rollup-linux-ppc64-musl": "4.59.0", "@rollup/rollup-linux-riscv64-gnu": "4.59.0", "@rollup/rollup-linux-riscv64-musl": "4.59.0", "@rollup/rollup-linux-s390x-gnu": "4.59.0", "@rollup/rollup-linux-x64-gnu": "4.59.0", "@rollup/rollup-linux-x64-musl": "4.59.0", "@rollup/rollup-openbsd-x64": "4.59.0", "@rollup/rollup-openharmony-arm64": "4.59.0", "@rollup/rollup-win32-arm64-msvc": "4.59.0", "@rollup/rollup-win32-ia32-msvc": "4.59.0", "@rollup/rollup-win32-x64-gnu": "4.59.0", "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg=="], + "rou3": ["rou3@0.7.12", "", {}, "sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg=="], "run-applescript": ["run-applescript@7.1.0", "", {}, "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q=="], @@ -1479,6 +1571,8 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], @@ -1513,6 +1607,8 @@ "sqlstring": ["sqlstring@2.3.3", "", {}, "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="], + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="], @@ -1529,6 +1625,8 @@ "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="], + "strip-literal": ["strip-literal@3.1.0", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg=="], + "strnum": ["strnum@2.1.2", "", {}, "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ=="], "strtok3": ["strtok3@10.3.4", "", { "dependencies": { "@tokenizer/token": "^0.3.0" } }, "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg=="], @@ -1553,10 +1651,18 @@ "tiny-invariant": ["tiny-invariant@1.3.3", "", {}, "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="], - "tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], + + "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], + "tinypool": ["tinypool@1.1.1", "", {}, "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg=="], + + "tinyrainbow": ["tinyrainbow@2.0.0", "", {}, "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw=="], + + "tinyspy": ["tinyspy@4.0.4", "", {}, "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], @@ -1609,6 +1715,12 @@ "victory-vendor": ["victory-vendor@37.3.6", "", { "dependencies": { "@types/d3-array": "^3.0.3", "@types/d3-ease": "^3.0.0", "@types/d3-interpolate": "^3.0.1", "@types/d3-scale": "^4.0.2", "@types/d3-shape": "^3.1.0", "@types/d3-time": "^3.0.0", "@types/d3-timer": "^3.0.0", "d3-array": "^3.1.6", "d3-ease": "^3.0.1", "d3-interpolate": "^3.0.1", "d3-scale": "^4.0.2", "d3-shape": "^3.1.0", "d3-time": "^3.0.0", "d3-timer": "^3.0.1" } }, "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ=="], + "vite": ["vite@7.3.1", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA=="], + + "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], + + "vitest": ["vitest@3.2.4", "", { "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.4", "@vitest/mocker": "3.2.4", "@vitest/pretty-format": "^3.2.4", "@vitest/runner": "3.2.4", "@vitest/snapshot": "3.2.4", "@vitest/spy": "3.2.4", "@vitest/utils": "3.2.4", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "picomatch": "^4.0.2", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.14", "tinypool": "^1.1.1", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", "vite-node": "3.2.4", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.2.4", "@vitest/ui": "3.2.4", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A=="], + "watchpack": ["watchpack@2.5.1", "", { "dependencies": { "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" } }, "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg=="], "wcwidth": ["wcwidth@1.0.1", "", { "dependencies": { "defaults": "^1.0.3" } }, "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg=="], @@ -1627,6 +1739,8 @@ "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "widest-line": ["widest-line@3.1.0", "", { "dependencies": { "string-width": "^4.0.0" } }, "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg=="], "wordwrap": ["wordwrap@1.0.0", "", {}, "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q=="], @@ -1675,6 +1789,8 @@ "@aws-sdk/core/@smithy/property-provider": ["@smithy/property-provider@4.2.11", "", { "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" } }, "sha512-14T1V64o6/ndyrnl1ze1ZhyLzIeYNN47oF/QU6P5m82AEtyOkMJTb0gO1dPubYjyyKuPD6OSVMPDKe+zioOnCg=="], + "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], "@mrleebo/prisma-ast/lilconfig": ["lilconfig@2.1.0", "", {}, "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ=="], @@ -1825,6 +1941,8 @@ "nypm/citty": ["citty@0.2.1", "", {}, "sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg=="], + "nypm/tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], + "ora/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], @@ -1839,6 +1957,8 @@ "react-router/set-cookie-parser": ["set-cookie-parser@2.7.2", "", {}, "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw=="], + "rollup/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "seek-bzip/commander": ["commander@6.2.1", "", {}, "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA=="], "send/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], @@ -1857,6 +1977,10 @@ "terminal-link/ansi-escapes": ["ansi-escapes@7.3.0", "", { "dependencies": { "environment": "^1.0.0" } }, "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="], + "vite/esbuild": ["esbuild@0.27.3", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.3", "@esbuild/android-arm": "0.27.3", "@esbuild/android-arm64": "0.27.3", "@esbuild/android-x64": "0.27.3", "@esbuild/darwin-arm64": "0.27.3", "@esbuild/darwin-x64": "0.27.3", "@esbuild/freebsd-arm64": "0.27.3", "@esbuild/freebsd-x64": "0.27.3", "@esbuild/linux-arm": "0.27.3", "@esbuild/linux-arm64": "0.27.3", "@esbuild/linux-ia32": "0.27.3", "@esbuild/linux-loong64": "0.27.3", "@esbuild/linux-mips64el": "0.27.3", "@esbuild/linux-ppc64": "0.27.3", "@esbuild/linux-riscv64": "0.27.3", "@esbuild/linux-s390x": "0.27.3", "@esbuild/linux-x64": "0.27.3", "@esbuild/netbsd-arm64": "0.27.3", "@esbuild/netbsd-x64": "0.27.3", "@esbuild/openbsd-arm64": "0.27.3", "@esbuild/openbsd-x64": "0.27.3", "@esbuild/openharmony-arm64": "0.27.3", "@esbuild/sunos-x64": "0.27.3", "@esbuild/win32-arm64": "0.27.3", "@esbuild/win32-ia32": "0.27.3", "@esbuild/win32-x64": "0.27.3" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg=="], + + "vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "whatwg-encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], "workflow/@workflow/errors": ["@workflow/errors@4.1.0-beta.17", "", { "dependencies": { "@workflow/utils": "4.1.0-beta.13", "ms": "2.1.3" } }, "sha512-ctDx9PrTCAkfsGqs6PgYAMGSaOmHESTMJEdj+d+RU0qEDfXWBZmM586hkf9hXw3jwXnw0VMp9X01jLsnWPyZcA=="], @@ -2065,6 +2189,58 @@ "send/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.3", "", { "os": "aix", "cpu": "ppc64" }, "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg=="], + + "vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.27.3", "", { "os": "android", "cpu": "arm" }, "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA=="], + + "vite/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.27.3", "", { "os": "android", "cpu": "arm64" }, "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg=="], + + "vite/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.27.3", "", { "os": "android", "cpu": "x64" }, "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ=="], + + "vite/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.27.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg=="], + + "vite/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.27.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg=="], + + "vite/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.27.3", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w=="], + + "vite/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.27.3", "", { "os": "freebsd", "cpu": "x64" }, "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA=="], + + "vite/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.27.3", "", { "os": "linux", "cpu": "arm" }, "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw=="], + + "vite/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.27.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg=="], + + "vite/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.27.3", "", { "os": "linux", "cpu": "ia32" }, "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg=="], + + "vite/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA=="], + + "vite/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw=="], + + "vite/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.27.3", "", { "os": "linux", "cpu": "ppc64" }, "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA=="], + + "vite/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.27.3", "", { "os": "linux", "cpu": "none" }, "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ=="], + + "vite/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.27.3", "", { "os": "linux", "cpu": "s390x" }, "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw=="], + + "vite/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.27.3", "", { "os": "linux", "cpu": "x64" }, "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA=="], + + "vite/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.27.3", "", { "os": "none", "cpu": "arm64" }, "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA=="], + + "vite/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.27.3", "", { "os": "none", "cpu": "x64" }, "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA=="], + + "vite/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.27.3", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw=="], + + "vite/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.27.3", "", { "os": "openbsd", "cpu": "x64" }, "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ=="], + + "vite/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.27.3", "", { "os": "none", "cpu": "arm64" }, "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g=="], + + "vite/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.27.3", "", { "os": "sunos", "cpu": "x64" }, "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA=="], + + "vite/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.27.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA=="], + + "vite/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.27.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q=="], + + "vite/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.3", "", { "os": "win32", "cpu": "x64" }, "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA=="], + "@aws-crypto/sha256-browser/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="], "@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="], diff --git a/drizzle/0014_taxonomy_computed_definitions.sql b/drizzle/0014_taxonomy_computed_definitions.sql new file mode 100644 index 0000000..25a0475 --- /dev/null +++ b/drizzle/0014_taxonomy_computed_definitions.sql @@ -0,0 +1,2 @@ +ALTER TABLE `filing_taxonomy_snapshot` +ADD `computed_definitions` text; diff --git a/lib/server/db/index.test.ts b/lib/server/db/index.test.ts index a8156ae..b5eeaca 100644 --- a/lib/server/db/index.test.ts +++ b/lib/server/db/index.test.ts @@ -1,96 +1,239 @@ -import { describe, expect, it } from 'bun:test'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { Database } from 'bun:sqlite'; -import { __dbInternals } from './index'; +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; +import { __dbInternals } from "./index"; function applyMigration(client: Database, fileName: string) { - const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); + const sql = readFileSync(join(process.cwd(), "drizzle", fileName), "utf8"); client.exec(sql); } -describe('sqlite schema compatibility bootstrap', () => { - it('adds missing watchlist columns and taxonomy tables for older local databases', () => { - const client = new Database(':memory:'); - client.exec('PRAGMA foreign_keys = ON;'); +describe("sqlite schema compatibility bootstrap", () => { + it("adds missing watchlist columns and taxonomy tables for older local databases", () => { + const client = new Database(":memory:"); + client.exec("PRAGMA foreign_keys = ON;"); - applyMigration(client, '0000_cold_silver_centurion.sql'); - applyMigration(client, '0001_glossy_statement_snapshots.sql'); - applyMigration(client, '0002_workflow_task_projection_metadata.sql'); - applyMigration(client, '0003_task_stage_event_timeline.sql'); - applyMigration(client, '0009_task_notification_context.sql'); + applyMigration(client, "0000_cold_silver_centurion.sql"); + applyMigration(client, "0001_glossy_statement_snapshots.sql"); + applyMigration(client, "0002_workflow_task_projection_metadata.sql"); + applyMigration(client, "0003_task_stage_event_timeline.sql"); + applyMigration(client, "0009_task_notification_context.sql"); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(false); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(false); - expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(false); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(false); - expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(false); - expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(false); - expect(__dbInternals.hasTable(client, 'research_memo')).toBe(false); + expect(__dbInternals.hasColumn(client, "watchlist_item", "category")).toBe( + false, + ); + expect(__dbInternals.hasColumn(client, "watchlist_item", "status")).toBe( + false, + ); + expect(__dbInternals.hasColumn(client, "holding", "company_name")).toBe( + false, + ); + expect(__dbInternals.hasTable(client, "filing_taxonomy_snapshot")).toBe( + false, + ); + expect(__dbInternals.hasTable(client, "research_journal_entry")).toBe( + false, + ); + expect(__dbInternals.hasTable(client, "research_artifact")).toBe(false); + expect(__dbInternals.hasTable(client, "research_memo")).toBe(false); __dbInternals.ensureLocalSqliteSchema(client); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'category')).toBe(true); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'tags')).toBe(true); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'status')).toBe(true); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'priority')).toBe(true); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'updated_at')).toBe(true); - expect(__dbInternals.hasColumn(client, 'watchlist_item', 'last_reviewed_at')).toBe(true); - expect(__dbInternals.hasColumn(client, 'holding', 'company_name')).toBe(true); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'faithful_rows')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'surface_rows')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'detail_rows')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'kpi_rows')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_fact')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'balance')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'period_type')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'data_type')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'authoritative_concept_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'mapping_method')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'surface_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'detail_parent_surface_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'kpi_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_concept', 'residual_flag')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'data_type')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'authoritative_concept_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'mapping_method')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'surface_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'detail_parent_surface_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'kpi_key')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'residual_flag')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'precision')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_fact', 'nil')).toBe(true); - expect(__dbInternals.hasColumn(client, 'task_run', 'stage_context')).toBe(true); - expect(__dbInternals.hasColumn(client, 'task_stage_event', 'stage_context')).toBe(true); - expect(__dbInternals.hasTable(client, 'research_journal_entry')).toBe(true); - expect(__dbInternals.hasTable(client, 'search_document')).toBe(true); - expect(__dbInternals.hasTable(client, 'search_chunk')).toBe(true); - expect(__dbInternals.hasTable(client, 'research_artifact')).toBe(true); - expect(__dbInternals.hasTable(client, 'research_memo')).toBe(true); - expect(__dbInternals.hasTable(client, 'research_memo_evidence')).toBe(true); - expect(__dbInternals.hasTable(client, 'company_overview_cache')).toBe(true); + expect(__dbInternals.hasColumn(client, "watchlist_item", "category")).toBe( + true, + ); + expect(__dbInternals.hasColumn(client, "watchlist_item", "tags")).toBe( + true, + ); + expect(__dbInternals.hasColumn(client, "watchlist_item", "status")).toBe( + true, + ); + expect(__dbInternals.hasColumn(client, "watchlist_item", "priority")).toBe( + true, + ); + expect( + __dbInternals.hasColumn(client, "watchlist_item", "updated_at"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "watchlist_item", "last_reviewed_at"), + ).toBe(true); + expect(__dbInternals.hasColumn(client, "holding", "company_name")).toBe( + true, + ); + expect(__dbInternals.hasTable(client, "filing_taxonomy_snapshot")).toBe( + true, + ); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "parser_engine", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "parser_version", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "taxonomy_regime", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "faithful_rows", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "surface_rows", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "detail_rows", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_snapshot", "kpi_rows"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "computed_definitions", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "normalization_summary", + ), + ).toBe(true); + expect(__dbInternals.hasTable(client, "filing_taxonomy_context")).toBe( + true, + ); + expect(__dbInternals.hasTable(client, "filing_taxonomy_fact")).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_concept", "balance"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_concept", "period_type"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_concept", "data_type"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_concept", + "authoritative_concept_key", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_concept", + "mapping_method", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_concept", "surface_key"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_concept", + "detail_parent_surface_key", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_concept", "kpi_key"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_concept", + "residual_flag", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "data_type"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_fact", + "authoritative_concept_key", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "mapping_method"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "surface_key"), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_fact", + "detail_parent_surface_key", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "kpi_key"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "residual_flag"), + ).toBe(true); + expect( + __dbInternals.hasColumn(client, "filing_taxonomy_fact", "precision"), + ).toBe(true); + expect(__dbInternals.hasColumn(client, "filing_taxonomy_fact", "nil")).toBe( + true, + ); + expect(__dbInternals.hasColumn(client, "task_run", "stage_context")).toBe( + true, + ); + expect( + __dbInternals.hasColumn(client, "task_stage_event", "stage_context"), + ).toBe(true); + expect(__dbInternals.hasTable(client, "research_journal_entry")).toBe(true); + expect(__dbInternals.hasTable(client, "search_document")).toBe(true); + expect(__dbInternals.hasTable(client, "search_chunk")).toBe(true); + expect(__dbInternals.hasTable(client, "research_artifact")).toBe(true); + expect(__dbInternals.hasTable(client, "research_memo")).toBe(true); + expect(__dbInternals.hasTable(client, "research_memo_evidence")).toBe(true); + expect(__dbInternals.hasTable(client, "company_overview_cache")).toBe(true); __dbInternals.loadSqliteExtensions(client); __dbInternals.ensureSearchVirtualTables(client); - expect(__dbInternals.hasTable(client, 'search_chunk_fts')).toBe(true); - expect(__dbInternals.hasTable(client, 'search_chunk_vec')).toBe(true); + expect(__dbInternals.hasTable(client, "search_chunk_fts")).toBe(true); + expect(__dbInternals.hasTable(client, "search_chunk_vec")).toBe(true); client.close(); }); - it('backfills legacy taxonomy snapshot sidecar columns and remains idempotent', () => { - const client = new Database(':memory:'); - client.exec('PRAGMA foreign_keys = ON;'); + it("backfills legacy taxonomy snapshot sidecar columns and remains idempotent", () => { + const client = new Database(":memory:"); + client.exec("PRAGMA foreign_keys = ON;"); - applyMigration(client, '0000_cold_silver_centurion.sql'); - applyMigration(client, '0005_financial_taxonomy_v3.sql'); + applyMigration(client, "0000_cold_silver_centurion.sql"); + applyMigration(client, "0005_financial_taxonomy_v3.sql"); client.exec(` INSERT INTO \`filing\` ( @@ -114,7 +257,8 @@ describe('sqlite schema compatibility bootstrap', () => { ); `); - const statementRows = '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; + const statementRows = + '{"income":[{"label":"Revenue","value":1}],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; client.exec(` INSERT INTO \`filing_taxonomy_snapshot\` ( @@ -143,7 +287,9 @@ describe('sqlite schema compatibility bootstrap', () => { __dbInternals.ensureLocalSqliteSchema(client); __dbInternals.ensureLocalSqliteSchema(client); - const row = client.query(` + const row = client + .query( + ` SELECT \`parser_engine\`, \`parser_version\`, @@ -152,10 +298,13 @@ describe('sqlite schema compatibility bootstrap', () => { \`surface_rows\`, \`detail_rows\`, \`kpi_rows\`, + \`computed_definitions\`, \`normalization_summary\` FROM \`filing_taxonomy_snapshot\` WHERE \`filing_id\` = 1 - `).get() as { + `, + ) + .get() as { parser_engine: string; parser_version: string; taxonomy_regime: string; @@ -163,66 +312,116 @@ describe('sqlite schema compatibility bootstrap', () => { surface_rows: string | null; detail_rows: string | null; kpi_rows: string | null; + computed_definitions: string | null; normalization_summary: string | null; }; - expect(row.parser_engine).toBe('fiscal-xbrl'); - expect(row.parser_version).toBe('unknown'); - expect(row.taxonomy_regime).toBe('unknown'); + expect(row.parser_engine).toBe("fiscal-xbrl"); + expect(row.parser_version).toBe("unknown"); + expect(row.taxonomy_regime).toBe("unknown"); expect(row.faithful_rows).toBe(statementRows); - expect(row.surface_rows).toBe('{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'); - expect(row.detail_rows).toBe('{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'); - expect(row.kpi_rows).toBe('[]'); + expect(row.surface_rows).toBe( + '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}', + ); + expect(row.detail_rows).toBe( + '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}', + ); + expect(row.kpi_rows).toBe("[]"); + expect(row.computed_definitions).toBe("[]"); expect(row.normalization_summary).toBeNull(); client.close(); }); - it('repairs partial taxonomy sidecar drift without requiring a table rebuild', () => { - const client = new Database(':memory:'); - client.exec('PRAGMA foreign_keys = ON;'); + it("repairs partial taxonomy sidecar drift without requiring a table rebuild", () => { + const client = new Database(":memory:"); + client.exec("PRAGMA foreign_keys = ON;"); - applyMigration(client, '0000_cold_silver_centurion.sql'); - applyMigration(client, '0005_financial_taxonomy_v3.sql'); - client.exec("ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';"); + applyMigration(client, "0000_cold_silver_centurion.sql"); + applyMigration(client, "0005_financial_taxonomy_v3.sql"); + client.exec( + "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'legacy-ts';", + ); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(false); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(false); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "parser_engine", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "normalization_summary", + ), + ).toBe(false); + expect(__dbInternals.hasTable(client, "filing_taxonomy_context")).toBe( + false, + ); __dbInternals.ensureLocalSqliteSchema(client); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_version')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'taxonomy_regime')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'normalization_summary')).toBe(true); - expect(__dbInternals.hasTable(client, 'filing_taxonomy_context')).toBe(true); - - client.close(); - }); - - it('throws on missing parser_engine column when verifyCriticalSchema is called', () => { - const client = new Database(':memory:'); - client.exec('PRAGMA foreign_keys = ON;'); - - applyMigration(client, '0000_cold_silver_centurion.sql'); - applyMigration(client, '0005_financial_taxonomy_v3.sql'); - - expect(__dbInternals.hasTable(client, 'filing_taxonomy_snapshot')).toBe(true); - expect(__dbInternals.hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(false); - - expect(() => __dbInternals.verifyCriticalSchema(client)).toThrow( - /filing_taxonomy_snapshot is missing columns: parser_engine/ + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "parser_version", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "taxonomy_regime", + ), + ).toBe(true); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "normalization_summary", + ), + ).toBe(true); + expect(__dbInternals.hasTable(client, "filing_taxonomy_context")).toBe( + true, ); client.close(); }); - it('verifyCriticalSchema passes when all required columns exist', () => { - const client = new Database(':memory:'); - client.exec('PRAGMA foreign_keys = ON;'); + it("throws on missing parser_engine column when verifyCriticalSchema is called", () => { + const client = new Database(":memory:"); + client.exec("PRAGMA foreign_keys = ON;"); - applyMigration(client, '0000_cold_silver_centurion.sql'); - applyMigration(client, '0005_financial_taxonomy_v3.sql'); + applyMigration(client, "0000_cold_silver_centurion.sql"); + applyMigration(client, "0005_financial_taxonomy_v3.sql"); + + expect(__dbInternals.hasTable(client, "filing_taxonomy_snapshot")).toBe( + true, + ); + expect( + __dbInternals.hasColumn( + client, + "filing_taxonomy_snapshot", + "parser_engine", + ), + ).toBe(false); + + expect(() => __dbInternals.verifyCriticalSchema(client)).toThrow( + /filing_taxonomy_snapshot is missing columns: parser_engine/, + ); + + client.close(); + }); + + it("verifyCriticalSchema passes when all required columns exist", () => { + const client = new Database(":memory:"); + client.exec("PRAGMA foreign_keys = ON;"); + + applyMigration(client, "0000_cold_silver_centurion.sql"); + applyMigration(client, "0005_financial_taxonomy_v3.sql"); __dbInternals.ensureLocalSqliteSchema(client); diff --git a/lib/server/db/schema.ts b/lib/server/db/schema.ts index 6189d9d..396c81a 100644 --- a/lib/server/db/schema.ts +++ b/lib/server/db/schema.ts @@ -1,13 +1,13 @@ -import { sql } from 'drizzle-orm'; +import { sql } from "drizzle-orm"; import { index, integer, numeric, sqliteTable, text, - uniqueIndex -} from 'drizzle-orm/sqlite-core'; -import type { TaskStageContext } from '@/lib/types'; + uniqueIndex, +} from "drizzle-orm/sqlite-core"; +import type { TaskStageContext } from "@/lib/types"; type FilingMetrics = { revenue: number | null; @@ -18,44 +18,57 @@ type FilingMetrics = { }; type TaxonomyAssetType = - | 'instance' - | 'schema' - | 'presentation' - | 'label' - | 'calculation' - | 'definition' - | 'pdf' - | 'other'; + | "instance" + | "schema" + | "presentation" + | "label" + | "calculation" + | "definition" + | "pdf" + | "other"; -type TaxonomyParseStatus = 'ready' | 'partial' | 'failed'; -type TaxonomyMetricValidationStatus = 'not_run' | 'matched' | 'mismatch' | 'error'; -type CoverageStatus = 'backlog' | 'active' | 'watch' | 'archive'; -type CoveragePriority = 'low' | 'medium' | 'high'; -type ResearchJournalEntryType = 'note' | 'filing_note' | 'status_change'; -type ResearchArtifactKind = 'filing' | 'ai_report' | 'note' | 'upload' | 'memo_snapshot' | 'status_change'; -type ResearchArtifactSource = 'system' | 'user'; -type ResearchVisibilityScope = 'private' | 'organization'; -type ResearchMemoRating = 'strong_buy' | 'buy' | 'hold' | 'sell'; -type ResearchMemoConviction = 'low' | 'medium' | 'high'; +type TaxonomyParseStatus = "ready" | "partial" | "failed"; +type TaxonomyMetricValidationStatus = + | "not_run" + | "matched" + | "mismatch" + | "error"; +type CoverageStatus = "backlog" | "active" | "watch" | "archive"; +type CoveragePriority = "low" | "medium" | "high"; +type ResearchJournalEntryType = "note" | "filing_note" | "status_change"; +type ResearchArtifactKind = + | "filing" + | "ai_report" + | "note" + | "upload" + | "memo_snapshot" + | "status_change"; +type ResearchArtifactSource = "system" | "user"; +type ResearchVisibilityScope = "private" | "organization"; +type ResearchMemoRating = "strong_buy" | "buy" | "hold" | "sell"; +type ResearchMemoConviction = "low" | "medium" | "high"; type ResearchMemoSection = - | 'thesis' - | 'variant_view' - | 'catalysts' - | 'risks' - | 'disconfirming_evidence' - | 'next_actions'; -type FinancialCadence = 'annual' | 'quarterly' | 'ltm'; -type SearchDocumentScope = 'global' | 'user'; -type SearchDocumentSourceKind = 'filing_document' | 'filing_brief' | 'research_note'; -type SearchIndexStatus = 'pending' | 'indexed' | 'failed'; + | "thesis" + | "variant_view" + | "catalysts" + | "risks" + | "disconfirming_evidence" + | "next_actions"; +type FinancialCadence = "annual" | "quarterly" | "ltm"; +type SearchDocumentScope = "global" | "user"; +type SearchDocumentSourceKind = + | "filing_document" + | "filing_brief" + | "research_note"; +type SearchIndexStatus = "pending" | "indexed" | "failed"; type FinancialSurfaceKind = - | 'income_statement' - | 'balance_sheet' - | 'cash_flow_statement' - | 'ratios' - | 'segments_kpis' - | 'adjusted' - | 'custom_metrics'; + | "income_statement" + | "balance_sheet" + | "cash_flow_statement" + | "ratios" + | "segments_kpis" + | "adjusted" + | "custom_metrics"; type FilingAnalysis = { provider?: string; @@ -77,12 +90,17 @@ type FilingAnalysis = { extractionMeta?: { provider: string; model: string; - source: 'primary_document' | 'metadata_fallback'; + source: "primary_document" | "metadata_fallback"; generatedAt: string; }; }; -type FinancialStatementKind = 'income' | 'balance' | 'cash_flow' | 'equity' | 'comprehensive_income'; +type FinancialStatementKind = + | "income" + | "balance" + | "cash_flow" + | "equity" + | "comprehensive_income"; type FilingStatementPeriod = { id: string; @@ -91,7 +109,7 @@ type FilingStatementPeriod = { filingDate: string; periodStart: string | null; periodEnd: string | null; - filingType: '10-K' | '10-Q'; + filingType: "10-K" | "10-Q"; periodLabel: string; }; @@ -128,12 +146,18 @@ type DimensionStatementSnapshotRow = { type FilingStatementBundle = { periods: FilingStatementPeriod[]; - statements: Record; + statements: Record< + FinancialStatementKind, + FilingFaithfulStatementSnapshotRow[] + >; }; type StandardizedStatementBundle = { periods: FilingStatementPeriod[]; - statements: Record; + statements: Record< + FinancialStatementKind, + StandardizedStatementSnapshotRow[] + >; }; type DimensionStatementBundle = { @@ -175,7 +199,7 @@ type TaxonomySurfaceSnapshotRow = { category: string; templateSection?: string; order: number; - unit: 'currency' | 'count' | 'shares' | 'percent' | 'ratio'; + unit: "currency" | "count" | "shares" | "percent" | "ratio"; values: Record; sourceConcepts: string[]; sourceRowKeys: string[]; @@ -183,7 +207,7 @@ type TaxonomySurfaceSnapshotRow = { formulaKey: string | null; hasDimensions: boolean; resolvedSourceRowKeys: Record; - statement?: 'income' | 'balance' | 'cash_flow'; + statement?: "income" | "balance" | "cash_flow"; detailCount?: number; }; @@ -209,7 +233,7 @@ type StructuredKpiSnapshotRow = { key: string; label: string; category: string; - unit: 'currency' | 'count' | 'shares' | 'percent' | 'ratio'; + unit: "currency" | "count" | "shares" | "percent" | "ratio"; order: number; segment: string | null; axis: string | null; @@ -217,10 +241,28 @@ type StructuredKpiSnapshotRow = { values: Record; sourceConcepts: string[]; sourceFactIds: number[]; - provenanceType: 'taxonomy' | 'structured_note'; + provenanceType: "taxonomy" | "structured_note"; hasDimensions: boolean; }; +type ComputationSpecSnapshot = + | { type: "ratio"; numerator: string; denominator: string } + | { type: "yoy_growth"; source: string } + | { type: "cagr"; source: string; years: number } + | { type: "per_share"; source: string; shares_key: string } + | { type: "simple"; formula: string }; + +type ComputedDefinitionSnapshotRow = { + key: string; + label: string; + category: string; + order: number; + unit: "currency" | "count" | "shares" | "percent" | "ratio"; + computation: ComputationSpecSnapshot; + supported_cadences?: FinancialCadence[]; + requires_external_data?: string[]; +}; + type TaxonomyNormalizationSummary = { surfaceRowCount: number; detailRowCount: number; @@ -251,422 +293,720 @@ type TaxonomyMetricValidationResult = { }; const authDateColumn = { - mode: 'timestamp_ms' + mode: "timestamp_ms", } as const; -export const user = sqliteTable('user', { - id: text('id').primaryKey().notNull(), - name: text('name').notNull(), - email: text('email').notNull(), - emailVerified: integer('emailVerified', { mode: 'boolean' }).notNull().default(false), - image: text('image'), - createdAt: integer('createdAt', authDateColumn).notNull(), - updatedAt: integer('updatedAt', authDateColumn).notNull(), - role: text('role'), - banned: integer('banned', { mode: 'boolean' }).default(false), - banReason: text('banReason'), - banExpires: integer('banExpires', authDateColumn) -}, (table) => ({ - userEmailUnique: uniqueIndex('user_email_uidx').on(table.email) -})); +export const user = sqliteTable( + "user", + { + id: text("id").primaryKey().notNull(), + name: text("name").notNull(), + email: text("email").notNull(), + emailVerified: integer("emailVerified", { mode: "boolean" }) + .notNull() + .default(false), + image: text("image"), + createdAt: integer("createdAt", authDateColumn).notNull(), + updatedAt: integer("updatedAt", authDateColumn).notNull(), + role: text("role"), + banned: integer("banned", { mode: "boolean" }).default(false), + banReason: text("banReason"), + banExpires: integer("banExpires", authDateColumn), + }, + (table) => ({ + userEmailUnique: uniqueIndex("user_email_uidx").on(table.email), + }), +); -export const organization = sqliteTable('organization', { - id: text('id').primaryKey().notNull(), - name: text('name').notNull(), - slug: text('slug').notNull(), - logo: text('logo'), - createdAt: integer('createdAt', authDateColumn).notNull(), - metadata: text('metadata') -}, (table) => ({ - organizationSlugUnique: uniqueIndex('organization_slug_uidx').on(table.slug) -})); +export const organization = sqliteTable( + "organization", + { + id: text("id").primaryKey().notNull(), + name: text("name").notNull(), + slug: text("slug").notNull(), + logo: text("logo"), + createdAt: integer("createdAt", authDateColumn).notNull(), + metadata: text("metadata"), + }, + (table) => ({ + organizationSlugUnique: uniqueIndex("organization_slug_uidx").on( + table.slug, + ), + }), +); -export const session = sqliteTable('session', { - id: text('id').primaryKey().notNull(), - expiresAt: integer('expiresAt', authDateColumn).notNull(), - token: text('token').notNull(), - createdAt: integer('createdAt', authDateColumn).notNull(), - updatedAt: integer('updatedAt', authDateColumn).notNull(), - ipAddress: text('ipAddress'), - userAgent: text('userAgent'), - userId: text('userId').notNull().references(() => user.id, { onDelete: 'cascade' }), - impersonatedBy: text('impersonatedBy'), - activeOrganizationId: text('activeOrganizationId') -}, (table) => ({ - sessionTokenUnique: uniqueIndex('session_token_uidx').on(table.token), - sessionUserIdIndex: index('session_userId_idx').on(table.userId) -})); +export const session = sqliteTable( + "session", + { + id: text("id").primaryKey().notNull(), + expiresAt: integer("expiresAt", authDateColumn).notNull(), + token: text("token").notNull(), + createdAt: integer("createdAt", authDateColumn).notNull(), + updatedAt: integer("updatedAt", authDateColumn).notNull(), + ipAddress: text("ipAddress"), + userAgent: text("userAgent"), + userId: text("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + impersonatedBy: text("impersonatedBy"), + activeOrganizationId: text("activeOrganizationId"), + }, + (table) => ({ + sessionTokenUnique: uniqueIndex("session_token_uidx").on(table.token), + sessionUserIdIndex: index("session_userId_idx").on(table.userId), + }), +); -export const account = sqliteTable('account', { - id: text('id').primaryKey().notNull(), - accountId: text('accountId').notNull(), - providerId: text('providerId').notNull(), - userId: text('userId').notNull().references(() => user.id, { onDelete: 'cascade' }), - accessToken: text('accessToken'), - refreshToken: text('refreshToken'), - idToken: text('idToken'), - accessTokenExpiresAt: integer('accessTokenExpiresAt', authDateColumn), - refreshTokenExpiresAt: integer('refreshTokenExpiresAt', authDateColumn), - scope: text('scope'), - password: text('password'), - createdAt: integer('createdAt', authDateColumn).notNull(), - updatedAt: integer('updatedAt', authDateColumn).notNull() -}, (table) => ({ - accountUserIdIndex: index('account_userId_idx').on(table.userId) -})); +export const account = sqliteTable( + "account", + { + id: text("id").primaryKey().notNull(), + accountId: text("accountId").notNull(), + providerId: text("providerId").notNull(), + userId: text("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + accessToken: text("accessToken"), + refreshToken: text("refreshToken"), + idToken: text("idToken"), + accessTokenExpiresAt: integer("accessTokenExpiresAt", authDateColumn), + refreshTokenExpiresAt: integer("refreshTokenExpiresAt", authDateColumn), + scope: text("scope"), + password: text("password"), + createdAt: integer("createdAt", authDateColumn).notNull(), + updatedAt: integer("updatedAt", authDateColumn).notNull(), + }, + (table) => ({ + accountUserIdIndex: index("account_userId_idx").on(table.userId), + }), +); -export const verification = sqliteTable('verification', { - id: text('id').primaryKey().notNull(), - identifier: text('identifier').notNull(), - value: text('value').notNull(), - expiresAt: integer('expiresAt', authDateColumn).notNull(), - createdAt: integer('createdAt', authDateColumn).notNull(), - updatedAt: integer('updatedAt', authDateColumn).notNull() -}, (table) => ({ - verificationIdentifierIndex: index('verification_identifier_idx').on(table.identifier) -})); +export const verification = sqliteTable( + "verification", + { + id: text("id").primaryKey().notNull(), + identifier: text("identifier").notNull(), + value: text("value").notNull(), + expiresAt: integer("expiresAt", authDateColumn).notNull(), + createdAt: integer("createdAt", authDateColumn).notNull(), + updatedAt: integer("updatedAt", authDateColumn).notNull(), + }, + (table) => ({ + verificationIdentifierIndex: index("verification_identifier_idx").on( + table.identifier, + ), + }), +); -export const member = sqliteTable('member', { - id: text('id').primaryKey().notNull(), - organizationId: text('organizationId').notNull().references(() => organization.id, { onDelete: 'cascade' }), - userId: text('userId').notNull().references(() => user.id, { onDelete: 'cascade' }), - role: text('role').notNull().default('member'), - createdAt: integer('createdAt', authDateColumn).notNull() -}, (table) => ({ - memberOrganizationIdIndex: index('member_organizationId_idx').on(table.organizationId), - memberUserIdIndex: index('member_userId_idx').on(table.userId) -})); +export const member = sqliteTable( + "member", + { + id: text("id").primaryKey().notNull(), + organizationId: text("organizationId") + .notNull() + .references(() => organization.id, { onDelete: "cascade" }), + userId: text("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + role: text("role").notNull().default("member"), + createdAt: integer("createdAt", authDateColumn).notNull(), + }, + (table) => ({ + memberOrganizationIdIndex: index("member_organizationId_idx").on( + table.organizationId, + ), + memberUserIdIndex: index("member_userId_idx").on(table.userId), + }), +); -export const invitation = sqliteTable('invitation', { - id: text('id').primaryKey().notNull(), - organizationId: text('organizationId').notNull().references(() => organization.id, { onDelete: 'cascade' }), - email: text('email').notNull(), - role: text('role'), - status: text('status').notNull().default('pending'), - expiresAt: integer('expiresAt', authDateColumn).notNull(), - createdAt: integer('createdAt', authDateColumn).notNull(), - inviterId: text('inviterId').notNull().references(() => user.id, { onDelete: 'cascade' }) -}, (table) => ({ - invitationOrganizationIdIndex: index('invitation_organizationId_idx').on(table.organizationId), - invitationEmailIndex: index('invitation_email_idx').on(table.email) -})); +export const invitation = sqliteTable( + "invitation", + { + id: text("id").primaryKey().notNull(), + organizationId: text("organizationId") + .notNull() + .references(() => organization.id, { onDelete: "cascade" }), + email: text("email").notNull(), + role: text("role"), + status: text("status").notNull().default("pending"), + expiresAt: integer("expiresAt", authDateColumn).notNull(), + createdAt: integer("createdAt", authDateColumn).notNull(), + inviterId: text("inviterId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + }, + (table) => ({ + invitationOrganizationIdIndex: index("invitation_organizationId_idx").on( + table.organizationId, + ), + invitationEmailIndex: index("invitation_email_idx").on(table.email), + }), +); -export const watchlistItem = sqliteTable('watchlist_item', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - company_name: text('company_name').notNull(), - sector: text('sector'), - category: text('category'), - tags: text('tags', { mode: 'json' }).$type(), - status: text('status').$type().notNull().default('backlog'), - priority: text('priority').$type().notNull().default('medium'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull(), - last_reviewed_at: text('last_reviewed_at') -}, (table) => ({ - watchlistUserTickerUnique: uniqueIndex('watchlist_user_ticker_uidx').on(table.user_id, table.ticker), - watchlistUserCreatedIndex: index('watchlist_user_created_idx').on(table.user_id, table.created_at), - watchlistUserUpdatedIndex: index('watchlist_user_updated_idx').on(table.user_id, table.updated_at) -})); +export const watchlistItem = sqliteTable( + "watchlist_item", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + company_name: text("company_name").notNull(), + sector: text("sector"), + category: text("category"), + tags: text("tags", { mode: "json" }).$type(), + status: text("status").$type().notNull().default("backlog"), + priority: text("priority") + .$type() + .notNull() + .default("medium"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + last_reviewed_at: text("last_reviewed_at"), + }, + (table) => ({ + watchlistUserTickerUnique: uniqueIndex("watchlist_user_ticker_uidx").on( + table.user_id, + table.ticker, + ), + watchlistUserCreatedIndex: index("watchlist_user_created_idx").on( + table.user_id, + table.created_at, + ), + watchlistUserUpdatedIndex: index("watchlist_user_updated_idx").on( + table.user_id, + table.updated_at, + ), + }), +); -export const holding = sqliteTable('holding', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - company_name: text('company_name'), - shares: numeric('shares').notNull(), - avg_cost: numeric('avg_cost').notNull(), - current_price: numeric('current_price'), - market_value: numeric('market_value').notNull(), - gain_loss: numeric('gain_loss').notNull(), - gain_loss_pct: numeric('gain_loss_pct').notNull(), - last_price_at: text('last_price_at'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - holdingUserTickerUnique: uniqueIndex('holding_user_ticker_uidx').on(table.user_id, table.ticker), - holdingUserIndex: index('holding_user_idx').on(table.user_id) -})); +export const holding = sqliteTable( + "holding", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + company_name: text("company_name"), + shares: numeric("shares").notNull(), + avg_cost: numeric("avg_cost").notNull(), + current_price: numeric("current_price"), + market_value: numeric("market_value").notNull(), + gain_loss: numeric("gain_loss").notNull(), + gain_loss_pct: numeric("gain_loss_pct").notNull(), + last_price_at: text("last_price_at"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + holdingUserTickerUnique: uniqueIndex("holding_user_ticker_uidx").on( + table.user_id, + table.ticker, + ), + holdingUserIndex: index("holding_user_idx").on(table.user_id), + }), +); -export const filing = sqliteTable('filing', { - id: integer('id').primaryKey({ autoIncrement: true }), - ticker: text('ticker').notNull(), - filing_type: text('filing_type').$type<'10-K' | '10-Q' | '8-K'>().notNull(), - filing_date: text('filing_date').notNull(), - accession_number: text('accession_number').notNull(), - cik: text('cik').notNull(), - company_name: text('company_name').notNull(), - filing_url: text('filing_url'), - submission_url: text('submission_url'), - primary_document: text('primary_document'), - metrics: text('metrics', { mode: 'json' }).$type(), - analysis: text('analysis', { mode: 'json' }).$type(), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - filingAccessionUnique: uniqueIndex('filing_accession_uidx').on(table.accession_number), - filingTickerDateIndex: index('filing_ticker_date_idx').on(table.ticker, table.filing_date), - filingDateIndex: index('filing_date_idx').on(table.filing_date) -})); +export const filing = sqliteTable( + "filing", + { + id: integer("id").primaryKey({ autoIncrement: true }), + ticker: text("ticker").notNull(), + filing_type: text("filing_type").$type<"10-K" | "10-Q" | "8-K">().notNull(), + filing_date: text("filing_date").notNull(), + accession_number: text("accession_number").notNull(), + cik: text("cik").notNull(), + company_name: text("company_name").notNull(), + filing_url: text("filing_url"), + submission_url: text("submission_url"), + primary_document: text("primary_document"), + metrics: text("metrics", { mode: "json" }).$type(), + analysis: text("analysis", { mode: "json" }).$type(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + filingAccessionUnique: uniqueIndex("filing_accession_uidx").on( + table.accession_number, + ), + filingTickerDateIndex: index("filing_ticker_date_idx").on( + table.ticker, + table.filing_date, + ), + filingDateIndex: index("filing_date_idx").on(table.filing_date), + }), +); -export const filingStatementSnapshot = sqliteTable('filing_statement_snapshot', { - id: integer('id').primaryKey({ autoIncrement: true }), - filing_id: integer('filing_id').notNull().references(() => filing.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - filing_date: text('filing_date').notNull(), - filing_type: text('filing_type').$type<'10-K' | '10-Q'>().notNull(), - period_end: text('period_end'), - statement_bundle: text('statement_bundle', { mode: 'json' }).$type(), - standardized_bundle: text('standardized_bundle', { mode: 'json' }).$type(), - dimension_bundle: text('dimension_bundle', { mode: 'json' }).$type(), - parse_status: text('parse_status').$type<'ready' | 'partial' | 'failed'>().notNull(), - parse_error: text('parse_error'), - source: text('source').$type<'sec_filing_summary' | 'xbrl_instance' | 'companyfacts_fallback'>().notNull(), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - filingStatementFilingUnique: uniqueIndex('filing_stmt_filing_uidx').on(table.filing_id), - filingStatementTickerDateIndex: index('filing_stmt_ticker_date_idx').on(table.ticker, table.filing_date), - filingStatementDateIndex: index('filing_stmt_date_idx').on(table.filing_date), - filingStatementStatusIndex: index('filing_stmt_status_idx').on(table.parse_status) -})); +export const filingStatementSnapshot = sqliteTable( + "filing_statement_snapshot", + { + id: integer("id").primaryKey({ autoIncrement: true }), + filing_id: integer("filing_id") + .notNull() + .references(() => filing.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + filing_date: text("filing_date").notNull(), + filing_type: text("filing_type").$type<"10-K" | "10-Q">().notNull(), + period_end: text("period_end"), + statement_bundle: text("statement_bundle", { + mode: "json", + }).$type(), + standardized_bundle: text("standardized_bundle", { + mode: "json", + }).$type(), + dimension_bundle: text("dimension_bundle", { + mode: "json", + }).$type(), + parse_status: text("parse_status") + .$type<"ready" | "partial" | "failed">() + .notNull(), + parse_error: text("parse_error"), + source: text("source") + .$type<"sec_filing_summary" | "xbrl_instance" | "companyfacts_fallback">() + .notNull(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + filingStatementFilingUnique: uniqueIndex("filing_stmt_filing_uidx").on( + table.filing_id, + ), + filingStatementTickerDateIndex: index("filing_stmt_ticker_date_idx").on( + table.ticker, + table.filing_date, + ), + filingStatementDateIndex: index("filing_stmt_date_idx").on( + table.filing_date, + ), + filingStatementStatusIndex: index("filing_stmt_status_idx").on( + table.parse_status, + ), + }), +); -export const filingTaxonomySnapshot = sqliteTable('filing_taxonomy_snapshot', { - id: integer('id').primaryKey({ autoIncrement: true }), - filing_id: integer('filing_id').notNull().references(() => filing.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - filing_date: text('filing_date').notNull(), - filing_type: text('filing_type').$type<'10-K' | '10-Q'>().notNull(), - parse_status: text('parse_status').$type().notNull(), - parse_error: text('parse_error'), - source: text('source').$type<'xbrl_instance' | 'xbrl_instance_with_linkbase' | 'legacy_html_fallback'>().notNull(), - parser_engine: text('parser_engine').notNull().default('fiscal-xbrl'), - parser_version: text('parser_version').notNull().default('unknown'), - taxonomy_regime: text('taxonomy_regime').$type<'us-gaap' | 'ifrs-full' | 'unknown'>().notNull().default('unknown'), - fiscal_pack: text('fiscal_pack'), - periods: text('periods', { mode: 'json' }).$type(), - faithful_rows: text('faithful_rows', { mode: 'json' }).$type(), - statement_rows: text('statement_rows', { mode: 'json' }).$type(), - surface_rows: text('surface_rows', { mode: 'json' }).$type | null>(), - detail_rows: text('detail_rows', { mode: 'json' }).$type | null>(), - kpi_rows: text('kpi_rows', { mode: 'json' }).$type(), - derived_metrics: text('derived_metrics', { mode: 'json' }).$type(), - validation_result: text('validation_result', { mode: 'json' }).$type(), - normalization_summary: text('normalization_summary', { mode: 'json' }).$type(), - facts_count: integer('facts_count').notNull().default(0), - concepts_count: integer('concepts_count').notNull().default(0), - dimensions_count: integer('dimensions_count').notNull().default(0), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - filingTaxonomySnapshotFilingUnique: uniqueIndex('filing_taxonomy_snapshot_filing_uidx').on(table.filing_id), - filingTaxonomySnapshotTickerDateIndex: index('filing_taxonomy_snapshot_ticker_date_idx').on(table.ticker, table.filing_date), - filingTaxonomySnapshotStatusIndex: index('filing_taxonomy_snapshot_status_idx').on(table.parse_status) -})); +export const filingTaxonomySnapshot = sqliteTable( + "filing_taxonomy_snapshot", + { + id: integer("id").primaryKey({ autoIncrement: true }), + filing_id: integer("filing_id") + .notNull() + .references(() => filing.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + filing_date: text("filing_date").notNull(), + filing_type: text("filing_type").$type<"10-K" | "10-Q">().notNull(), + parse_status: text("parse_status").$type().notNull(), + parse_error: text("parse_error"), + source: text("source") + .$type< + "xbrl_instance" | "xbrl_instance_with_linkbase" | "legacy_html_fallback" + >() + .notNull(), + parser_engine: text("parser_engine").notNull().default("fiscal-xbrl"), + parser_version: text("parser_version").notNull().default("unknown"), + taxonomy_regime: text("taxonomy_regime") + .$type<"us-gaap" | "ifrs-full" | "unknown">() + .notNull() + .default("unknown"), + fiscal_pack: text("fiscal_pack"), + periods: text("periods", { mode: "json" }).$type(), + faithful_rows: text("faithful_rows", { mode: "json" }).$type< + TaxonomyStatementBundle["statements"] | null + >(), + statement_rows: text("statement_rows", { mode: "json" }).$type< + TaxonomyStatementBundle["statements"] | null + >(), + surface_rows: text("surface_rows", { mode: "json" }).$type | null>(), + detail_rows: text("detail_rows", { mode: "json" }).$type | null>(), + kpi_rows: text("kpi_rows", { mode: "json" }).$type< + StructuredKpiSnapshotRow[] | null + >(), + computed_definitions: text("computed_definitions", { mode: "json" }).$type< + ComputedDefinitionSnapshotRow[] | null + >(), + derived_metrics: text("derived_metrics", { + mode: "json", + }).$type(), + validation_result: text("validation_result", { + mode: "json", + }).$type(), + normalization_summary: text("normalization_summary", { + mode: "json", + }).$type(), + facts_count: integer("facts_count").notNull().default(0), + concepts_count: integer("concepts_count").notNull().default(0), + dimensions_count: integer("dimensions_count").notNull().default(0), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + filingTaxonomySnapshotFilingUnique: uniqueIndex( + "filing_taxonomy_snapshot_filing_uidx", + ).on(table.filing_id), + filingTaxonomySnapshotTickerDateIndex: index( + "filing_taxonomy_snapshot_ticker_date_idx", + ).on(table.ticker, table.filing_date), + filingTaxonomySnapshotStatusIndex: index( + "filing_taxonomy_snapshot_status_idx", + ).on(table.parse_status), + }), +); -export const filingTaxonomyContext = sqliteTable('filing_taxonomy_context', { - id: integer('id').primaryKey({ autoIncrement: true }), - snapshot_id: integer('snapshot_id').notNull().references(() => filingTaxonomySnapshot.id, { onDelete: 'cascade' }), - context_id: text('context_id').notNull(), - entity_identifier: text('entity_identifier'), - entity_scheme: text('entity_scheme'), - period_start: text('period_start'), - period_end: text('period_end'), - period_instant: text('period_instant'), - segment_json: text('segment_json', { mode: 'json' }).$type | null>(), - scenario_json: text('scenario_json', { mode: 'json' }).$type | null>(), - created_at: text('created_at').notNull() -}, (table) => ({ - filingTaxonomyContextSnapshotIndex: index('filing_taxonomy_context_snapshot_idx').on(table.snapshot_id), - filingTaxonomyContextUnique: uniqueIndex('filing_taxonomy_context_uidx').on(table.snapshot_id, table.context_id) -})); +export const filingTaxonomyContext = sqliteTable( + "filing_taxonomy_context", + { + id: integer("id").primaryKey({ autoIncrement: true }), + snapshot_id: integer("snapshot_id") + .notNull() + .references(() => filingTaxonomySnapshot.id, { onDelete: "cascade" }), + context_id: text("context_id").notNull(), + entity_identifier: text("entity_identifier"), + entity_scheme: text("entity_scheme"), + period_start: text("period_start"), + period_end: text("period_end"), + period_instant: text("period_instant"), + segment_json: text("segment_json", { mode: "json" }).$type | null>(), + scenario_json: text("scenario_json", { mode: "json" }).$type | null>(), + created_at: text("created_at").notNull(), + }, + (table) => ({ + filingTaxonomyContextSnapshotIndex: index( + "filing_taxonomy_context_snapshot_idx", + ).on(table.snapshot_id), + filingTaxonomyContextUnique: uniqueIndex("filing_taxonomy_context_uidx").on( + table.snapshot_id, + table.context_id, + ), + }), +); -export const filingTaxonomyAsset = sqliteTable('filing_taxonomy_asset', { - id: integer('id').primaryKey({ autoIncrement: true }), - snapshot_id: integer('snapshot_id').notNull().references(() => filingTaxonomySnapshot.id, { onDelete: 'cascade' }), - asset_type: text('asset_type').$type().notNull(), - name: text('name').notNull(), - url: text('url').notNull(), - size_bytes: integer('size_bytes'), - score: numeric('score'), - is_selected: integer('is_selected', { mode: 'boolean' }).notNull().default(false), - created_at: text('created_at').notNull() -}, (table) => ({ - filingTaxonomyAssetSnapshotIndex: index('filing_taxonomy_asset_snapshot_idx').on(table.snapshot_id), - filingTaxonomyAssetTypeIndex: index('filing_taxonomy_asset_type_idx').on(table.snapshot_id, table.asset_type) -})); +export const filingTaxonomyAsset = sqliteTable( + "filing_taxonomy_asset", + { + id: integer("id").primaryKey({ autoIncrement: true }), + snapshot_id: integer("snapshot_id") + .notNull() + .references(() => filingTaxonomySnapshot.id, { onDelete: "cascade" }), + asset_type: text("asset_type").$type().notNull(), + name: text("name").notNull(), + url: text("url").notNull(), + size_bytes: integer("size_bytes"), + score: numeric("score"), + is_selected: integer("is_selected", { mode: "boolean" }) + .notNull() + .default(false), + created_at: text("created_at").notNull(), + }, + (table) => ({ + filingTaxonomyAssetSnapshotIndex: index( + "filing_taxonomy_asset_snapshot_idx", + ).on(table.snapshot_id), + filingTaxonomyAssetTypeIndex: index("filing_taxonomy_asset_type_idx").on( + table.snapshot_id, + table.asset_type, + ), + }), +); -export const filingTaxonomyConcept = sqliteTable('filing_taxonomy_concept', { - id: integer('id').primaryKey({ autoIncrement: true }), - snapshot_id: integer('snapshot_id').notNull().references(() => filingTaxonomySnapshot.id, { onDelete: 'cascade' }), - concept_key: text('concept_key').notNull(), - qname: text('qname').notNull(), - namespace_uri: text('namespace_uri').notNull(), - local_name: text('local_name').notNull(), - label: text('label'), - is_extension: integer('is_extension', { mode: 'boolean' }).notNull().default(false), - balance: text('balance'), - period_type: text('period_type'), - data_type: text('data_type'), - statement_kind: text('statement_kind').$type(), - role_uri: text('role_uri'), - authoritative_concept_key: text('authoritative_concept_key'), - mapping_method: text('mapping_method'), - surface_key: text('surface_key'), - detail_parent_surface_key: text('detail_parent_surface_key'), - kpi_key: text('kpi_key'), - residual_flag: integer('residual_flag', { mode: 'boolean' }).notNull().default(false), - presentation_order: numeric('presentation_order'), - presentation_depth: integer('presentation_depth'), - parent_concept_key: text('parent_concept_key'), - is_abstract: integer('is_abstract', { mode: 'boolean' }).notNull().default(false), - created_at: text('created_at').notNull() -}, (table) => ({ - filingTaxonomyConceptSnapshotIndex: index('filing_taxonomy_concept_snapshot_idx').on(table.snapshot_id), - filingTaxonomyConceptStatementIndex: index('filing_taxonomy_concept_statement_idx').on(table.snapshot_id, table.statement_kind), - filingTaxonomyConceptUnique: uniqueIndex('filing_taxonomy_concept_uidx').on( - table.snapshot_id, - table.concept_key, - table.role_uri, - table.presentation_order - ) -})); +export const filingTaxonomyConcept = sqliteTable( + "filing_taxonomy_concept", + { + id: integer("id").primaryKey({ autoIncrement: true }), + snapshot_id: integer("snapshot_id") + .notNull() + .references(() => filingTaxonomySnapshot.id, { onDelete: "cascade" }), + concept_key: text("concept_key").notNull(), + qname: text("qname").notNull(), + namespace_uri: text("namespace_uri").notNull(), + local_name: text("local_name").notNull(), + label: text("label"), + is_extension: integer("is_extension", { mode: "boolean" }) + .notNull() + .default(false), + balance: text("balance"), + period_type: text("period_type"), + data_type: text("data_type"), + statement_kind: text("statement_kind").$type(), + role_uri: text("role_uri"), + authoritative_concept_key: text("authoritative_concept_key"), + mapping_method: text("mapping_method"), + surface_key: text("surface_key"), + detail_parent_surface_key: text("detail_parent_surface_key"), + kpi_key: text("kpi_key"), + residual_flag: integer("residual_flag", { mode: "boolean" }) + .notNull() + .default(false), + presentation_order: numeric("presentation_order"), + presentation_depth: integer("presentation_depth"), + parent_concept_key: text("parent_concept_key"), + is_abstract: integer("is_abstract", { mode: "boolean" }) + .notNull() + .default(false), + created_at: text("created_at").notNull(), + }, + (table) => ({ + filingTaxonomyConceptSnapshotIndex: index( + "filing_taxonomy_concept_snapshot_idx", + ).on(table.snapshot_id), + filingTaxonomyConceptStatementIndex: index( + "filing_taxonomy_concept_statement_idx", + ).on(table.snapshot_id, table.statement_kind), + filingTaxonomyConceptUnique: uniqueIndex("filing_taxonomy_concept_uidx").on( + table.snapshot_id, + table.concept_key, + table.role_uri, + table.presentation_order, + ), + }), +); -export const filingTaxonomyFact = sqliteTable('filing_taxonomy_fact', { - id: integer('id').primaryKey({ autoIncrement: true }), - snapshot_id: integer('snapshot_id').notNull().references(() => filingTaxonomySnapshot.id, { onDelete: 'cascade' }), - concept_key: text('concept_key').notNull(), - qname: text('qname').notNull(), - namespace_uri: text('namespace_uri').notNull(), - local_name: text('local_name').notNull(), - data_type: text('data_type'), - statement_kind: text('statement_kind').$type(), - role_uri: text('role_uri'), - authoritative_concept_key: text('authoritative_concept_key'), - mapping_method: text('mapping_method'), - surface_key: text('surface_key'), - detail_parent_surface_key: text('detail_parent_surface_key'), - kpi_key: text('kpi_key'), - residual_flag: integer('residual_flag', { mode: 'boolean' }).notNull().default(false), - context_id: text('context_id').notNull(), - unit: text('unit'), - decimals: text('decimals'), - precision: text('precision'), - nil: integer('nil', { mode: 'boolean' }).notNull().default(false), - value_num: numeric('value_num').notNull(), - period_start: text('period_start'), - period_end: text('period_end'), - period_instant: text('period_instant'), - dimensions: text('dimensions', { mode: 'json' }).$type().notNull(), - is_dimensionless: integer('is_dimensionless', { mode: 'boolean' }).notNull().default(true), - source_file: text('source_file'), - created_at: text('created_at').notNull() -}, (table) => ({ - filingTaxonomyFactSnapshotIndex: index('filing_taxonomy_fact_snapshot_idx').on(table.snapshot_id), - filingTaxonomyFactConceptIndex: index('filing_taxonomy_fact_concept_idx').on(table.snapshot_id, table.concept_key), - filingTaxonomyFactPeriodIndex: index('filing_taxonomy_fact_period_idx').on(table.snapshot_id, table.period_end, table.period_instant), - filingTaxonomyFactStatementIndex: index('filing_taxonomy_fact_statement_idx').on(table.snapshot_id, table.statement_kind) -})); +export const filingTaxonomyFact = sqliteTable( + "filing_taxonomy_fact", + { + id: integer("id").primaryKey({ autoIncrement: true }), + snapshot_id: integer("snapshot_id") + .notNull() + .references(() => filingTaxonomySnapshot.id, { onDelete: "cascade" }), + concept_key: text("concept_key").notNull(), + qname: text("qname").notNull(), + namespace_uri: text("namespace_uri").notNull(), + local_name: text("local_name").notNull(), + data_type: text("data_type"), + statement_kind: text("statement_kind").$type(), + role_uri: text("role_uri"), + authoritative_concept_key: text("authoritative_concept_key"), + mapping_method: text("mapping_method"), + surface_key: text("surface_key"), + detail_parent_surface_key: text("detail_parent_surface_key"), + kpi_key: text("kpi_key"), + residual_flag: integer("residual_flag", { mode: "boolean" }) + .notNull() + .default(false), + context_id: text("context_id").notNull(), + unit: text("unit"), + decimals: text("decimals"), + precision: text("precision"), + nil: integer("nil", { mode: "boolean" }).notNull().default(false), + value_num: numeric("value_num").notNull(), + period_start: text("period_start"), + period_end: text("period_end"), + period_instant: text("period_instant"), + dimensions: text("dimensions", { mode: "json" }) + .$type() + .notNull(), + is_dimensionless: integer("is_dimensionless", { mode: "boolean" }) + .notNull() + .default(true), + source_file: text("source_file"), + created_at: text("created_at").notNull(), + }, + (table) => ({ + filingTaxonomyFactSnapshotIndex: index( + "filing_taxonomy_fact_snapshot_idx", + ).on(table.snapshot_id), + filingTaxonomyFactConceptIndex: index( + "filing_taxonomy_fact_concept_idx", + ).on(table.snapshot_id, table.concept_key), + filingTaxonomyFactPeriodIndex: index("filing_taxonomy_fact_period_idx").on( + table.snapshot_id, + table.period_end, + table.period_instant, + ), + filingTaxonomyFactStatementIndex: index( + "filing_taxonomy_fact_statement_idx", + ).on(table.snapshot_id, table.statement_kind), + }), +); -export const filingTaxonomyMetricValidation = sqliteTable('filing_taxonomy_metric_validation', { - id: integer('id').primaryKey({ autoIncrement: true }), - snapshot_id: integer('snapshot_id').notNull().references(() => filingTaxonomySnapshot.id, { onDelete: 'cascade' }), - metric_key: text('metric_key').$type().notNull(), - taxonomy_value: numeric('taxonomy_value'), - llm_value: numeric('llm_value'), - absolute_diff: numeric('absolute_diff'), - relative_diff: numeric('relative_diff'), - status: text('status').$type().notNull(), - evidence_pages: text('evidence_pages', { mode: 'json' }).$type().notNull(), - pdf_url: text('pdf_url'), - provider: text('provider'), - model: text('model'), - error: text('error'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - filingTaxonomyMetricValidationSnapshotIndex: index('filing_taxonomy_metric_validation_snapshot_idx').on(table.snapshot_id), - filingTaxonomyMetricValidationStatusIndex: index('filing_taxonomy_metric_validation_status_idx').on(table.snapshot_id, table.status), - filingTaxonomyMetricValidationUnique: uniqueIndex('filing_taxonomy_metric_validation_uidx').on(table.snapshot_id, table.metric_key) -})); +export const filingTaxonomyMetricValidation = sqliteTable( + "filing_taxonomy_metric_validation", + { + id: integer("id").primaryKey({ autoIncrement: true }), + snapshot_id: integer("snapshot_id") + .notNull() + .references(() => filingTaxonomySnapshot.id, { onDelete: "cascade" }), + metric_key: text("metric_key").$type().notNull(), + taxonomy_value: numeric("taxonomy_value"), + llm_value: numeric("llm_value"), + absolute_diff: numeric("absolute_diff"), + relative_diff: numeric("relative_diff"), + status: text("status").$type().notNull(), + evidence_pages: text("evidence_pages", { mode: "json" }) + .$type() + .notNull(), + pdf_url: text("pdf_url"), + provider: text("provider"), + model: text("model"), + error: text("error"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + filingTaxonomyMetricValidationSnapshotIndex: index( + "filing_taxonomy_metric_validation_snapshot_idx", + ).on(table.snapshot_id), + filingTaxonomyMetricValidationStatusIndex: index( + "filing_taxonomy_metric_validation_status_idx", + ).on(table.snapshot_id, table.status), + filingTaxonomyMetricValidationUnique: uniqueIndex( + "filing_taxonomy_metric_validation_uidx", + ).on(table.snapshot_id, table.metric_key), + }), +); -export const companyFinancialBundle = sqliteTable('company_financial_bundle', { - id: integer('id').primaryKey({ autoIncrement: true }), - ticker: text('ticker').notNull(), - surface_kind: text('surface_kind').$type().notNull(), - cadence: text('cadence').$type().notNull(), - bundle_version: integer('bundle_version').notNull(), - source_snapshot_ids: text('source_snapshot_ids', { mode: 'json' }).$type().notNull(), - source_signature: text('source_signature').notNull(), - payload: text('payload', { mode: 'json' }).$type>().notNull(), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - companyFinancialBundleUnique: uniqueIndex('company_financial_bundle_uidx').on(table.ticker, table.surface_kind, table.cadence), - companyFinancialBundleTickerIndex: index('company_financial_bundle_ticker_idx').on(table.ticker, table.updated_at) -})); +export const companyFinancialBundle = sqliteTable( + "company_financial_bundle", + { + id: integer("id").primaryKey({ autoIncrement: true }), + ticker: text("ticker").notNull(), + surface_kind: text("surface_kind").$type().notNull(), + cadence: text("cadence").$type().notNull(), + bundle_version: integer("bundle_version").notNull(), + source_snapshot_ids: text("source_snapshot_ids", { mode: "json" }) + .$type() + .notNull(), + source_signature: text("source_signature").notNull(), + payload: text("payload", { mode: "json" }) + .$type>() + .notNull(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + companyFinancialBundleUnique: uniqueIndex( + "company_financial_bundle_uidx", + ).on(table.ticker, table.surface_kind, table.cadence), + companyFinancialBundleTickerIndex: index( + "company_financial_bundle_ticker_idx", + ).on(table.ticker, table.updated_at), + }), +); -export const companyOverviewCache = sqliteTable('company_overview_cache', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - cache_version: integer('cache_version').notNull(), - source_signature: text('source_signature').notNull(), - payload: text('payload', { mode: 'json' }).$type>().notNull(), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - companyOverviewCacheUnique: uniqueIndex('company_overview_cache_uidx').on(table.user_id, table.ticker), - companyOverviewCacheLookupIndex: index('company_overview_cache_lookup_idx').on(table.user_id, table.ticker, table.updated_at) -})); +export const companyOverviewCache = sqliteTable( + "company_overview_cache", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + cache_version: integer("cache_version").notNull(), + source_signature: text("source_signature").notNull(), + payload: text("payload", { mode: "json" }) + .$type>() + .notNull(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + companyOverviewCacheUnique: uniqueIndex("company_overview_cache_uidx").on( + table.user_id, + table.ticker, + ), + companyOverviewCacheLookupIndex: index( + "company_overview_cache_lookup_idx", + ).on(table.user_id, table.ticker, table.updated_at), + }), +); -export const filingLink = sqliteTable('filing_link', { - id: integer('id').primaryKey({ autoIncrement: true }), - filing_id: integer('filing_id').notNull().references(() => filing.id, { onDelete: 'cascade' }), - link_type: text('link_type').notNull(), - url: text('url').notNull(), - source: text('source').notNull().default('sec'), - created_at: text('created_at').notNull() -}, (table) => ({ - filingLinkUnique: uniqueIndex('filing_link_unique_uidx').on(table.filing_id, table.url), - filingLinkFilingIndex: index('filing_link_filing_idx').on(table.filing_id) -})); +export const filingLink = sqliteTable( + "filing_link", + { + id: integer("id").primaryKey({ autoIncrement: true }), + filing_id: integer("filing_id") + .notNull() + .references(() => filing.id, { onDelete: "cascade" }), + link_type: text("link_type").notNull(), + url: text("url").notNull(), + source: text("source").notNull().default("sec"), + created_at: text("created_at").notNull(), + }, + (table) => ({ + filingLinkUnique: uniqueIndex("filing_link_unique_uidx").on( + table.filing_id, + table.url, + ), + filingLinkFilingIndex: index("filing_link_filing_idx").on(table.filing_id), + }), +); -export const taskRun = sqliteTable('task_run', { - id: text('id').primaryKey().notNull(), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - task_type: text('task_type').$type<'sync_filings' | 'refresh_prices' | 'analyze_filing' | 'portfolio_insights' | 'index_search'>().notNull(), - status: text('status').$type<'queued' | 'running' | 'completed' | 'failed'>().notNull(), - stage: text('stage').notNull(), - stage_detail: text('stage_detail'), - stage_context: text('stage_context', { mode: 'json' }).$type(), - resource_key: text('resource_key'), - notification_read_at: text('notification_read_at'), - notification_silenced_at: text('notification_silenced_at'), - priority: integer('priority').notNull(), - payload: text('payload', { mode: 'json' }).$type>().notNull(), - result: text('result', { mode: 'json' }).$type | null>(), - error: text('error'), - attempts: integer('attempts').notNull(), - max_attempts: integer('max_attempts').notNull(), - workflow_run_id: text('workflow_run_id'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull(), - finished_at: text('finished_at') -}, (table) => ({ - taskUserCreatedIndex: index('task_user_created_idx').on(table.user_id, table.created_at), - taskUserUpdatedIndex: index('task_user_updated_idx').on(table.user_id, table.updated_at), - taskStatusIndex: index('task_status_idx').on(table.status), - taskUserResourceStatusIndex: index('task_user_resource_status_idx').on( - table.user_id, - table.task_type, - table.resource_key, - table.status, - table.created_at - ), - taskWorkflowRunUnique: uniqueIndex('task_workflow_run_uidx').on(table.workflow_run_id) -})); +export const taskRun = sqliteTable( + "task_run", + { + id: text("id").primaryKey().notNull(), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + task_type: text("task_type") + .$type< + | "sync_filings" + | "refresh_prices" + | "analyze_filing" + | "portfolio_insights" + | "index_search" + >() + .notNull(), + status: text("status") + .$type<"queued" | "running" | "completed" | "failed">() + .notNull(), + stage: text("stage").notNull(), + stage_detail: text("stage_detail"), + stage_context: text("stage_context", { + mode: "json", + }).$type(), + resource_key: text("resource_key"), + notification_read_at: text("notification_read_at"), + notification_silenced_at: text("notification_silenced_at"), + priority: integer("priority").notNull(), + payload: text("payload", { mode: "json" }) + .$type>() + .notNull(), + result: text("result", { mode: "json" }).$type | null>(), + error: text("error"), + attempts: integer("attempts").notNull(), + max_attempts: integer("max_attempts").notNull(), + workflow_run_id: text("workflow_run_id"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + finished_at: text("finished_at"), + }, + (table) => ({ + taskUserCreatedIndex: index("task_user_created_idx").on( + table.user_id, + table.created_at, + ), + taskUserUpdatedIndex: index("task_user_updated_idx").on( + table.user_id, + table.updated_at, + ), + taskStatusIndex: index("task_status_idx").on(table.status), + taskUserResourceStatusIndex: index("task_user_resource_status_idx").on( + table.user_id, + table.task_type, + table.resource_key, + table.status, + table.created_at, + ), + taskWorkflowRunUnique: uniqueIndex("task_workflow_run_uidx").on( + table.workflow_run_id, + ), + }), +); // Note: Partial unique index for active resource-scoped task deduplication is created via // migration 0013_task_active_resource_unique.sql. SQLite does not support partial indexes @@ -674,161 +1014,289 @@ export const taskRun = sqliteTable('task_run', { // CREATE UNIQUE INDEX task_active_resource_uidx ON task_run (user_id, task_type, resource_key) // WHERE resource_key IS NOT NULL AND status IN ('queued', 'running'); -export const taskStageEvent = sqliteTable('task_stage_event', { - id: integer('id').primaryKey({ autoIncrement: true }), - task_id: text('task_id').notNull().references(() => taskRun.id, { onDelete: 'cascade' }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - stage: text('stage').notNull(), - stage_detail: text('stage_detail'), - stage_context: text('stage_context', { mode: 'json' }).$type(), - status: text('status').$type<'queued' | 'running' | 'completed' | 'failed'>().notNull(), - created_at: text('created_at').notNull() -}, (table) => ({ - taskStageEventTaskCreatedIndex: index('task_stage_event_task_created_idx').on(table.task_id, table.created_at), - taskStageEventUserCreatedIndex: index('task_stage_event_user_created_idx').on(table.user_id, table.created_at) -})); +export const taskStageEvent = sqliteTable( + "task_stage_event", + { + id: integer("id").primaryKey({ autoIncrement: true }), + task_id: text("task_id") + .notNull() + .references(() => taskRun.id, { onDelete: "cascade" }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + stage: text("stage").notNull(), + stage_detail: text("stage_detail"), + stage_context: text("stage_context", { + mode: "json", + }).$type(), + status: text("status") + .$type<"queued" | "running" | "completed" | "failed">() + .notNull(), + created_at: text("created_at").notNull(), + }, + (table) => ({ + taskStageEventTaskCreatedIndex: index( + "task_stage_event_task_created_idx", + ).on(table.task_id, table.created_at), + taskStageEventUserCreatedIndex: index( + "task_stage_event_user_created_idx", + ).on(table.user_id, table.created_at), + }), +); -export const portfolioInsight = sqliteTable('portfolio_insight', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - provider: text('provider').notNull(), - model: text('model').notNull(), - content: text('content').notNull(), - created_at: text('created_at').notNull() -}, (table) => ({ - insightUserCreatedIndex: index('insight_user_created_idx').on(table.user_id, table.created_at) -})); +export const portfolioInsight = sqliteTable( + "portfolio_insight", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + provider: text("provider").notNull(), + model: text("model").notNull(), + content: text("content").notNull(), + created_at: text("created_at").notNull(), + }, + (table) => ({ + insightUserCreatedIndex: index("insight_user_created_idx").on( + table.user_id, + table.created_at, + ), + }), +); -export const researchJournalEntry = sqliteTable('research_journal_entry', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - ticker: text('ticker').notNull(), - accession_number: text('accession_number'), - entry_type: text('entry_type').$type().notNull(), - title: text('title'), - body_markdown: text('body_markdown').notNull(), - metadata: text('metadata', { mode: 'json' }).$type | null>(), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - researchJournalTickerIndex: index('research_journal_ticker_idx').on(table.user_id, table.ticker, table.created_at), - researchJournalAccessionIndex: index('research_journal_accession_idx').on(table.user_id, table.accession_number) -})); +export const researchJournalEntry = sqliteTable( + "research_journal_entry", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + ticker: text("ticker").notNull(), + accession_number: text("accession_number"), + entry_type: text("entry_type").$type().notNull(), + title: text("title"), + body_markdown: text("body_markdown").notNull(), + metadata: text("metadata", { mode: "json" }).$type | null>(), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + researchJournalTickerIndex: index("research_journal_ticker_idx").on( + table.user_id, + table.ticker, + table.created_at, + ), + researchJournalAccessionIndex: index("research_journal_accession_idx").on( + table.user_id, + table.accession_number, + ), + }), +); -export const searchDocument = sqliteTable('search_document', { - id: integer('id').primaryKey({ autoIncrement: true }), - source_kind: text('source_kind').$type().notNull(), - source_ref: text('source_ref').notNull(), - scope: text('scope').$type().notNull(), - user_id: text('user_id').references(() => user.id, { onDelete: 'cascade' }), - ticker: text('ticker'), - accession_number: text('accession_number'), - title: text('title'), - content_text: text('content_text').notNull(), - content_hash: text('content_hash').notNull(), - metadata: text('metadata', { mode: 'json' }).$type | null>(), - index_status: text('index_status').$type().notNull(), - indexed_at: text('indexed_at'), - last_error: text('last_error'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - searchDocumentSourceUnique: uniqueIndex('search_document_source_uidx').on( - table.scope, - sql`ifnull(${table.user_id}, '')`, - table.source_kind, - table.source_ref - ), - searchDocumentScopeIndex: index('search_document_scope_idx').on( - table.scope, - table.source_kind, - table.ticker, - table.updated_at - ), - searchDocumentAccessionIndex: index('search_document_accession_idx').on(table.accession_number, table.source_kind) -})); +export const searchDocument = sqliteTable( + "search_document", + { + id: integer("id").primaryKey({ autoIncrement: true }), + source_kind: text("source_kind") + .$type() + .notNull(), + source_ref: text("source_ref").notNull(), + scope: text("scope").$type().notNull(), + user_id: text("user_id").references(() => user.id, { onDelete: "cascade" }), + ticker: text("ticker"), + accession_number: text("accession_number"), + title: text("title"), + content_text: text("content_text").notNull(), + content_hash: text("content_hash").notNull(), + metadata: text("metadata", { mode: "json" }).$type | null>(), + index_status: text("index_status").$type().notNull(), + indexed_at: text("indexed_at"), + last_error: text("last_error"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + searchDocumentSourceUnique: uniqueIndex("search_document_source_uidx").on( + table.scope, + sql`ifnull(${table.user_id}, '')`, + table.source_kind, + table.source_ref, + ), + searchDocumentScopeIndex: index("search_document_scope_idx").on( + table.scope, + table.source_kind, + table.ticker, + table.updated_at, + ), + searchDocumentAccessionIndex: index("search_document_accession_idx").on( + table.accession_number, + table.source_kind, + ), + }), +); -export const searchChunk = sqliteTable('search_chunk', { - id: integer('id').primaryKey({ autoIncrement: true }), - document_id: integer('document_id').notNull().references(() => searchDocument.id, { onDelete: 'cascade' }), - chunk_index: integer('chunk_index').notNull(), - chunk_text: text('chunk_text').notNull(), - char_count: integer('char_count').notNull(), - start_offset: integer('start_offset').notNull(), - end_offset: integer('end_offset').notNull(), - heading_path: text('heading_path'), - citation_label: text('citation_label').notNull(), - created_at: text('created_at').notNull() -}, (table) => ({ - searchChunkUnique: uniqueIndex('search_chunk_document_chunk_uidx').on(table.document_id, table.chunk_index), - searchChunkDocumentIndex: index('search_chunk_document_idx').on(table.document_id) -})); +export const searchChunk = sqliteTable( + "search_chunk", + { + id: integer("id").primaryKey({ autoIncrement: true }), + document_id: integer("document_id") + .notNull() + .references(() => searchDocument.id, { onDelete: "cascade" }), + chunk_index: integer("chunk_index").notNull(), + chunk_text: text("chunk_text").notNull(), + char_count: integer("char_count").notNull(), + start_offset: integer("start_offset").notNull(), + end_offset: integer("end_offset").notNull(), + heading_path: text("heading_path"), + citation_label: text("citation_label").notNull(), + created_at: text("created_at").notNull(), + }, + (table) => ({ + searchChunkUnique: uniqueIndex("search_chunk_document_chunk_uidx").on( + table.document_id, + table.chunk_index, + ), + searchChunkDocumentIndex: index("search_chunk_document_idx").on( + table.document_id, + ), + }), +); -export const researchArtifact = sqliteTable('research_artifact', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - organization_id: text('organization_id').references(() => organization.id, { onDelete: 'set null' }), - ticker: text('ticker').notNull(), - accession_number: text('accession_number'), - kind: text('kind').$type().notNull(), - source: text('source').$type().notNull().default('user'), - subtype: text('subtype'), - title: text('title'), - summary: text('summary'), - body_markdown: text('body_markdown'), - search_text: text('search_text'), - visibility_scope: text('visibility_scope').$type().notNull().default('private'), - tags: text('tags', { mode: 'json' }).$type(), - metadata: text('metadata', { mode: 'json' }).$type | null>(), - file_name: text('file_name'), - mime_type: text('mime_type'), - file_size_bytes: integer('file_size_bytes'), - storage_path: text('storage_path'), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - researchArtifactTickerIndex: index('research_artifact_ticker_idx').on(table.user_id, table.ticker, table.updated_at), - researchArtifactKindIndex: index('research_artifact_kind_idx').on(table.user_id, table.kind, table.updated_at), - researchArtifactAccessionIndex: index('research_artifact_accession_idx').on(table.user_id, table.accession_number), - researchArtifactSourceIndex: index('research_artifact_source_idx').on(table.user_id, table.source, table.updated_at) -})); +export const researchArtifact = sqliteTable( + "research_artifact", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + organization_id: text("organization_id").references(() => organization.id, { + onDelete: "set null", + }), + ticker: text("ticker").notNull(), + accession_number: text("accession_number"), + kind: text("kind").$type().notNull(), + source: text("source") + .$type() + .notNull() + .default("user"), + subtype: text("subtype"), + title: text("title"), + summary: text("summary"), + body_markdown: text("body_markdown"), + search_text: text("search_text"), + visibility_scope: text("visibility_scope") + .$type() + .notNull() + .default("private"), + tags: text("tags", { mode: "json" }).$type(), + metadata: text("metadata", { mode: "json" }).$type | null>(), + file_name: text("file_name"), + mime_type: text("mime_type"), + file_size_bytes: integer("file_size_bytes"), + storage_path: text("storage_path"), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + researchArtifactTickerIndex: index("research_artifact_ticker_idx").on( + table.user_id, + table.ticker, + table.updated_at, + ), + researchArtifactKindIndex: index("research_artifact_kind_idx").on( + table.user_id, + table.kind, + table.updated_at, + ), + researchArtifactAccessionIndex: index("research_artifact_accession_idx").on( + table.user_id, + table.accession_number, + ), + researchArtifactSourceIndex: index("research_artifact_source_idx").on( + table.user_id, + table.source, + table.updated_at, + ), + }), +); -export const researchMemo = sqliteTable('research_memo', { - id: integer('id').primaryKey({ autoIncrement: true }), - user_id: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }), - organization_id: text('organization_id').references(() => organization.id, { onDelete: 'set null' }), - ticker: text('ticker').notNull(), - rating: text('rating').$type(), - conviction: text('conviction').$type(), - time_horizon_months: integer('time_horizon_months'), - packet_title: text('packet_title'), - packet_subtitle: text('packet_subtitle'), - thesis_markdown: text('thesis_markdown').notNull().default(''), - variant_view_markdown: text('variant_view_markdown').notNull().default(''), - catalysts_markdown: text('catalysts_markdown').notNull().default(''), - risks_markdown: text('risks_markdown').notNull().default(''), - disconfirming_evidence_markdown: text('disconfirming_evidence_markdown').notNull().default(''), - next_actions_markdown: text('next_actions_markdown').notNull().default(''), - created_at: text('created_at').notNull(), - updated_at: text('updated_at').notNull() -}, (table) => ({ - researchMemoTickerUnique: uniqueIndex('research_memo_ticker_uidx').on(table.user_id, table.ticker), - researchMemoUpdatedIndex: index('research_memo_updated_idx').on(table.user_id, table.updated_at) -})); +export const researchMemo = sqliteTable( + "research_memo", + { + id: integer("id").primaryKey({ autoIncrement: true }), + user_id: text("user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + organization_id: text("organization_id").references(() => organization.id, { + onDelete: "set null", + }), + ticker: text("ticker").notNull(), + rating: text("rating").$type(), + conviction: text("conviction").$type(), + time_horizon_months: integer("time_horizon_months"), + packet_title: text("packet_title"), + packet_subtitle: text("packet_subtitle"), + thesis_markdown: text("thesis_markdown").notNull().default(""), + variant_view_markdown: text("variant_view_markdown").notNull().default(""), + catalysts_markdown: text("catalysts_markdown").notNull().default(""), + risks_markdown: text("risks_markdown").notNull().default(""), + disconfirming_evidence_markdown: text("disconfirming_evidence_markdown") + .notNull() + .default(""), + next_actions_markdown: text("next_actions_markdown").notNull().default(""), + created_at: text("created_at").notNull(), + updated_at: text("updated_at").notNull(), + }, + (table) => ({ + researchMemoTickerUnique: uniqueIndex("research_memo_ticker_uidx").on( + table.user_id, + table.ticker, + ), + researchMemoUpdatedIndex: index("research_memo_updated_idx").on( + table.user_id, + table.updated_at, + ), + }), +); -export const researchMemoEvidence = sqliteTable('research_memo_evidence', { - id: integer('id').primaryKey({ autoIncrement: true }), - memo_id: integer('memo_id').notNull().references(() => researchMemo.id, { onDelete: 'cascade' }), - artifact_id: integer('artifact_id').notNull().references(() => researchArtifact.id, { onDelete: 'cascade' }), - section: text('section').$type().notNull(), - annotation: text('annotation'), - sort_order: integer('sort_order').notNull().default(0), - created_at: text('created_at').notNull() -}, (table) => ({ - researchMemoEvidenceMemoIndex: index('research_memo_evidence_memo_idx').on(table.memo_id, table.section, table.sort_order), - researchMemoEvidenceArtifactIndex: index('research_memo_evidence_artifact_idx').on(table.artifact_id), - researchMemoEvidenceUnique: uniqueIndex('research_memo_evidence_unique_uidx').on(table.memo_id, table.artifact_id, table.section) -})); +export const researchMemoEvidence = sqliteTable( + "research_memo_evidence", + { + id: integer("id").primaryKey({ autoIncrement: true }), + memo_id: integer("memo_id") + .notNull() + .references(() => researchMemo.id, { onDelete: "cascade" }), + artifact_id: integer("artifact_id") + .notNull() + .references(() => researchArtifact.id, { onDelete: "cascade" }), + section: text("section").$type().notNull(), + annotation: text("annotation"), + sort_order: integer("sort_order").notNull().default(0), + created_at: text("created_at").notNull(), + }, + (table) => ({ + researchMemoEvidenceMemoIndex: index("research_memo_evidence_memo_idx").on( + table.memo_id, + table.section, + table.sort_order, + ), + researchMemoEvidenceArtifactIndex: index( + "research_memo_evidence_artifact_idx", + ).on(table.artifact_id), + researchMemoEvidenceUnique: uniqueIndex( + "research_memo_evidence_unique_uidx", + ).on(table.memo_id, table.artifact_id, table.section), + }), +); export const authSchema = { user, @@ -837,7 +1305,7 @@ export const authSchema = { verification, organization, member, - invitation + invitation, }; export const appSchema = { @@ -861,10 +1329,10 @@ export const appSchema = { searchChunk, researchArtifact, researchMemo, - researchMemoEvidence + researchMemoEvidence, }; export const schema = { ...authSchema, - ...appSchema + ...appSchema, }; diff --git a/lib/server/db/sqlite-schema-compat.ts b/lib/server/db/sqlite-schema-compat.ts index 8bbcb49..5a5f75f 100644 --- a/lib/server/db/sqlite-schema-compat.ts +++ b/lib/server/db/sqlite-schema-compat.ts @@ -1,9 +1,11 @@ -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import type { Database } from 'bun:sqlite'; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { Database } from "bun:sqlite"; -const DEFAULT_SURFACE_ROWS_JSON = '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; -const DEFAULT_DETAIL_ROWS_JSON = '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'; +const DEFAULT_SURFACE_ROWS_JSON = + '{"income":[],"balance":[],"cash_flow":[],"equity":[],"comprehensive_income":[]}'; +const DEFAULT_DETAIL_ROWS_JSON = + '{"income":{},"balance":{},"cash_flow":{},"equity":{},"comprehensive_income":{}}'; type MissingColumnDefinition = { name: string; @@ -12,36 +14,49 @@ type MissingColumnDefinition = { export function hasTable(client: Database, tableName: string) { const row = client - .query('SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1') - .get('table', tableName) as { name: string } | null; + .query("SELECT name FROM sqlite_master WHERE type = ? AND name = ? LIMIT 1") + .get("table", tableName) as { name: string } | null; return row !== null; } -export function hasColumn(client: Database, tableName: string, columnName: string) { +export function hasColumn( + client: Database, + tableName: string, + columnName: string, +) { if (!hasTable(client, tableName)) { return false; } - const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>; + const rows = client.query(`PRAGMA table_info(${tableName})`).all() as Array<{ + name: string; + }>; return rows.some((row) => row.name === columnName); } export function applySqlFile(client: Database, fileName: string) { - const sql = readFileSync(join(process.cwd(), 'drizzle', fileName), 'utf8'); + const sql = readFileSync(join(process.cwd(), "drizzle", fileName), "utf8"); client.exec(sql); } export function applyBaseSchemaCompat(client: Database) { - const sql = readFileSync(join(process.cwd(), 'drizzle', '0000_cold_silver_centurion.sql'), 'utf8') - .replaceAll('CREATE TABLE `', 'CREATE TABLE IF NOT EXISTS `') - .replaceAll('CREATE UNIQUE INDEX `', 'CREATE UNIQUE INDEX IF NOT EXISTS `') - .replaceAll('CREATE INDEX `', 'CREATE INDEX IF NOT EXISTS `'); + const sql = readFileSync( + join(process.cwd(), "drizzle", "0000_cold_silver_centurion.sql"), + "utf8", + ) + .replaceAll("CREATE TABLE `", "CREATE TABLE IF NOT EXISTS `") + .replaceAll("CREATE UNIQUE INDEX `", "CREATE UNIQUE INDEX IF NOT EXISTS `") + .replaceAll("CREATE INDEX `", "CREATE INDEX IF NOT EXISTS `"); client.exec(sql); } -function ensureColumns(client: Database, tableName: string, columns: MissingColumnDefinition[]) { +function ensureColumns( + client: Database, + tableName: string, + columns: MissingColumnDefinition[], +) { if (!hasTable(client, tableName)) { return; } @@ -54,7 +69,7 @@ function ensureColumns(client: Database, tableName: string, columns: MissingColu } function ensureResearchWorkspaceSchema(client: Database) { - if (!hasTable(client, 'research_artifact')) { + if (!hasTable(client, "research_artifact")) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_artifact\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -84,7 +99,7 @@ function ensureResearchWorkspaceSchema(client: Database) { `); } - if (!hasTable(client, 'research_memo')) { + if (!hasTable(client, "research_memo")) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_memo\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -110,7 +125,7 @@ function ensureResearchWorkspaceSchema(client: Database) { `); } - if (!hasTable(client, 'research_memo_evidence')) { + if (!hasTable(client, "research_memo_evidence")) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_memo_evidence\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -126,15 +141,33 @@ function ensureResearchWorkspaceSchema(client: Database) { `); } - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);'); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);'); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);'); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_artifact_ticker_idx` ON `research_artifact` (`user_id`, `ticker`, `updated_at`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_artifact_kind_idx` ON `research_artifact` (`user_id`, `kind`, `updated_at`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_artifact_accession_idx` ON `research_artifact` (`user_id`, `accession_number`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_artifact_source_idx` ON `research_artifact` (`user_id`, `source`, `updated_at`);", + ); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_ticker_uidx` ON `research_memo` (`user_id`, `ticker`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_memo_updated_idx` ON `research_memo` (`user_id`, `updated_at`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_memo_evidence_memo_idx` ON `research_memo_evidence` (`memo_id`, `section`, `sort_order`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_memo_evidence_artifact_idx` ON `research_memo_evidence` (`artifact_id`);", + ); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `research_memo_evidence_unique_uidx` ON `research_memo_evidence` (`memo_id`, `artifact_id`, `section`);", + ); client.exec(` CREATE VIRTUAL TABLE IF NOT EXISTS \`research_artifact_fts\` USING fts5( artifact_id UNINDEXED, @@ -268,7 +301,7 @@ function ensureResearchWorkspaceSchema(client: Database) { ); `); - client.exec('DELETE FROM `research_artifact_fts`;'); + client.exec("DELETE FROM `research_artifact_fts`;"); client.exec(` INSERT INTO \`research_artifact_fts\` ( \`artifact_id\`, @@ -297,39 +330,71 @@ function ensureResearchWorkspaceSchema(client: Database) { } const TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS = [ - 'parser_engine', - 'parser_version', - 'taxonomy_regime', - 'fiscal_pack', - 'faithful_rows', - 'surface_rows', - 'detail_rows', - 'kpi_rows', - 'normalization_summary' + "parser_engine", + "parser_version", + "taxonomy_regime", + "fiscal_pack", + "faithful_rows", + "surface_rows", + "detail_rows", + "kpi_rows", + "computed_definitions", + "normalization_summary", ] as const; function ensureTaxonomySnapshotCompat(client: Database) { - if (!hasTable(client, 'filing_taxonomy_snapshot')) { + if (!hasTable(client, "filing_taxonomy_snapshot")) { return; } - ensureColumns(client, 'filing_taxonomy_snapshot', [ - { name: 'parser_engine', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'fiscal-xbrl';" }, - { name: 'parser_version', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text NOT NULL DEFAULT 'unknown';" }, - { name: 'taxonomy_regime', sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text NOT NULL DEFAULT 'unknown';" }, - { name: 'fiscal_pack', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;' }, - { name: 'faithful_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;' }, - { name: 'surface_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;' }, - { name: 'detail_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;' }, - { name: 'kpi_rows', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;' }, - { name: 'normalization_summary', sql: 'ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;' } + ensureColumns(client, "filing_taxonomy_snapshot", [ + { + name: "parser_engine", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_engine` text NOT NULL DEFAULT 'fiscal-xbrl';", + }, + { + name: "parser_version", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `parser_version` text NOT NULL DEFAULT 'unknown';", + }, + { + name: "taxonomy_regime", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `taxonomy_regime` text NOT NULL DEFAULT 'unknown';", + }, + { + name: "fiscal_pack", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `fiscal_pack` text;", + }, + { + name: "faithful_rows", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `faithful_rows` text;", + }, + { + name: "surface_rows", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `surface_rows` text;", + }, + { + name: "detail_rows", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `detail_rows` text;", + }, + { + name: "kpi_rows", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `kpi_rows` text;", + }, + { + name: "computed_definitions", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `computed_definitions` text;", + }, + { + name: "normalization_summary", + sql: "ALTER TABLE `filing_taxonomy_snapshot` ADD `normalization_summary` text;", + }, ]); for (const columnName of TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS) { - if (!hasColumn(client, 'filing_taxonomy_snapshot', columnName)) { + if (!hasColumn(client, "filing_taxonomy_snapshot", columnName)) { throw new Error( `Schema compat failed: filing_taxonomy_snapshot missing required column '${columnName}'. ` + - `Delete the database file and restart to rebuild schema.` + `Delete the database file and restart to rebuild schema.`, ); } } @@ -340,12 +405,13 @@ function ensureTaxonomySnapshotCompat(client: Database) { \`faithful_rows\` = COALESCE(\`faithful_rows\`, \`statement_rows\`), \`surface_rows\` = COALESCE(\`surface_rows\`, '${DEFAULT_SURFACE_ROWS_JSON}'), \`detail_rows\` = COALESCE(\`detail_rows\`, '${DEFAULT_DETAIL_ROWS_JSON}'), - \`kpi_rows\` = COALESCE(\`kpi_rows\`, '[]'); + \`kpi_rows\` = COALESCE(\`kpi_rows\`, '[]'), + \`computed_definitions\` = COALESCE(\`computed_definitions\`, '[]'); `); } function ensureTaxonomyContextCompat(client: Database) { - if (!hasTable(client, 'filing_taxonomy_context')) { + if (!hasTable(client, "filing_taxonomy_context")) { client.exec(` CREATE TABLE IF NOT EXISTS \`filing_taxonomy_context\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -364,35 +430,93 @@ function ensureTaxonomyContextCompat(client: Database) { `); } - client.exec('CREATE INDEX IF NOT EXISTS `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);'); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);'); + client.exec( + "CREATE INDEX IF NOT EXISTS `filing_taxonomy_context_snapshot_idx` ON `filing_taxonomy_context` (`snapshot_id`);", + ); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `filing_taxonomy_context_uidx` ON `filing_taxonomy_context` (`snapshot_id`,`context_id`);", + ); } function ensureTaxonomyConceptCompat(client: Database) { - ensureColumns(client, 'filing_taxonomy_concept', [ - { name: 'balance', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;' }, - { name: 'period_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;' }, - { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;' }, - { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;' }, - { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;' }, - { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;' }, - { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;' }, - { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;' }, - { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer NOT NULL DEFAULT false;' } + ensureColumns(client, "filing_taxonomy_concept", [ + { + name: "balance", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `balance` text;", + }, + { + name: "period_type", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `period_type` text;", + }, + { + name: "data_type", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `data_type` text;", + }, + { + name: "authoritative_concept_key", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `authoritative_concept_key` text;", + }, + { + name: "mapping_method", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `mapping_method` text;", + }, + { + name: "surface_key", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `surface_key` text;", + }, + { + name: "detail_parent_surface_key", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `detail_parent_surface_key` text;", + }, + { + name: "kpi_key", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `kpi_key` text;", + }, + { + name: "residual_flag", + sql: "ALTER TABLE `filing_taxonomy_concept` ADD `residual_flag` integer NOT NULL DEFAULT false;", + }, ]); } function ensureTaxonomyFactCompat(client: Database) { - ensureColumns(client, 'filing_taxonomy_fact', [ - { name: 'data_type', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;' }, - { name: 'authoritative_concept_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;' }, - { name: 'mapping_method', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;' }, - { name: 'surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;' }, - { name: 'detail_parent_surface_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;' }, - { name: 'kpi_key', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;' }, - { name: 'residual_flag', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer NOT NULL DEFAULT false;' }, - { name: 'precision', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;' }, - { name: 'nil', sql: 'ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer NOT NULL DEFAULT false;' } + ensureColumns(client, "filing_taxonomy_fact", [ + { + name: "data_type", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `data_type` text;", + }, + { + name: "authoritative_concept_key", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `authoritative_concept_key` text;", + }, + { + name: "mapping_method", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `mapping_method` text;", + }, + { + name: "surface_key", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `surface_key` text;", + }, + { + name: "detail_parent_surface_key", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `detail_parent_surface_key` text;", + }, + { + name: "kpi_key", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `kpi_key` text;", + }, + { + name: "residual_flag", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `residual_flag` integer NOT NULL DEFAULT false;", + }, + { + name: "precision", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `precision` text;", + }, + { + name: "nil", + sql: "ALTER TABLE `filing_taxonomy_fact` ADD `nil` integer NOT NULL DEFAULT false;", + }, ]); } @@ -405,18 +529,18 @@ function ensureTaxonomyCompat(client: Database) { export function ensureLocalSqliteSchema(client: Database) { const missingBaseSchema = [ - 'filing', - 'watchlist_item', - 'holding', - 'task_run', - 'portfolio_insight' + "filing", + "watchlist_item", + "holding", + "task_run", + "portfolio_insight", ].some((tableName) => !hasTable(client, tableName)); if (missingBaseSchema) { applyBaseSchemaCompat(client); } - if (!hasTable(client, 'user')) { + if (!hasTable(client, "user")) { client.exec(` CREATE TABLE IF NOT EXISTS \`user\` ( \`id\` text PRIMARY KEY NOT NULL, @@ -432,10 +556,12 @@ export function ensureLocalSqliteSchema(client: Database) { \`banExpires\` integer ); `); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);'); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `user_email_uidx` ON `user` (`email`);", + ); } - if (!hasTable(client, 'organization')) { + if (!hasTable(client, "organization")) { client.exec(` CREATE TABLE IF NOT EXISTS \`organization\` ( \`id\` text PRIMARY KEY NOT NULL, @@ -446,46 +572,86 @@ export function ensureLocalSqliteSchema(client: Database) { \`metadata\` text ); `); - client.exec('CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);'); + client.exec( + "CREATE UNIQUE INDEX IF NOT EXISTS `organization_slug_uidx` ON `organization` (`slug`);", + ); } - if (!hasTable(client, 'filing_statement_snapshot')) { - applySqlFile(client, '0001_glossy_statement_snapshots.sql'); + if (!hasTable(client, "filing_statement_snapshot")) { + applySqlFile(client, "0001_glossy_statement_snapshots.sql"); } - ensureColumns(client, 'task_run', [ - { name: 'stage', sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';" }, - { name: 'stage_detail', sql: 'ALTER TABLE `task_run` ADD `stage_detail` text;' }, - { name: 'stage_context', sql: 'ALTER TABLE `task_run` ADD `stage_context` text;' }, - { name: 'resource_key', sql: 'ALTER TABLE `task_run` ADD `resource_key` text;' }, - { name: 'notification_read_at', sql: 'ALTER TABLE `task_run` ADD `notification_read_at` text;' }, - { name: 'notification_silenced_at', sql: 'ALTER TABLE `task_run` ADD `notification_silenced_at` text;' } + ensureColumns(client, "task_run", [ + { + name: "stage", + sql: "ALTER TABLE `task_run` ADD `stage` text NOT NULL DEFAULT 'queued';", + }, + { + name: "stage_detail", + sql: "ALTER TABLE `task_run` ADD `stage_detail` text;", + }, + { + name: "stage_context", + sql: "ALTER TABLE `task_run` ADD `stage_context` text;", + }, + { + name: "resource_key", + sql: "ALTER TABLE `task_run` ADD `resource_key` text;", + }, + { + name: "notification_read_at", + sql: "ALTER TABLE `task_run` ADD `notification_read_at` text;", + }, + { + name: "notification_silenced_at", + sql: "ALTER TABLE `task_run` ADD `notification_silenced_at` text;", + }, ]); - if (!hasTable(client, 'task_stage_event')) { - applySqlFile(client, '0003_task_stage_event_timeline.sql'); + if (!hasTable(client, "task_stage_event")) { + applySqlFile(client, "0003_task_stage_event_timeline.sql"); } - if (hasTable(client, 'task_stage_event') && !hasColumn(client, 'task_stage_event', 'stage_context')) { - client.exec('ALTER TABLE `task_stage_event` ADD `stage_context` text;'); + if ( + hasTable(client, "task_stage_event") && + !hasColumn(client, "task_stage_event", "stage_context") + ) { + client.exec("ALTER TABLE `task_stage_event` ADD `stage_context` text;"); } - client.exec('CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);'); + client.exec( + "CREATE INDEX IF NOT EXISTS `task_user_updated_idx` ON `task_run` (`user_id`, `updated_at`);", + ); client.exec(`CREATE UNIQUE INDEX IF NOT EXISTS task_active_resource_uidx ON task_run (user_id, task_type, resource_key) WHERE resource_key IS NOT NULL AND status IN ('queued', 'running');`); - ensureColumns(client, 'watchlist_item', [ - { name: 'category', sql: 'ALTER TABLE `watchlist_item` ADD `category` text;' }, - { name: 'tags', sql: 'ALTER TABLE `watchlist_item` ADD `tags` text;' }, - { name: 'status', sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';" }, - { name: 'priority', sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';" }, - { name: 'updated_at', sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';" }, - { name: 'last_reviewed_at', sql: 'ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;' } + ensureColumns(client, "watchlist_item", [ + { + name: "category", + sql: "ALTER TABLE `watchlist_item` ADD `category` text;", + }, + { name: "tags", sql: "ALTER TABLE `watchlist_item` ADD `tags` text;" }, + { + name: "status", + sql: "ALTER TABLE `watchlist_item` ADD `status` text NOT NULL DEFAULT 'backlog';", + }, + { + name: "priority", + sql: "ALTER TABLE `watchlist_item` ADD `priority` text NOT NULL DEFAULT 'medium';", + }, + { + name: "updated_at", + sql: "ALTER TABLE `watchlist_item` ADD `updated_at` text NOT NULL DEFAULT '';", + }, + { + name: "last_reviewed_at", + sql: "ALTER TABLE `watchlist_item` ADD `last_reviewed_at` text;", + }, ]); - if (hasTable(client, 'watchlist_item')) { + if (hasTable(client, "watchlist_item")) { client.exec(` UPDATE \`watchlist_item\` SET @@ -503,27 +669,32 @@ WHERE resource_key IS NOT NULL AND status IN ('queued', 'running');`); END; `); - client.exec('CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);'); + client.exec( + "CREATE INDEX IF NOT EXISTS `watchlist_user_updated_idx` ON `watchlist_item` (`user_id`, `updated_at`);", + ); } - if (hasTable(client, 'holding') && !hasColumn(client, 'holding', 'company_name')) { - client.exec('ALTER TABLE `holding` ADD `company_name` text;'); + if ( + hasTable(client, "holding") && + !hasColumn(client, "holding", "company_name") + ) { + client.exec("ALTER TABLE `holding` ADD `company_name` text;"); } - if (!hasTable(client, 'filing_taxonomy_snapshot')) { - applySqlFile(client, '0005_financial_taxonomy_v3.sql'); + if (!hasTable(client, "filing_taxonomy_snapshot")) { + applySqlFile(client, "0005_financial_taxonomy_v3.sql"); } ensureTaxonomyCompat(client); - if (!hasTable(client, 'company_financial_bundle')) { - applySqlFile(client, '0007_company_financial_bundles.sql'); + if (!hasTable(client, "company_financial_bundle")) { + applySqlFile(client, "0007_company_financial_bundles.sql"); } - if (!hasTable(client, 'company_overview_cache')) { - applySqlFile(client, '0012_company_overview_cache.sql'); + if (!hasTable(client, "company_overview_cache")) { + applySqlFile(client, "0012_company_overview_cache.sql"); } - if (!hasTable(client, 'research_journal_entry')) { + if (!hasTable(client, "research_journal_entry")) { client.exec(` CREATE TABLE IF NOT EXISTS \`research_journal_entry\` ( \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -539,12 +710,16 @@ WHERE resource_key IS NOT NULL AND status IN ('queued', 'running');`); FOREIGN KEY (\`user_id\`) REFERENCES \`user\`(\`id\`) ON UPDATE no action ON DELETE cascade ); `); - client.exec('CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);'); - client.exec('CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);'); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_journal_ticker_idx` ON `research_journal_entry` (`user_id`, `ticker`, `created_at`);", + ); + client.exec( + "CREATE INDEX IF NOT EXISTS `research_journal_accession_idx` ON `research_journal_entry` (`user_id`, `accession_number`);", + ); } - if (!hasTable(client, 'search_document')) { - applySqlFile(client, '0008_search_rag.sql'); + if (!hasTable(client, "search_document")) { + applySqlFile(client, "0008_search_rag.sql"); } ensureResearchWorkspaceSchema(client); @@ -555,7 +730,7 @@ export const __sqliteSchemaCompatInternals = { applySqlFile, hasColumn, hasTable, - TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS + TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS, }; export { TAXONOMY_SNAPSHOT_REQUIRED_COLUMNS }; diff --git a/lib/server/repos/filing-taxonomy.test.ts b/lib/server/repos/filing-taxonomy.test.ts index eb2e329..041cb61 100644 --- a/lib/server/repos/filing-taxonomy.test.ts +++ b/lib/server/repos/filing-taxonomy.test.ts @@ -1,120 +1,146 @@ -import { describe, expect, it } from 'bun:test'; -import { __filingTaxonomyInternals } from './filing-taxonomy'; +import { describe, expect, it } from "bun:test"; +import { __filingTaxonomyInternals } from "./filing-taxonomy"; -describe('filing taxonomy snapshot normalization', () => { - it('normalizes legacy snake_case nested snapshot payloads in toSnapshotRecord', () => { +describe("filing taxonomy snapshot normalization", () => { + it("normalizes legacy snake_case nested snapshot payloads in toSnapshotRecord", () => { const record = __filingTaxonomyInternals.toSnapshotRecord({ id: 1, filing_id: 10, - ticker: 'MSFT', - filing_date: '2026-01-28', - filing_type: '10-Q', - parse_status: 'ready', + ticker: "MSFT", + filing_date: "2026-01-28", + filing_type: "10-Q", + parse_status: "ready", parse_error: null, - source: 'xbrl_instance', - parser_engine: 'fiscal-xbrl', - parser_version: '0.1.0', - taxonomy_regime: 'us-gaap', - fiscal_pack: 'core', - periods: [{ - id: 'fy-2025', - filing_id: 10, - accession_number: '0001', - filing_date: '2026-01-28', - period_start: '2025-01-01', - period_end: '2025-12-31', - filing_type: '10-Q', - period_label: 'FY 2025' - }], + source: "xbrl_instance", + parser_engine: "fiscal-xbrl", + parser_version: "0.1.0", + taxonomy_regime: "us-gaap", + fiscal_pack: "core", + periods: [ + { + id: "fy-2025", + filing_id: 10, + accession_number: "0001", + filing_date: "2026-01-28", + period_start: "2025-01-01", + period_end: "2025-12-31", + filing_type: "10-Q", + period_label: "FY 2025", + }, + ], faithful_rows: { - income: [{ - key: 'revenue', - label: 'Revenue', - concept_key: 'us-gaap:Revenue', - qname: 'us-gaap:Revenue', - namespace_uri: 'http://fasb.org/us-gaap/2025', - local_name: 'Revenue', - is_extension: false, - statement: 'income', - role_uri: 'income', - order: 10, - depth: 0, - parent_key: null, - values: { 'fy-2025': 10 }, - units: { 'fy-2025': 'iso4217:USD' }, - has_dimensions: false, - source_fact_ids: [1] - }], + income: [ + { + key: "revenue", + label: "Revenue", + concept_key: "us-gaap:Revenue", + qname: "us-gaap:Revenue", + namespace_uri: "http://fasb.org/us-gaap/2025", + local_name: "Revenue", + is_extension: false, + statement: "income", + role_uri: "income", + order: 10, + depth: 0, + parent_key: null, + values: { "fy-2025": 10 }, + units: { "fy-2025": "iso4217:USD" }, + has_dimensions: false, + source_fact_ids: [1], + }, + ], balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }, statement_rows: null, surface_rows: { - income: [{ - key: 'revenue', - label: 'Revenue', - category: 'revenue', - template_section: 'revenue', - order: 10, - unit: 'currency', - values: { 'fy-2025': 10 }, - source_concepts: ['us-gaap:Revenue'], - source_row_keys: ['revenue'], - source_fact_ids: [1], - formula_key: null, - has_dimensions: false, - resolved_source_row_keys: { 'fy-2025': 'revenue' }, - statement: 'income', - detail_count: 1, - resolution_method: 'direct', - confidence: 'high', - warning_codes: ['legacy_surface'] - }], + income: [ + { + key: "revenue", + label: "Revenue", + category: "revenue", + template_section: "revenue", + order: 10, + unit: "currency", + values: { "fy-2025": 10 }, + source_concepts: ["us-gaap:Revenue"], + source_row_keys: ["revenue"], + source_fact_ids: [1], + formula_key: null, + has_dimensions: false, + resolved_source_row_keys: { "fy-2025": "revenue" }, + statement: "income", + detail_count: 1, + resolution_method: "direct", + confidence: "high", + warning_codes: ["legacy_surface"], + }, + ], balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }, detail_rows: { income: { - revenue: [{ - key: 'revenue_detail', - parent_surface_key: 'revenue', - label: 'Revenue Detail', - concept_key: 'us-gaap:RevenueDetail', - qname: 'us-gaap:RevenueDetail', - namespace_uri: 'http://fasb.org/us-gaap/2025', - local_name: 'RevenueDetail', - unit: 'iso4217:USD', - values: { 'fy-2025': 10 }, - source_fact_ids: [2], - is_extension: false, - dimensions_summary: ['region:americas'], - residual_flag: false - }] + revenue: [ + { + key: "revenue_detail", + parent_surface_key: "revenue", + label: "Revenue Detail", + concept_key: "us-gaap:RevenueDetail", + qname: "us-gaap:RevenueDetail", + namespace_uri: "http://fasb.org/us-gaap/2025", + local_name: "RevenueDetail", + unit: "iso4217:USD", + values: { "fy-2025": 10 }, + source_fact_ids: [2], + is_extension: false, + dimensions_summary: ["region:americas"], + residual_flag: false, + }, + ], }, balance: {}, cash_flow: {}, equity: {}, - comprehensive_income: {} + comprehensive_income: {}, }, - kpi_rows: [{ - key: 'cloud_growth', - label: 'Cloud Growth', - category: 'operating_kpi', - unit: 'percent', - order: 10, - segment: null, - axis: null, - member: null, - values: { 'fy-2025': 0.25 }, - source_concepts: ['msft:CloudGrowth'], - source_fact_ids: [3], - provenance_type: 'taxonomy', - has_dimensions: false - }], + kpi_rows: [ + { + key: "cloud_growth", + label: "Cloud Growth", + category: "operating_kpi", + unit: "percent", + order: 10, + segment: null, + axis: null, + member: null, + values: { "fy-2025": 0.25 }, + source_concepts: ["msft:CloudGrowth"], + source_fact_ids: [3], + provenance_type: "taxonomy", + has_dimensions: false, + }, + ], + computed_definitions: [ + { + key: "gross_margin", + label: "Gross Margin", + category: "margins", + order: 10, + unit: "percent", + computation: { + type: "ratio", + numerator: "gross_profit", + denominator: "revenue", + }, + supported_cadences: ["annual", "quarterly"], + requires_external_data: [], + }, + ], derived_metrics: null, validation_result: null, normalization_summary: { @@ -123,169 +149,228 @@ describe('filing taxonomy snapshot normalization', () => { kpi_row_count: 1, unmapped_row_count: 0, material_unmapped_row_count: 0, - warnings: ['legacy_warning'] + warnings: ["legacy_warning"], }, facts_count: 3, concepts_count: 3, dimensions_count: 1, - created_at: '2026-01-28T00:00:00.000Z', - updated_at: '2026-01-28T00:00:00.000Z' + created_at: "2026-01-28T00:00:00.000Z", + updated_at: "2026-01-28T00:00:00.000Z", } as never); expect(record.periods[0]).toMatchObject({ filingId: 10, - accessionNumber: '0001', - filingDate: '2026-01-28', - periodStart: '2025-01-01', - periodEnd: '2025-12-31', - periodLabel: 'FY 2025' + accessionNumber: "0001", + filingDate: "2026-01-28", + periodStart: "2025-01-01", + periodEnd: "2025-12-31", + periodLabel: "FY 2025", }); expect(record.faithful_rows.income[0]).toMatchObject({ - conceptKey: 'us-gaap:Revenue', - namespaceUri: 'http://fasb.org/us-gaap/2025', - localName: 'Revenue', - roleUri: 'income', + conceptKey: "us-gaap:Revenue", + namespaceUri: "http://fasb.org/us-gaap/2025", + localName: "Revenue", + roleUri: "income", parentKey: null, hasDimensions: false, - sourceFactIds: [1] + sourceFactIds: [1], }); expect(record.surface_rows.income[0]).toMatchObject({ - templateSection: 'revenue', - sourceConcepts: ['us-gaap:Revenue'], - sourceRowKeys: ['revenue'], + templateSection: "revenue", + sourceConcepts: ["us-gaap:Revenue"], + sourceRowKeys: ["revenue"], sourceFactIds: [1], formulaKey: null, hasDimensions: false, - resolvedSourceRowKeys: { 'fy-2025': 'revenue' }, + resolvedSourceRowKeys: { "fy-2025": "revenue" }, detailCount: 1, - resolutionMethod: 'direct', - warningCodes: ['legacy_surface'] + resolutionMethod: "direct", + warningCodes: ["legacy_surface"], }); expect(record.detail_rows.income.revenue?.[0]).toMatchObject({ - parentSurfaceKey: 'revenue', - conceptKey: 'us-gaap:RevenueDetail', - namespaceUri: 'http://fasb.org/us-gaap/2025', + parentSurfaceKey: "revenue", + conceptKey: "us-gaap:RevenueDetail", + namespaceUri: "http://fasb.org/us-gaap/2025", sourceFactIds: [2], - dimensionsSummary: ['region:americas'], - residualFlag: false + dimensionsSummary: ["region:americas"], + residualFlag: false, }); expect(record.kpi_rows[0]).toMatchObject({ - sourceConcepts: ['msft:CloudGrowth'], + sourceConcepts: ["msft:CloudGrowth"], sourceFactIds: [3], - provenanceType: 'taxonomy', - hasDimensions: false + provenanceType: "taxonomy", + hasDimensions: false, }); + expect(record.computed_definitions).toEqual([ + { + key: "gross_margin", + label: "Gross Margin", + category: "margins", + order: 10, + unit: "percent", + computation: { + type: "ratio", + numerator: "gross_profit", + denominator: "revenue", + }, + supported_cadences: ["annual", "quarterly"], + requires_external_data: [], + }, + ]); expect(record.normalization_summary).toEqual({ surfaceRowCount: 1, detailRowCount: 1, kpiRowCount: 1, unmappedRowCount: 0, materialUnmappedRowCount: 0, - warnings: ['legacy_warning'] + warnings: ["legacy_warning"], }); }); - it('keeps mixed camelCase and snake_case payloads compatible', () => { - const normalized = __filingTaxonomyInternals.normalizeFilingTaxonomySnapshotPayload({ - periods: [{ - id: 'fy-2025', - filingId: 10, - accessionNumber: '0001', - filingDate: '2026-01-28', - periodStart: '2025-01-01', - periodEnd: '2025-12-31', - filingType: '10-K', - periodLabel: 'FY 2025' - }], - faithful_rows: { - income: [{ - key: 'revenue', - label: 'Revenue', - conceptKey: 'us-gaap:Revenue', - qname: 'us-gaap:Revenue', - namespaceUri: 'http://fasb.org/us-gaap/2025', - localName: 'Revenue', - isExtension: false, - statement: 'income', - roleUri: 'income', - order: 10, - depth: 0, - parentKey: null, - values: { 'fy-2025': 10 }, - units: { 'fy-2025': 'iso4217:USD' }, - hasDimensions: false, - sourceFactIds: [1] - }], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [] - }, - statement_rows: null, - surface_rows: { - income: [{ - key: 'revenue', - label: 'Revenue', - category: 'revenue', - order: 10, - unit: 'currency', - values: { 'fy-2025': 10 }, - source_concepts: ['us-gaap:Revenue'], - source_row_keys: ['revenue'], - source_fact_ids: [1], - formula_key: null, - has_dimensions: false, - resolved_source_row_keys: { 'fy-2025': 'revenue' } - }], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [] - }, - detail_rows: { - income: { - revenue: [{ - key: 'revenue_detail', - parentSurfaceKey: 'revenue', - label: 'Revenue Detail', - conceptKey: 'us-gaap:RevenueDetail', - qname: 'us-gaap:RevenueDetail', - namespaceUri: 'http://fasb.org/us-gaap/2025', - localName: 'RevenueDetail', - unit: 'iso4217:USD', - values: { 'fy-2025': 10 }, - sourceFactIds: [2], - isExtension: false, - dimensionsSummary: [], - residualFlag: false - }] + it("keeps mixed camelCase and snake_case payloads compatible", () => { + const normalized = + __filingTaxonomyInternals.normalizeFilingTaxonomySnapshotPayload({ + periods: [ + { + id: "fy-2025", + filingId: 10, + accessionNumber: "0001", + filingDate: "2026-01-28", + periodStart: "2025-01-01", + periodEnd: "2025-12-31", + filingType: "10-K", + periodLabel: "FY 2025", + }, + ], + faithful_rows: { + income: [ + { + key: "revenue", + label: "Revenue", + conceptKey: "us-gaap:Revenue", + qname: "us-gaap:Revenue", + namespaceUri: "http://fasb.org/us-gaap/2025", + localName: "Revenue", + isExtension: false, + statement: "income", + roleUri: "income", + order: 10, + depth: 0, + parentKey: null, + values: { "fy-2025": 10 }, + units: { "fy-2025": "iso4217:USD" }, + hasDimensions: false, + sourceFactIds: [1], + }, + ], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], }, - balance: {}, - cash_flow: {}, - equity: {}, - comprehensive_income: {} - }, - kpi_rows: [], - normalization_summary: { - surfaceRowCount: 1, - detail_row_count: 1, - kpiRowCount: 0, - unmapped_row_count: 0, - materialUnmappedRowCount: 0, - warnings: [] - } - }); + statement_rows: null, + surface_rows: { + income: [ + { + key: "revenue", + label: "Revenue", + category: "revenue", + order: 10, + unit: "currency", + values: { "fy-2025": 10 }, + source_concepts: ["us-gaap:Revenue"], + source_row_keys: ["revenue"], + source_fact_ids: [1], + formula_key: null, + has_dimensions: false, + resolved_source_row_keys: { "fy-2025": "revenue" }, + }, + ], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + detail_rows: { + income: { + revenue: [ + { + key: "revenue_detail", + parentSurfaceKey: "revenue", + label: "Revenue Detail", + conceptKey: "us-gaap:RevenueDetail", + qname: "us-gaap:RevenueDetail", + namespaceUri: "http://fasb.org/us-gaap/2025", + localName: "RevenueDetail", + unit: "iso4217:USD", + values: { "fy-2025": 10 }, + sourceFactIds: [2], + isExtension: false, + dimensionsSummary: [], + residualFlag: false, + }, + ], + }, + balance: {}, + cash_flow: {}, + equity: {}, + comprehensive_income: {}, + }, + kpi_rows: [], + computed_definitions: [ + { + key: "revenue_yoy", + label: "Revenue YoY", + category: "growth", + order: 20, + unit: "percent", + computation: { + type: "yoy_growth", + source: "revenue", + }, + supportedCadences: ["annual"], + requiresExternalData: [], + }, + ], + normalization_summary: { + surfaceRowCount: 1, + detail_row_count: 1, + kpiRowCount: 0, + unmapped_row_count: 0, + materialUnmappedRowCount: 0, + warnings: [], + }, + }); expect(normalized.periods[0]?.filingId).toBe(10); - expect(normalized.surface_rows.income[0]?.sourceConcepts).toEqual(['us-gaap:Revenue']); - expect(normalized.detail_rows.income.revenue?.[0]?.parentSurfaceKey).toBe('revenue'); + expect(normalized.surface_rows.income[0]?.sourceConcepts).toEqual([ + "us-gaap:Revenue", + ]); + expect(normalized.detail_rows.income.revenue?.[0]?.parentSurfaceKey).toBe( + "revenue", + ); expect(normalized.normalization_summary).toEqual({ surfaceRowCount: 1, detailRowCount: 1, kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, - warnings: [] + warnings: [], }); + expect(normalized.computed_definitions).toEqual([ + { + key: "revenue_yoy", + label: "Revenue YoY", + category: "growth", + order: 20, + unit: "percent", + computation: { + type: "yoy_growth", + source: "revenue", + }, + supported_cadences: ["annual"], + requires_external_data: [], + }, + ]); }); }); diff --git a/lib/server/repos/filing-taxonomy.ts b/lib/server/repos/filing-taxonomy.ts index 667fa7c..9eb7456 100644 --- a/lib/server/repos/filing-taxonomy.ts +++ b/lib/server/repos/filing-taxonomy.ts @@ -1,4 +1,5 @@ -import { and, desc, eq, gte, inArray, lt, sql } from 'drizzle-orm'; +import { and, desc, eq, gte, inArray, lt, sql } from "drizzle-orm"; +import type { ComputedDefinition } from "@/lib/generated"; import type { DetailFinancialRow, Filing, @@ -10,30 +11,33 @@ import type { SurfaceFinancialRow, TaxonomyDimensionMember, TaxonomyFactRow, - TaxonomyStatementRow -} from '@/lib/types'; -import { db, getSqliteClient } from '@/lib/server/db'; -import { withFinancialIngestionSchemaRetry } from '@/lib/server/db/financial-ingestion-schema'; + TaxonomyStatementRow, +} from "@/lib/types"; +import { db, getSqliteClient } from "@/lib/server/db"; +import { withFinancialIngestionSchemaRetry } from "@/lib/server/db/financial-ingestion-schema"; import { filingTaxonomyAsset, filingTaxonomyConcept, filingTaxonomyContext, filingTaxonomyFact, filingTaxonomyMetricValidation, - filingTaxonomySnapshot -} from '@/lib/server/db/schema'; + filingTaxonomySnapshot, +} from "@/lib/server/db/schema"; -export type FilingTaxonomyParseStatus = 'ready' | 'partial' | 'failed'; -export type FilingTaxonomySource = 'xbrl_instance' | 'xbrl_instance_with_linkbase' | 'legacy_html_fallback'; +export type FilingTaxonomyParseStatus = "ready" | "partial" | "failed"; +export type FilingTaxonomySource = + | "xbrl_instance" + | "xbrl_instance_with_linkbase" + | "legacy_html_fallback"; export type FilingTaxonomyAssetType = - | 'instance' - | 'schema' - | 'presentation' - | 'label' - | 'calculation' - | 'definition' - | 'pdf' - | 'other'; + | "instance" + | "schema" + | "presentation" + | "label" + | "calculation" + | "definition" + | "pdf" + | "other"; export type FilingTaxonomyPeriod = { id: string; @@ -42,7 +46,7 @@ export type FilingTaxonomyPeriod = { filingDate: string; periodStart: string | null; periodEnd: string | null; - filingType: '10-K' | '10-Q'; + filingType: "10-K" | "10-Q"; periodLabel: string; }; @@ -51,13 +55,13 @@ export type FilingTaxonomySnapshotRecord = { filing_id: number; ticker: string; filing_date: string; - filing_type: '10-K' | '10-Q'; + filing_type: "10-K" | "10-Q"; parse_status: FilingTaxonomyParseStatus; parse_error: string | null; source: FilingTaxonomySource; parser_engine: string; parser_version: string; - taxonomy_regime: 'us-gaap' | 'ifrs-full' | 'unknown'; + taxonomy_regime: "us-gaap" | "ifrs-full" | "unknown"; fiscal_pack: string | null; periods: FilingTaxonomyPeriod[]; faithful_rows: Record; @@ -65,7 +69,8 @@ export type FilingTaxonomySnapshotRecord = { surface_rows: Record; detail_rows: Record; kpi_rows: StructuredKpiRow[]; - derived_metrics: Filing['metrics']; + computed_definitions: ComputedDefinition[]; + derived_metrics: Filing["metrics"]; validation_result: MetricValidationResult | null; normalization_summary: NormalizationSummary | null; facts_count: number; @@ -162,12 +167,12 @@ export type FilingTaxonomyFactRecord = { export type FilingTaxonomyMetricValidationRecord = { id: number; snapshot_id: number; - metric_key: keyof NonNullable; + metric_key: keyof NonNullable; taxonomy_value: number | null; llm_value: number | null; absolute_diff: number | null; relative_diff: number | null; - status: 'not_run' | 'matched' | 'mismatch' | 'error'; + status: "not_run" | "matched" | "mismatch" | "error"; evidence_pages: number[]; pdf_url: string | null; provider: string | null; @@ -181,13 +186,13 @@ export type UpsertFilingTaxonomySnapshotInput = { filing_id: number; ticker: string; filing_date: string; - filing_type: '10-K' | '10-Q'; + filing_type: "10-K" | "10-Q"; parse_status: FilingTaxonomyParseStatus; parse_error: string | null; source: FilingTaxonomySource; parser_engine: string; parser_version: string; - taxonomy_regime: 'us-gaap' | 'ifrs-full' | 'unknown'; + taxonomy_regime: "us-gaap" | "ifrs-full" | "unknown"; fiscal_pack: string | null; periods: FilingTaxonomyPeriod[]; faithful_rows: Record; @@ -195,7 +200,8 @@ export type UpsertFilingTaxonomySnapshotInput = { surface_rows: Record; detail_rows: Record; kpi_rows: StructuredKpiRow[]; - derived_metrics: Filing['metrics']; + computed_definitions: ComputedDefinition[]; + derived_metrics: Filing["metrics"]; validation_result: MetricValidationResult | null; normalization_summary: NormalizationSummary | null; facts_count: number; @@ -270,12 +276,12 @@ export type UpsertFilingTaxonomySnapshotInput = { source_file: string | null; }>; metric_validations: Array<{ - metric_key: keyof NonNullable; + metric_key: keyof NonNullable; taxonomy_value: number | null; llm_value: number | null; absolute_diff: number | null; relative_diff: number | null; - status: 'not_run' | 'matched' | 'mismatch' | 'error'; + status: "not_run" | "matched" | "mismatch" | "error"; evidence_pages: number[]; pdf_url: string | null; provider: string | null; @@ -285,11 +291,11 @@ export type UpsertFilingTaxonomySnapshotInput = { }; const FINANCIAL_STATEMENT_KINDS = [ - 'income', - 'balance', - 'cash_flow', - 'equity', - 'comprehensive_income' + "income", + "balance", + "cash_flow", + "equity", + "comprehensive_income", ] as const satisfies FinancialStatementKind[]; type StatementRowMap = Record; @@ -303,11 +309,11 @@ function tenYearsAgoIso() { } function asNumber(value: unknown) { - if (typeof value === 'number') { + if (typeof value === "number") { return Number.isFinite(value) ? value : null; } - if (typeof value === 'string') { + if (typeof value === "string") { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } @@ -324,33 +330,29 @@ function asNumericText(value: number | null) { } function asObject(value: unknown) { - return value !== null && typeof value === 'object' && !Array.isArray(value) - ? value as Record + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) : null; } function asString(value: unknown) { - return typeof value === 'string' ? value : null; + return typeof value === "string" ? value : null; } function asNullableString(value: unknown) { - return typeof value === 'string' - ? value - : value === null - ? null - : null; + return typeof value === "string" ? value : value === null ? null : null; } function asBoolean(value: unknown) { - return typeof value === 'boolean' ? value : Boolean(value); + return typeof value === "boolean" ? value : Boolean(value); } function asStatementKind(value: unknown): FinancialStatementKind | null { - return value === 'income' - || value === 'balance' - || value === 'cash_flow' - || value === 'equity' - || value === 'comprehensive_income' + return value === "income" || + value === "balance" || + value === "cash_flow" || + value === "equity" || + value === "comprehensive_income" ? value : null; } @@ -362,7 +364,7 @@ function normalizeNumberMap(value: unknown) { } return Object.fromEntries( - Object.entries(object).map(([key, entry]) => [key, asNumber(entry)]) + Object.entries(object).map(([key, entry]) => [key, asNumber(entry)]), ); } @@ -373,13 +375,16 @@ function normalizeNullableStringMap(value: unknown) { } return Object.fromEntries( - Object.entries(object).map(([key, entry]) => [key, asNullableString(entry)]) + Object.entries(object).map(([key, entry]) => [ + key, + asNullableString(entry), + ]), ); } function normalizeStringArray(value: unknown) { return Array.isArray(value) - ? value.filter((entry): entry is string => typeof entry === 'string') + ? value.filter((entry): entry is string => typeof entry === "string") : []; } @@ -407,16 +412,26 @@ function normalizePeriods(value: unknown): FilingTaxonomyPeriod[] { const id = asString(row.id); const filingId = asNumber(row.filingId ?? row.filing_id); - const accessionNumber = asString(row.accessionNumber ?? row.accession_number); + const accessionNumber = asString( + row.accessionNumber ?? row.accession_number, + ); const filingDate = asString(row.filingDate ?? row.filing_date); - const filingType = row.filingType === '10-K' || row.filing_type === '10-K' - ? '10-K' - : row.filingType === '10-Q' || row.filing_type === '10-Q' - ? '10-Q' - : null; + const filingType = + row.filingType === "10-K" || row.filing_type === "10-K" + ? "10-K" + : row.filingType === "10-Q" || row.filing_type === "10-Q" + ? "10-Q" + : null; const periodLabel = asString(row.periodLabel ?? row.period_label); - if (!id || filingId === null || !accessionNumber || !filingDate || !filingType || !periodLabel) { + if ( + !id || + filingId === null || + !accessionNumber || + !filingDate || + !filingType || + !periodLabel + ) { return null; } @@ -428,7 +443,7 @@ function normalizePeriods(value: unknown): FilingTaxonomyPeriod[] { periodStart: asNullableString(row.periodStart ?? row.period_start), periodEnd: asNullableString(row.periodEnd ?? row.period_end), filingType, - periodLabel + periodLabel, } satisfies FilingTaxonomyPeriod; }) .filter((entry): entry is FilingTaxonomyPeriod => entry !== null); @@ -436,7 +451,7 @@ function normalizePeriods(value: unknown): FilingTaxonomyPeriod[] { function normalizeStatementRows( value: unknown, - fallbackRows: StatementRowMap = emptyStatementRows() + fallbackRows: StatementRowMap = emptyStatementRows(), ): StatementRowMap { const object = asObject(value); if (!object) { @@ -453,13 +468,21 @@ function normalizeStatementRows( return null; } - const key = asString(row.key) ?? asString(row.conceptKey ?? row.concept_key); + const key = + asString(row.key) ?? asString(row.conceptKey ?? row.concept_key); const label = asString(row.label); const conceptKey = asString(row.conceptKey ?? row.concept_key); const qname = asString(row.qname); const namespaceUri = asString(row.namespaceUri ?? row.namespace_uri); const localName = asString(row.localName ?? row.local_name); - if (!key || !label || !conceptKey || !qname || !namespaceUri || !localName) { + if ( + !key || + !label || + !conceptKey || + !qname || + !namespaceUri || + !localName + ) { return null; } @@ -479,7 +502,9 @@ function normalizeStatementRows( values: normalizeNumberMap(row.values), units: normalizeNullableStringMap(row.units), hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions), - sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids) + sourceFactIds: normalizeNumberArray( + row.sourceFactIds ?? row.source_fact_ids, + ), }; }) .filter((entry): entry is TaxonomyStatementRow => entry !== null); @@ -490,7 +515,7 @@ function normalizeStatementRows( function normalizeSurfaceRows( value: unknown, - fallbackRows: SurfaceRowMap = emptySurfaceRows() + fallbackRows: SurfaceRowMap = emptySurfaceRows(), ): SurfaceRowMap { const object = asObject(value); if (!object) { @@ -521,23 +546,38 @@ function normalizeSurfaceRows( const normalizedRow: SurfaceFinancialRow = { key, label, - category: category as SurfaceFinancialRow['category'], + category: category as SurfaceFinancialRow["category"], order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER, - unit: unit as SurfaceFinancialRow['unit'], + unit: unit as SurfaceFinancialRow["unit"], values: normalizeNumberMap(row.values), - sourceConcepts: normalizeStringArray(row.sourceConcepts ?? row.source_concepts), - sourceRowKeys: normalizeStringArray(row.sourceRowKeys ?? row.source_row_keys), - sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids), + sourceConcepts: normalizeStringArray( + row.sourceConcepts ?? row.source_concepts, + ), + sourceRowKeys: normalizeStringArray( + row.sourceRowKeys ?? row.source_row_keys, + ), + sourceFactIds: normalizeNumberArray( + row.sourceFactIds ?? row.source_fact_ids, + ), formulaKey: asNullableString(row.formulaKey ?? row.formula_key), hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions), - resolvedSourceRowKeys: normalizeNullableStringMap(row.resolvedSourceRowKeys ?? row.resolved_source_row_keys) + resolvedSourceRowKeys: normalizeNullableStringMap( + row.resolvedSourceRowKeys ?? row.resolved_source_row_keys, + ), }; - const templateSection = asString(row.templateSection ?? row.template_section); + const templateSection = asString( + row.templateSection ?? row.template_section, + ); if (templateSection) { - normalizedRow.templateSection = templateSection as SurfaceFinancialRow['templateSection']; + normalizedRow.templateSection = + templateSection as SurfaceFinancialRow["templateSection"]; } - if (normalizedStatement === 'income' || normalizedStatement === 'balance' || normalizedStatement === 'cash_flow') { + if ( + normalizedStatement === "income" || + normalizedStatement === "balance" || + normalizedStatement === "cash_flow" + ) { normalizedRow.statement = normalizedStatement; } @@ -547,19 +587,25 @@ function normalizeSurfaceRows( } if ( - resolutionMethod === 'direct' - || resolutionMethod === 'surface_bridge' - || resolutionMethod === 'formula_derived' - || resolutionMethod === 'not_meaningful' + resolutionMethod === "direct" || + resolutionMethod === "surface_bridge" || + resolutionMethod === "formula_derived" || + resolutionMethod === "not_meaningful" ) { normalizedRow.resolutionMethod = resolutionMethod; } - if (confidence === 'high' || confidence === 'medium' || confidence === 'low') { + if ( + confidence === "high" || + confidence === "medium" || + confidence === "low" + ) { normalizedRow.confidence = confidence; } - const warningCodes = normalizeStringArray(row.warningCodes ?? row.warning_codes); + const warningCodes = normalizeStringArray( + row.warningCodes ?? row.warning_codes, + ); if (warningCodes.length > 0) { normalizedRow.warningCodes = warningCodes; } @@ -574,7 +620,7 @@ function normalizeSurfaceRows( function normalizeDetailRows( value: unknown, - fallbackRows: DetailRowMap = emptyDetailRows() + fallbackRows: DetailRowMap = emptyDetailRows(), ): DetailRowMap { const object = asObject(value); if (!object) { @@ -588,43 +634,62 @@ function normalizeDetailRows( Object.entries(groups).map(([surfaceKey, rows]) => { const normalizedRows = Array.isArray(rows) ? rows - .map((entry) => { - const row = asObject(entry); - if (!row) { - return null; - } + .map((entry) => { + const row = asObject(entry); + if (!row) { + return null; + } - const key = asString(row.key) ?? asString(row.conceptKey ?? row.concept_key); - const label = asString(row.label); - const conceptKey = asString(row.conceptKey ?? row.concept_key); - const qname = asString(row.qname); - const namespaceUri = asString(row.namespaceUri ?? row.namespace_uri); - const localName = asString(row.localName ?? row.local_name); - if (!key || !label || !conceptKey || !qname || !namespaceUri || !localName) { - return null; - } + const key = + asString(row.key) ?? + asString(row.conceptKey ?? row.concept_key); + const label = asString(row.label); + const conceptKey = asString(row.conceptKey ?? row.concept_key); + const qname = asString(row.qname); + const namespaceUri = asString( + row.namespaceUri ?? row.namespace_uri, + ); + const localName = asString(row.localName ?? row.local_name); + if ( + !key || + !label || + !conceptKey || + !qname || + !namespaceUri || + !localName + ) { + return null; + } - return { - key, - parentSurfaceKey: asString(row.parentSurfaceKey ?? row.parent_surface_key) ?? surfaceKey, - label, - conceptKey, - qname, - namespaceUri, - localName, - unit: asNullableString(row.unit), - values: normalizeNumberMap(row.values), - sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids), - isExtension: asBoolean(row.isExtension ?? row.is_extension), - dimensionsSummary: normalizeStringArray(row.dimensionsSummary ?? row.dimensions_summary), - residualFlag: asBoolean(row.residualFlag ?? row.residual_flag) - }; - }) - .filter((entry): entry is DetailFinancialRow => entry !== null) + return { + key, + parentSurfaceKey: + asString(row.parentSurfaceKey ?? row.parent_surface_key) ?? + surfaceKey, + label, + conceptKey, + qname, + namespaceUri, + localName, + unit: asNullableString(row.unit), + values: normalizeNumberMap(row.values), + sourceFactIds: normalizeNumberArray( + row.sourceFactIds ?? row.source_fact_ids, + ), + isExtension: asBoolean(row.isExtension ?? row.is_extension), + dimensionsSummary: normalizeStringArray( + row.dimensionsSummary ?? row.dimensions_summary, + ), + residualFlag: asBoolean( + row.residualFlag ?? row.residual_flag, + ), + }; + }) + .filter((entry): entry is DetailFinancialRow => entry !== null) : []; return [surfaceKey, normalizedRows]; - }) + }), ); } @@ -648,29 +713,132 @@ function normalizeKpiRows(value: unknown) { const category = asString(row.category); const unit = asString(row.unit); const provenanceType = row.provenanceType ?? row.provenance_type; - if (!key || !label || !category || !unit || (provenanceType !== 'taxonomy' && provenanceType !== 'structured_note')) { + if ( + !key || + !label || + !category || + !unit || + (provenanceType !== "taxonomy" && provenanceType !== "structured_note") + ) { return null; } return { key, label, - category: category as StructuredKpiRow['category'], - unit: unit as StructuredKpiRow['unit'], + category: category as StructuredKpiRow["category"], + unit: unit as StructuredKpiRow["unit"], order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER, segment: asNullableString(row.segment), axis: asNullableString(row.axis), member: asNullableString(row.member), values: normalizeNumberMap(row.values), - sourceConcepts: normalizeStringArray(row.sourceConcepts ?? row.source_concepts), - sourceFactIds: normalizeNumberArray(row.sourceFactIds ?? row.source_fact_ids), + sourceConcepts: normalizeStringArray( + row.sourceConcepts ?? row.source_concepts, + ), + sourceFactIds: normalizeNumberArray( + row.sourceFactIds ?? row.source_fact_ids, + ), provenanceType, - hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions) + hasDimensions: asBoolean(row.hasDimensions ?? row.has_dimensions), } satisfies StructuredKpiRow; }) .filter((entry): entry is StructuredKpiRow => entry !== null); } +function normalizeComputedDefinitions(value: unknown): ComputedDefinition[] { + if (!Array.isArray(value)) { + return []; + } + + return value + .map((entry) => { + const row = asObject(entry); + if (!row) { + return null; + } + + const key = asString(row.key); + const label = asString(row.label); + const category = asString(row.category); + const unit = asString(row.unit); + const computation = asObject(row.computation); + const computationType = asString(computation?.type); + if ( + !key || + !label || + !category || + !unit || + !computation || + !computationType + ) { + return null; + } + + const normalizedComputation = (() => { + if (computationType === "ratio") { + const numerator = asString(computation.numerator); + const denominator = asString(computation.denominator); + return numerator && denominator + ? ({ type: "ratio", numerator, denominator } as const) + : null; + } + + if (computationType === "yoy_growth") { + const source = asString(computation.source); + return source ? ({ type: "yoy_growth", source } as const) : null; + } + + if (computationType === "cagr") { + const source = asString(computation.source); + const years = asNumber(computation.years); + return source && years !== null + ? ({ type: "cagr", source, years } as const) + : null; + } + + if (computationType === "per_share") { + const source = asString(computation.source); + const shares_key = asString( + computation.shares_key ?? computation.sharesKey, + ); + return source && shares_key + ? ({ type: "per_share", source, shares_key } as const) + : null; + } + + if (computationType === "simple") { + const formula = asString(computation.formula); + return formula ? ({ type: "simple", formula } as const) : null; + } + + return null; + })(); + + if (!normalizedComputation) { + return null; + } + + const normalizedDefinition: ComputedDefinition = { + key, + label, + category, + order: asNumber(row.order) ?? Number.MAX_SAFE_INTEGER, + unit: unit as ComputedDefinition["unit"], + computation: normalizedComputation, + supported_cadences: normalizeStringArray( + row.supported_cadences ?? row.supportedCadences, + ) as ComputedDefinition["supported_cadences"], + requires_external_data: normalizeStringArray( + row.requires_external_data ?? row.requiresExternalData, + ), + }; + + return normalizedDefinition; + }) + .filter((entry): entry is ComputedDefinition => entry !== null); +} + function normalizeNormalizationSummary(value: unknown) { const row = asObject(value); if (!row) { @@ -678,12 +846,17 @@ function normalizeNormalizationSummary(value: unknown) { } return { - surfaceRowCount: asNumber(row.surfaceRowCount ?? row.surface_row_count) ?? 0, + surfaceRowCount: + asNumber(row.surfaceRowCount ?? row.surface_row_count) ?? 0, detailRowCount: asNumber(row.detailRowCount ?? row.detail_row_count) ?? 0, kpiRowCount: asNumber(row.kpiRowCount ?? row.kpi_row_count) ?? 0, - unmappedRowCount: asNumber(row.unmappedRowCount ?? row.unmapped_row_count) ?? 0, - materialUnmappedRowCount: asNumber(row.materialUnmappedRowCount ?? row.material_unmapped_row_count) ?? 0, - warnings: normalizeStringArray(row.warnings) + unmappedRowCount: + asNumber(row.unmappedRowCount ?? row.unmapped_row_count) ?? 0, + materialUnmappedRowCount: + asNumber( + row.materialUnmappedRowCount ?? row.material_unmapped_row_count, + ) ?? 0, + warnings: normalizeStringArray(row.warnings), } satisfies NormalizationSummary; } @@ -694,10 +867,14 @@ export function normalizeFilingTaxonomySnapshotPayload(input: { surface_rows: unknown; detail_rows: unknown; kpi_rows: unknown; + computed_definitions: unknown; normalization_summary: unknown; }) { const faithfulRows = normalizeStatementRows(input.faithful_rows); - const statementRows = normalizeStatementRows(input.statement_rows, faithfulRows); + const statementRows = normalizeStatementRows( + input.statement_rows, + faithfulRows, + ); return { periods: normalizePeriods(input.periods), @@ -706,7 +883,12 @@ export function normalizeFilingTaxonomySnapshotPayload(input: { surface_rows: normalizeSurfaceRows(input.surface_rows), detail_rows: normalizeDetailRows(input.detail_rows), kpi_rows: normalizeKpiRows(input.kpi_rows), - normalization_summary: normalizeNormalizationSummary(input.normalization_summary) + computed_definitions: normalizeComputedDefinitions( + input.computed_definitions, + ), + normalization_summary: normalizeNormalizationSummary( + input.normalization_summary, + ), }; } @@ -716,7 +898,7 @@ function emptyStatementRows(): StatementRowMap { balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }; } @@ -726,7 +908,7 @@ function emptySurfaceRows(): SurfaceRowMap { balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }; } @@ -736,11 +918,13 @@ function emptyDetailRows(): DetailRowMap { balance: {}, cash_flow: {}, equity: {}, - comprehensive_income: {} + comprehensive_income: {}, }; } -function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): FilingTaxonomySnapshotRecord { +function toSnapshotRecord( + row: typeof filingTaxonomySnapshot.$inferSelect, +): FilingTaxonomySnapshotRecord { const normalized = normalizeFilingTaxonomySnapshotPayload({ periods: row.periods, faithful_rows: row.faithful_rows, @@ -748,7 +932,8 @@ function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): Fili surface_rows: row.surface_rows, detail_rows: row.detail_rows, kpi_rows: row.kpi_rows, - normalization_summary: row.normalization_summary + computed_definitions: row.computed_definitions, + normalization_summary: row.normalization_summary, }); return { @@ -770,6 +955,7 @@ function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): Fili surface_rows: normalized.surface_rows, detail_rows: normalized.detail_rows, kpi_rows: normalized.kpi_rows, + computed_definitions: normalized.computed_definitions, derived_metrics: row.derived_metrics ?? null, validation_result: row.validation_result ?? null, normalization_summary: normalized.normalization_summary, @@ -777,11 +963,13 @@ function toSnapshotRecord(row: typeof filingTaxonomySnapshot.$inferSelect): Fili concepts_count: row.concepts_count, dimensions_count: row.dimensions_count, created_at: row.created_at, - updated_at: row.updated_at + updated_at: row.updated_at, }; } -function toContextRecord(row: typeof filingTaxonomyContext.$inferSelect): FilingTaxonomyContextRecord { +function toContextRecord( + row: typeof filingTaxonomyContext.$inferSelect, +): FilingTaxonomyContextRecord { return { id: row.id, snapshot_id: row.snapshot_id, @@ -793,11 +981,13 @@ function toContextRecord(row: typeof filingTaxonomyContext.$inferSelect): Filing period_instant: row.period_instant, segment_json: row.segment_json ?? null, scenario_json: row.scenario_json ?? null, - created_at: row.created_at + created_at: row.created_at, }; } -function toAssetRecord(row: typeof filingTaxonomyAsset.$inferSelect): FilingTaxonomyAssetRecord { +function toAssetRecord( + row: typeof filingTaxonomyAsset.$inferSelect, +): FilingTaxonomyAssetRecord { return { id: row.id, snapshot_id: row.snapshot_id, @@ -807,11 +997,13 @@ function toAssetRecord(row: typeof filingTaxonomyAsset.$inferSelect): FilingTaxo size_bytes: row.size_bytes, score: asNumber(row.score), is_selected: row.is_selected, - created_at: row.created_at + created_at: row.created_at, }; } -function toConceptRecord(row: typeof filingTaxonomyConcept.$inferSelect): FilingTaxonomyConceptRecord { +function toConceptRecord( + row: typeof filingTaxonomyConcept.$inferSelect, +): FilingTaxonomyConceptRecord { return { id: row.id, snapshot_id: row.snapshot_id, @@ -836,11 +1028,13 @@ function toConceptRecord(row: typeof filingTaxonomyConcept.$inferSelect): Filing presentation_depth: row.presentation_depth, parent_concept_key: row.parent_concept_key, is_abstract: row.is_abstract, - created_at: row.created_at + created_at: row.created_at, }; } -function toFactRecord(row: typeof filingTaxonomyFact.$inferSelect): FilingTaxonomyFactRecord { +function toFactRecord( + row: typeof filingTaxonomyFact.$inferSelect, +): FilingTaxonomyFactRecord { const value = asNumber(row.value_num); if (value === null) { throw new Error(`Invalid value_num for taxonomy fact row ${row.id}`); @@ -874,11 +1068,13 @@ function toFactRecord(row: typeof filingTaxonomyFact.$inferSelect): FilingTaxono dimensions: row.dimensions, is_dimensionless: row.is_dimensionless, source_file: row.source_file, - created_at: row.created_at + created_at: row.created_at, }; } -function toMetricValidationRecord(row: typeof filingTaxonomyMetricValidation.$inferSelect): FilingTaxonomyMetricValidationRecord { +function toMetricValidationRecord( + row: typeof filingTaxonomyMetricValidation.$inferSelect, +): FilingTaxonomyMetricValidationRecord { return { id: row.id, snapshot_id: row.snapshot_id, @@ -894,7 +1090,7 @@ function toMetricValidationRecord(row: typeof filingTaxonomyMetricValidation.$in model: row.model, error: row.error, created_at: row.created_at, - updated_at: row.updated_at + updated_at: row.updated_at, }; } @@ -958,7 +1154,9 @@ export async function listFilingTaxonomyMetricValidations(snapshotId: number) { return rows.map(toMetricValidationRecord); } -export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySnapshotInput) { +export async function upsertFilingTaxonomySnapshot( + input: UpsertFilingTaxonomySnapshotInput, +) { const now = new Date().toISOString(); const normalized = normalizeFilingTaxonomySnapshotPayload(input); @@ -983,6 +1181,7 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn surface_rows: normalized.surface_rows, detail_rows: normalized.detail_rows, kpi_rows: normalized.kpi_rows, + computed_definitions: normalized.computed_definitions, derived_metrics: input.derived_metrics, validation_result: input.validation_result, normalization_summary: normalized.normalization_summary, @@ -990,7 +1189,7 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn concepts_count: input.concepts_count, dimensions_count: input.dimensions_count, created_at: now, - updated_at: now + updated_at: now, }) .onConflictDoUpdate({ target: filingTaxonomySnapshot.filing_id, @@ -1011,153 +1210,186 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn surface_rows: normalized.surface_rows, detail_rows: normalized.detail_rows, kpi_rows: normalized.kpi_rows, + computed_definitions: normalized.computed_definitions, derived_metrics: input.derived_metrics, validation_result: input.validation_result, normalization_summary: normalized.normalization_summary, facts_count: input.facts_count, concepts_count: input.concepts_count, dimensions_count: input.dimensions_count, - updated_at: now - } + updated_at: now, + }, }) .returning(); const snapshotId = saved.id; try { - await tx.delete(filingTaxonomyAsset).where(eq(filingTaxonomyAsset.snapshot_id, snapshotId)); - await tx.delete(filingTaxonomyContext).where(eq(filingTaxonomyContext.snapshot_id, snapshotId)); - await tx.delete(filingTaxonomyConcept).where(eq(filingTaxonomyConcept.snapshot_id, snapshotId)); - await tx.delete(filingTaxonomyFact).where(eq(filingTaxonomyFact.snapshot_id, snapshotId)); - await tx.delete(filingTaxonomyMetricValidation).where(eq(filingTaxonomyMetricValidation.snapshot_id, snapshotId)); + await tx + .delete(filingTaxonomyAsset) + .where(eq(filingTaxonomyAsset.snapshot_id, snapshotId)); + await tx + .delete(filingTaxonomyContext) + .where(eq(filingTaxonomyContext.snapshot_id, snapshotId)); + await tx + .delete(filingTaxonomyConcept) + .where(eq(filingTaxonomyConcept.snapshot_id, snapshotId)); + await tx + .delete(filingTaxonomyFact) + .where(eq(filingTaxonomyFact.snapshot_id, snapshotId)); + await tx + .delete(filingTaxonomyMetricValidation) + .where(eq(filingTaxonomyMetricValidation.snapshot_id, snapshotId)); } catch (error) { - throw new Error(`Failed to delete child records for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to delete child records for snapshot ${snapshotId}: ${error}`, + ); } if (input.contexts.length > 0) { try { - await tx.insert(filingTaxonomyContext).values(input.contexts.map((context) => ({ - snapshot_id: snapshotId, - context_id: context.context_id, - entity_identifier: context.entity_identifier, - entity_scheme: context.entity_scheme, - period_start: context.period_start, - period_end: context.period_end, - period_instant: context.period_instant, - segment_json: context.segment_json, - scenario_json: context.scenario_json, - created_at: now - }))); + await tx.insert(filingTaxonomyContext).values( + input.contexts.map((context) => ({ + snapshot_id: snapshotId, + context_id: context.context_id, + entity_identifier: context.entity_identifier, + entity_scheme: context.entity_scheme, + period_start: context.period_start, + period_end: context.period_end, + period_instant: context.period_instant, + segment_json: context.segment_json, + scenario_json: context.scenario_json, + created_at: now, + })), + ); } catch (error) { - throw new Error(`Failed to insert ${input.contexts.length} contexts for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to insert ${input.contexts.length} contexts for snapshot ${snapshotId}: ${error}`, + ); } } if (input.assets.length > 0) { try { - await tx.insert(filingTaxonomyAsset).values(input.assets.map((asset) => ({ - snapshot_id: snapshotId, - asset_type: asset.asset_type, - name: asset.name, - url: asset.url, - size_bytes: asset.size_bytes, - score: asNumericText(asset.score), - is_selected: asset.is_selected, - created_at: now - }))); + await tx.insert(filingTaxonomyAsset).values( + input.assets.map((asset) => ({ + snapshot_id: snapshotId, + asset_type: asset.asset_type, + name: asset.name, + url: asset.url, + size_bytes: asset.size_bytes, + score: asNumericText(asset.score), + is_selected: asset.is_selected, + created_at: now, + })), + ); } catch (error) { - throw new Error(`Failed to insert ${input.assets.length} assets for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to insert ${input.assets.length} assets for snapshot ${snapshotId}: ${error}`, + ); } } if (input.concepts.length > 0) { try { - await tx.insert(filingTaxonomyConcept).values(input.concepts.map((concept) => ({ - snapshot_id: snapshotId, - concept_key: concept.concept_key, - qname: concept.qname, - namespace_uri: concept.namespace_uri, - local_name: concept.local_name, - label: concept.label, - is_extension: concept.is_extension, - balance: concept.balance, - period_type: concept.period_type, - data_type: concept.data_type, - statement_kind: concept.statement_kind, - role_uri: concept.role_uri, - authoritative_concept_key: concept.authoritative_concept_key, - mapping_method: concept.mapping_method, - surface_key: concept.surface_key, - detail_parent_surface_key: concept.detail_parent_surface_key, - kpi_key: concept.kpi_key, - residual_flag: concept.residual_flag, - presentation_order: asNumericText(concept.presentation_order), - presentation_depth: concept.presentation_depth, - parent_concept_key: concept.parent_concept_key, - is_abstract: concept.is_abstract, - created_at: now - }))); + await tx.insert(filingTaxonomyConcept).values( + input.concepts.map((concept) => ({ + snapshot_id: snapshotId, + concept_key: concept.concept_key, + qname: concept.qname, + namespace_uri: concept.namespace_uri, + local_name: concept.local_name, + label: concept.label, + is_extension: concept.is_extension, + balance: concept.balance, + period_type: concept.period_type, + data_type: concept.data_type, + statement_kind: concept.statement_kind, + role_uri: concept.role_uri, + authoritative_concept_key: concept.authoritative_concept_key, + mapping_method: concept.mapping_method, + surface_key: concept.surface_key, + detail_parent_surface_key: concept.detail_parent_surface_key, + kpi_key: concept.kpi_key, + residual_flag: concept.residual_flag, + presentation_order: asNumericText(concept.presentation_order), + presentation_depth: concept.presentation_depth, + parent_concept_key: concept.parent_concept_key, + is_abstract: concept.is_abstract, + created_at: now, + })), + ); } catch (error) { - throw new Error(`Failed to insert ${input.concepts.length} concepts for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to insert ${input.concepts.length} concepts for snapshot ${snapshotId}: ${error}`, + ); } } if (input.facts.length > 0) { try { - await tx.insert(filingTaxonomyFact).values(input.facts.map((fact) => ({ - snapshot_id: snapshotId, - concept_key: fact.concept_key, - qname: fact.qname, - namespace_uri: fact.namespace_uri, - local_name: fact.local_name, - data_type: fact.data_type, - statement_kind: fact.statement_kind, - role_uri: fact.role_uri, - authoritative_concept_key: fact.authoritative_concept_key, - mapping_method: fact.mapping_method, - surface_key: fact.surface_key, - detail_parent_surface_key: fact.detail_parent_surface_key, - kpi_key: fact.kpi_key, - residual_flag: fact.residual_flag, - context_id: fact.context_id, - unit: fact.unit, - decimals: fact.decimals, - precision: fact.precision, - nil: fact.nil, - value_num: String(fact.value_num), - period_start: fact.period_start, - period_end: fact.period_end, - period_instant: fact.period_instant, - dimensions: fact.dimensions, - is_dimensionless: fact.is_dimensionless, - source_file: fact.source_file, - created_at: now - }))); + await tx.insert(filingTaxonomyFact).values( + input.facts.map((fact) => ({ + snapshot_id: snapshotId, + concept_key: fact.concept_key, + qname: fact.qname, + namespace_uri: fact.namespace_uri, + local_name: fact.local_name, + data_type: fact.data_type, + statement_kind: fact.statement_kind, + role_uri: fact.role_uri, + authoritative_concept_key: fact.authoritative_concept_key, + mapping_method: fact.mapping_method, + surface_key: fact.surface_key, + detail_parent_surface_key: fact.detail_parent_surface_key, + kpi_key: fact.kpi_key, + residual_flag: fact.residual_flag, + context_id: fact.context_id, + unit: fact.unit, + decimals: fact.decimals, + precision: fact.precision, + nil: fact.nil, + value_num: String(fact.value_num), + period_start: fact.period_start, + period_end: fact.period_end, + period_instant: fact.period_instant, + dimensions: fact.dimensions, + is_dimensionless: fact.is_dimensionless, + source_file: fact.source_file, + created_at: now, + })), + ); } catch (error) { - throw new Error(`Failed to insert ${input.facts.length} facts for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to insert ${input.facts.length} facts for snapshot ${snapshotId}: ${error}`, + ); } } if (input.metric_validations.length > 0) { try { - await tx.insert(filingTaxonomyMetricValidation).values(input.metric_validations.map((check) => ({ - snapshot_id: snapshotId, - metric_key: check.metric_key, - taxonomy_value: asNumericText(check.taxonomy_value), - llm_value: asNumericText(check.llm_value), - absolute_diff: asNumericText(check.absolute_diff), - relative_diff: asNumericText(check.relative_diff), - status: check.status, - evidence_pages: check.evidence_pages, - pdf_url: check.pdf_url, - provider: check.provider, - model: check.model, - error: check.error, - created_at: now, - updated_at: now - }))); + await tx.insert(filingTaxonomyMetricValidation).values( + input.metric_validations.map((check) => ({ + snapshot_id: snapshotId, + metric_key: check.metric_key, + taxonomy_value: asNumericText(check.taxonomy_value), + llm_value: asNumericText(check.llm_value), + absolute_diff: asNumericText(check.absolute_diff), + relative_diff: asNumericText(check.relative_diff), + status: check.status, + evidence_pages: check.evidence_pages, + pdf_url: check.pdf_url, + provider: check.provider, + model: check.model, + error: check.error, + created_at: now, + updated_at: now, + })), + ); } catch (error) { - throw new Error(`Failed to insert ${input.metric_validations.length} metric validations for snapshot ${snapshotId}: ${error}`); + throw new Error( + `Failed to insert ${input.metric_validations.length} metric validations for snapshot ${snapshotId}: ${error}`, + ); } } @@ -1167,16 +1399,18 @@ export async function upsertFilingTaxonomySnapshot(input: UpsertFilingTaxonomySn export async function listFilingTaxonomySnapshotsByTicker(input: { ticker: string; - window: '10y' | 'all'; - filingTypes?: Array<'10-K' | '10-Q'>; + window: "10y" | "all"; + filingTypes?: Array<"10-K" | "10-Q">; limit?: number; cursor?: string | null; }) { const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 40), 1), 120); const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null; - const constraints = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())]; + const constraints = [ + eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase()), + ]; - if (input.window === '10y') { + if (input.window === "10y") { constraints.push(gte(filingTaxonomySnapshot.filing_date, tenYearsAgoIso())); } @@ -1185,25 +1419,30 @@ export async function listFilingTaxonomySnapshotsByTicker(input: { } if (input.filingTypes && input.filingTypes.length > 0) { - constraints.push(inArray(filingTaxonomySnapshot.filing_type, input.filingTypes)); + constraints.push( + inArray(filingTaxonomySnapshot.filing_type, input.filingTypes), + ); } const rows = await db .select() .from(filingTaxonomySnapshot) .where(and(...constraints)) - .orderBy(desc(filingTaxonomySnapshot.filing_date), desc(filingTaxonomySnapshot.id)) + .orderBy( + desc(filingTaxonomySnapshot.filing_date), + desc(filingTaxonomySnapshot.id), + ) .limit(safeLimit + 1); const hasMore = rows.length > safeLimit; const usedRows = hasMore ? rows.slice(0, safeLimit) : rows; const nextCursor = hasMore - ? String(usedRows[usedRows.length - 1]?.id ?? '') + ? String(usedRows[usedRows.length - 1]?.id ?? "") : null; return { snapshots: usedRows.map(toSnapshotRecord), - nextCursor + nextCursor, }; } @@ -1211,35 +1450,43 @@ export async function countFilingTaxonomySnapshotStatuses(ticker: string) { const rows = await db .select({ status: filingTaxonomySnapshot.parse_status, - count: sql`count(*)` + count: sql`count(*)`, }) .from(filingTaxonomySnapshot) .where(eq(filingTaxonomySnapshot.ticker, ticker.trim().toUpperCase())) .groupBy(filingTaxonomySnapshot.parse_status); - return rows.reduce>((acc, row) => { - acc[row.status] = Number(row.count); - return acc; - }, { - ready: 0, - partial: 0, - failed: 0 - }); + return rows.reduce>( + (acc, row) => { + acc[row.status] = Number(row.count); + return acc; + }, + { + ready: 0, + partial: 0, + failed: 0, + }, + ); } export async function listTaxonomyFactsByTicker(input: { ticker: string; - window: '10y' | 'all'; + window: "10y" | "all"; statement?: FinancialStatementKind; - filingTypes?: Array<'10-K' | '10-Q'>; + filingTypes?: Array<"10-K" | "10-Q">; cursor?: string | null; limit?: number; }) { - const safeLimit = Math.min(Math.max(Math.trunc(input.limit ?? 500), 1), 10000); + const safeLimit = Math.min( + Math.max(Math.trunc(input.limit ?? 500), 1), + 10000, + ); const cursorId = input.cursor ? Number.parseInt(input.cursor, 10) : null; - const conditions = [eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase())]; + const conditions = [ + eq(filingTaxonomySnapshot.ticker, input.ticker.trim().toUpperCase()), + ]; - if (input.window === '10y') { + if (input.window === "10y") { conditions.push(gte(filingTaxonomySnapshot.filing_date, tenYearsAgoIso())); } @@ -1248,7 +1495,9 @@ export async function listTaxonomyFactsByTicker(input: { } if (input.filingTypes && input.filingTypes.length > 0) { - conditions.push(inArray(filingTaxonomySnapshot.filing_type, input.filingTypes)); + conditions.push( + inArray(filingTaxonomySnapshot.filing_type, input.filingTypes), + ); } if (cursorId && Number.isFinite(cursorId) && cursorId > 0) { @@ -1276,17 +1525,20 @@ export async function listTaxonomyFactsByTicker(input: { period_instant: filingTaxonomyFact.period_instant, dimensions: filingTaxonomyFact.dimensions, is_dimensionless: filingTaxonomyFact.is_dimensionless, - source_file: filingTaxonomyFact.source_file + source_file: filingTaxonomyFact.source_file, }) .from(filingTaxonomyFact) - .innerJoin(filingTaxonomySnapshot, eq(filingTaxonomyFact.snapshot_id, filingTaxonomySnapshot.id)) + .innerJoin( + filingTaxonomySnapshot, + eq(filingTaxonomyFact.snapshot_id, filingTaxonomySnapshot.id), + ) .where(and(...conditions)) .orderBy(desc(filingTaxonomyFact.id)) .limit(safeLimit + 1); const hasMore = rows.length > safeLimit; const used = hasMore ? rows.slice(0, safeLimit) : rows; - const nextCursor = hasMore ? String(used[used.length - 1]?.id ?? '') : null; + const nextCursor = hasMore ? String(used[used.length - 1]?.id ?? "") : null; const facts: TaxonomyFactRow[] = used.map((row) => { const value = asNumber(row.value_num); @@ -1314,13 +1566,13 @@ export async function listTaxonomyFactsByTicker(input: { periodInstant: row.period_instant, dimensions: row.dimensions, isDimensionless: row.is_dimensionless, - sourceFile: row.source_file + sourceFile: row.source_file, }; }); return { facts, - nextCursor + nextCursor, }; } @@ -1340,5 +1592,5 @@ export async function listTaxonomyAssetsBySnapshotIds(snapshotIds: number[]) { export const __filingTaxonomyInternals = { normalizeFilingTaxonomySnapshotPayload, - toSnapshotRecord + toSnapshotRecord, }; diff --git a/lib/server/task-processors.outcomes.test.ts b/lib/server/task-processors.outcomes.test.ts index 6398d9f..83b2358 100644 --- a/lib/server/task-processors.outcomes.test.ts +++ b/lib/server/task-processors.outcomes.test.ts @@ -1,11 +1,5 @@ -import { - beforeEach, - describe, - expect, - it, - mock -} from 'bun:test'; -import type { Filing, Holding, Task } from '@/lib/types'; +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import type { Filing, Holding, Task } from "@/lib/types"; const stageUpdates: Array<{ taskId: string; @@ -14,290 +8,373 @@ const stageUpdates: Array<{ context: Record | null; }> = []; -const mockRunAiAnalysis = mock(async (_prompt: string, _instruction: string, options?: { workload?: string }) => { - if (options?.workload === 'extraction') { - return { - provider: 'zhipu', - model: 'glm-extract', - text: JSON.stringify({ - summary: 'Revenue growth remained resilient despite FX pressure.', - keyPoints: ['Revenue up year-over-year'], - redFlags: ['Debt service burden is rising'], - followUpQuestions: ['Is margin guidance sustainable?'], - portfolioSignals: ['Monitor leverage trend'], - segmentSpecificData: ['Services segment outgrew hardware segment.'], - geographicRevenueBreakdown: ['EMEA revenue grew faster than Americas.'], - companySpecificData: ['Same-store sales increased 4.2%.'], - secApiCrossChecks: ['Revenue from SEC API aligns with filing narrative.'], - confidence: 0.72 - }) - }; - } +const mockRunAiAnalysis = mock( + async ( + _prompt: string, + _instruction: string, + options?: { workload?: string }, + ) => { + if (options?.workload === "extraction") { + return { + provider: "zhipu", + model: "glm-extract", + text: JSON.stringify({ + summary: "Revenue growth remained resilient despite FX pressure.", + keyPoints: ["Revenue up year-over-year"], + redFlags: ["Debt service burden is rising"], + followUpQuestions: ["Is margin guidance sustainable?"], + portfolioSignals: ["Monitor leverage trend"], + segmentSpecificData: ["Services segment outgrew hardware segment."], + geographicRevenueBreakdown: [ + "EMEA revenue grew faster than Americas.", + ], + companySpecificData: ["Same-store sales increased 4.2%."], + secApiCrossChecks: [ + "Revenue from SEC API aligns with filing narrative.", + ], + confidence: 0.72, + }), + }; + } - return { - provider: 'zhipu', - model: options?.workload === 'report' ? 'glm-report' : 'glm-generic', - text: 'Structured output' - }; -}); + return { + provider: "zhipu", + model: options?.workload === "report" ? "glm-report" : "glm-generic", + text: "Structured output", + }; + }, +); const mockBuildPortfolioSummary = mock((_holdings: Holding[]) => ({ positions: 14, - total_value: '100000', - total_gain_loss: '1000', - total_cost_basis: '99000', - avg_return_pct: '0.01' + total_value: "100000", + total_gain_loss: "1000", + total_cost_basis: "99000", + avg_return_pct: "0.01", })); const mockGetQuote = mock(async (ticker: string) => { - return ticker === 'MSFT' ? 410 : 205; + return ticker === "MSFT" ? 410 : 205; }); -const mockIndexSearchDocuments = mock(async (input: { - onStage?: (stage: 'collect' | 'fetch' | 'chunk' | 'embed' | 'persist', detail: string, context?: Record | null) => Promise | void; -}) => { - await input.onStage?.('collect', 'Collected 12 source records for search indexing', { - counters: { +const mockIndexSearchDocuments = mock( + async (input: { + onStage?: ( + stage: "collect" | "fetch" | "chunk" | "embed" | "persist", + detail: string, + context?: Record | null, + ) => Promise | void; + }) => { + await input.onStage?.( + "collect", + "Collected 12 source records for search indexing", + { + counters: { + sourcesCollected: 12, + deleted: 3, + }, + }, + ); + await input.onStage?.( + "fetch", + "Preparing filing_brief 0000320193-26-000001", + { + progress: { + current: 1, + total: 12, + unit: "sources", + }, + subject: { + ticker: "AAPL", + accessionNumber: "0000320193-26-000001", + }, + }, + ); + await input.onStage?.( + "embed", + "Embedding 248 chunks for 0000320193-26-000001", + { + progress: { + current: 1, + total: 12, + unit: "sources", + }, + counters: { + chunksEmbedded: 248, + }, + }, + ); + + return { sourcesCollected: 12, - deleted: 3 - } - }); - await input.onStage?.('fetch', 'Preparing filing_brief 0000320193-26-000001', { - progress: { - current: 1, - total: 12, - unit: 'sources' - }, - subject: { - ticker: 'AAPL', - accessionNumber: '0000320193-26-000001' - } - }); - await input.onStage?.('embed', 'Embedding 248 chunks for 0000320193-26-000001', { - progress: { - current: 1, - total: 12, - unit: 'sources' - }, - counters: { - chunksEmbedded: 248 - } - }); - - return { - sourcesCollected: 12, - indexed: 12, - skipped: 1, - deleted: 3, - chunksEmbedded: 248 - }; -}); + indexed: 12, + skipped: 1, + deleted: 3, + chunksEmbedded: 248, + }; + }, +); const sampleFiling = (): Filing => ({ id: 1, - ticker: 'AAPL', - filing_type: '10-Q', - filing_date: '2026-01-30', - accession_number: '0000320193-26-000001', - cik: '0000320193', - company_name: 'Apple Inc.', - filing_url: 'https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/a10q.htm', - submission_url: 'https://data.sec.gov/submissions/CIK0000320193.json', - primary_document: 'a10q.htm', + ticker: "AAPL", + filing_type: "10-Q", + filing_date: "2026-01-30", + accession_number: "0000320193-26-000001", + cik: "0000320193", + company_name: "Apple Inc.", + filing_url: + "https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/a10q.htm", + submission_url: "https://data.sec.gov/submissions/CIK0000320193.json", + primary_document: "a10q.htm", metrics: { revenue: 120_000_000_000, netIncome: 25_000_000_000, totalAssets: 410_000_000_000, cash: 70_000_000_000, - debt: 98_000_000_000 + debt: 98_000_000_000, }, analysis: null, - created_at: '2026-01-30T00:00:00.000Z', - updated_at: '2026-01-30T00:00:00.000Z' + created_at: "2026-01-30T00:00:00.000Z", + updated_at: "2026-01-30T00:00:00.000Z", }); const mockGetFilingByAccession = mock(async () => sampleFiling()); -const mockListFilingsRecords = mock(async () => [sampleFiling(), { - ...sampleFiling(), - id: 2, - accession_number: '0000320193-26-000002', - filing_date: '2026-02-28' -}]); +const mockListFilingsRecords = mock(async () => [ + sampleFiling(), + { + ...sampleFiling(), + id: 2, + accession_number: "0000320193-26-000002", + filing_date: "2026-02-28", + }, +]); const mockSaveFilingAnalysis = mock(async () => {}); const mockUpdateFilingMetricsById = mock(async () => {}); const mockUpsertFilingsRecords = mock(async () => ({ inserted: 2, - updated: 0 + updated: 0, })); const mockDeleteCompanyFinancialBundlesForTicker = mock(async () => {}); const mockGetFilingTaxonomySnapshotByFilingId = mock(async () => null); const mockUpsertFilingTaxonomySnapshot = mock(async () => {}); +const mockValidateMetricsWithPdfLlm = mock(async () => ({ + validation_result: { + status: "matched" as const, + checks: [], + validatedAt: "2026-03-09T00:00:00.000Z", + }, + metric_validations: [], +})); const mockApplyRefreshedPrices = mock(async () => 24); const mockListHoldingsForPriceRefresh = mock(async () => [ { id: 1, - user_id: 'user-1', - ticker: 'AAPL', - company_name: 'Apple Inc.', - shares: '10', - avg_cost: '150', - current_price: '200', - market_value: '2000', - gain_loss: '500', - gain_loss_pct: '0.33', + user_id: "user-1", + ticker: "AAPL", + company_name: "Apple Inc.", + shares: "10", + avg_cost: "150", + current_price: "200", + market_value: "2000", + gain_loss: "500", + gain_loss_pct: "0.33", last_price_at: null, - created_at: '2026-03-09T00:00:00.000Z', - updated_at: '2026-03-09T00:00:00.000Z' + created_at: "2026-03-09T00:00:00.000Z", + updated_at: "2026-03-09T00:00:00.000Z", }, { id: 2, - user_id: 'user-1', - ticker: 'MSFT', - company_name: 'Microsoft Corporation', - shares: '4', - avg_cost: '300', - current_price: '400', - market_value: '1600', - gain_loss: '400', - gain_loss_pct: '0.25', + user_id: "user-1", + ticker: "MSFT", + company_name: "Microsoft Corporation", + shares: "4", + avg_cost: "300", + current_price: "400", + market_value: "1600", + gain_loss: "400", + gain_loss_pct: "0.25", last_price_at: null, - created_at: '2026-03-09T00:00:00.000Z', - updated_at: '2026-03-09T00:00:00.000Z' - } + created_at: "2026-03-09T00:00:00.000Z", + updated_at: "2026-03-09T00:00:00.000Z", + }, ]); -const mockListUserHoldings = mock(async () => await mockListHoldingsForPriceRefresh()); +const mockListUserHoldings = mock( + async () => await mockListHoldingsForPriceRefresh(), +); const mockCreatePortfolioInsight = mock(async () => {}); -const mockUpdateTaskStage = mock(async (taskId: string, stage: string, detail: string | null, context?: Record | null) => { - stageUpdates.push({ - taskId, - stage, - detail, - context: context ?? null - }); -}); +const mockUpdateTaskStage = mock( + async ( + taskId: string, + stage: string, + detail: string | null, + context?: Record | null, + ) => { + stageUpdates.push({ + taskId, + stage, + detail, + context: context ?? null, + }); + }, +); const mockFetchPrimaryFilingText = mock(async () => ({ - text: 'Revenue accelerated in services and margins improved.', - source: 'primary_document' as const + text: "Revenue accelerated in services and margins improved.", + source: "primary_document" as const, })); -const mockFetchRecentFilings = mock(async () => ([ +const mockFetchRecentFilings = mock(async () => [ { - ticker: 'AAPL', - filingType: '10-Q', - filingDate: '2026-01-30', - accessionNumber: '0000320193-26-000001', - cik: '0000320193', - companyName: 'Apple Inc.', - filingUrl: 'https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/a10q.htm', - submissionUrl: 'https://data.sec.gov/submissions/CIK0000320193.json', - primaryDocument: 'a10q.htm' + ticker: "AAPL", + filingType: "10-Q", + filingDate: "2026-01-30", + accessionNumber: "0000320193-26-000001", + cik: "0000320193", + companyName: "Apple Inc.", + filingUrl: + "https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/a10q.htm", + submissionUrl: "https://data.sec.gov/submissions/CIK0000320193.json", + primaryDocument: "a10q.htm", }, { - ticker: 'AAPL', - filingType: '10-K', - filingDate: '2025-10-30', - accessionNumber: '0000320193-25-000001', - cik: '0000320193', - companyName: 'Apple Inc.', - filingUrl: 'https://www.sec.gov/Archives/edgar/data/320193/000032019325000001/a10k.htm', - submissionUrl: 'https://data.sec.gov/submissions/CIK0000320193.json', - primaryDocument: 'a10k.htm' - } -])); + ticker: "AAPL", + filingType: "10-K", + filingDate: "2025-10-30", + accessionNumber: "0000320193-25-000001", + cik: "0000320193", + companyName: "Apple Inc.", + filingUrl: + "https://www.sec.gov/Archives/edgar/data/320193/000032019325000001/a10k.htm", + submissionUrl: "https://data.sec.gov/submissions/CIK0000320193.json", + primaryDocument: "a10k.htm", + }, +]); const mockEnqueueTask = mock(async () => ({ - id: 'search-task-1' -})); -const mockHydrateFilingTaxonomySnapshot = mock(async (input: { filingId: number }) => ({ - filing_id: input.filingId, - ticker: 'AAPL', - filing_date: '2026-01-30', - filing_type: '10-Q', - parse_status: 'ready', - parse_error: null, - source: 'xbrl_instance', - periods: [], - statement_rows: { - income: [], - balance: [], - cash_flow: [], - equity: [], - comprehensive_income: [] - }, - derived_metrics: { - revenue: 120_000_000_000 - }, - validation_result: { - status: 'matched', - checks: [], - validatedAt: '2026-03-09T00:00:00.000Z' - }, - facts_count: 1, - concepts_count: 1, - dimensions_count: 0, - assets: [], - concepts: [], - facts: [], - metric_validations: [] + id: "search-task-1", })); +const mockHydrateFilingTaxonomySnapshot = mock( + async (input: { filingId: number }) => ({ + filing_id: input.filingId, + ticker: "AAPL", + filing_date: "2026-01-30", + filing_type: "10-Q", + parse_status: "ready", + parse_error: null, + source: "xbrl_instance", + periods: [], + statement_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + faithful_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + surface_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + detail_rows: { + income: {}, + balance: {}, + cash_flow: {}, + equity: {}, + comprehensive_income: {}, + }, + kpi_rows: [], + computed_definitions: [], + derived_metrics: { + revenue: 120_000_000_000, + }, + validation_result: { + status: "matched", + checks: [], + validatedAt: "2026-03-09T00:00:00.000Z", + }, + facts_count: 1, + concepts_count: 1, + dimensions_count: 0, + assets: [], + concepts: [], + facts: [], + metric_validations: [], + xbrl_validation: { + status: "passed", + }, + }), +); -mock.module('@/lib/server/ai', () => ({ - runAiAnalysis: mockRunAiAnalysis +mock.module("@/lib/server/ai", () => ({ + runAiAnalysis: mockRunAiAnalysis, })); -mock.module('@/lib/server/portfolio', () => ({ - buildPortfolioSummary: mockBuildPortfolioSummary +mock.module("@/lib/server/portfolio", () => ({ + buildPortfolioSummary: mockBuildPortfolioSummary, })); -mock.module('@/lib/server/prices', () => ({ - getQuote: mockGetQuote +mock.module("@/lib/server/prices", () => ({ + getQuote: mockGetQuote, })); -mock.module('@/lib/server/search', () => ({ - indexSearchDocuments: mockIndexSearchDocuments +mock.module("@/lib/server/search", () => ({ + indexSearchDocuments: mockIndexSearchDocuments, })); -mock.module('@/lib/server/repos/filings', () => ({ +mock.module("@/lib/server/repos/filings", () => ({ getFilingByAccession: mockGetFilingByAccession, listFilingsRecords: mockListFilingsRecords, saveFilingAnalysis: mockSaveFilingAnalysis, updateFilingMetricsById: mockUpdateFilingMetricsById, - upsertFilingsRecords: mockUpsertFilingsRecords + upsertFilingsRecords: mockUpsertFilingsRecords, })); -mock.module('@/lib/server/repos/company-financial-bundles', () => ({ - deleteCompanyFinancialBundlesForTicker: mockDeleteCompanyFinancialBundlesForTicker +mock.module("@/lib/server/repos/company-financial-bundles", () => ({ + deleteCompanyFinancialBundlesForTicker: + mockDeleteCompanyFinancialBundlesForTicker, })); -mock.module('@/lib/server/repos/filing-taxonomy', () => ({ +mock.module("@/lib/server/repos/filing-taxonomy", () => ({ getFilingTaxonomySnapshotByFilingId: mockGetFilingTaxonomySnapshotByFilingId, - upsertFilingTaxonomySnapshot: mockUpsertFilingTaxonomySnapshot + upsertFilingTaxonomySnapshot: mockUpsertFilingTaxonomySnapshot, })); -mock.module('@/lib/server/repos/holdings', () => ({ +mock.module("@/lib/server/repos/holdings", () => ({ applyRefreshedPrices: mockApplyRefreshedPrices, listHoldingsForPriceRefresh: mockListHoldingsForPriceRefresh, - listUserHoldings: mockListUserHoldings + listUserHoldings: mockListUserHoldings, })); -mock.module('@/lib/server/repos/insights', () => ({ - createPortfolioInsight: mockCreatePortfolioInsight +mock.module("@/lib/server/repos/insights", () => ({ + createPortfolioInsight: mockCreatePortfolioInsight, })); -mock.module('@/lib/server/repos/tasks', () => ({ - updateTaskStage: mockUpdateTaskStage +mock.module("@/lib/server/repos/tasks", () => ({ + updateTaskStage: mockUpdateTaskStage, })); -mock.module('@/lib/server/sec', () => ({ +mock.module("@/lib/server/sec", () => ({ fetchPrimaryFilingText: mockFetchPrimaryFilingText, - fetchRecentFilings: mockFetchRecentFilings + fetchRecentFilings: mockFetchRecentFilings, })); -mock.module('@/lib/server/tasks', () => ({ - enqueueTask: mockEnqueueTask +mock.module("@/lib/server/tasks", () => ({ + enqueueTask: mockEnqueueTask, })); -mock.module('@/lib/server/taxonomy/engine', () => ({ - hydrateFilingTaxonomySnapshot: mockHydrateFilingTaxonomySnapshot +mock.module("@/lib/server/taxonomy/engine", () => ({ + hydrateFilingTaxonomySnapshot: mockHydrateFilingTaxonomySnapshot, +})); +mock.module("@/lib/server/taxonomy/pdf-validation", () => ({ + validateMetricsWithPdfLlm: mockValidateMetricsWithPdfLlm, })); -const { runTaskProcessor } = await import('./task-processors'); +const { runTaskProcessor } = await import("./task-processors"); function taskFactory(overrides: Partial = {}): Task { return { - id: 'task-1', - user_id: 'user-1', - task_type: 'sync_filings', - status: 'running', - stage: 'running', - stage_detail: 'Running', + id: "task-1", + user_id: "user-1", + task_type: "sync_filings", + status: "running", + stage: "running", + stage_detail: "Running", stage_context: null, resource_key: null, notification_read_at: null, @@ -308,24 +385,24 @@ function taskFactory(overrides: Partial = {}): Task { error: null, attempts: 1, max_attempts: 3, - workflow_run_id: 'run-1', - created_at: '2026-03-09T00:00:00.000Z', - updated_at: '2026-03-09T00:00:00.000Z', + workflow_run_id: "run-1", + created_at: "2026-03-09T00:00:00.000Z", + updated_at: "2026-03-09T00:00:00.000Z", finished_at: null, notification: { - title: 'Task', - statusLine: 'Running', + title: "Task", + statusLine: "Running", detailLine: null, - tone: 'info', + tone: "info", progress: null, stats: [], - actions: [] + actions: [], }, - ...overrides + ...overrides, }; } -describe('task processor outcomes', () => { +describe("task processor outcomes", () => { beforeEach(() => { stageUpdates.length = 0; mockRunAiAnalysis.mockClear(); @@ -335,78 +412,108 @@ describe('task processor outcomes', () => { mockCreatePortfolioInsight.mockClear(); mockUpdateTaskStage.mockClear(); mockEnqueueTask.mockClear(); + mockValidateMetricsWithPdfLlm.mockClear(); }); - it('returns sync filing completion detail and progress context', async () => { - const outcome = await runTaskProcessor(taskFactory({ - task_type: 'sync_filings', - payload: { - ticker: 'AAPL', - limit: 2 - } - })); + it("returns sync filing completion detail and progress context", async () => { + const outcome = await runTaskProcessor( + taskFactory({ + task_type: "sync_filings", + payload: { + ticker: "AAPL", + limit: 2, + }, + }), + ); - expect(outcome.completionDetail).toContain('Synced 2 filings for AAPL'); + expect(outcome.completionDetail).toContain("Synced 2 filings for AAPL"); expect(outcome.result.fetched).toBe(2); - expect(outcome.result.searchTaskId).toBe('search-task-1'); + expect(outcome.result.searchTaskId).toBe("search-task-1"); expect(outcome.completionContext?.counters?.hydrated).toBe(2); - expect(stageUpdates.some((entry) => entry.stage === 'sync.extract_taxonomy' && entry.context?.subject)).toBe(true); + expect( + stageUpdates.some( + (entry) => + entry.stage === "sync.extract_taxonomy" && entry.context?.subject, + ), + ).toBe(true); + expect(mockValidateMetricsWithPdfLlm).toHaveBeenCalled(); + expect(mockUpsertFilingTaxonomySnapshot).toHaveBeenCalled(); }); - it('returns refresh price completion detail with live quote progress', async () => { - const outcome = await runTaskProcessor(taskFactory({ - task_type: 'refresh_prices' - })); + it("returns refresh price completion detail with live quote progress", async () => { + const outcome = await runTaskProcessor( + taskFactory({ + task_type: "refresh_prices", + }), + ); - expect(outcome.completionDetail).toBe('Refreshed prices for 2 tickers across 2 holdings.'); + expect(outcome.completionDetail).toBe( + "Refreshed prices for 2/2 tickers across 2 holdings.", + ); expect(outcome.result.updatedCount).toBe(24); - expect(stageUpdates.filter((entry) => entry.stage === 'refresh.fetch_quotes')).toHaveLength(3); + expect( + stageUpdates.filter((entry) => entry.stage === "refresh.fetch_quotes"), + ).toHaveLength(3); expect(stageUpdates.at(-1)?.context?.counters).toBeDefined(); }); - it('returns analyze filing completion detail with report metadata', async () => { - const outcome = await runTaskProcessor(taskFactory({ - task_type: 'analyze_filing', - payload: { - accessionNumber: '0000320193-26-000001' - } - })); + it("returns analyze filing completion detail with report metadata", async () => { + const outcome = await runTaskProcessor( + taskFactory({ + task_type: "analyze_filing", + payload: { + accessionNumber: "0000320193-26-000001", + }, + }), + ); - expect(outcome.completionDetail).toBe('Analysis report generated for AAPL 10-Q 0000320193-26-000001.'); - expect(outcome.result.ticker).toBe('AAPL'); - expect(outcome.result.filingType).toBe('10-Q'); - expect(outcome.result.model).toBe('glm-report'); + expect(outcome.completionDetail).toBe( + "Analysis report generated for AAPL 10-Q 0000320193-26-000001.", + ); + expect(outcome.result.ticker).toBe("AAPL"); + expect(outcome.result.filingType).toBe("10-Q"); + expect(outcome.result.model).toBe("glm-report"); expect(mockSaveFilingAnalysis).toHaveBeenCalled(); }); - it('returns index search completion detail and counters', async () => { - const outcome = await runTaskProcessor(taskFactory({ - task_type: 'index_search', - payload: { - ticker: 'AAPL', - sourceKinds: ['filing_brief'] - } - })); + it("returns index search completion detail and counters", async () => { + const outcome = await runTaskProcessor( + taskFactory({ + task_type: "index_search", + payload: { + ticker: "AAPL", + sourceKinds: ["filing_brief"], + }, + }), + ); - expect(outcome.completionDetail).toBe('Indexed 12 sources, embedded 248 chunks, skipped 1, deleted 3 stale documents.'); + expect(outcome.completionDetail).toBe( + "Indexed 12 sources, embedded 248 chunks, skipped 1, deleted 3 stale documents.", + ); expect(outcome.result.indexed).toBe(12); expect(outcome.completionContext?.counters?.chunksEmbedded).toBe(248); - expect(stageUpdates.some((entry) => entry.stage === 'search.embed')).toBe(true); + expect(stageUpdates.some((entry) => entry.stage === "search.embed")).toBe( + true, + ); }); - it('returns portfolio insight completion detail and summary payload', async () => { - const outcome = await runTaskProcessor(taskFactory({ - task_type: 'portfolio_insights' - })); + it("returns portfolio insight completion detail and summary payload", async () => { + const outcome = await runTaskProcessor( + taskFactory({ + task_type: "portfolio_insights", + }), + ); - expect(outcome.completionDetail).toBe('Generated portfolio insight for 14 holdings.'); - expect(outcome.result.provider).toBe('zhipu'); + expect(outcome.completionDetail).toBe( + "Generated portfolio insight for 14 holdings.", + ); + expect(outcome.result.provider).toBe("zhipu"); expect(outcome.result.summary).toEqual({ positions: 14, - total_value: '100000', - total_gain_loss: '1000', - total_cost_basis: '99000', - avg_return_pct: '0.01' + total_value: "100000", + total_gain_loss: "1000", + total_cost_basis: "99000", + avg_return_pct: "0.01", }); expect(mockCreatePortfolioInsight).toHaveBeenCalled(); }); diff --git a/lib/server/task-processors.ts b/lib/server/task-processors.ts index bef9308..ce96955 100644 --- a/lib/server/task-processors.ts +++ b/lib/server/task-processors.ts @@ -5,52 +5,48 @@ import type { Holding, Task, TaskStage, - TaskStageContext -} from '@/lib/types'; -import { runAiAnalysis } from '@/lib/server/ai'; -import { buildPortfolioSummary } from '@/lib/server/portfolio'; -import { getQuote } from '@/lib/server/prices'; -import { indexSearchDocuments } from '@/lib/server/search'; + TaskStageContext, +} from "@/lib/types"; +import { runAiAnalysis } from "@/lib/server/ai"; +import { buildPortfolioSummary } from "@/lib/server/portfolio"; +import { getQuote } from "@/lib/server/prices"; +import { indexSearchDocuments } from "@/lib/server/search"; import { getFilingByAccession, listFilingsRecords, saveFilingAnalysis, updateFilingMetricsById, - upsertFilingsRecords -} from '@/lib/server/repos/filings'; -import { - deleteCompanyFinancialBundlesForTicker -} from '@/lib/server/repos/company-financial-bundles'; + upsertFilingsRecords, +} from "@/lib/server/repos/filings"; +import { deleteCompanyFinancialBundlesForTicker } from "@/lib/server/repos/company-financial-bundles"; import { getFilingTaxonomySnapshotByFilingId, normalizeFilingTaxonomySnapshotPayload, - upsertFilingTaxonomySnapshot -} from '@/lib/server/repos/filing-taxonomy'; + upsertFilingTaxonomySnapshot, +} from "@/lib/server/repos/filing-taxonomy"; import { applyRefreshedPrices, listHoldingsForPriceRefresh, - listUserHoldings -} from '@/lib/server/repos/holdings'; -import { createPortfolioInsight } from '@/lib/server/repos/insights'; -import { updateTaskStage } from '@/lib/server/repos/tasks'; -import { - fetchPrimaryFilingText, - fetchRecentFilings -} from '@/lib/server/sec'; -import { enqueueTask } from '@/lib/server/tasks'; -import { hydrateFilingTaxonomySnapshot } from '@/lib/server/taxonomy/engine'; + listUserHoldings, +} from "@/lib/server/repos/holdings"; +import { createPortfolioInsight } from "@/lib/server/repos/insights"; +import { updateTaskStage } from "@/lib/server/repos/tasks"; +import { fetchPrimaryFilingText, fetchRecentFilings } from "@/lib/server/sec"; +import { enqueueTask } from "@/lib/server/tasks"; +import { hydrateFilingTaxonomySnapshot } from "@/lib/server/taxonomy/engine"; +import { validateMetricsWithPdfLlm } from "@/lib/server/taxonomy/pdf-validation"; const EXTRACTION_REQUIRED_KEYS = [ - 'summary', - 'keyPoints', - 'redFlags', - 'followUpQuestions', - 'portfolioSignals', - 'segmentSpecificData', - 'geographicRevenueBreakdown', - 'companySpecificData', - 'secApiCrossChecks', - 'confidence' + "summary", + "keyPoints", + "redFlags", + "followUpQuestions", + "portfolioSignals", + "segmentSpecificData", + "geographicRevenueBreakdown", + "companySpecificData", + "secApiCrossChecks", + "confidence", ] as const; const EXTRACTION_MAX_ITEMS = 6; const EXTRACTION_ITEM_MAX_LENGTH = 280; @@ -63,7 +59,7 @@ const SEGMENT_PATTERNS = [ /\bsegment margin\b/i, /\bsegment profit\b/i, /\bbusiness segment\b/i, - /\breportable segment\b/i + /\breportable segment\b/i, ]; const GEOGRAPHIC_PATTERNS = [ /\bgeographic\b/i, @@ -74,7 +70,7 @@ const GEOGRAPHIC_PATTERNS = [ /\bnorth america\b/i, /\beurope\b/i, /\bchina\b/i, - /\binternational\b/i + /\binternational\b/i, ]; const COMPANY_SPECIFIC_PATTERNS = [ /\bsame[- ]store\b/i, @@ -90,13 +86,15 @@ const COMPANY_SPECIFIC_PATTERNS = [ /\boccupancy\b/i, /\brevpar\b/i, /\bretention\b/i, - /\bchurn\b/i + /\bchurn\b/i, ]; -type FilingMetricKey = keyof NonNullable; +type FilingMetricKey = keyof NonNullable; -function isFinancialMetricsForm(filingType: string): filingType is '10-K' | '10-Q' { - return filingType === '10-K' || filingType === '10-Q'; +function isFinancialMetricsForm( + filingType: string, +): filingType is "10-K" | "10-Q" { + return filingType === "10-K" || filingType === "10-Q"; } const METRIC_CHECK_PATTERNS: Array<{ @@ -105,34 +103,34 @@ const METRIC_CHECK_PATTERNS: Array<{ patterns: RegExp[]; }> = [ { - key: 'revenue', - label: 'Revenue', - patterns: [/\brevenue\b/i, /\bsales\b/i] + key: "revenue", + label: "Revenue", + patterns: [/\brevenue\b/i, /\bsales\b/i], }, { - key: 'netIncome', - label: 'Net income', - patterns: [/\bnet income\b/i, /\bprofit\b/i] + key: "netIncome", + label: "Net income", + patterns: [/\bnet income\b/i, /\bprofit\b/i], }, { - key: 'totalAssets', - label: 'Total assets', - patterns: [/\btotal assets\b/i, /\bassets\b/i] + key: "totalAssets", + label: "Total assets", + patterns: [/\btotal assets\b/i, /\bassets\b/i], }, { - key: 'cash', - label: 'Cash', - patterns: [/\bcash\b/i, /\bcash equivalents\b/i] + key: "cash", + label: "Cash", + patterns: [/\bcash\b/i, /\bcash equivalents\b/i], }, { - key: 'debt', - label: 'Debt', - patterns: [/\bdebt\b/i, /\bborrowings\b/i, /\bliabilit(?:y|ies)\b/i] - } + key: "debt", + label: "Debt", + patterns: [/\bdebt\b/i, /\bborrowings\b/i, /\bliabilit(?:y|ies)\b/i], + }, ]; function toTaskResult(value: unknown): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) { + if (!value || typeof value !== "object" || Array.isArray(value)) { return { value }; } @@ -148,12 +146,12 @@ export type TaskExecutionOutcome = { function buildTaskOutcome( result: unknown, completionDetail: string, - completionContext: TaskStageContext | null = null + completionContext: TaskStageContext | null = null, ): TaskExecutionOutcome { return { result: toTaskResult(result), completionDetail, - completionContext + completionContext, }; } @@ -161,7 +159,7 @@ async function setProjectionStage( task: Task, stage: TaskStage, detail: string | null = null, - context: TaskStageContext | null = null + context: TaskStageContext | null = null, ) { await updateTaskStage(task.id, stage, detail, context); } @@ -171,29 +169,29 @@ function buildProgressContext(input: { total: number; unit: string; counters?: Record; - subject?: TaskStageContext['subject']; + subject?: TaskStageContext["subject"]; }): TaskStageContext { return { progress: { current: input.current, total: input.total, - unit: input.unit + unit: input.unit, }, counters: input.counters, - subject: input.subject + subject: input.subject, }; } function parseTicker(raw: unknown) { - if (typeof raw !== 'string' || raw.trim().length < 1) { - throw new Error('Ticker is required'); + if (typeof raw !== "string" || raw.trim().length < 1) { + throw new Error("Ticker is required"); } return raw.trim().toUpperCase(); } function parseLimit(raw: unknown, fallback: number, min: number, max: number) { - const numberValue = typeof raw === 'number' ? raw : Number(raw); + const numberValue = typeof raw === "number" ? raw : Number(raw); if (!Number.isFinite(numberValue)) { return fallback; @@ -204,7 +202,7 @@ function parseLimit(raw: unknown, fallback: number, min: number, max: number) { } function parseOptionalText(raw: unknown) { - if (typeof raw !== 'string') { + if (typeof raw !== "string") { return null; } @@ -218,7 +216,7 @@ function parseOptionalStringArray(raw: unknown) { } return raw - .filter((entry): entry is string => typeof entry === 'string') + .filter((entry): entry is string => typeof entry === "string") .map((entry) => entry.trim()) .filter((entry) => entry.length > 0); } @@ -230,7 +228,7 @@ function parseTags(raw: unknown) { const unique = new Set(); for (const entry of raw) { - if (typeof entry !== 'string') { + if (typeof entry !== "string") { continue; } @@ -246,11 +244,11 @@ function parseTags(raw: unknown) { } function sanitizeExtractionText(value: unknown, maxLength: number) { - if (typeof value !== 'string') { + if (typeof value !== "string") { return null; } - const collapsed = value.replace(/\s+/g, ' ').trim(); + const collapsed = value.replace(/\s+/g, " ").trim(); if (!collapsed) { return null; } @@ -266,7 +264,10 @@ function sanitizeExtractionList(value: unknown) { const cleaned: string[] = []; for (const entry of value) { - const normalized = sanitizeExtractionText(entry, EXTRACTION_ITEM_MAX_LENGTH); + const normalized = sanitizeExtractionText( + entry, + EXTRACTION_ITEM_MAX_LENGTH, + ); if (!normalized) { continue; } @@ -308,9 +309,9 @@ function uniqueExtractionList(items: Array) { function collectTextSignals(filingText: string, patterns: RegExp[]) { const lines = filingText - .replace(/\r/g, '\n') + .replace(/\r/g, "\n") .split(/\n+/) - .map((line) => line.replace(/\s+/g, ' ').trim()) + .map((line) => line.replace(/\s+/g, " ").trim()) .filter((line) => line.length >= 24); const matches: string[] = []; @@ -331,11 +332,13 @@ function collectTextSignals(filingText: string, patterns: RegExp[]) { function parseExtractionPayload(raw: string): FilingExtraction | null { const fencedJson = raw.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1]; - const candidate = fencedJson ?? (() => { - const start = raw.indexOf('{'); - const end = raw.lastIndexOf('}'); - return start >= 0 && end > start ? raw.slice(start, end + 1) : null; - })(); + const candidate = + fencedJson ?? + (() => { + const start = raw.indexOf("{"); + const end = raw.lastIndexOf("}"); + return start >= 0 && end > start ? raw.slice(start, end + 1) : null; + })(); if (!candidate) { return null; @@ -348,7 +351,7 @@ function parseExtractionPayload(raw: string): FilingExtraction | null { return null; } - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { return null; } @@ -365,35 +368,49 @@ function parseExtractionPayload(raw: string): FilingExtraction | null { } for (const key of keys) { - if (!EXTRACTION_REQUIRED_KEYS.includes(key as (typeof EXTRACTION_REQUIRED_KEYS)[number])) { + if ( + !EXTRACTION_REQUIRED_KEYS.includes( + key as (typeof EXTRACTION_REQUIRED_KEYS)[number], + ) + ) { return null; } } - const summary = sanitizeExtractionText(payload.summary, EXTRACTION_SUMMARY_MAX_LENGTH); + const summary = sanitizeExtractionText( + payload.summary, + EXTRACTION_SUMMARY_MAX_LENGTH, + ); const keyPoints = sanitizeExtractionList(payload.keyPoints); const redFlags = sanitizeExtractionList(payload.redFlags); const followUpQuestions = sanitizeExtractionList(payload.followUpQuestions); const portfolioSignals = sanitizeExtractionList(payload.portfolioSignals); - const segmentSpecificData = sanitizeExtractionList(payload.segmentSpecificData); - const geographicRevenueBreakdown = sanitizeExtractionList(payload.geographicRevenueBreakdown); - const companySpecificData = sanitizeExtractionList(payload.companySpecificData); + const segmentSpecificData = sanitizeExtractionList( + payload.segmentSpecificData, + ); + const geographicRevenueBreakdown = sanitizeExtractionList( + payload.geographicRevenueBreakdown, + ); + const companySpecificData = sanitizeExtractionList( + payload.companySpecificData, + ); const secApiCrossChecks = sanitizeExtractionList(payload.secApiCrossChecks); - const confidenceRaw = typeof payload.confidence === 'number' - ? payload.confidence - : Number(payload.confidence); + const confidenceRaw = + typeof payload.confidence === "number" + ? payload.confidence + : Number(payload.confidence); if ( - !summary - || !keyPoints - || !redFlags - || !followUpQuestions - || !portfolioSignals - || !segmentSpecificData - || !geographicRevenueBreakdown - || !companySpecificData - || !secApiCrossChecks - || !Number.isFinite(confidenceRaw) + !summary || + !keyPoints || + !redFlags || + !followUpQuestions || + !portfolioSignals || + !segmentSpecificData || + !geographicRevenueBreakdown || + !companySpecificData || + !secApiCrossChecks || + !Number.isFinite(confidenceRaw) ) { return null; } @@ -408,7 +425,7 @@ function parseExtractionPayload(raw: string): FilingExtraction | null { geographicRevenueBreakdown, companySpecificData, secApiCrossChecks, - confidence: Math.min(Math.max(confidenceRaw, 0), 1) + confidence: Math.min(Math.max(confidenceRaw, 0), 1), }; } @@ -417,7 +434,7 @@ function metricSnapshotLine(label: string, value: number | null | undefined) { return `${label}: not reported`; } - return `${label}: ${Math.round(value).toLocaleString('en-US')}`; + return `${label}: ${Math.round(value).toLocaleString("en-US")}`; } function buildSecApiCrossChecks(filing: Filing, filingText: string) { @@ -427,18 +444,22 @@ function buildSecApiCrossChecks(filing: Filing, filingText: string) { for (const descriptor of METRIC_CHECK_PATTERNS) { const value = filing.metrics?.[descriptor.key]; if (value === null || value === undefined || !Number.isFinite(value)) { - checks.push(`${descriptor.label}: SEC API metric unavailable for this filing.`); + checks.push( + `${descriptor.label}: SEC API metric unavailable for this filing.`, + ); continue; } - const hasMention = descriptor.patterns.some((pattern) => pattern.test(normalizedText)); + const hasMention = descriptor.patterns.some((pattern) => + pattern.test(normalizedText), + ); if (hasMention) { checks.push( - `${descriptor.label}: SEC API value ${Math.round(value).toLocaleString('en-US')} appears referenced in filing narrative.` + `${descriptor.label}: SEC API value ${Math.round(value).toLocaleString("en-US")} appears referenced in filing narrative.`, ); } else { checks.push( - `${descriptor.label}: SEC API value ${Math.round(value).toLocaleString('en-US')} was not confidently located in sampled filing text.` + `${descriptor.label}: SEC API value ${Math.round(value).toLocaleString("en-US")} was not confidently located in sampled filing text.`, ); } } @@ -453,52 +474,69 @@ function deterministicExtractionFallback(filing: Filing): FilingExtraction { summary: `${filing.company_name} ${filing.filing_type} filed on ${filing.filing_date}. Deterministic extraction fallback was used because filing text parsing was unavailable or invalid.`, keyPoints: [ `${filing.filing_type} filing recorded for ${filing.ticker}.`, - metricSnapshotLine('Revenue', metrics?.revenue), - metricSnapshotLine('Net income', metrics?.netIncome), - metricSnapshotLine('Total assets', metrics?.totalAssets) + metricSnapshotLine("Revenue", metrics?.revenue), + metricSnapshotLine("Net income", metrics?.netIncome), + metricSnapshotLine("Total assets", metrics?.totalAssets), ], redFlags: [ - metricSnapshotLine('Cash', metrics?.cash), - metricSnapshotLine('Debt', metrics?.debt), - filing.primary_document ? 'Primary document is indexed and available for review.' : 'Primary document reference is unavailable in current filing metadata.' + metricSnapshotLine("Cash", metrics?.cash), + metricSnapshotLine("Debt", metrics?.debt), + filing.primary_document + ? "Primary document is indexed and available for review." + : "Primary document reference is unavailable in current filing metadata.", ], followUpQuestions: [ - 'What changed versus the prior filing in guidance, margins, or liquidity?', - 'Are any material risks under-emphasized relative to historical filings?', - 'Should portfolio exposure be adjusted before the next reporting cycle?' + "What changed versus the prior filing in guidance, margins, or liquidity?", + "Are any material risks under-emphasized relative to historical filings?", + "Should portfolio exposure be adjusted before the next reporting cycle?", ], portfolioSignals: [ - 'Validate trend direction using at least two prior filings.', - 'Cross-check leverage and liquidity metrics against position sizing rules.', - 'Track language shifts around guidance or demand assumptions.' + "Validate trend direction using at least two prior filings.", + "Cross-check leverage and liquidity metrics against position sizing rules.", + "Track language shifts around guidance or demand assumptions.", ], segmentSpecificData: [ - 'Segment-level disclosures were not parsed in deterministic fallback mode.' + "Segment-level disclosures were not parsed in deterministic fallback mode.", ], geographicRevenueBreakdown: [ - 'Geographic revenue disclosures were not parsed in deterministic fallback mode.' + "Geographic revenue disclosures were not parsed in deterministic fallback mode.", ], companySpecificData: [ - 'Company-specific operating KPIs (for example same-store sales) were not parsed in deterministic fallback mode.' + "Company-specific operating KPIs (for example same-store sales) were not parsed in deterministic fallback mode.", ], secApiCrossChecks: [ - `${metricSnapshotLine('Revenue', metrics?.revenue)} (SEC API baseline; text verification unavailable).`, - `${metricSnapshotLine('Net income', metrics?.netIncome)} (SEC API baseline; text verification unavailable).` + `${metricSnapshotLine("Revenue", metrics?.revenue)} (SEC API baseline; text verification unavailable).`, + `${metricSnapshotLine("Net income", metrics?.netIncome)} (SEC API baseline; text verification unavailable).`, ], - confidence: 0.2 + confidence: 0.2, }; } -function buildRuleBasedExtraction(filing: Filing, filingText: string): FilingExtraction { +function buildRuleBasedExtraction( + filing: Filing, + filingText: string, +): FilingExtraction { const baseline = deterministicExtractionFallback(filing); const segmentSpecificData = collectTextSignals(filingText, SEGMENT_PATTERNS); - const geographicRevenueBreakdown = collectTextSignals(filingText, GEOGRAPHIC_PATTERNS); - const companySpecificData = collectTextSignals(filingText, COMPANY_SPECIFIC_PATTERNS); + const geographicRevenueBreakdown = collectTextSignals( + filingText, + GEOGRAPHIC_PATTERNS, + ); + const companySpecificData = collectTextSignals( + filingText, + COMPANY_SPECIFIC_PATTERNS, + ); const secApiCrossChecks = buildSecApiCrossChecks(filing, filingText); - const segmentLead = segmentSpecificData[0] ? `Segment detail: ${segmentSpecificData[0]}` : null; - const geographicLead = geographicRevenueBreakdown[0] ? `Geographic detail: ${geographicRevenueBreakdown[0]}` : null; - const companyLead = companySpecificData[0] ? `Company-specific KPI: ${companySpecificData[0]}` : null; + const segmentLead = segmentSpecificData[0] + ? `Segment detail: ${segmentSpecificData[0]}` + : null; + const geographicLead = geographicRevenueBreakdown[0] + ? `Geographic detail: ${geographicRevenueBreakdown[0]}` + : null; + const companyLead = companySpecificData[0] + ? `Company-specific KPI: ${companySpecificData[0]}` + : null; return { summary: `${filing.company_name} ${filing.filing_type} filed on ${filing.filing_date}. SEC API metrics were retained as the baseline and filing text was scanned for segment and company-specific disclosures.`, @@ -506,33 +544,47 @@ function buildRuleBasedExtraction(filing: Filing, filingText: string): FilingExt ...baseline.keyPoints, segmentLead, geographicLead, - companyLead + companyLead, ]), redFlags: uniqueExtractionList([ ...baseline.redFlags, - secApiCrossChecks.find((line) => /not confidently located/i.test(line)) + secApiCrossChecks.find((line) => /not confidently located/i.test(line)), ]), followUpQuestions: uniqueExtractionList([ ...baseline.followUpQuestions, - segmentSpecificData.length > 0 ? 'How do segment trends change the consolidated margin outlook?' : 'Does management provide segment-level KPIs in supplemental exhibits?' + segmentSpecificData.length > 0 + ? "How do segment trends change the consolidated margin outlook?" + : "Does management provide segment-level KPIs in supplemental exhibits?", ]), portfolioSignals: uniqueExtractionList([ ...baseline.portfolioSignals, - companySpecificData.length > 0 ? 'Incorporate company-specific KPI direction into near-term position sizing.' : 'Track future filings for explicit operating KPI disclosures.' + companySpecificData.length > 0 + ? "Incorporate company-specific KPI direction into near-term position sizing." + : "Track future filings for explicit operating KPI disclosures.", ]), - segmentSpecificData: segmentSpecificData.length > 0 - ? segmentSpecificData - : baseline.segmentSpecificData, - geographicRevenueBreakdown: geographicRevenueBreakdown.length > 0 - ? geographicRevenueBreakdown - : baseline.geographicRevenueBreakdown, - companySpecificData: companySpecificData.length > 0 - ? companySpecificData - : baseline.companySpecificData, - secApiCrossChecks: secApiCrossChecks.length > 0 - ? secApiCrossChecks - : baseline.secApiCrossChecks, - confidence: segmentSpecificData.length + geographicRevenueBreakdown.length + companySpecificData.length > 0 ? 0.4 : 0.3 + segmentSpecificData: + segmentSpecificData.length > 0 + ? segmentSpecificData + : baseline.segmentSpecificData, + geographicRevenueBreakdown: + geographicRevenueBreakdown.length > 0 + ? geographicRevenueBreakdown + : baseline.geographicRevenueBreakdown, + companySpecificData: + companySpecificData.length > 0 + ? companySpecificData + : baseline.companySpecificData, + secApiCrossChecks: + secApiCrossChecks.length > 0 + ? secApiCrossChecks + : baseline.secApiCrossChecks, + confidence: + segmentSpecificData.length + + geographicRevenueBreakdown.length + + companySpecificData.length > + 0 + ? 0.4 + : 0.3, }; } @@ -540,53 +592,74 @@ function preferExtractionList(primary: string[], fallback: string[]) { return primary.length > 0 ? primary : fallback; } -function mergeExtractionWithFallback(primary: FilingExtraction, fallback: FilingExtraction): FilingExtraction { +function mergeExtractionWithFallback( + primary: FilingExtraction, + fallback: FilingExtraction, +): FilingExtraction { return { summary: primary.summary || fallback.summary, keyPoints: preferExtractionList(primary.keyPoints, fallback.keyPoints), redFlags: preferExtractionList(primary.redFlags, fallback.redFlags), - followUpQuestions: preferExtractionList(primary.followUpQuestions, fallback.followUpQuestions), - portfolioSignals: preferExtractionList(primary.portfolioSignals, fallback.portfolioSignals), - segmentSpecificData: preferExtractionList(primary.segmentSpecificData, fallback.segmentSpecificData), - geographicRevenueBreakdown: preferExtractionList(primary.geographicRevenueBreakdown, fallback.geographicRevenueBreakdown), - companySpecificData: preferExtractionList(primary.companySpecificData, fallback.companySpecificData), - secApiCrossChecks: preferExtractionList(primary.secApiCrossChecks, fallback.secApiCrossChecks), - confidence: Math.min(Math.max(primary.confidence, 0), 1) + followUpQuestions: preferExtractionList( + primary.followUpQuestions, + fallback.followUpQuestions, + ), + portfolioSignals: preferExtractionList( + primary.portfolioSignals, + fallback.portfolioSignals, + ), + segmentSpecificData: preferExtractionList( + primary.segmentSpecificData, + fallback.segmentSpecificData, + ), + geographicRevenueBreakdown: preferExtractionList( + primary.geographicRevenueBreakdown, + fallback.geographicRevenueBreakdown, + ), + companySpecificData: preferExtractionList( + primary.companySpecificData, + fallback.companySpecificData, + ), + secApiCrossChecks: preferExtractionList( + primary.secApiCrossChecks, + fallback.secApiCrossChecks, + ), + confidence: Math.min(Math.max(primary.confidence, 0), 1), }; } function extractionPrompt(filing: Filing, filingText: string) { return [ - 'Extract structured signals from the SEC filing text.', + "Extract structured signals from the SEC filing text.", `Company: ${filing.company_name} (${filing.ticker})`, `Form: ${filing.filing_type}`, `Filed: ${filing.filing_date}`, `SEC API baseline metrics: ${JSON.stringify(filing.metrics ?? {})}`, - 'Use SEC API metrics as canonical numeric values and validate whether each appears consistent with filing text context.', - 'Prioritize company-specific and segment-specific disclosures not covered by SEC endpoint fields (for example same-store sales, geographic mix, segment margin).', - 'Return ONLY valid JSON with exactly these keys and no extra keys:', + "Use SEC API metrics as canonical numeric values and validate whether each appears consistent with filing text context.", + "Prioritize company-specific and segment-specific disclosures not covered by SEC endpoint fields (for example same-store sales, geographic mix, segment margin).", + "Return ONLY valid JSON with exactly these keys and no extra keys:", '{"summary":"string","keyPoints":["string"],"redFlags":["string"],"followUpQuestions":["string"],"portfolioSignals":["string"],"segmentSpecificData":["string"],"geographicRevenueBreakdown":["string"],"companySpecificData":["string"],"secApiCrossChecks":["string"],"confidence":0}', `Rules: every array max ${EXTRACTION_MAX_ITEMS} items; each item <= ${EXTRACTION_ITEM_MAX_LENGTH} chars; summary <= ${EXTRACTION_SUMMARY_MAX_LENGTH} chars; confidence between 0 and 1.`, - 'Filing text follows:', - filingText - ].join('\n\n'); + "Filing text follows:", + filingText, + ].join("\n\n"); } function reportPrompt( filing: Filing, extraction: FilingExtraction, - extractionMeta: FilingExtractionMeta + extractionMeta: FilingExtractionMeta, ) { return [ - 'You are a fiscal research assistant focused on regulatory signals.', + "You are a fiscal research assistant focused on regulatory signals.", `Analyze this SEC filing from ${filing.company_name} (${filing.ticker}).`, `Form: ${filing.filing_type}`, `Filed: ${filing.filing_date}`, `SEC API baseline metrics: ${JSON.stringify(filing.metrics ?? {})}`, `Structured extraction context (${extractionMeta.source}): ${JSON.stringify(extraction)}`, - 'Use SEC API values as the baseline financials and explicitly reference segment/company-specific details from extraction.', - 'Return concise sections: Thesis, Red Flags, Follow-up Questions, Portfolio Impact.' - ].join('\n'); + "Use SEC API values as the baseline financials and explicitly reference segment/company-specific details from extraction.", + "Return concise sections: Thesis, Red Flags, Follow-up Questions, Portfolio Impact.", + ].join("\n"); } function filingLinks(filing: { @@ -596,11 +669,11 @@ function filingLinks(filing: { const links: Array<{ link_type: string; url: string }> = []; if (filing.filingUrl) { - links.push({ link_type: 'primary_document', url: filing.filingUrl }); + links.push({ link_type: "primary_document", url: filing.filingUrl }); } if (filing.submissionUrl) { - links.push({ link_type: 'submission_index', url: filing.submissionUrl }); + links.push({ link_type: "submission_index", url: filing.submissionUrl }); } return links; @@ -613,30 +686,30 @@ async function processSyncFilings(task: Task) { const tags = parseTags(task.payload.tags); const scopeLabel = [ category, - tags.length > 0 ? `tags: ${tags.join(', ')}` : null + tags.length > 0 ? `tags: ${tags.join(", ")}` : null, ] .filter((entry): entry is string => Boolean(entry)) - .join(' | '); + .join(" | "); let searchTaskId: string | null = null; const tickerSubject = { ticker }; await setProjectionStage( task, - 'sync.fetch_filings', - `Fetching up to ${limit} filings for ${ticker}${scopeLabel ? ` (${scopeLabel})` : ''}`, - { subject: tickerSubject } + "sync.fetch_filings", + `Fetching up to ${limit} filings for ${ticker}${scopeLabel ? ` (${scopeLabel})` : ""}`, + { subject: tickerSubject }, ); const filings = await fetchRecentFilings(ticker, limit); await setProjectionStage( task, - 'sync.persist_filings', + "sync.persist_filings", `Persisting ${filings.length} filings and source links`, { counters: { fetched: filings.length }, - subject: tickerSubject - } + subject: tickerSubject, + }, ); const saveResult = await upsertFilingsRecords( filings.map((filing) => ({ @@ -650,72 +723,77 @@ async function processSyncFilings(task: Task) { submission_url: filing.submissionUrl, primary_document: filing.primaryDocument, metrics: null, - links: filingLinks(filing) - })) + links: filingLinks(filing), + })), ); let taxonomySnapshotsHydrated = 0; let taxonomySnapshotsFailed = 0; - const hydrateCandidates = (await listFilingsRecords({ - ticker, - limit: Math.min(Math.max(limit * 3, 40), STATEMENT_HYDRATION_MAX_FILINGS) - })) - .filter((filing): filing is Filing & { filing_type: '10-K' | '10-Q' } => { - return isFinancialMetricsForm(filing.filing_type); - }); + const hydrateCandidates = ( + await listFilingsRecords({ + ticker, + limit: Math.min(Math.max(limit * 3, 40), STATEMENT_HYDRATION_MAX_FILINGS), + }) + ).filter((filing): filing is Filing & { filing_type: "10-K" | "10-Q" } => { + return isFinancialMetricsForm(filing.filing_type); + }); await setProjectionStage( task, - 'sync.discover_assets', + "sync.discover_assets", `Discovering taxonomy assets for ${hydrateCandidates.length} candidate filings`, buildProgressContext({ current: 0, total: hydrateCandidates.length, - unit: 'filings', + unit: "filings", counters: { fetched: filings.length, inserted: saveResult.inserted, updated: saveResult.updated, hydrated: 0, - failed: 0 + failed: 0, }, - subject: tickerSubject - }) + subject: tickerSubject, + }), ); for (let index = 0; index < hydrateCandidates.length; index += 1) { const filing = hydrateCandidates[index]; - const existingSnapshot = await getFilingTaxonomySnapshotByFilingId(filing.id); - const shouldRefresh = !existingSnapshot - || Date.parse(existingSnapshot.updated_at) < Date.parse(filing.updated_at); + const existingSnapshot = await getFilingTaxonomySnapshotByFilingId( + filing.id, + ); + const shouldRefresh = + !existingSnapshot || + Date.parse(existingSnapshot.updated_at) < Date.parse(filing.updated_at); if (!shouldRefresh) { continue; } - const stageContext = (stage: TaskStage) => buildProgressContext({ - current: index + 1, - total: hydrateCandidates.length, - unit: 'filings', - counters: { - fetched: filings.length, - inserted: saveResult.inserted, - updated: saveResult.updated, - hydrated: taxonomySnapshotsHydrated, - failed: taxonomySnapshotsFailed - }, - subject: { - ticker, - accessionNumber: filing.accession_number, - label: stage - } - }); + const stageContext = (stage: TaskStage) => + buildProgressContext({ + current: index + 1, + total: hydrateCandidates.length, + unit: "filings", + counters: { + fetched: filings.length, + inserted: saveResult.inserted, + updated: saveResult.updated, + hydrated: taxonomySnapshotsHydrated, + failed: taxonomySnapshotsFailed, + }, + subject: { + ticker, + accessionNumber: filing.accession_number, + label: stage, + }, + }); try { await setProjectionStage( task, - 'sync.extract_taxonomy', + "sync.extract_taxonomy", `Extracting XBRL taxonomy for ${filing.accession_number}`, - stageContext('sync.extract_taxonomy') + stageContext("sync.extract_taxonomy"), ); const snapshot = await hydrateFilingTaxonomySnapshot({ filingId: filing.id, @@ -725,40 +803,73 @@ async function processSyncFilings(task: Task) { filingDate: filing.filing_date, filingType: filing.filing_type, filingUrl: filing.filing_url, - primaryDocument: filing.primary_document ?? null + primaryDocument: filing.primary_document ?? null, }); + let pdfValidation = { + validation_result: snapshot.validation_result, + metric_validations: snapshot.metric_validations, + }; + + try { + pdfValidation = await validateMetricsWithPdfLlm({ + metrics: snapshot.derived_metrics, + assets: snapshot.assets, + }); + } catch (error) { + const message = + error instanceof Error + ? error.message + : "PDF metric validation failed"; + pdfValidation = { + validation_result: { + status: "error", + checks: [], + validatedAt: new Date().toISOString(), + }, + metric_validations: snapshot.metric_validations.map((check) => ({ + ...check, + error: check.error ?? message, + })), + }; + } + const normalizedSnapshot = { ...snapshot, - ...normalizeFilingTaxonomySnapshotPayload(snapshot) + validation_result: pdfValidation.validation_result, + metric_validations: pdfValidation.metric_validations, + ...normalizeFilingTaxonomySnapshotPayload(snapshot), }; await setProjectionStage( task, - 'sync.normalize_taxonomy', + "sync.normalize_taxonomy", `Materializing statements for ${filing.accession_number}`, - stageContext('sync.normalize_taxonomy') + stageContext("sync.normalize_taxonomy"), ); await setProjectionStage( task, - 'sync.derive_metrics', + "sync.derive_metrics", `Deriving taxonomy metrics for ${filing.accession_number}`, - stageContext('sync.derive_metrics') + stageContext("sync.derive_metrics"), ); await setProjectionStage( task, - 'sync.validate_pdf_metrics', + "sync.validate_pdf_metrics", `Validating metrics via PDF + LLM for ${filing.accession_number}`, - stageContext('sync.validate_pdf_metrics') + stageContext("sync.validate_pdf_metrics"), ); await setProjectionStage( task, - 'sync.persist_taxonomy', + "sync.persist_taxonomy", `Persisting taxonomy snapshot for ${filing.accession_number}`, - stageContext('sync.persist_taxonomy') + stageContext("sync.persist_taxonomy"), ); await upsertFilingTaxonomySnapshot(normalizedSnapshot); - await updateFilingMetricsById(filing.id, normalizedSnapshot.derived_metrics); + await updateFilingMetricsById( + filing.id, + normalizedSnapshot.derived_metrics, + ); await deleteCompanyFinancialBundlesForTicker(filing.ticker); taxonomySnapshotsHydrated += 1; } catch (error) { @@ -768,49 +879,51 @@ async function processSyncFilings(task: Task) { ticker: filing.ticker, filing_date: filing.filing_date, filing_type: filing.filing_type, - parse_status: 'failed', - parse_error: error instanceof Error ? error.message : 'Taxonomy hydration failed', - source: 'legacy_html_fallback', - parser_engine: 'fiscal-xbrl', - parser_version: 'unknown', - taxonomy_regime: 'unknown', - fiscal_pack: 'core', + parse_status: "failed", + parse_error: + error instanceof Error ? error.message : "Taxonomy hydration failed", + source: "legacy_html_fallback", + parser_engine: "fiscal-xbrl", + parser_version: "unknown", + taxonomy_regime: "unknown", + fiscal_pack: "core", periods: [], faithful_rows: { income: [], balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }, statement_rows: { income: [], balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }, surface_rows: { income: [], balance: [], cash_flow: [], equity: [], - comprehensive_income: [] + comprehensive_income: [], }, detail_rows: { income: {}, balance: {}, cash_flow: {}, equity: {}, - comprehensive_income: {} + comprehensive_income: {}, }, kpi_rows: [], + computed_definitions: [], contexts: [], derived_metrics: filing.metrics ?? null, validation_result: { - status: 'error', + status: "error", checks: [], - validatedAt: now + validatedAt: now, }, normalization_summary: { surfaceRowCount: 0, @@ -818,7 +931,7 @@ async function processSyncFilings(task: Task) { kpiRowCount: 0, unmappedRowCount: 0, materialUnmappedRowCount: 0, - warnings: [] + warnings: [], }, facts_count: 0, concepts_count: 0, @@ -826,7 +939,7 @@ async function processSyncFilings(task: Task) { assets: [], concepts: [], facts: [], - metric_validations: [] + metric_validations: [], }); await deleteCompanyFinancialBundlesForTicker(filing.ticker); taxonomySnapshotsFailed += 1; @@ -838,13 +951,13 @@ async function processSyncFilings(task: Task) { try { const searchTask = await enqueueTask({ userId: task.user_id, - taskType: 'index_search', + taskType: "index_search", payload: { ticker, - sourceKinds: ['filing_document', 'filing_brief'] + sourceKinds: ["filing_document", "filing_brief"], }, priority: 55, - resourceKey: `index_search:ticker:${ticker}` + resourceKey: `index_search:ticker:${ticker}`, }); searchTaskId = searchTask.id; } catch (error) { @@ -860,7 +973,7 @@ async function processSyncFilings(task: Task) { updated: saveResult.updated, taxonomySnapshotsHydrated, taxonomySnapshotsFailed, - searchTaskId + searchTaskId, }; return buildTaskOutcome( @@ -869,26 +982,30 @@ async function processSyncFilings(task: Task) { buildProgressContext({ current: hydrateCandidates.length, total: hydrateCandidates.length || 1, - unit: 'filings', + unit: "filings", counters: { fetched: filings.length, inserted: saveResult.inserted, updated: saveResult.updated, hydrated: taxonomySnapshotsHydrated, - failed: taxonomySnapshotsFailed + failed: taxonomySnapshotsFailed, }, - subject: tickerSubject - }) + subject: tickerSubject, + }), ); } async function processRefreshPrices(task: Task) { const userId = task.user_id; if (!userId) { - throw new Error('Task is missing user scope'); + throw new Error("Task is missing user scope"); } - await setProjectionStage(task, 'refresh.load_holdings', 'Loading holdings for price refresh'); + await setProjectionStage( + task, + "refresh.load_holdings", + "Loading holdings for price refresh", + ); const userHoldings = await listHoldingsForPriceRefresh(userId); const tickers = [...new Set(userHoldings.map((entry) => entry.ticker))]; const quotes = new Map(); @@ -896,29 +1013,29 @@ async function processRefreshPrices(task: Task) { const staleTickers: string[] = []; const baseContext = { counters: { - holdings: userHoldings.length - } + holdings: userHoldings.length, + }, } satisfies TaskStageContext; await setProjectionStage( task, - 'refresh.load_holdings', + "refresh.load_holdings", `Loaded ${userHoldings.length} holdings across ${tickers.length} tickers`, - baseContext + baseContext, ); await setProjectionStage( task, - 'refresh.fetch_quotes', + "refresh.fetch_quotes", `Fetching quotes for ${tickers.length} tickers`, buildProgressContext({ current: 0, total: tickers.length, - unit: 'tickers', + unit: "tickers", counters: { - holdings: userHoldings.length - } - }) + holdings: userHoldings.length, + }, + }), ); for (let index = 0; index < tickers.length; index += 1) { const ticker = tickers[index]; @@ -933,44 +1050,50 @@ async function processRefreshPrices(task: Task) { } await setProjectionStage( task, - 'refresh.fetch_quotes', + "refresh.fetch_quotes", `Fetching quotes for ${tickers.length} tickers`, buildProgressContext({ current: index + 1, total: tickers.length, - unit: 'tickers', + unit: "tickers", counters: { holdings: userHoldings.length, failed: failedTickers.length, - stale: staleTickers.length + stale: staleTickers.length, }, - subject: { ticker } - }) + subject: { ticker }, + }), ); } await setProjectionStage( task, - 'refresh.persist_prices', + "refresh.persist_prices", `Writing refreshed prices for ${quotes.size} tickers across ${userHoldings.length} holdings`, { counters: { holdings: userHoldings.length, failed: failedTickers.length, - stale: staleTickers.length - } - } + stale: staleTickers.length, + }, + }, + ); + const updatedCount = await applyRefreshedPrices( + userId, + quotes, + new Date().toISOString(), ); - const updatedCount = await applyRefreshedPrices(userId, quotes, new Date().toISOString()); const result = { updatedCount, totalTickers: tickers.length, failedTickers, - staleTickers + staleTickers, }; - const messageParts = [`Refreshed prices for ${quotes.size}/${tickers.length} tickers`]; + const messageParts = [ + `Refreshed prices for ${quotes.size}/${tickers.length} tickers`, + ]; if (failedTickers.length > 0) { messageParts.push(`(${failedTickers.length} unavailable)`); } @@ -980,37 +1103,43 @@ async function processRefreshPrices(task: Task) { return buildTaskOutcome( result, - `${messageParts.join(' ')} across ${userHoldings.length} holdings.`, + `${messageParts.join(" ")} across ${userHoldings.length} holdings.`, { progress: { current: tickers.length, total: tickers.length || 1, - unit: 'tickers' + unit: "tickers", }, counters: { holdings: userHoldings.length, updatedCount, failed: failedTickers.length, - stale: staleTickers.length - } - } + stale: staleTickers.length, + }, + }, ); } async function processAnalyzeFiling(task: Task) { - const accessionNumber = typeof task.payload.accessionNumber === 'string' - ? task.payload.accessionNumber - : ''; + const accessionNumber = + typeof task.payload.accessionNumber === "string" + ? task.payload.accessionNumber + : ""; if (!accessionNumber) { - throw new Error('accessionNumber is required'); + throw new Error("accessionNumber is required"); } - await setProjectionStage(task, 'analyze.load_filing', `Loading filing ${accessionNumber}`, { - subject: { - accessionNumber - } - }); + await setProjectionStage( + task, + "analyze.load_filing", + `Loading filing ${accessionNumber}`, + { + subject: { + accessionNumber, + }, + }, + ); const filing = await getFilingByAccession(accessionNumber); if (!filing) { @@ -1020,93 +1149,121 @@ async function processAnalyzeFiling(task: Task) { const analyzeSubject = { ticker: filing.ticker, accessionNumber, - label: filing.filing_type + label: filing.filing_type, }; const defaultExtraction = deterministicExtractionFallback(filing); let extraction = defaultExtraction; let extractionMeta: FilingExtractionMeta = { - provider: 'deterministic-fallback', - model: 'metadata-fallback', - source: 'metadata_fallback', - generatedAt: new Date().toISOString() + provider: "deterministic-fallback", + model: "metadata-fallback", + source: "metadata_fallback", + generatedAt: new Date().toISOString(), }; - let filingDocument: Awaited> | null = null; + let filingDocument: Awaited< + ReturnType + > | null = null; try { - await setProjectionStage(task, 'analyze.fetch_document', 'Fetching primary filing document', { - subject: analyzeSubject - }); + await setProjectionStage( + task, + "analyze.fetch_document", + "Fetching primary filing document", + { + subject: analyzeSubject, + }, + ); filingDocument = await fetchPrimaryFilingText({ filingUrl: filing.filing_url, cik: filing.cik, accessionNumber: filing.accession_number, - primaryDocument: filing.primary_document ?? null + primaryDocument: filing.primary_document ?? null, }); } catch { filingDocument = null; } if (filingDocument?.text) { - await setProjectionStage(task, 'analyze.extract', 'Generating extraction context from filing text', { - subject: analyzeSubject - }); - const ruleBasedExtraction = buildRuleBasedExtraction(filing, filingDocument.text); + await setProjectionStage( + task, + "analyze.extract", + "Generating extraction context from filing text", + { + subject: analyzeSubject, + }, + ); + const ruleBasedExtraction = buildRuleBasedExtraction( + filing, + filingDocument.text, + ); const extractionResult = await runAiAnalysis( extractionPrompt(filing, filingDocument.text), - 'Return strict JSON only.', - { workload: 'extraction' } + "Return strict JSON only.", + { workload: "extraction" }, ); const parsed = parseExtractionPayload(extractionResult.text); if (!parsed) { - throw new Error('Extraction output invalid JSON schema'); + throw new Error("Extraction output invalid JSON schema"); } extraction = mergeExtractionWithFallback(parsed, ruleBasedExtraction); extractionMeta = { - provider: 'zhipu', + provider: "zhipu", model: extractionResult.model, source: filingDocument.source, - generatedAt: new Date().toISOString() + generatedAt: new Date().toISOString(), }; } - await setProjectionStage(task, 'analyze.generate_report', 'Generating final filing analysis report', { - subject: analyzeSubject - }); + await setProjectionStage( + task, + "analyze.generate_report", + "Generating final filing analysis report", + { + subject: analyzeSubject, + }, + ); const analysis = await runAiAnalysis( reportPrompt(filing, extraction, extractionMeta), - 'Use concise institutional analyst language.', - { workload: 'report' } + "Use concise institutional analyst language.", + { workload: "report" }, ); - await setProjectionStage(task, 'analyze.persist_report', 'Persisting filing analysis output', { - subject: analyzeSubject - }); + await setProjectionStage( + task, + "analyze.persist_report", + "Persisting filing analysis output", + { + subject: analyzeSubject, + }, + ); await saveFilingAnalysis(accessionNumber, { provider: analysis.provider, model: analysis.model, text: analysis.text, extraction, - extractionMeta + extractionMeta, }); let searchTaskId: string | null = null; try { const searchTask = await enqueueTask({ userId: task.user_id, - taskType: 'index_search', + taskType: "index_search", payload: { accessionNumber, - sourceKinds: ['filing_brief'] + sourceKinds: ["filing_brief"], }, priority: 58, - resourceKey: `index_search:filing_brief:${accessionNumber}` + resourceKey: `index_search:filing_brief:${accessionNumber}`, }); searchTaskId = searchTask.id; } catch (error) { - console.error(`[search-index-analyze] failed for ${accessionNumber}:`, error); + console.error( + `[search-index-analyze] failed for ${accessionNumber}:`, + error, + ); } const result = { @@ -1117,54 +1274,69 @@ async function processAnalyzeFiling(task: Task) { model: analysis.model, extractionProvider: extractionMeta.provider, extractionModel: extractionMeta.model, - searchTaskId + searchTaskId, }; return buildTaskOutcome( result, `Analysis report generated for ${filing.ticker} ${filing.filing_type} ${accessionNumber}.`, { - subject: analyzeSubject - } + subject: analyzeSubject, + }, ); } async function processIndexSearch(task: Task) { - await setProjectionStage(task, 'search.collect_sources', 'Collecting source records for search indexing'); + await setProjectionStage( + task, + "search.collect_sources", + "Collecting source records for search indexing", + ); const ticker = parseOptionalText(task.payload.ticker); const accessionNumber = parseOptionalText(task.payload.accessionNumber); - const journalEntryId = task.payload.journalEntryId === undefined - ? null - : Number(task.payload.journalEntryId); + const journalEntryId = + task.payload.journalEntryId === undefined + ? null + : Number(task.payload.journalEntryId); const deleteSourceRefs = Array.isArray(task.payload.deleteSourceRefs) - ? task.payload.deleteSourceRefs - .filter((entry): entry is { - sourceKind: string; - sourceRef: string; - scope: string; - userId?: string | null; - } => { - return Boolean( - entry - && typeof entry === 'object' - && typeof (entry as { sourceKind?: unknown }).sourceKind === 'string' - && typeof (entry as { sourceRef?: unknown }).sourceRef === 'string' - && typeof (entry as { scope?: unknown }).scope === 'string' - ); - }) + ? task.payload.deleteSourceRefs.filter( + ( + entry, + ): entry is { + sourceKind: string; + sourceRef: string; + scope: string; + userId?: string | null; + } => { + return Boolean( + entry && + typeof entry === "object" && + typeof (entry as { sourceKind?: unknown }).sourceKind === + "string" && + typeof (entry as { sourceRef?: unknown }).sourceRef === "string" && + typeof (entry as { scope?: unknown }).scope === "string", + ); + }, + ) : []; - const sourceKinds = parseOptionalStringArray(task.payload.sourceKinds) - .filter((sourceKind): sourceKind is 'filing_document' | 'filing_brief' | 'research_note' => { - return sourceKind === 'filing_document' - || sourceKind === 'filing_brief' - || sourceKind === 'research_note'; - }); - const validatedJournalEntryId = typeof journalEntryId === 'number' - && Number.isInteger(journalEntryId) - && journalEntryId > 0 - ? journalEntryId - : null; + const sourceKinds = parseOptionalStringArray(task.payload.sourceKinds).filter( + ( + sourceKind, + ): sourceKind is "filing_document" | "filing_brief" | "research_note" => { + return ( + sourceKind === "filing_document" || + sourceKind === "filing_brief" || + sourceKind === "research_note" + ); + }, + ); + const validatedJournalEntryId = + typeof journalEntryId === "number" && + Number.isInteger(journalEntryId) && + journalEntryId > 0 + ? journalEntryId + : null; const result = await indexSearchDocuments({ userId: task.user_id, @@ -1173,39 +1345,71 @@ async function processIndexSearch(task: Task) { journalEntryId: validatedJournalEntryId, sourceKinds: sourceKinds.length > 0 ? sourceKinds : undefined, deleteSourceRefs: deleteSourceRefs.map((entry) => ({ - sourceKind: entry.sourceKind as 'filing_document' | 'filing_brief' | 'research_note', + sourceKind: entry.sourceKind as + | "filing_document" + | "filing_brief" + | "research_note", sourceRef: entry.sourceRef, - scope: entry.scope === 'user' ? 'user' : 'global', - userId: typeof entry.userId === 'string' ? entry.userId : null + scope: entry.scope === "user" ? "user" : "global", + userId: typeof entry.userId === "string" ? entry.userId : null, })), onStage: async (stage, detail, context) => { switch (stage) { - case 'collect': - await setProjectionStage(task, 'search.collect_sources', detail, context ?? { - subject: ticker ? { ticker } : accessionNumber ? { accessionNumber } : null - }); + case "collect": + await setProjectionStage( + task, + "search.collect_sources", + detail, + context ?? { + subject: ticker + ? { ticker } + : accessionNumber + ? { accessionNumber } + : null, + }, + ); break; - case 'fetch': - await setProjectionStage(task, 'search.fetch_documents', detail, context ?? null); + case "fetch": + await setProjectionStage( + task, + "search.fetch_documents", + detail, + context ?? null, + ); break; - case 'chunk': - await setProjectionStage(task, 'search.chunk', detail, context ?? null); + case "chunk": + await setProjectionStage( + task, + "search.chunk", + detail, + context ?? null, + ); break; - case 'embed': - await setProjectionStage(task, 'search.embed', detail, context ?? null); + case "embed": + await setProjectionStage( + task, + "search.embed", + detail, + context ?? null, + ); break; - case 'persist': - await setProjectionStage(task, 'search.persist', detail, context ?? null); + case "persist": + await setProjectionStage( + task, + "search.persist", + detail, + context ?? null, + ); break; } - } + }, }); const taskResult = { ticker, accessionNumber, journalEntryId: validatedJournalEntryId, - ...result + ...result, }; return buildTaskOutcome( @@ -1215,17 +1419,21 @@ async function processIndexSearch(task: Task) { progress: { current: result.sourcesCollected, total: result.sourcesCollected || 1, - unit: 'sources' + unit: "sources", }, counters: { sourcesCollected: result.sourcesCollected, indexed: result.indexed, chunksEmbedded: result.chunksEmbedded, skipped: result.skipped, - deleted: result.deleted + deleted: result.deleted, }, - subject: ticker ? { ticker } : accessionNumber ? { accessionNumber } : null - } + subject: ticker + ? { ticker } + : accessionNumber + ? { accessionNumber } + : null, + }, ); } @@ -1237,58 +1445,72 @@ function holdingDigest(holdings: Holding[]) { currentPrice: holding.current_price, marketValue: holding.market_value, gainLoss: holding.gain_loss, - gainLossPct: holding.gain_loss_pct + gainLossPct: holding.gain_loss_pct, })); } async function processPortfolioInsights(task: Task) { const userId = task.user_id; if (!userId) { - throw new Error('Task is missing user scope'); + throw new Error("Task is missing user scope"); } - await setProjectionStage(task, 'insights.load_holdings', 'Loading holdings for portfolio insight generation'); + await setProjectionStage( + task, + "insights.load_holdings", + "Loading holdings for portfolio insight generation", + ); const userHoldings = await listUserHoldings(userId); const summary = buildPortfolioSummary(userHoldings); const holdingsContext = { counters: { - holdings: userHoldings.length - } + holdings: userHoldings.length, + }, } satisfies TaskStageContext; await setProjectionStage( task, - 'insights.load_holdings', + "insights.load_holdings", `Loaded ${userHoldings.length} holdings for portfolio insight generation`, - holdingsContext + holdingsContext, ); const prompt = [ - 'Generate portfolio intelligence with actionable recommendations.', + "Generate portfolio intelligence with actionable recommendations.", `Portfolio summary: ${JSON.stringify(summary)}`, `Holdings: ${JSON.stringify(holdingDigest(userHoldings))}`, - 'Respond with: 1) health score (0-100), 2) top 3 risks, 3) top 3 opportunities, 4) next actions in 7 days.' - ].join('\n'); + "Respond with: 1) health score (0-100), 2) top 3 risks, 3) top 3 opportunities, 4) next actions in 7 days.", + ].join("\n"); - await setProjectionStage(task, 'insights.generate', 'Generating portfolio AI insight', holdingsContext); + await setProjectionStage( + task, + "insights.generate", + "Generating portfolio AI insight", + holdingsContext, + ); const analysis = await runAiAnalysis( prompt, - 'Act as a risk-aware buy-side analyst.', - { workload: 'report' } + "Act as a risk-aware buy-side analyst.", + { workload: "report" }, ); - await setProjectionStage(task, 'insights.persist', 'Persisting generated portfolio insight', holdingsContext); + await setProjectionStage( + task, + "insights.persist", + "Persisting generated portfolio insight", + holdingsContext, + ); await createPortfolioInsight({ userId, provider: analysis.provider, model: analysis.model, - content: analysis.text + content: analysis.text, }); const result = { provider: analysis.provider, model: analysis.model, - summary + summary, }; return buildTaskOutcome( @@ -1296,29 +1518,29 @@ async function processPortfolioInsights(task: Task) { `Generated portfolio insight for ${summary.positions} holdings.`, { counters: { - holdings: summary.positions - } - } + holdings: summary.positions, + }, + }, ); } export const __taskProcessorInternals = { parseExtractionPayload, deterministicExtractionFallback, - isFinancialMetricsForm + isFinancialMetricsForm, }; export async function runTaskProcessor(task: Task) { switch (task.task_type) { - case 'sync_filings': + case "sync_filings": return await processSyncFilings(task); - case 'refresh_prices': + case "refresh_prices": return await processRefreshPrices(task); - case 'analyze_filing': + case "analyze_filing": return await processAnalyzeFiling(task); - case 'portfolio_insights': + case "portfolio_insights": return await processPortfolioInsights(task); - case 'index_search': + case "index_search": return await processIndexSearch(task); default: throw new Error(`Unsupported task type: ${task.task_type}`); diff --git a/lib/server/taxonomy/engine.test.ts b/lib/server/taxonomy/engine.test.ts index 8086be4..4f7b98a 100644 --- a/lib/server/taxonomy/engine.test.ts +++ b/lib/server/taxonomy/engine.test.ts @@ -1,37 +1,43 @@ -import { beforeEach, describe, expect, it, mock } from 'bun:test'; +import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { FinancialStatementKind } from '@/lib/types'; -import type { TaxonomyHydrationInput, TaxonomyHydrationResult } from '@/lib/server/taxonomy/types'; +import type { FinancialStatementKind } from "@/lib/types"; +import type { + TaxonomyHydrationInput, + TaxonomyHydrationResult, +} from "@/lib/server/taxonomy/types"; -function createStatementRecord(factory: () => T): Record { +function createStatementRecord( + factory: () => T, +): Record { return { income: factory(), balance: factory(), cash_flow: factory(), equity: factory(), - comprehensive_income: factory() + comprehensive_income: factory(), }; } function createHydrationResult(): TaxonomyHydrationResult { return { filing_id: 1, - ticker: 'TEST', - filing_date: '2025-12-31', - filing_type: '10-K', - parse_status: 'ready', + ticker: "TEST", + filing_date: "2025-12-31", + filing_type: "10-K", + parse_status: "ready", parse_error: null, - source: 'xbrl_instance_with_linkbase', - parser_engine: 'fiscal-xbrl', - parser_version: '0.1.0', - taxonomy_regime: 'us-gaap', - fiscal_pack: 'core', + source: "xbrl_instance_with_linkbase", + parser_engine: "fiscal-xbrl", + parser_version: "0.1.0", + taxonomy_regime: "us-gaap", + fiscal_pack: "core", periods: [], faithful_rows: createStatementRecord(() => []), statement_rows: createStatementRecord(() => []), surface_rows: createStatementRecord(() => []), detail_rows: createStatementRecord(() => ({})), kpi_rows: [], + computed_definitions: [], contexts: [], derived_metrics: null, validation_result: null, @@ -48,42 +54,44 @@ function createHydrationResult(): TaxonomyHydrationResult { kpi_row_count: 0, unmapped_row_count: 0, material_unmapped_row_count: 0, - warnings: ['rust_warning'] + warnings: ["rust_warning"], }, xbrl_validation: { - status: 'passed' - } + status: "passed", + }, }; } const mockHydrateFromSidecar = mock(async () => createHydrationResult()); -mock.module('@/lib/server/taxonomy/parser-client', () => ({ - hydrateFilingTaxonomySnapshotFromSidecar: mockHydrateFromSidecar +mock.module("@/lib/server/taxonomy/parser-client", () => ({ + hydrateFilingTaxonomySnapshotFromSidecar: mockHydrateFromSidecar, })); -describe('taxonomy engine rust path', () => { +describe("taxonomy engine rust path", () => { beforeEach(() => { mockHydrateFromSidecar.mockClear(); }); - it('returns sidecar output directly from the Rust sidecar', async () => { - const { hydrateFilingTaxonomySnapshot } = await import('@/lib/server/taxonomy/engine'); + it("returns sidecar output directly from the Rust sidecar", async () => { + const { hydrateFilingTaxonomySnapshot } = + await import("@/lib/server/taxonomy/engine"); const input: TaxonomyHydrationInput = { filingId: 1, - ticker: 'TEST', - cik: '0000000001', - accessionNumber: '0000000001-25-000001', - filingDate: '2025-12-31', - filingType: '10-K', - filingUrl: 'https://www.sec.gov/Archives/edgar/data/1/000000000125000001/', - primaryDocument: 'test-20251231.htm' + ticker: "TEST", + cik: "0000000001", + accessionNumber: "0000000001-25-000001", + filingDate: "2025-12-31", + filingType: "10-K", + filingUrl: + "https://www.sec.gov/Archives/edgar/data/1/000000000125000001/", + primaryDocument: "test-20251231.htm", }; const result = await hydrateFilingTaxonomySnapshot(input); expect(mockHydrateFromSidecar).toHaveBeenCalledTimes(1); - expect(result.parser_engine).toBe('fiscal-xbrl'); - expect(result.normalization_summary.warnings).toEqual(['rust_warning']); + expect(result.parser_engine).toBe("fiscal-xbrl"); + expect(result.normalization_summary.warnings).toEqual(["rust_warning"]); }); }); diff --git a/lib/server/taxonomy/parser-client.test.ts b/lib/server/taxonomy/parser-client.test.ts new file mode 100644 index 0000000..3efd645 --- /dev/null +++ b/lib/server/taxonomy/parser-client.test.ts @@ -0,0 +1,286 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +import type { + TaxonomyHydrationInput, + TaxonomyHydrationResult, +} from "@/lib/server/taxonomy/types"; +import { __parserClientInternals } from "@/lib/server/taxonomy/parser-client"; + +function streamFromText(text: string) { + const encoded = new TextEncoder().encode(text); + + return new ReadableStream({ + start(controller) { + controller.enqueue(encoded); + controller.close(); + }, + }); +} + +function sampleHydrationResult(): TaxonomyHydrationResult { + return { + filing_id: 1, + ticker: "AAPL", + filing_date: "2026-01-30", + filing_type: "10-Q", + parse_status: "ready", + parse_error: null, + source: "xbrl_instance", + parser_engine: "fiscal-xbrl", + parser_version: "0.1.0", + taxonomy_regime: "us-gaap", + fiscal_pack: "core", + periods: [], + faithful_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + statement_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + surface_rows: { + income: [], + balance: [], + cash_flow: [], + equity: [], + comprehensive_income: [], + }, + detail_rows: { + income: {}, + balance: {}, + cash_flow: {}, + equity: {}, + comprehensive_income: {}, + }, + kpi_rows: [], + computed_definitions: [], + contexts: [], + derived_metrics: null, + validation_result: null, + facts_count: 0, + concepts_count: 0, + dimensions_count: 0, + assets: [], + concepts: [], + facts: [], + metric_validations: [], + normalization_summary: { + surface_row_count: 0, + detail_row_count: 0, + kpi_row_count: 0, + unmapped_row_count: 0, + material_unmapped_row_count: 0, + warnings: [], + }, + xbrl_validation: { + status: "passed", + }, + }; +} + +function sampleInput(): TaxonomyHydrationInput { + return { + filingId: 1, + ticker: "AAPL", + cik: "0000320193", + accessionNumber: "0000320193-26-000001", + filingDate: "2026-01-30", + filingType: "10-Q", + filingUrl: + "https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/", + primaryDocument: "a10q.htm", + }; +} + +const passThroughTimeout = ((handler: TimerHandler, timeout?: number) => + globalThis.setTimeout( + handler, + timeout, + )) as unknown as typeof globalThis.setTimeout; +const immediateTimeout = ((handler: TimerHandler) => { + if (typeof handler === "function") { + handler(); + } + + return 1 as unknown as ReturnType; +}) as unknown as typeof globalThis.setTimeout; + +describe("parser client", () => { + beforeEach(() => { + delete process.env.FISCAL_XBRL_BIN; + delete process.env.XBRL_ENGINE_TIMEOUT_MS; + }); + + it("throws when the sidecar binary cannot be resolved", () => { + expect(() => + __parserClientInternals.resolveFiscalXbrlBinary({ + existsSync: () => false, + }), + ).toThrow(/Rust XBRL sidecar binary is required/); + }); + + it("returns parsed sidecar JSON on success", async () => { + const stdinWrite = mock(() => {}); + const stdinEnd = mock(() => {}); + + const result = await __parserClientInternals.hydrateFromSidecarImpl( + sampleInput(), + { + existsSync: () => true, + spawn: mock(() => ({ + stdin: { + write: stdinWrite, + end: stdinEnd, + }, + stdout: streamFromText(JSON.stringify(sampleHydrationResult())), + stderr: streamFromText(""), + exited: Promise.resolve(0), + kill: mock(() => {}), + })) as never, + setTimeout: passThroughTimeout, + clearTimeout, + }, + ); + + expect(result.parser_engine).toBe("fiscal-xbrl"); + expect(stdinWrite).toHaveBeenCalledTimes(1); + expect(stdinEnd).toHaveBeenCalledTimes(1); + }); + + it("throws when the sidecar exits non-zero", async () => { + await expect( + __parserClientInternals.hydrateFromSidecarImpl(sampleInput(), { + existsSync: () => true, + spawn: mock(() => ({ + stdin: { + write: () => {}, + end: () => {}, + }, + stdout: streamFromText(""), + stderr: streamFromText("fatal parse error"), + exited: Promise.resolve(3), + kill: mock(() => {}), + })) as never, + setTimeout: passThroughTimeout, + clearTimeout, + }), + ).rejects.toThrow(/exit code 3/); + }); + + it("throws on invalid JSON stdout", async () => { + await expect( + __parserClientInternals.hydrateFromSidecarImpl(sampleInput(), { + existsSync: () => true, + spawn: mock(() => ({ + stdin: { + write: () => {}, + end: () => {}, + }, + stdout: streamFromText("{not json"), + stderr: streamFromText(""), + exited: Promise.resolve(0), + kill: mock(() => {}), + })) as never, + setTimeout: passThroughTimeout, + clearTimeout, + }), + ).rejects.toThrow(); + }); + + it("kills the sidecar when the timeout fires", async () => { + const kill = mock(() => {}); + + await expect( + __parserClientInternals.hydrateFromSidecarImpl(sampleInput(), { + existsSync: () => true, + spawn: mock(() => ({ + stdin: { + write: () => {}, + end: () => {}, + }, + stdout: streamFromText(""), + stderr: streamFromText("killed"), + exited: Promise.resolve(137), + kill, + })) as never, + setTimeout: immediateTimeout, + clearTimeout: () => {}, + }), + ).rejects.toThrow(/exit code 137/); + + expect(kill).toHaveBeenCalledTimes(1); + }); + + it("retries retryable sidecar failures but not invalid requests", async () => { + let attempts = 0; + const spawn = mock(() => { + attempts += 1; + + const exitCode = attempts < 3 ? 1 : 0; + const stdout = + exitCode === 0 ? JSON.stringify(sampleHydrationResult()) : ""; + const stderr = exitCode === 0 ? "" : "process killed"; + + return { + stdin: { + write: () => {}, + end: () => {}, + }, + stdout: streamFromText(stdout), + stderr: streamFromText(stderr), + exited: Promise.resolve(exitCode), + kill: mock(() => {}), + }; + }); + + const result = + await __parserClientInternals.hydrateFilingTaxonomySnapshotFromSidecarWithDeps( + sampleInput(), + { + existsSync: () => true, + spawn: spawn as never, + setTimeout: passThroughTimeout, + clearTimeout, + }, + ); + + expect(result.parser_version).toBe("0.1.0"); + expect(attempts).toBe(3); + + attempts = 0; + const invalidRequestSpawn = mock(() => { + attempts += 1; + + return { + stdin: { + write: () => {}, + end: () => {}, + }, + stdout: streamFromText(""), + stderr: streamFromText("invalid request: bad command"), + exited: Promise.resolve(6), + kill: mock(() => {}), + }; + }); + + await expect( + __parserClientInternals.hydrateFilingTaxonomySnapshotFromSidecarWithDeps( + sampleInput(), + { + existsSync: () => true, + spawn: invalidRequestSpawn as never, + setTimeout: passThroughTimeout, + clearTimeout, + }, + ), + ).rejects.toThrow(/invalid request/); + expect(attempts).toBe(1); + }); +}); diff --git a/lib/server/taxonomy/parser-client.ts b/lib/server/taxonomy/parser-client.ts index d1fdcbc..916b60d 100644 --- a/lib/server/taxonomy/parser-client.ts +++ b/lib/server/taxonomy/parser-client.ts @@ -1,36 +1,89 @@ -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; -import type { TaxonomyHydrationInput, TaxonomyHydrationResult } from '@/lib/server/taxonomy/types'; -import { withRetry } from '@/lib/server/utils/retry'; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import type { + TaxonomyHydrationInput, + TaxonomyHydrationResult, +} from "@/lib/server/taxonomy/types"; +import { withRetry } from "@/lib/server/utils/retry"; + +type SpawnedSidecar = { + stdin: { write: (chunk: Uint8Array) => void; end: () => void }; + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill: () => void; +}; + +type SidecarDeps = { + existsSync: typeof existsSync; + spawn: typeof Bun.spawn; + setTimeout: typeof globalThis.setTimeout; + clearTimeout: typeof globalThis.clearTimeout; +}; function candidateBinaryPaths() { return [ process.env.FISCAL_XBRL_BIN?.trim(), - join(process.cwd(), 'bin', 'fiscal-xbrl'), - join(process.cwd(), 'rust', 'target', 'release', 'fiscal-xbrl'), - join(process.cwd(), 'rust', 'target', 'debug', 'fiscal-xbrl') - ].filter((value): value is string => typeof value === 'string' && value.length > 0); + join(process.cwd(), "bin", "fiscal-xbrl"), + join(process.cwd(), "rust", "target", "release", "fiscal-xbrl"), + join(process.cwd(), "rust", "target", "debug", "fiscal-xbrl"), + ].filter( + (value): value is string => typeof value === "string" && value.length > 0, + ); } export function resolveFiscalXbrlBinary() { - const resolved = candidateBinaryPaths().find((path) => existsSync(path)); + return resolveFiscalXbrlBinaryWithDeps({ + existsSync, + }); +} + +function resolveFiscalXbrlBinaryWithDeps( + deps: Pick, +) { + const resolved = candidateBinaryPaths().find((path) => deps.existsSync(path)); if (!resolved) { - throw new Error('Rust XBRL sidecar binary is required but was not found. Set FISCAL_XBRL_BIN or build `fiscal-xbrl` under rust/target.'); + throw new Error( + "Rust XBRL sidecar binary is required but was not found. Set FISCAL_XBRL_BIN or build `fiscal-xbrl` under rust/target.", + ); } return resolved; } export async function hydrateFilingTaxonomySnapshotFromSidecar( - input: TaxonomyHydrationInput + input: TaxonomyHydrationInput, ): Promise { - return withRetry(() => hydrateFromSidecarImpl(input)); + return hydrateFilingTaxonomySnapshotFromSidecarWithDeps(input, { + existsSync, + spawn: Bun.spawn, + setTimeout: globalThis.setTimeout, + clearTimeout: globalThis.clearTimeout, + }); } -async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise { - const binary = resolveFiscalXbrlBinary(); - const timeoutMs = Math.max(Number(process.env.XBRL_ENGINE_TIMEOUT_MS ?? 45_000), 1_000); - const command = [binary, 'hydrate-filing']; +async function hydrateFilingTaxonomySnapshotFromSidecarWithDeps( + input: TaxonomyHydrationInput, + deps: SidecarDeps, +): Promise { + return withRetry(() => hydrateFromSidecarImpl(input, deps)); +} + +async function hydrateFromSidecarImpl( + input: TaxonomyHydrationInput, + deps: SidecarDeps = { + existsSync, + spawn: Bun.spawn, + setTimeout: globalThis.setTimeout, + clearTimeout: globalThis.clearTimeout, + }, +): Promise { + const binary = resolveFiscalXbrlBinaryWithDeps(deps); + const timeoutMs = Math.max( + Number(process.env.XBRL_ENGINE_TIMEOUT_MS ?? 45_000), + 1_000, + ); + const command = [binary, "hydrate-filing"]; const requestBody = JSON.stringify({ filingId: input.filingId, ticker: input.ticker, @@ -40,22 +93,24 @@ async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise { + const timeout = deps.setTimeout(() => { child.kill(); }, timeoutMs); @@ -63,7 +118,7 @@ async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise 0) { @@ -71,11 +126,20 @@ async function hydrateFromSidecarImpl(input: TaxonomyHydrationInput): Promise; + metric_key: keyof NonNullable; taxonomy_value: number | null; llm_value: number | null; absolute_diff: number | null; relative_diff: number | null; - status: 'not_run' | 'matched' | 'mismatch' | 'error'; + status: "not_run" | "matched" | "mismatch" | "error"; evidence_pages: number[]; pdf_url: string | null; provider: string | null; @@ -119,7 +120,7 @@ export type TaxonomyHydrationPeriod = { filing_date: string; period_start: string | null; period_end: string | null; - filing_type: '10-K' | '10-Q'; + filing_type: "10-K" | "10-Q"; period_label: string; }; @@ -148,7 +149,7 @@ export type TaxonomyHydrationSurfaceRow = { category: string; template_section?: string; order: number; - unit: 'currency' | 'count' | 'shares' | 'percent' | 'ratio'; + unit: "currency" | "count" | "shares" | "percent" | "ratio"; values: Record; source_concepts: string[]; source_row_keys: string[]; @@ -156,10 +157,14 @@ export type TaxonomyHydrationSurfaceRow = { formula_key: string | null; has_dimensions: boolean; resolved_source_row_keys: Record; - statement?: 'income' | 'balance' | 'cash_flow'; + statement?: "income" | "balance" | "cash_flow"; detail_count?: number; - resolution_method?: 'direct' | 'surface_bridge' | 'formula_derived' | 'not_meaningful'; - confidence?: 'high' | 'medium' | 'low'; + resolution_method?: + | "direct" + | "surface_bridge" + | "formula_derived" + | "not_meaningful"; + confidence?: "high" | "medium" | "low"; warning_codes?: string[]; }; @@ -183,7 +188,7 @@ export type TaxonomyHydrationStructuredKpiRow = { key: string; label: string; category: string; - unit: 'currency' | 'count' | 'shares' | 'percent' | 'ratio'; + unit: "currency" | "count" | "shares" | "percent" | "ratio"; order: number; segment: string | null; axis: string | null; @@ -191,7 +196,7 @@ export type TaxonomyHydrationStructuredKpiRow = { values: Record; source_concepts: string[]; source_fact_ids: number[]; - provenance_type: 'taxonomy' | 'structured_note'; + provenance_type: "taxonomy" | "structured_note"; has_dimensions: boolean; }; @@ -205,7 +210,7 @@ export type TaxonomyHydrationNormalizationSummary = { }; export type XbrlValidationResult = { - status: 'passed' | 'warning' | 'error'; + status: "passed" | "warning" | "error"; message?: string; }; @@ -215,7 +220,7 @@ export type TaxonomyHydrationInput = { cik: string; accessionNumber: string; filingDate: string; - filingType: '10-K' | '10-Q'; + filingType: "10-K" | "10-Q"; filingUrl: string | null; primaryDocument: string | null; }; @@ -224,20 +229,30 @@ export type TaxonomyHydrationResult = { filing_id: number; ticker: string; filing_date: string; - filing_type: '10-K' | '10-Q'; + filing_type: "10-K" | "10-Q"; parse_status: FilingTaxonomyParseStatus; parse_error: string | null; source: FilingTaxonomySource; parser_engine: string; parser_version: string; - taxonomy_regime: 'us-gaap' | 'ifrs-full' | 'unknown'; + taxonomy_regime: "us-gaap" | "ifrs-full" | "unknown"; fiscal_pack: string | null; periods: TaxonomyHydrationPeriod[]; - faithful_rows: Record; - statement_rows: Record; + faithful_rows: Record< + FinancialStatementKind, + TaxonomyHydrationStatementRow[] + >; + statement_rows: Record< + FinancialStatementKind, + TaxonomyHydrationStatementRow[] + >; surface_rows: Record; - detail_rows: Record>; + detail_rows: Record< + FinancialStatementKind, + Record + >; kpi_rows: TaxonomyHydrationStructuredKpiRow[]; + computed_definitions: ComputedDefinition[]; contexts: Array<{ context_id: string; entity_identifier: string | null; @@ -248,7 +263,7 @@ export type TaxonomyHydrationResult = { segment_json: Record | null; scenario_json: Record | null; }>; - derived_metrics: Filing['metrics']; + derived_metrics: Filing["metrics"]; validation_result: MetricValidationResult | null; facts_count: number; concepts_count: number; diff --git a/package.json b/package.json index c36efe5..0e22dda 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,15 @@ "build": "bun run generate && bun --bun next build --turbopack", "bootstrap:prod": "bun run scripts/bootstrap-production.ts", "check:sidecar": "cargo check --manifest-path rust/Cargo.toml", + "fmt": "cargo fmt --manifest-path rust/Cargo.toml --all --check && bun x prettier --check --ignore-unknown README.md package.json lib/server/db/index.test.ts lib/server/db/schema.ts lib/server/db/sqlite-schema-compat.ts lib/server/repos/filing-taxonomy.test.ts lib/server/repos/filing-taxonomy.ts lib/server/task-processors.outcomes.test.ts lib/server/task-processors.ts lib/server/taxonomy/engine.test.ts lib/server/taxonomy/parser-client.ts lib/server/taxonomy/parser-client.test.ts lib/server/taxonomy/types.ts scripts/e2e-prepare.test.ts test/bun-test-shim.ts test/vitest.setup.ts vitest.config.mts", + "typecheck": "bun x tsc --noEmit", "validate:taxonomy-packs": "bun run scripts/validate-taxonomy-packs.ts", "start": "bun --bun next start", - "lint": "bun run generate && bun x tsc --noEmit", + "lint": "bun run generate && bun run typecheck", "e2e:prepare": "bun run scripts/e2e-prepare.ts", "e2e:webserver": "bun run scripts/e2e-webserver.ts", "workflow:setup": "workflow-postgres-setup", + "test": "bun x vitest run", "backfill:filing-metrics": "bun run scripts/backfill-filing-metrics.ts", "backfill:filing-statements": "bun run scripts/backfill-filing-statements.ts", "backfill:search-index": "bun run scripts/backfill-search-index.ts", @@ -65,7 +68,9 @@ "bun-types": "^1.3.10", "drizzle-kit": "^0.31.9", "postcss": "^8.5.8", + "prettier": "^3.6.2", "tailwindcss": "^4.2.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^3.2.4" } } diff --git a/rust/fiscal-xbrl-core/src/crabrl_adapter.rs b/rust/fiscal-xbrl-core/src/crabrl_adapter.rs new file mode 100644 index 0000000..d88d2f4 --- /dev/null +++ b/rust/fiscal-xbrl-core/src/crabrl_adapter.rs @@ -0,0 +1,231 @@ +use anyhow::{Context, Result}; +use crabrl::{Document, FactValue, Measure, Parser, Period, UnitType}; +use serde_json::json; + +use crate::{ + is_xbrl_infrastructure_prefix, ContextOutput, DimensionOutput, ParsedFact, ParsedInstance, +}; + +pub(crate) fn parse_xbrl_instance( + raw: &str, + source_file: Option, +) -> Result { + let document = Parser::new() + .parse_bytes(raw.as_bytes()) + .context("crabrl failed to parse XBRL instance")?; + + Ok(ParsedInstance { + contexts: build_contexts(&document), + facts: build_facts(&document, source_file), + }) +} + +fn build_contexts(document: &Document) -> Vec { + document + .contexts + .iter() + .map(|context| { + let (period_start, period_end, period_instant) = convert_period(&context.period); + + ContextOutput { + context_id: context.id.to_string(), + entity_identifier: Some(context.entity.identifier.to_string()), + entity_scheme: Some(context.entity.scheme.to_string()), + period_start, + period_end, + period_instant, + segment_json: context.entity.segment.as_ref().map(segment_to_json), + scenario_json: context.scenario.as_ref().map(scenario_to_json), + } + }) + .collect() +} + +fn build_facts(document: &Document, source_file: Option) -> Vec { + document + .facts + .concept_ids + .iter() + .enumerate() + .filter_map(|(index, concept_id)| { + let qname = document + .concept_names + .get(*concept_id as usize)? + .to_string(); + let (prefix, local_name) = split_qname(&qname)?; + if is_xbrl_infrastructure_prefix(&prefix) { + return None; + } + + let value = numeric_fact_value(document.facts.values.get(index)?)?; + let context = document + .contexts + .get(*document.facts.context_ids.get(index)? as usize)?; + let namespace_uri = document + .namespaces + .get(prefix.as_str()) + .map(|value| value.to_string()) + .unwrap_or_else(|| format!("urn:unknown:{prefix}")); + let (period_start, period_end, period_instant) = convert_period(&context.period); + let dimensions = context_dimensions(context); + + Some(ParsedFact { + concept_key: format!("{namespace_uri}#{local_name}"), + qname, + namespace_uri, + local_name, + data_type: None, + context_id: context.id.to_string(), + unit: unit_for_fact(document, index), + decimals: document + .facts + .decimals + .get(index) + .and_then(|value| value.map(|entry| entry.to_string())), + precision: None, + nil: matches!(document.facts.values.get(index), Some(FactValue::Nil)), + value, + period_start, + period_end, + period_instant, + is_dimensionless: dimensions.is_empty(), + dimensions, + source_file: source_file.clone(), + }) + }) + .collect() +} + +fn numeric_fact_value(value: &FactValue) -> Option { + match value { + FactValue::Decimal(value) => Some(*value), + FactValue::Integer(value) => Some(*value as f64), + _ => None, + } +} + +fn split_qname(qname: &str) -> Option<(String, String)> { + let (prefix, local_name) = qname.split_once(':')?; + let prefix = prefix.trim().to_string(); + let local_name = local_name.trim().to_string(); + if prefix.is_empty() || local_name.is_empty() { + return None; + } + + Some((prefix, local_name)) +} + +fn convert_period(period: &Period) -> (Option, Option, Option) { + match period { + Period::Instant { date } => (None, None, Some(date.to_string())), + Period::Duration { start, end } => (Some(start.to_string()), Some(end.to_string()), None), + Period::Forever => (None, None, None), + } +} + +fn context_dimensions(context: &crabrl::Context) -> Vec { + let mut dimensions = Vec::new(); + + if let Some(segment) = context.entity.segment.as_ref() { + dimensions.extend( + segment + .explicit_members + .iter() + .map(|member| DimensionOutput { + axis: member.dimension.to_string(), + member: member.member.to_string(), + }), + ); + } + + if let Some(scenario) = context.scenario.as_ref() { + dimensions.extend( + scenario + .explicit_members + .iter() + .map(|member| DimensionOutput { + axis: member.dimension.to_string(), + member: member.member.to_string(), + }), + ); + } + + dimensions +} + +fn unit_for_fact(document: &Document, fact_index: usize) -> Option { + let unit_id = *document.facts.unit_ids.get(fact_index)?; + if unit_id == 0 { + return None; + } + + document + .units + .get((unit_id - 1) as usize) + .map(|unit| unit_type_to_string(&unit.unit_type)) +} + +fn unit_type_to_string(unit_type: &UnitType) -> String { + match unit_type { + UnitType::Simple(measures) => join_measures(measures, "/"), + UnitType::Multiply(measures) => join_measures(measures, "*"), + UnitType::Divide { + numerator, + denominator, + } => format!( + "{}/{}", + join_measures(numerator, "*"), + join_measures(denominator, "*") + ), + } +} + +fn join_measures(measures: &[Measure], separator: &str) -> String { + measures + .iter() + .map(measure_to_string) + .collect::>() + .join(separator) +} + +fn measure_to_string(measure: &Measure) -> String { + if measure.namespace.is_empty() { + measure.name.to_string() + } else { + format!("{}:{}", measure.namespace, measure.name) + } +} + +fn segment_to_json(segment: &crabrl::Segment) -> serde_json::Value { + json!({ + "explicitMembers": segment.explicit_members.iter().map(|member| { + json!({ + "axis": member.dimension.to_string(), + "member": member.member.to_string(), + }) + }).collect::>(), + "typedMembers": segment.typed_members.iter().map(|member| { + json!({ + "axis": member.dimension.to_string(), + "value": member.value.to_string(), + }) + }).collect::>(), + }) +} + +fn scenario_to_json(scenario: &crabrl::Scenario) -> serde_json::Value { + json!({ + "explicitMembers": scenario.explicit_members.iter().map(|member| { + json!({ + "axis": member.dimension.to_string(), + "member": member.member.to_string(), + }) + }).collect::>(), + "typedMembers": scenario.typed_members.iter().map(|member| { + json!({ + "axis": member.dimension.to_string(), + "value": member.value.to_string(), + }) + }).collect::>(), + }) +} diff --git a/rust/fiscal-xbrl-core/src/lib.rs b/rust/fiscal-xbrl-core/src/lib.rs index 65473cf..3d2f8dc 100644 --- a/rust/fiscal-xbrl-core/src/lib.rs +++ b/rust/fiscal-xbrl-core/src/lib.rs @@ -9,6 +9,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Mutex; use std::time::{Duration, Instant}; +mod crabrl_adapter; mod kpi_mapper; mod metrics; mod pack_selector; @@ -54,44 +55,6 @@ where fetch_fn() } -static CONTEXT_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?context\b[^>]*\bid=["']([^"']+)["'][^>]*>(.*?)"#).unwrap() -}); -static UNIT_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?unit\b[^>]*\bid=["']([^"']+)["'][^>]*>(.*?)"#).unwrap() -}); -static FACT_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<([a-zA-Z0-9_\-]+):([a-zA-Z0-9_\-.]+)\b([^>]*\bcontextRef=["'][^"']+["'][^>]*)>(.*?)"#).unwrap() -}); -static EXPLICIT_MEMBER_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?explicitMember\b[^>]*\bdimension=["']([^"']+)["'][^>]*>(.*?)"#).unwrap() -}); -static TYPED_MEMBER_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?typedMember\b[^>]*\bdimension=["']([^"']+)["'][^>]*>(.*?)"#).unwrap() -}); -static IDENTIFIER_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?identifier\b[^>]*\bscheme=["']([^"']+)["'][^>]*>(.*?)"#).unwrap() -}); -static SEGMENT_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?segment\b[^>]*>(.*?)"#) - .unwrap() -}); -static SCENARIO_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?scenario\b[^>]*>(.*?)"#) - .unwrap() -}); -static START_DATE_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?startDate>(.*?)"#).unwrap() -}); -static END_DATE_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?endDate>(.*?)"#).unwrap() -}); -static INSTANT_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?instant>(.*?)"#).unwrap() -}); -static MEASURE_RE: Lazy = Lazy::new(|| { - Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?measure>(.*?)"#).unwrap() -}); static LABEL_LINK_RE: Lazy = Lazy::new(|| { Regex::new(r#"(?is)<(?:[a-z0-9_\-]+:)?labelLink\b[^>]*>(.*?)"#) .unwrap() @@ -465,25 +428,7 @@ pub type SurfaceRowMap = BTreeMap>; pub type DetailRowStatementMap = BTreeMap>>; #[derive(Debug, Clone)] -struct ParsedContext { - id: String, - entity_identifier: Option, - entity_scheme: Option, - period_start: Option, - period_end: Option, - period_instant: Option, - dimensions: Vec, - segment: Option, - scenario: Option, -} - -#[derive(Debug, Clone)] -struct ParsedUnit { - measure: Option, -} - -#[derive(Debug, Clone)] -struct ParsedFact { +pub(crate) struct ParsedFact { concept_key: String, qname: String, namespace_uri: String, @@ -593,7 +538,8 @@ pub fn hydrate_filing(input: HydrateFilingRequest) -> Result) -> XbrlValidati } } -struct ParsedInstance { +pub(crate) struct ParsedInstance { contexts: Vec, facts: Vec, } -fn parse_xbrl_instance(raw: &str, source_file: Option) -> ParsedInstance { - let namespaces = parse_namespace_map(raw, "xbrl"); - let context_by_id = parse_contexts(raw); - let unit_by_id = parse_units(raw); - let mut facts = Vec::new(); - - for captures in FACT_RE.captures_iter(raw) { - let prefix = captures - .get(1) - .map(|value| value.as_str().trim()) - .unwrap_or_default(); - let local_name = captures - .get(2) - .map(|value| value.as_str().trim()) - .unwrap_or_default(); - let attrs = captures - .get(3) - .map(|value| value.as_str()) - .unwrap_or_default(); - let body = decode_xml_entities( - captures - .get(4) - .map(|value| value.as_str()) - .unwrap_or_default() - .trim(), - ); - - if prefix.is_empty() || local_name.is_empty() || is_xbrl_infrastructure_prefix(prefix) { - continue; - } - - let attr_map = parse_attrs(attrs); - let Some(context_id) = attr_map - .get("contextRef") - .cloned() - .or_else(|| attr_map.get("contextref").cloned()) - else { - continue; - }; - - let Some(value) = parse_number(&body) else { - continue; - }; - - let namespace_uri = namespaces - .get(prefix) - .cloned() - .unwrap_or_else(|| format!("urn:unknown:{prefix}")); - let context = context_by_id.get(&context_id); - let unit_ref = attr_map - .get("unitRef") - .cloned() - .or_else(|| attr_map.get("unitref").cloned()); - let unit = unit_ref - .as_ref() - .and_then(|unit_ref| unit_by_id.get(unit_ref)) - .and_then(|unit| unit.measure.clone()) - .or(unit_ref); - - facts.push(ParsedFact { - concept_key: format!("{namespace_uri}#{local_name}"), - qname: format!("{prefix}:{local_name}"), - namespace_uri, - local_name: local_name.to_string(), - data_type: None, - context_id: context_id.clone(), - unit, - decimals: attr_map.get("decimals").cloned(), - precision: attr_map.get("precision").cloned(), - nil: attr_map - .get("xsi:nil") - .or_else(|| attr_map.get("nil")) - .map(|value| value.eq_ignore_ascii_case("true")) - .unwrap_or(false), - value, - period_start: context.and_then(|value| value.period_start.clone()), - period_end: context.and_then(|value| value.period_end.clone()), - period_instant: context.and_then(|value| value.period_instant.clone()), - dimensions: context - .map(|value| value.dimensions.clone()) - .unwrap_or_default(), - is_dimensionless: context - .map(|value| value.dimensions.is_empty()) - .unwrap_or(true), - source_file: source_file.clone(), - }); - } - - let contexts = context_by_id - .values() - .map(|context| ContextOutput { - context_id: context.id.clone(), - entity_identifier: context.entity_identifier.clone(), - entity_scheme: context.entity_scheme.clone(), - period_start: context.period_start.clone(), - period_end: context.period_end.clone(), - period_instant: context.period_instant.clone(), - segment_json: context.segment.clone(), - scenario_json: context.scenario.clone(), - }) - .collect::>(); - - ParsedInstance { contexts, facts } +fn parse_xbrl_instance(raw: &str, source_file: Option) -> Result { + crabrl_adapter::parse_xbrl_instance(raw, source_file) } fn parse_namespace_map(raw: &str, root_tag_hint: &str) -> HashMap { @@ -1277,173 +1122,7 @@ fn parse_namespace_map(raw: &str, root_tag_hint: &str) -> HashMap HashMap { - let mut contexts = HashMap::new(); - - for captures in CONTEXT_RE.captures_iter(raw) { - let Some(context_id) = captures - .get(1) - .map(|value| value.as_str().trim().to_string()) - else { - continue; - }; - let block = captures - .get(2) - .map(|value| value.as_str()) - .unwrap_or_default(); - let (entity_identifier, entity_scheme) = IDENTIFIER_RE - .captures(block) - .map(|captures| { - ( - captures - .get(2) - .map(|value| decode_xml_entities(value.as_str().trim())), - captures - .get(1) - .map(|value| decode_xml_entities(value.as_str().trim())), - ) - }) - .unwrap_or((None, None)); - - let period_start = START_DATE_RE - .captures(block) - .and_then(|captures| captures.get(1)) - .map(|value| decode_xml_entities(value.as_str().trim())); - let period_end = END_DATE_RE - .captures(block) - .and_then(|captures| captures.get(1)) - .map(|value| decode_xml_entities(value.as_str().trim())); - let period_instant = INSTANT_RE - .captures(block) - .and_then(|captures| captures.get(1)) - .map(|value| decode_xml_entities(value.as_str().trim())); - - let segment = SEGMENT_RE - .captures(block) - .and_then(|captures| captures.get(1)) - .map(|value| parse_dimension_container(value.as_str())); - let scenario = SCENARIO_RE - .captures(block) - .and_then(|captures| captures.get(1)) - .map(|value| parse_dimension_container(value.as_str())); - - let mut dimensions = Vec::new(); - if let Some(segment_value) = segment.as_ref() { - if let Some(members) = segment_value - .get("explicitMembers") - .and_then(|value| value.as_array()) - { - for member in members { - if let (Some(axis), Some(member_value)) = ( - member.get("axis").and_then(|value| value.as_str()), - member.get("member").and_then(|value| value.as_str()), - ) { - dimensions.push(DimensionOutput { - axis: axis.to_string(), - member: member_value.to_string(), - }); - } - } - } - } - if let Some(scenario_value) = scenario.as_ref() { - if let Some(members) = scenario_value - .get("explicitMembers") - .and_then(|value| value.as_array()) - { - for member in members { - if let (Some(axis), Some(member_value)) = ( - member.get("axis").and_then(|value| value.as_str()), - member.get("member").and_then(|value| value.as_str()), - ) { - dimensions.push(DimensionOutput { - axis: axis.to_string(), - member: member_value.to_string(), - }); - } - } - } - } - - contexts.insert( - context_id.clone(), - ParsedContext { - id: context_id, - entity_identifier, - entity_scheme, - period_start, - period_end, - period_instant, - dimensions, - segment, - scenario, - }, - ); - } - - contexts -} - -fn parse_dimension_container(raw: &str) -> serde_json::Value { - let explicit_members = EXPLICIT_MEMBER_RE - .captures_iter(raw) - .filter_map(|captures| { - Some(serde_json::json!({ - "axis": decode_xml_entities(captures.get(1)?.as_str().trim()), - "member": decode_xml_entities(captures.get(2)?.as_str().trim()) - })) - }) - .collect::>(); - let typed_members = TYPED_MEMBER_RE - .captures_iter(raw) - .filter_map(|captures| { - Some(serde_json::json!({ - "axis": decode_xml_entities(captures.get(1)?.as_str().trim()), - "value": decode_xml_entities(captures.get(2)?.as_str().trim()) - })) - }) - .collect::>(); - - serde_json::json!({ - "explicitMembers": explicit_members, - "typedMembers": typed_members - }) -} - -fn parse_units(raw: &str) -> HashMap { - let mut units = HashMap::new(); - for captures in UNIT_RE.captures_iter(raw) { - let Some(id) = captures - .get(1) - .map(|value| value.as_str().trim().to_string()) - else { - continue; - }; - let block = captures - .get(2) - .map(|value| value.as_str()) - .unwrap_or_default(); - let measures = MEASURE_RE - .captures_iter(block) - .filter_map(|captures| captures.get(1)) - .map(|value| decode_xml_entities(value.as_str().trim())) - .filter(|value| !value.is_empty()) - .collect::>(); - - let measure = if measures.len() == 1 { - measures.first().cloned() - } else if measures.len() > 1 { - Some(measures.join("/")) - } else { - None - }; - - units.insert(id, ParsedUnit { measure }); - } - units -} - -fn is_xbrl_infrastructure_prefix(prefix: &str) -> bool { +pub(crate) fn is_xbrl_infrastructure_prefix(prefix: &str) -> bool { matches!( prefix.to_ascii_lowercase().as_str(), "xbrli" | "xlink" | "link" | "xbrldi" | "xbrldt" @@ -1474,25 +1153,6 @@ fn decode_xml_entities(value: &str) -> String { .replace(" ", " ") } -fn parse_number(raw: &str) -> Option { - let trimmed = raw.trim(); - if trimmed.is_empty() || trimmed.chars().all(|char| char == '-') { - return None; - } - let negative = trimmed.starts_with('(') && trimmed.ends_with(')'); - let normalized = Regex::new(r#"<[^>]+>"#) - .unwrap() - .replace_all(trimmed, " ") - .replace(',', "") - .replace('$', "") - .replace(['(', ')'], "") - .replace('\u{2212}', "-") - .split_whitespace() - .collect::(); - let parsed = normalized.parse::().ok()?; - Some(if negative { -parsed.abs() } else { parsed }) -} - fn parse_label_linkbase(raw: &str) -> HashMap { let namespaces = parse_namespace_map(raw, "linkbase"); let mut preferred = HashMap::::new(); @@ -2543,7 +2203,8 @@ mod tests { "#; - let parsed = parse_xbrl_instance(raw, Some("test.xml".to_string())); + let parsed = parse_xbrl_instance(raw, Some("test.xml".to_string())) + .expect("crabrl parser should parse test instance"); assert_eq!(parsed.facts.len(), 1); assert_eq!( parsed.facts[0].qname, diff --git a/rust/fiscal-xbrl-core/src/surface_mapper.rs b/rust/fiscal-xbrl-core/src/surface_mapper.rs index 1f1c5eb..6001201 100644 --- a/rust/fiscal-xbrl-core/src/surface_mapper.rs +++ b/rust/fiscal-xbrl-core/src/surface_mapper.rs @@ -3,8 +3,8 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::pack_selector::FiscalPack; use crate::taxonomy_loader::{ - load_crosswalk, load_surface_pack, CrosswalkFile, SurfaceDefinition, SurfaceFormula, - SurfaceFormulaOp, SurfaceSignTransform, + load_crosswalk, load_income_bridge, load_surface_pack, CrosswalkFile, IncomeBridgeFile, + IncomeBridgeRow, SurfaceDefinition, SurfaceFormula, SurfaceFormulaOp, SurfaceSignTransform, }; use crate::{ ConceptOutput, DetailRowOutput, DetailRowStatementMap, FactOutput, NormalizationSummaryOutput, @@ -114,6 +114,7 @@ pub fn build_compact_surface_model( ) -> Result { let pack = load_surface_pack(fiscal_pack)?; let crosswalk = load_crosswalk(taxonomy_regime)?; + let income_bridge = load_income_bridge(fiscal_pack).ok(); let mut surface_rows = empty_surface_row_map(); let mut detail_rows = empty_detail_row_map(); let mut concept_mappings = HashMap::::new(); @@ -157,14 +158,20 @@ pub fn build_compact_surface_model( .filter(|matched| matched.match_role == MatchRole::Detail) .cloned() .collect::>(); + let bridge_detail_matches = collect_income_bridge_detail_matches( + definition, + &rows, + crosswalk.as_ref(), + income_bridge.as_ref(), + ); let detail_matches = if definition.detail_grouping_policy == "group_all_children" { - if detail_component_matches.is_empty() - && definition.rollup_policy == "aggregate_children" - { + let detail_matches = + merge_detail_matches(&detail_component_matches, &bridge_detail_matches); + if detail_matches.is_empty() && definition.rollup_policy == "aggregate_children" { Vec::new() } else { - detail_component_matches.clone() + detail_matches } } else { Vec::new() @@ -758,28 +765,123 @@ fn match_statement_row<'a>( None } +fn collect_income_bridge_detail_matches<'a>( + definition: &SurfaceDefinition, + rows: &'a [StatementRowOutput], + crosswalk: Option<&CrosswalkFile>, + income_bridge: Option<&IncomeBridgeFile>, +) -> Vec> { + if definition.statement != "income" + || definition.rollup_policy != "aggregate_children" + || definition.detail_grouping_policy != "group_all_children" + { + return Vec::new(); + } + + let Some(bridge_row) = + income_bridge.and_then(|bridge| bridge.rows.get(&definition.surface_key)) + else { + return Vec::new(); + }; + + rows.iter() + .filter(|row| has_any_value(&row.values)) + .filter_map(|row| match_income_bridge_detail_row(row, bridge_row, crosswalk)) + .collect() +} + +fn match_income_bridge_detail_row<'a>( + row: &'a StatementRowOutput, + bridge_row: &IncomeBridgeRow, + crosswalk: Option<&CrosswalkFile>, +) -> Option> { + let authoritative_concept_key = crosswalk + .and_then(|crosswalk| crosswalk.mappings.get(&row.qname)) + .map(|mapping| mapping.authoritative_concept_key.clone()) + .or_else(|| { + if !row.is_extension { + Some(row.qname.clone()) + } else { + None + } + }); + + let matches_group = bridge_row + .component_concept_groups + .positive + .iter() + .chain(bridge_row.component_concept_groups.negative.iter()) + .any(|group| { + group.concepts.iter().any(|candidate| { + candidate_matches(candidate, &row.qname) + || candidate_matches(candidate, &row.local_name) + || authoritative_concept_key + .as_ref() + .map(|concept| candidate_matches(candidate, concept)) + .unwrap_or(false) + }) + }); + + if !matches_group { + return None; + } + + Some(MatchedStatementRow { + row, + authoritative_concept_key, + mapping_method: MappingMethod::AggregateChildren, + match_role: MatchRole::Detail, + rank: 2, + }) +} + +fn merge_detail_matches<'a>( + direct_matches: &[MatchedStatementRow<'a>], + bridge_matches: &[MatchedStatementRow<'a>], +) -> Vec> { + let mut merged = HashMap::>::new(); + + for matched in direct_matches.iter().chain(bridge_matches.iter()) { + merged + .entry(matched.row.key.clone()) + .and_modify(|existing| { + if compare_statement_matches(matched, existing).is_lt() { + *existing = matched.clone(); + } + }) + .or_insert_with(|| matched.clone()); + } + + merged.into_values().collect() +} + fn pick_best_match<'a>(matches: &'a [MatchedStatementRow<'a>]) -> &'a MatchedStatementRow<'a> { matches .iter() - .min_by(|left, right| { - left.rank - .cmp(&right.rank) - .then_with(|| { - let left_dimension_rank = if left.row.has_dimensions { 1 } else { 0 }; - let right_dimension_rank = if right.row.has_dimensions { 1 } else { 0 }; - left_dimension_rank.cmp(&right_dimension_rank) - }) - .then_with(|| left.row.order.cmp(&right.row.order)) - .then_with(|| { - max_abs_value(&right.row.values) - .partial_cmp(&max_abs_value(&left.row.values)) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .then_with(|| left.row.label.cmp(&right.row.label)) - }) + .min_by(|left, right| compare_statement_matches(left, right)) .expect("pick_best_match requires at least one match") } +fn compare_statement_matches( + left: &MatchedStatementRow<'_>, + right: &MatchedStatementRow<'_>, +) -> std::cmp::Ordering { + left.rank + .cmp(&right.rank) + .then_with(|| { + let left_dimension_rank = if left.row.has_dimensions { 1 } else { 0 }; + let right_dimension_rank = if right.row.has_dimensions { 1 } else { 0 }; + left_dimension_rank.cmp(&right_dimension_rank) + }) + .then_with(|| left.row.order.cmp(&right.row.order)) + .then_with(|| { + max_abs_value(&right.row.values) + .partial_cmp(&max_abs_value(&left.row.values)) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .then_with(|| left.row.label.cmp(&right.row.label)) +} + fn build_surface_values( periods: &[PeriodOutput], matches: &[MatchedStatementRow<'_>], diff --git a/rust/fiscal-xbrl-core/src/universal_income.rs b/rust/fiscal-xbrl-core/src/universal_income.rs index 4881030..58d6652 100644 --- a/rust/fiscal-xbrl-core/src/universal_income.rs +++ b/rust/fiscal-xbrl-core/src/universal_income.rs @@ -336,22 +336,26 @@ fn build_formula_row( .positive .iter() .filter_map(|surface_key| { - income_surface_rows - .iter() - .find(|row| row.key == *surface_key) + resolve_component_surface_source( + surface_key, + income_statement_rows, + income_surface_rows, + crosswalk, + ) }) - .map(surface_source) .collect::>(); let negative_surface_sources = bridge_row .component_surfaces .negative .iter() .filter_map(|surface_key| { - income_surface_rows - .iter() - .find(|row| row.key == *surface_key) + resolve_component_surface_source( + surface_key, + income_statement_rows, + income_surface_rows, + crosswalk, + ) }) - .map(surface_source) .collect::>(); let (positive_group_sources, positive_group_rows) = collect_group_sources( @@ -810,6 +814,44 @@ fn collect_group_sources<'a>( (sources, rows) } +fn resolve_component_surface_source( + surface_key: &str, + income_statement_rows: &[StatementRowOutput], + income_surface_rows: &[SurfaceRowOutput], + crosswalk: Option<&CrosswalkFile>, +) -> Option { + if let Some(surface_row) = income_surface_rows + .iter() + .find(|row| row.key == surface_key) + { + return Some(surface_source(surface_row)); + } + + let matches = income_statement_rows + .iter() + .filter(|row| has_any_value(&row.values)) + .filter(|row| row_matches_surface_key(row, surface_key, crosswalk)) + .map(statement_row_source) + .collect::>(); + + if matches.is_empty() { + return None; + } + + Some(merge_value_sources(&matches)) +} + +fn row_matches_surface_key( + row: &StatementRowOutput, + surface_key: &str, + crosswalk: Option<&CrosswalkFile>, +) -> bool { + crosswalk + .and_then(|crosswalk| crosswalk.mappings.get(&row.qname)) + .map(|mapping| mapping.surface_key.eq_ignore_ascii_case(surface_key)) + .unwrap_or(false) +} + fn match_direct_authoritative<'a>( row: &'a StatementRowOutput, candidates: &[String], @@ -1024,6 +1066,52 @@ fn surface_source(row: &SurfaceRowOutput) -> ValueSource { } } +fn merge_value_sources(sources: &[ValueSource]) -> ValueSource { + let mut values = BTreeMap::>::new(); + + for period_id in sources.iter().flat_map(|source| source.values.keys()) { + values.entry(period_id.clone()).or_insert_with(|| { + let period_values = sources + .iter() + .map(|source| source.values.get(period_id).copied().flatten()) + .collect::>(); + if period_values.iter().all(|value| value.is_none()) { + None + } else { + Some( + period_values + .into_iter() + .map(|value| value.unwrap_or(0.0)) + .sum(), + ) + } + }); + } + + ValueSource { + values, + source_concepts: unique_sorted_strings( + sources + .iter() + .flat_map(|source| source.source_concepts.clone()) + .collect(), + ), + source_row_keys: unique_sorted_strings( + sources + .iter() + .flat_map(|source| source.source_row_keys.clone()) + .collect(), + ), + source_fact_ids: unique_sorted_i64( + sources + .iter() + .flat_map(|source| source.source_fact_ids.clone()) + .collect(), + ), + has_dimensions: sources.iter().any(|source| source.has_dimensions), + } +} + fn fact_matches_period(fact: &FactOutput, period: &PeriodOutput) -> bool { if fact.period_end != period.period_end { return false; diff --git a/scripts/e2e-prepare.test.ts b/scripts/e2e-prepare.test.ts index 45abb06..9d59b7a 100644 --- a/scripts/e2e-prepare.test.ts +++ b/scripts/e2e-prepare.test.ts @@ -1,17 +1,17 @@ -import { describe, expect, it } from 'bun:test'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { Database } from 'bun:sqlite'; -import { ensureFinancialIngestionSchemaHealthy } from '../lib/server/db/financial-ingestion-schema'; -import { hasColumn, hasTable } from '../lib/server/db/sqlite-schema-compat'; -import { prepareE2eDatabase } from './e2e-prepare'; +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; +import { ensureFinancialIngestionSchemaHealthy } from "../lib/server/db/financial-ingestion-schema"; +import { hasColumn, hasTable } from "../lib/server/db/sqlite-schema-compat"; +import { prepareE2eDatabase } from "./e2e-prepare"; -describe('prepareE2eDatabase', () => { - it('bootstraps a fresh e2e database with the current taxonomy schema shape', () => { - const tempDir = mkdtempSync(join(tmpdir(), 'fiscal-e2e-prepare-')); - const databasePath = join(tempDir, 'e2e.sqlite'); - const workflowDataDir = join(tempDir, 'workflow-data'); +describe("prepareE2eDatabase", () => { + it("bootstraps a fresh e2e database with the current taxonomy schema shape", () => { + const tempDir = mkdtempSync(join(tmpdir(), "fiscal-e2e-prepare-")); + const databasePath = join(tempDir, "e2e.sqlite"); + const workflowDataDir = join(tempDir, "workflow-data"); try { prepareE2eDatabase({ databasePath, workflowDataDir }); @@ -19,12 +19,19 @@ describe('prepareE2eDatabase', () => { const client = new Database(databasePath, { create: true }); try { - client.exec('PRAGMA foreign_keys = ON;'); + client.exec("PRAGMA foreign_keys = ON;"); - expect(hasColumn(client, 'filing_taxonomy_snapshot', 'parser_engine')).toBe(true); - expect(hasTable(client, 'filing_taxonomy_context')).toBe(true); + expect( + hasColumn(client, "filing_taxonomy_snapshot", "parser_engine"), + ).toBe(true); + expect( + hasColumn(client, "filing_taxonomy_snapshot", "computed_definitions"), + ).toBe(true); + expect(hasTable(client, "filing_taxonomy_context")).toBe(true); - const health = ensureFinancialIngestionSchemaHealthy(client, { mode: 'auto' }); + const health = ensureFinancialIngestionSchemaHealthy(client, { + mode: "auto", + }); expect(health.ok).toBe(true); } finally { client.close(); diff --git a/test/bun-test-shim.ts b/test/bun-test-shim.ts new file mode 100644 index 0000000..56893ac --- /dev/null +++ b/test/bun-test-shim.ts @@ -0,0 +1,33 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +type Mock = typeof vi.fn & { + module: typeof vi.mock; +}; + +const mock = Object.assign( + ((implementation?: Parameters[0]) => + vi.fn(implementation)) as typeof vi.fn, + { + module: vi.mock, + }, +) as Mock; + +export { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + mock, +}; diff --git a/test/vitest.setup.ts b/test/vitest.setup.ts new file mode 100644 index 0000000..01f0b7b --- /dev/null +++ b/test/vitest.setup.ts @@ -0,0 +1,6 @@ +const bunGlobal = ((globalThis as { Bun?: Record }).Bun ??= + {}); + +if (typeof bunGlobal.sleep !== "function") { + bunGlobal.sleep = async (_ms: number) => {}; +} diff --git a/vitest.config.mts b/vitest.config.mts new file mode 100644 index 0000000..5c1fe33 --- /dev/null +++ b/vitest.config.mts @@ -0,0 +1,15 @@ +import { resolve } from "node:path"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + resolve: { + alias: { + "@": resolve(process.cwd(), "."), + "bun:test": resolve(process.cwd(), "test/bun-test-shim.ts"), + }, + }, + test: { + environment: "node", + setupFiles: ["./test/vitest.setup.ts"], + }, +});