Francy51/Neon-Desk
2
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects Releases Wiki Activity

Automate issuer overlay creation from ticker searches

Browse Source
This commit is contained in:
francy51
2026-03-19 20:44:58 -04:00
parent 17de3dd72d
commit 391d6d34ce
79 changed files with 4746 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

573
rust/fiscal-xbrl-core/src/lib.rs
View File

@@ -6,6 +6,7 @@ use regex::Regex;
use reqwest::blocking::Client;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs;
use std::sync::Mutex;
use std::time::{Duration, Instant};
@@ -17,7 +18,7 @@ mod surface_mapper;
mod taxonomy_loader;
mod universal_income;
use taxonomy_loader::{ComputationSpec, ComputedDefinition};
use taxonomy_loader::{ComputationSpec, ComputedDefinition, RuntimeIssuerOverlay, SurfacePackFile};
pub const PARSER_ENGINE: &str = "fiscal-xbrl";
pub const PARSER_VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -26,6 +27,316 @@ const DEFAULT_SEC_RATE_LIMIT_MS: u64 = 100;
const HTTP_TIMEOUT_SECS: u64 = 30;
static RATE_LIMITER: Lazy<Mutex<Instant>> = Lazy::new(|| Mutex::new(Instant::now()));
static PRIMARY_STATEMENT_CONCEPT_INDEX: Lazy<PrimaryStatementConceptIndex> =
Lazy::new(load_primary_statement_concept_index);
#[derive(Debug, Default, Clone)]
struct SurfaceConceptIndex {
primary_by_qname: HashMap<String, HashSet<String>>,
primary_by_local_name: HashMap<String, HashSet<String>>,
disclosure_by_qname: HashMap<String, HashSet<String>>,
disclosure_by_local_name: HashMap<String, HashSet<String>>,
}
#[derive(Debug, Default, Clone)]
struct SurfaceConceptMembership {
primary_statements: HashSet<String>,
disclosure_surfaces: HashSet<String>,
}
impl SurfaceConceptIndex {
fn insert(&mut self, concept: &str, statement: &str, surface_key: &str) {
let Some(normalized_concept) = normalize_taxonomy_identity(concept) else {
return;
};
let Some(local_name) = normalize_taxonomy_local_name(concept) else {
return;
};
if statement == "disclosure" {
self.disclosure_by_qname
.entry(normalized_concept)
.or_default()
.insert(surface_key.to_string());
self.disclosure_by_local_name
.entry(local_name)
.or_default()
.insert(surface_key.to_string());
return;
}
if !is_primary_statement_kind(statement) {
return;
}
self.primary_by_qname
.entry(normalized_concept)
.or_default()
.insert(statement.to_string());
self.primary_by_local_name
.entry(local_name)
.or_default()
.insert(statement.to_string());
}
fn membership_for(&self, qname: &str, local_name: &str) -> SurfaceConceptMembership {
let mut membership = SurfaceConceptMembership::default();
if let Some(statement) = statement_overlap_override(local_name) {
membership.primary_statements.insert(statement.to_string());
}
for value in self
.resolve_from_map(&self.primary_by_qname, qname)
.into_iter()
.flatten()
{
membership.primary_statements.insert(value);
}
for value in self
.resolve_from_map(&self.primary_by_local_name, local_name)
.into_iter()
.flatten()
{
membership.primary_statements.insert(value);
}
for value in self
.resolve_from_map(&self.disclosure_by_qname, qname)
.into_iter()
.flatten()
{
membership.disclosure_surfaces.insert(value);
}
for value in self
.resolve_from_map(&self.disclosure_by_local_name, local_name)
.into_iter()
.flatten()
{
membership.disclosure_surfaces.insert(value);
}
membership
}
fn resolve_primary_statement(&self, qname: &str, local_name: &str) -> Option<String> {
if let Some(statement) = statement_overlap_override(local_name) {
return Some(statement.to_string());
}
self.resolve_unique_from_map(&self.primary_by_qname, qname)
.or_else(|| self.resolve_unique_from_map(&self.primary_by_local_name, local_name))
}
fn resolve_unique_from_map(
&self,
map: &HashMap<String, HashSet<String>>,
value: &str,
) -> Option<String> {
let normalized = normalize_taxonomy_identity(value)?;
let statements = map.get(&normalized)?;
if statements.len() != 1 {
return None;
}
statements.iter().next().cloned()
}
fn resolve_from_map(
&self,
map: &HashMap<String, HashSet<String>>,
value: &str,
) -> Option<HashSet<String>> {
let normalized = normalize_taxonomy_identity(value)?;
map.get(&normalized).cloned()
}
}
#[derive(Debug, Default)]
struct PrimaryStatementConceptIndex {
by_qname: HashMap<String, HashSet<String>>,
by_local_name: HashMap<String, HashSet<String>>,
}
impl PrimaryStatementConceptIndex {
fn insert(&mut self, concept: &str, statement: &str) {
if !is_primary_statement_kind(statement) {
return;
}
let Some(normalized_concept) = normalize_taxonomy_identity(concept) else {
return;
};
self.by_qname
.entry(normalized_concept)
.or_default()
.insert(statement.to_string());
let Some(local_name) = normalize_taxonomy_local_name(concept) else {
return;
};
self.by_local_name
.entry(local_name)
.or_default()
.insert(statement.to_string());
}
fn resolve(&self, qname: &str, local_name: &str) -> Option<String> {
if let Some(statement) = statement_overlap_override(local_name) {
return Some(statement.to_string());
}
self.resolve_from_map(&self.by_qname, qname)
.or_else(|| self.resolve_from_map(&self.by_local_name, local_name))
}
fn resolve_from_map(
&self,
map: &HashMap<String, HashSet<String>>,
value: &str,
) -> Option<String> {
let normalized = normalize_taxonomy_identity(value)?;
let statements = map.get(&normalized)?;
if statements.len() != 1 {
return None;
}
statements.iter().next().cloned()
}
}
fn is_primary_statement_kind(statement: &str) -> bool {
matches!(
statement,
"income" | "balance" | "cash_flow" | "equity" | "comprehensive_income"
)
}
fn normalize_taxonomy_identity(value: &str) -> Option<String> {
let trimmed = value.trim().to_ascii_lowercase();
if trimmed.is_empty() {
None
} else {
Some(trimmed)
}
}
fn normalize_taxonomy_local_name(value: &str) -> Option<String> {
let normalized = normalize_taxonomy_identity(value)?;
let local_name = normalized
.rsplit_once(':')
.map(|(_, local_name)| local_name)
.unwrap_or(normalized.as_str())
.trim()
.to_string();
if local_name.is_empty() {
None
} else {
Some(local_name)
}
}
fn statement_overlap_override(local_name: &str) -> Option<&'static str> {
match normalize_taxonomy_local_name(local_name).as_deref() {
Some("stockholdersequity") => Some("equity"),
Some("retainedearningsaccumulateddeficit") => Some("equity"),
Some("commonstocksincludingadditionalpaidincapital") => Some("equity"),
Some("accumulatedothercomprehensiveincomelossnetoftax") => Some("equity"),
Some("stockholdersequityother") => Some("equity"),
Some("liabilitiesandstockholdersequity") => Some("balance"),
_ => None,
}
}
fn load_primary_statement_concept_index() -> PrimaryStatementConceptIndex {
let mut index = PrimaryStatementConceptIndex::default();
let taxonomy_dir = match taxonomy_loader::resolve_taxonomy_dir() {
Ok(path) => path,
Err(error) => {
eprintln!("[fiscal-xbrl] primary statement taxonomy index unavailable: {error}");
return index;
}
};
let fiscal_dir = taxonomy_dir.join("fiscal").join("v1");
let entries = match fs::read_dir(&fiscal_dir) {
Ok(entries) => entries,
Err(error) => {
eprintln!(
"[fiscal-xbrl] unable to read taxonomy surface directory {}: {error}",
fiscal_dir.display()
);
return index;
}
};
for entry in entries.flatten() {
let path = entry.path();
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("");
if !file_name.ends_with(".surface.json") || file_name == "universal_income.surface.json" {
continue;
}
let raw = match fs::read_to_string(&path) {
Ok(raw) => raw,
Err(error) => {
eprintln!(
"[fiscal-xbrl] unable to read surface taxonomy {}: {error}",
path.display()
);
continue;
}
};
let pack = match serde_json::from_str::<SurfacePackFile>(&raw) {
Ok(pack) => pack,
Err(error) => {
eprintln!(
"[fiscal-xbrl] unable to parse surface taxonomy {}: {error}",
path.display()
);
continue;
}
};
for surface in pack
.surfaces
.iter()
.filter(|surface| is_primary_statement_kind(&surface.statement))
{
for concept in surface
.allowed_source_concepts
.iter()
.chain(surface.allowed_authoritative_concepts.iter())
{
index.insert(concept, &surface.statement);
}
}
}
index
}
fn build_surface_concept_index(surface_pack: &SurfacePackFile) -> SurfaceConceptIndex {
let mut index = SurfaceConceptIndex::default();
for surface in &surface_pack.surfaces {
for concept in surface
.allowed_source_concepts
.iter()
.chain(surface.allowed_authoritative_concepts.iter())
.chain(surface.issuer_overlay_source_concepts.iter())
.chain(surface.issuer_overlay_authoritative_concepts.iter())
{
index.insert(concept, &surface.statement, &surface.surface_key);
}
}
index
}
fn sec_rate_limit_delay() -> u64 {
std::env::var("SEC_RATE_LIMIT_MS")
@@ -89,6 +400,8 @@ pub struct HydrateFilingRequest {
pub filing_type: String,
pub filing_url: Option<String>,
pub primary_document: Option<String>,
#[serde(default)]
pub issuer_overlay: Option<RuntimeIssuerOverlay>,
pub cache_dir: Option<String>,
}
@@ -420,6 +733,10 @@ pub struct NormalizationSummaryOutput {
pub kpi_row_count: usize,
pub unmapped_row_count: usize,
pub material_unmapped_row_count: usize,
pub residual_primary_count: usize,
pub residual_disclosure_count: usize,
pub unsupported_concept_count: usize,
pub issuer_overlay_match_count: usize,
pub warnings: Vec<String>,
}
@@ -520,6 +837,10 @@ pub fn hydrate_filing(input: HydrateFilingRequest) -> Result<HydrateFilingRespon
kpi_row_count: 0,
unmapped_row_count: 0,
material_unmapped_row_count: 0,
residual_primary_count: 0,
residual_disclosure_count: 0,
unsupported_concept_count: 0,
issuer_overlay_match_count: 0,
warnings: vec![],
},
});
@@ -571,6 +892,27 @@ pub fn hydrate_filing(input: HydrateFilingRequest) -> Result<HydrateFilingRespon
}
}
let initial_materialized = materialize_taxonomy_statements(
input.filing_id,
&input.accession_number,
&input.filing_date,
&input.filing_type,
&parsed_instance.facts,
&presentation,
&label_by_concept,
None,
);
let taxonomy_regime = infer_taxonomy_regime(&parsed_instance.facts);
let pack_selection = pack_selector::select_fiscal_pack(
&initial_materialized.statement_rows,
&initial_materialized.facts,
);
let surface_pack = taxonomy_loader::load_surface_pack_with_overlays(
pack_selection.pack,
Some(&input.ticker),
input.issuer_overlay.as_ref(),
)?;
let surface_concept_index = build_surface_concept_index(&surface_pack);
let materialized = materialize_taxonomy_statements(
input.filing_id,
&input.accession_number,
@@ -579,19 +921,23 @@ pub fn hydrate_filing(input: HydrateFilingRequest) -> Result<HydrateFilingRespon
&parsed_instance.facts,
&presentation,
&label_by_concept,
Some(&surface_concept_index),
);
let taxonomy_regime = infer_taxonomy_regime(&parsed_instance.facts);
let mut concepts = materialized.concepts;
let mut facts = materialized.facts;
let pack_selection = pack_selector::select_fiscal_pack(&materialized.statement_rows, &facts);
let fiscal_pack = pack_selection.pack.as_str().to_string();
let mut compact_model = surface_mapper::build_compact_surface_model(
&materialized.periods,
&materialized.statement_rows,
&taxonomy_regime,
pack_selection.pack,
Some(&input.ticker),
input.issuer_overlay.as_ref(),
pack_selection.warnings,
)?;
compact_model
.normalization_summary
.unsupported_concept_count = materialized.unsupported_concept_count;
universal_income::apply_universal_income_rows(
&materialized.periods,
&materialized.statement_rows,
@@ -1465,6 +1811,7 @@ struct MaterializedStatements {
statement_rows: StatementRowMap,
concepts: Vec<ConceptOutput>,
facts: Vec<FactOutput>,
unsupported_concept_count: usize,
}
fn materialize_taxonomy_statements(
@@ -1475,6 +1822,7 @@ fn materialize_taxonomy_statements(
facts: &[ParsedFact],
presentation: &[PresentationNode],
label_by_concept: &HashMap<String, String>,
surface_index: Option<&SurfaceConceptIndex>,
) -> MaterializedStatements {
let compact_accession = accession_number.replace('-', "");
let mut period_by_signature = HashMap::<String, PeriodOutput>::new();
@@ -1551,6 +1899,7 @@ fn materialize_taxonomy_statements(
let mut grouped_by_statement = empty_parsed_fact_map();
let mut enriched_facts = Vec::new();
let mut unsupported_concepts = HashSet::<String>::new();
for (index, fact) in facts.iter().enumerate() {
let nodes = presentation_by_concept
@@ -1558,9 +1907,28 @@ fn materialize_taxonomy_statements(
.cloned()
.unwrap_or_default();
let best_node = nodes.first().copied();
let statement_kind = best_node
.and_then(|node| classify_statement_role(&node.role_uri))
.or_else(|| concept_statement_fallback(&fact.local_name));
let role_statement_kind =
best_node.and_then(|node| classify_statement_role(&node.role_uri));
let disclosure_role_family =
best_node.and_then(|node| classify_disclosure_role(&node.role_uri));
let statement_kind = if let Some(statement_kind) = role_statement_kind {
Some(statement_kind)
} else if disclosure_role_family.is_some() {
Some("disclosure".to_string())
} else if let Some(index) = surface_index {
let membership = index.membership_for(&fact.qname, &fact.local_name);
if !membership.disclosure_surfaces.is_empty() {
Some("disclosure".to_string())
} else {
index.resolve_primary_statement(&fact.qname, &fact.local_name)
}
} else {
concept_statement_fallback(&fact.qname, &fact.local_name)
};
if statement_kind.is_none() {
unsupported_concepts.insert(fact.concept_key.clone());
}
let fact_output = FactOutput {
concept_key: fact.concept_key.clone(),
@@ -1762,6 +2130,7 @@ fn materialize_taxonomy_statements(
statement_rows,
concepts,
facts: enriched_facts,
unsupported_concept_count: unsupported_concepts.len(),
}
}
@@ -1795,11 +2164,12 @@ fn empty_detail_row_map() -> DetailRowStatementMap {
.collect()
}
fn statement_keys() -> [&'static str; 5] {
fn statement_keys() -> [&'static str; 6] {
[
"income",
"balance",
"cash_flow",
"disclosure",
"equity",
"comprehensive_income",
]
@@ -1810,6 +2180,7 @@ fn statement_key_ref(value: &str) -> Option<&'static str> {
"income" => Some("income"),
"balance" => Some("balance"),
"cash_flow" => Some("cash_flow"),
"disclosure" => Some("disclosure"),
"equity" => Some("equity"),
"comprehensive_income" => Some("comprehensive_income"),
_ => None,
@@ -1913,52 +2284,62 @@ fn classify_statement_role(role_uri: &str) -> Option<String> {
None
}
fn concept_statement_fallback(local_name: &str) -> Option<String> {
let normalized = local_name.to_ascii_lowercase();
if Regex::new(r#"equity|retainedearnings|additionalpaidincapital"#)
.unwrap()
.is_match(&normalized)
pub fn classify_disclosure_role(role_uri: &str) -> Option<String> {
let normalized = role_uri.to_ascii_lowercase();
if normalized.contains("tax") {
return Some("income_tax_disclosure".to_string());
}
if normalized.contains("debt")
|| normalized.contains("borrow")
|| normalized.contains("notespayable")
{
return Some("equity".to_string());
return Some("debt_disclosure".to_string());
}
if normalized.contains("comprehensiveincome") {
return Some("comprehensive_income".to_string());
if normalized.contains("lease") {
return Some("lease_disclosure".to_string());
}
if Regex::new(
r#"deferredpolicyacquisitioncosts(andvalueofbusinessacquired)?$|supplementaryinsuranceinformationdeferredpolicyacquisitioncosts$|deferredacquisitioncosts$"#,
)
.unwrap()
.is_match(&normalized)
if normalized.contains("derivative") || normalized.contains("hedg") {
return Some("derivative_instruments_disclosure".to_string());
}
if normalized.contains("intangible") || normalized.contains("goodwill") {
return Some("intangible_assets_disclosure".to_string());
}
if normalized.contains("revenue")
|| normalized.contains("performanceobligation")
|| normalized.contains("contractwithcustomer")
{
return Some("balance".to_string());
return Some("revenue_disclosure".to_string());
}
if Regex::new(
r#"netcashprovidedbyusedin.*activities|increasedecreasein|paymentstoacquire|paymentsforcapitalimprovements$|paymentsfordepositsonrealestateacquisitions$|paymentsforrepurchase|paymentsofdividends|dividendscommonstockcash$|proceedsfrom|repaymentsofdebt|sharebasedcompensation$|allocatedsharebasedcompensationexpense$|depreciationdepletionandamortization$|depreciationamortizationandaccretionnet$|depreciationandamortization$|depreciationamortizationandother$|otheradjustmentstoreconcilenetincomelosstocashprovidedbyusedinoperatingactivities"#,
)
.unwrap()
.is_match(&normalized)
if normalized.contains("restrictedcash") {
return Some("cash_flow_disclosure".to_string());
}
if normalized.contains("businesscombination")
|| normalized.contains("acquisition")
|| normalized.contains("purchaseprice")
{
return Some("cash_flow".to_string());
return Some("business_combinations_disclosure".to_string());
}
if Regex::new(
r#"asset|liabilit|debt|financingreceivable|loansreceivable|deposits|allowanceforcreditloss|futurepolicybenefits|policyholderaccountbalances|unearnedpremiums|realestateinvestmentproperty|grossatcarryingvalue|investmentproperty"#,
)
.unwrap()
.is_match(&normalized)
{
return Some("balance".to_string());
if normalized.contains("sharebased") || normalized.contains("stockcompensation") {
return Some("share_based_compensation_disclosure".to_string());
}
if Regex::new(
r#"revenue|income|profit|expense|costof|leaseincome|rental|premiums|claims|underwriting|policyacquisition|interestincome|interestexpense|noninterest|leasedandrentedproperty"#,
)
.unwrap()
.is_match(&normalized)
{
return Some("income".to_string());
if normalized.contains("equitymethod") || normalized.contains("equitysecurity") {
return Some("equity_investments_disclosure".to_string());
}
if normalized.contains("deferredtax") {
return Some("deferred_tax_balance_disclosure".to_string());
}
if normalized.contains("othercomprehensiveincome") || normalized.contains("oci") {
return Some("other_comprehensive_income_disclosure".to_string());
}
None
}
fn concept_statement_fallback(qname: &str, local_name: &str) -> Option<String> {
PRIMARY_STATEMENT_CONCEPT_INDEX.resolve(qname, local_name)
}
fn is_standard_namespace(namespace_uri: &str) -> bool {
let lower = namespace_uri.to_ascii_lowercase();
lower.contains("us-gaap")
@@ -2110,6 +2491,8 @@ mod tests {
&statement_rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("core pack should load and map");
@@ -2218,60 +2601,124 @@ mod tests {
fn classifies_pack_specific_concepts_without_presentation_roles() {
assert_eq!(
concept_statement_fallback(
"us-gaap:FinancingReceivableExcludingAccruedInterestAfterAllowanceForCreditLoss",
"FinancingReceivableExcludingAccruedInterestAfterAllowanceForCreditLoss"
)
.as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback("Deposits").as_deref(),
concept_statement_fallback("us-gaap:Deposits", "Deposits").as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback("RealEstateInvestmentPropertyNet").as_deref(),
concept_statement_fallback(
"us-gaap:RealEstateInvestmentPropertyNet",
"RealEstateInvestmentPropertyNet"
)
.as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback("DeferredPolicyAcquisitionCosts").as_deref(),
concept_statement_fallback(
"us-gaap:DeferredPolicyAcquisitionCosts",
"DeferredPolicyAcquisitionCosts"
)
.as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback("DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired")
concept_statement_fallback(
"us-gaap:DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired",
"DeferredPolicyAcquisitionCostsAndValueOfBusinessAcquired"
)
.as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback(
"us-gaap:IncreaseDecreaseInAccountsReceivable",
"IncreaseDecreaseInAccountsReceivable"
)
.as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("us-gaap:PaymentsOfDividends", "PaymentsOfDividends")
.as_deref(),
Some("balance")
);
assert_eq!(
concept_statement_fallback("IncreaseDecreaseInAccountsReceivable").as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("PaymentsOfDividends").as_deref(),
concept_statement_fallback("us-gaap:RepaymentsOfDebt", "RepaymentsOfDebt").as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("RepaymentsOfDebt").as_deref(),
concept_statement_fallback("us-gaap:ShareBasedCompensation", "ShareBasedCompensation")
.as_deref(),
None
);
assert_eq!(
concept_statement_fallback(
"us-gaap:PaymentsForCapitalImprovements",
"PaymentsForCapitalImprovements"
)
.as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("ShareBasedCompensation").as_deref(),
concept_statement_fallback(
"us-gaap:PaymentsForDepositsOnRealEstateAcquisitions",
"PaymentsForDepositsOnRealEstateAcquisitions"
)
.as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("PaymentsForCapitalImprovements").as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("PaymentsForDepositsOnRealEstateAcquisitions").as_deref(),
Some("cash_flow")
);
assert_eq!(
concept_statement_fallback("LeaseIncome").as_deref(),
concept_statement_fallback("us-gaap:LeaseIncome", "LeaseIncome").as_deref(),
Some("income")
);
assert_eq!(
concept_statement_fallback("DirectCostsOfLeasedAndRentedPropertyOrEquipment")
.as_deref(),
concept_statement_fallback(
"us-gaap:DirectCostsOfLeasedAndRentedPropertyOrEquipment",
"DirectCostsOfLeasedAndRentedPropertyOrEquipment"
)
.as_deref(),
Some("income")
);
}
#[test]
fn keeps_disclosure_only_concepts_out_of_primary_statement_fallback() {
assert_eq!(
concept_statement_fallback(
"us-gaap:RevenueRemainingPerformanceObligation",
"RevenueRemainingPerformanceObligation"
),
None
);
assert_eq!(
concept_statement_fallback(
"us-gaap:EquitySecuritiesFVNINoncurrent",
"EquitySecuritiesFVNINoncurrent"
),
None
);
}
#[test]
fn applies_explicit_balance_equity_overlap_overrides() {
assert_eq!(
concept_statement_fallback("us-gaap:StockholdersEquity", "StockholdersEquity")
.as_deref(),
Some("equity")
);
assert_eq!(
concept_statement_fallback(
"us-gaap:LiabilitiesAndStockholdersEquity",
"LiabilitiesAndStockholdersEquity"
)
.as_deref(),
Some("balance")
);
}
}

100
rust/fiscal-xbrl-core/src/surface_mapper.rs
View File

@@ -3,8 +3,9 @@ use std::collections::{BTreeMap, HashMap, HashSet};
use crate::pack_selector::FiscalPack;
use crate::taxonomy_loader::{
load_crosswalk, load_income_bridge, load_surface_pack, CrosswalkFile, IncomeBridgeFile,
IncomeBridgeRow, SurfaceDefinition, SurfaceFormula, SurfaceFormulaOp, SurfaceSignTransform,
load_crosswalk, load_income_bridge, load_surface_pack_with_overlays, CrosswalkFile,
IncomeBridgeFile, IncomeBridgeRow, RuntimeIssuerOverlay, SurfaceDefinition, SurfaceFormula,
SurfaceFormulaOp, SurfaceOrigin, SurfaceSignTransform,
};
use crate::{
ConceptOutput, DetailRowOutput, DetailRowStatementMap, FactOutput, NormalizationSummaryOutput,
@@ -63,6 +64,7 @@ struct MatchedStatementRow<'a> {
mapping_method: MappingMethod,
match_role: MatchRole,
rank: i64,
issuer_overlay_match: bool,
}
#[derive(Debug, Default, Clone)]
@@ -110,9 +112,11 @@ pub fn build_compact_surface_model(
statement_rows: &StatementRowMap,
taxonomy_regime: &str,
fiscal_pack: FiscalPack,
ticker: Option<&str>,
runtime_overlay: Option<&RuntimeIssuerOverlay>,
warnings: Vec<String>,
) -> Result<CompactSurfaceModel> {
let pack = load_surface_pack(fiscal_pack)?;
let pack = load_surface_pack_with_overlays(fiscal_pack, ticker, runtime_overlay)?;
let crosswalk = load_crosswalk(taxonomy_regime)?;
let income_bridge = load_income_bridge(fiscal_pack).ok();
let mut surface_rows = empty_surface_row_map();
@@ -122,6 +126,9 @@ pub fn build_compact_surface_model(
let mut detail_row_count = 0usize;
let mut unmapped_row_count = 0usize;
let mut material_unmapped_row_count = 0usize;
let mut residual_primary_count = 0usize;
let mut residual_disclosure_count = 0usize;
let mut issuer_overlay_match_keys = HashSet::<String>::new();
for statement in statement_keys() {
let rows = statement_rows.get(statement).cloned().unwrap_or_default();
@@ -130,6 +137,11 @@ pub fn build_compact_surface_model(
.iter()
.filter(|definition| definition.statement == statement)
.collect::<Vec<_>>();
if statement_definitions.is_empty() {
surface_rows.insert(statement.to_string(), Vec::new());
detail_rows.insert(statement.to_string(), BTreeMap::new());
continue;
}
statement_definitions.sort_by(|left, right| {
left.order
.cmp(&right.order)
@@ -203,6 +215,9 @@ pub fn build_compact_surface_model(
residual_flag: false,
},
);
if matched.issuer_overlay_match {
issuer_overlay_match_keys.insert(matched.row.concept_key.clone());
}
}
let details = detail_matches
@@ -221,6 +236,9 @@ pub fn build_compact_surface_model(
residual_flag: false,
},
);
if matched.issuer_overlay_match {
issuer_overlay_match_keys.insert(matched.row.concept_key.clone());
}
build_detail_row(
matched.row,
&definition.surface_key,
@@ -310,6 +328,11 @@ pub fn build_compact_surface_model(
if !residual_rows.is_empty() {
unmapped_row_count += residual_rows.len();
if statement == "disclosure" {
residual_disclosure_count += residual_rows.len();
} else {
residual_primary_count += residual_rows.len();
}
material_unmapped_row_count += residual_rows
.iter()
.filter(|row| max_abs_value(&row.values) >= threshold)
@@ -331,6 +354,10 @@ pub fn build_compact_surface_model(
kpi_row_count: 0,
unmapped_row_count,
material_unmapped_row_count,
residual_primary_count,
residual_disclosure_count,
unsupported_concept_count: 0,
issuer_overlay_match_count: issuer_overlay_match_keys.len(),
warnings,
},
concept_mappings,
@@ -734,21 +761,38 @@ fn match_statement_row<'a>(
}) || authoritative_mapping
.map(|mapping| mapping.surface_key == definition.surface_key)
.unwrap_or(false);
let matches_overlay_authoritative =
authoritative_concept_key.as_ref().map_or(false, |concept| {
definition
.issuer_overlay_authoritative_concepts
.iter()
.any(|candidate| candidate_matches(candidate, concept))
});
if matches_authoritative {
if matches_authoritative || matches_overlay_authoritative {
return Some(MatchedStatementRow {
row,
authoritative_concept_key,
mapping_method: MappingMethod::AuthoritativeDirect,
match_role: MatchRole::Surface,
rank: 0,
issuer_overlay_match: matches_overlay_authoritative
|| definition.origin == SurfaceOrigin::IssuerOverlay,
});
}
let matches_source = definition.allowed_source_concepts.iter().any(|candidate| {
candidate_matches(candidate, &row.qname) || candidate_matches(candidate, &row.local_name)
});
if matches_source {
let matches_overlay_source =
definition
.issuer_overlay_source_concepts
.iter()
.any(|candidate| {
candidate_matches(candidate, &row.qname)
|| candidate_matches(candidate, &row.local_name)
});
if matches_source || matches_overlay_source {
return Some(MatchedStatementRow {
row,
authoritative_concept_key,
@@ -759,6 +803,8 @@ fn match_statement_row<'a>(
MatchRole::Surface
},
rank: 1,
issuer_overlay_match: matches_overlay_source
|| definition.origin == SurfaceOrigin::IssuerOverlay,
});
}
@@ -832,6 +878,7 @@ fn match_income_bridge_detail_row<'a>(
mapping_method: MappingMethod::AggregateChildren,
match_role: MatchRole::Detail,
rank: 2,
issuer_overlay_match: false,
})
}
@@ -1033,11 +1080,12 @@ fn candidate_matches(candidate: &str, actual: &str) -> bool {
.unwrap_or(false)
}
fn statement_keys() -> [&'static str; 5] {
fn statement_keys() -> [&'static str; 6] {
[
"income",
"balance",
"cash_flow",
"disclosure",
"equity",
"comprehensive_income",
]
@@ -1122,6 +1170,7 @@ mod tests {
("income".to_string(), Vec::new()),
("balance".to_string(), Vec::new()),
("cash_flow".to_string(), Vec::new()),
("disclosure".to_string(), Vec::new()),
("equity".to_string(), Vec::new()),
("comprehensive_income".to_string(), Vec::new()),
])
@@ -1151,6 +1200,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1178,6 +1229,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1220,6 +1273,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1290,9 +1345,16 @@ mod tests {
},
];
let model =
build_compact_surface_model(&periods, &rows, "us-gaap", FiscalPack::Core, vec![])
.expect("compact model should build");
let model = build_compact_surface_model(
&periods,
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
let receivables = model
.surface_rows
@@ -1365,6 +1427,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1430,6 +1494,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::BankLender,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1473,6 +1539,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Insurance,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1557,6 +1625,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1640,6 +1710,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Core,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1686,6 +1758,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Insurance,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1742,6 +1816,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::ReitRealEstate,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1786,6 +1862,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::Software,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1825,6 +1903,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::EntertainmentBroadcasters,
None,
None,
vec![],
)
.expect("compact model should build");
@@ -1866,6 +1946,8 @@ mod tests {
&rows,
"us-gaap",
FiscalPack::PlanDefinedBenefit,
None,
None,
vec![],
)
.expect("compact model should build");

286
rust/fiscal-xbrl-core/src/taxonomy_loader.rs
View File

@@ -11,6 +11,19 @@ fn default_include_in_output() -> bool {
true
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SurfaceOrigin {
PackPrimary,
PackDisclosure,
CorePrimary,
CoreDisclosure,
IssuerOverlay,
}
fn default_surface_origin() -> SurfaceOrigin {
SurfaceOrigin::CorePrimary
}
#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum SurfaceSignTransform {
@@ -25,6 +38,25 @@ pub struct SurfacePackFile {
pub surfaces: Vec<SurfaceDefinition>,
}
#[derive(Debug, Deserialize, Clone)]
pub struct RuntimeIssuerOverlay {
pub version: String,
pub ticker: String,
pub pack: Option<String>,
#[serde(default)]
pub mappings: Vec<RuntimeIssuerOverlayMapping>,
}
#[derive(Debug, Deserialize, Clone)]
pub struct RuntimeIssuerOverlayMapping {
pub surface_key: String,
pub statement: String,
#[serde(default)]
pub allowed_source_concepts: Vec<String>,
#[serde(default)]
pub allowed_authoritative_concepts: Vec<String>,
}
#[derive(Debug, Deserialize, Clone)]
pub struct SurfaceDefinition {
pub surface_key: String,
@@ -43,6 +75,12 @@ pub struct SurfaceDefinition {
pub include_in_output: bool,
#[serde(default)]
pub sign_transform: Option<SurfaceSignTransform>,
#[serde(skip, default = "default_surface_origin")]
pub origin: SurfaceOrigin,
#[serde(skip, default)]
pub issuer_overlay_source_concepts: Vec<String>,
#[serde(skip, default)]
pub issuer_overlay_authoritative_concepts: Vec<String>,
}
#[derive(Debug, Deserialize, Clone)]
@@ -244,44 +282,114 @@ pub fn resolve_taxonomy_dir() -> Result<PathBuf> {
}
pub fn load_surface_pack(pack: FiscalPack) -> Result<SurfacePackFile> {
load_surface_pack_with_overlays(pack, None, None)
}
pub fn load_surface_pack_with_overlays(
pack: FiscalPack,
ticker: Option<&str>,
runtime_overlay: Option<&RuntimeIssuerOverlay>,
) -> Result<SurfacePackFile> {
let taxonomy_dir = resolve_taxonomy_dir()?;
let path = taxonomy_dir
.join("fiscal")
.join("v1")
.join(format!("{}.surface.json", pack.as_str()));
let mut file = load_surface_pack_file(&path)?;
let fiscal_dir = taxonomy_dir.join("fiscal").join("v1");
let mut file = SurfacePackFile {
version: "fiscal-v1".to_string(),
pack: pack.as_str().to_string(),
surfaces: Vec::new(),
};
merge_surface_pack_file(
&mut file,
load_surface_pack_file(&fiscal_dir.join(format!("{}.surface.json", pack.as_str())))?,
if matches!(pack, FiscalPack::Core) {
SurfaceOrigin::CorePrimary
} else {
SurfaceOrigin::PackPrimary
},
);
merge_optional_surface_pack_file(
&mut file,
load_optional_surface_pack_file(
&fiscal_dir.join(format!("{}.disclosure.surface.json", pack.as_str())),
)?,
if matches!(pack, FiscalPack::Core) {
SurfaceOrigin::CoreDisclosure
} else {
SurfaceOrigin::PackDisclosure
},
);
if !matches!(pack, FiscalPack::Core) {
let core_path = taxonomy_dir
.join("fiscal")
.join("v1")
.join("core.surface.json");
let core_file = load_surface_pack_file(&core_path)?;
let pack_inherited_keys = file
.surfaces
.iter()
.filter(|surface| surface.statement == "balance" || surface.statement == "cash_flow")
.map(|surface| (surface.statement.clone(), surface.surface_key.clone()))
.collect::<std::collections::HashSet<_>>();
file.surfaces.extend(
core_file
.surfaces
.into_iter()
.filter(|surface| {
surface.statement == "balance" || surface.statement == "cash_flow"
})
.filter(|surface| {
!pack_inherited_keys
.contains(&(surface.statement.clone(), surface.surface_key.clone()))
}),
merge_surface_pack_file(
&mut file,
load_surface_pack_file(&fiscal_dir.join("core.surface.json"))?,
SurfaceOrigin::CorePrimary,
);
}
merge_optional_surface_pack_file(
&mut file,
load_optional_surface_pack_file(&fiscal_dir.join("core.disclosure.surface.json"))?,
SurfaceOrigin::CoreDisclosure,
);
if let Some(normalized_ticker) = ticker
.map(|value| value.trim().to_ascii_lowercase())
.filter(|value| !value.is_empty())
{
merge_optional_surface_pack_file(
&mut file,
load_optional_surface_pack_file(
&fiscal_dir
.join("issuers")
.join(format!("{normalized_ticker}.surface.json")),
)?,
SurfaceOrigin::IssuerOverlay,
);
}
if let Some(runtime_overlay) = runtime_overlay {
merge_runtime_issuer_overlay(&mut file, runtime_overlay);
}
let _ = (&file.version, &file.pack);
Ok(file)
}
fn merge_runtime_issuer_overlay(target: &mut SurfacePackFile, overlay: &RuntimeIssuerOverlay) {
for mapping in &overlay.mappings {
let Some(existing) = target
.surfaces
.iter_mut()
.find(|candidate| candidate.surface_key == mapping.surface_key)
else {
continue;
};
if existing.statement != mapping.statement {
continue;
}
existing.issuer_overlay_source_concepts.extend(
mapping
.allowed_source_concepts
.iter()
.cloned()
.filter(|concept| !concept.trim().is_empty()),
);
existing.issuer_overlay_authoritative_concepts.extend(
mapping
.allowed_authoritative_concepts
.iter()
.cloned()
.filter(|concept| !concept.trim().is_empty()),
);
existing.issuer_overlay_source_concepts.sort();
existing.issuer_overlay_source_concepts.dedup();
existing.issuer_overlay_authoritative_concepts.sort();
existing.issuer_overlay_authoritative_concepts.dedup();
}
}
fn load_surface_pack_file(path: &PathBuf) -> Result<SurfacePackFile> {
let raw = fs::read_to_string(path).with_context(|| {
format!(
@@ -297,6 +405,128 @@ fn load_surface_pack_file(path: &PathBuf) -> Result<SurfacePackFile> {
})
}
fn load_optional_surface_pack_file(path: &PathBuf) -> Result<Option<SurfacePackFile>> {
if !path.exists() {
return Ok(None);
}
load_surface_pack_file(path).map(Some)
}
fn merge_optional_surface_pack_file(
target: &mut SurfacePackFile,
file: Option<SurfacePackFile>,
origin: SurfaceOrigin,
) {
if let Some(file) = file {
merge_surface_pack_file(target, file, origin);
}
}
fn merge_surface_pack_file(
target: &mut SurfacePackFile,
mut incoming: SurfacePackFile,
origin: SurfaceOrigin,
) {
for mut surface in incoming.surfaces.drain(..) {
surface.origin = origin;
if let Some(existing) = target
.surfaces
.iter_mut()
.find(|candidate| candidate.surface_key == surface.surface_key)
{
if surface.origin == SurfaceOrigin::IssuerOverlay {
merge_surface_definition(existing, surface);
} else if matches!(
existing.origin,
SurfaceOrigin::PackPrimary | SurfaceOrigin::PackDisclosure
) && matches!(
surface.origin,
SurfaceOrigin::CorePrimary | SurfaceOrigin::CoreDisclosure
) {
continue;
} else {
merge_surface_definition(existing, surface);
}
} else {
target.surfaces.push(surface);
}
}
}
fn merge_surface_definition(existing: &mut SurfaceDefinition, incoming: SurfaceDefinition) {
if incoming.origin == SurfaceOrigin::IssuerOverlay {
existing.issuer_overlay_source_concepts.extend(
incoming
.allowed_source_concepts
.iter()
.cloned()
.filter(|concept| !concept.trim().is_empty()),
);
existing.issuer_overlay_authoritative_concepts.extend(
incoming
.allowed_authoritative_concepts
.iter()
.cloned()
.filter(|concept| !concept.trim().is_empty()),
);
if existing.statement.is_empty() {
existing.statement = incoming.statement;
}
if existing.label.is_empty() {
existing.label = incoming.label;
}
if existing.category.is_empty() {
existing.category = incoming.category;
}
if existing.unit.is_empty() {
existing.unit = incoming.unit;
}
if existing.formula_fallback.is_none() {
existing.formula_fallback = incoming.formula_fallback;
}
if existing.sign_transform.is_none() {
existing.sign_transform = incoming.sign_transform;
}
existing.include_in_output = existing.include_in_output || incoming.include_in_output;
existing.materiality_policy = if existing.materiality_policy.is_empty() {
incoming.materiality_policy
} else {
existing.materiality_policy.clone()
};
existing.detail_grouping_policy = if existing.detail_grouping_policy.is_empty() {
incoming.detail_grouping_policy
} else {
existing.detail_grouping_policy.clone()
};
existing.rollup_policy = if existing.rollup_policy.is_empty() {
incoming.rollup_policy
} else {
existing.rollup_policy.clone()
};
return;
}
existing.allowed_source_concepts.extend(
incoming
.allowed_source_concepts
.into_iter()
.filter(|concept| !concept.trim().is_empty()),
);
existing.allowed_authoritative_concepts.extend(
incoming
.allowed_authoritative_concepts
.into_iter()
.filter(|concept| !concept.trim().is_empty()),
);
existing.allowed_source_concepts.sort();
existing.allowed_source_concepts.dedup();
existing.allowed_authoritative_concepts.sort();
existing.allowed_authoritative_concepts.dedup();
}
pub fn load_crosswalk(regime: &str) -> Result<Option<CrosswalkFile>> {
let file_name = match regime {
"us-gaap" => "us-gaap.json",