mirror of
https://github.com/stefanoamorelli/crabrl.git
synced 2026-04-18 07:10:42 +00:00
feat: add core XBRL parser implementation
- High-performance parser with zero-copy design - SIMD optimizations for text processing - Memory-mapped file support - SEC EDGAR validation rules - Linkbase and schema support - Custom memory allocator with mimalloc
This commit is contained in:
177
src/allocator.rs
Normal file
177
src/allocator.rs
Normal file
@@ -0,0 +1,177 @@
|
||||
use bumpalo::Bump;
|
||||
use std::cell::RefCell;
|
||||
use std::mem::MaybeUninit;
|
||||
use std::ptr::NonNull;
|
||||
use std::sync::Arc;
|
||||
use parking_lot::Mutex;
|
||||
use string_interner::{DefaultBackend, Symbol};
|
||||
use string_interner::symbol::SymbolU32;
|
||||
|
||||
const ARENA_SIZE: usize = 64 * 1024 * 1024; // 64MB arenas
|
||||
const POOL_SIZE: usize = 1024;
|
||||
|
||||
#[repr(align(64))]
|
||||
pub struct ArenaAllocator {
|
||||
current: RefCell<Bump>,
|
||||
arenas: RefCell<Vec<Bump>>,
|
||||
string_interner: Arc<Mutex<string_interner::StringInterner<DefaultBackend>>>,
|
||||
}
|
||||
|
||||
impl ArenaAllocator {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current: RefCell::new(Bump::with_capacity(ARENA_SIZE)),
|
||||
arenas: RefCell::new(Vec::with_capacity(16)),
|
||||
string_interner: Arc::new(Mutex::new(string_interner::StringInterner::new())),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn alloc<T>(&self, val: T) -> &T {
|
||||
unsafe {
|
||||
let ptr = self.current.borrow().alloc(val) as *const T;
|
||||
&*ptr
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn alloc_slice<T: Copy>(&self, slice: &[T]) -> &[T] {
|
||||
unsafe {
|
||||
let ptr = self.current.borrow().alloc_slice_copy(slice) as *const [T];
|
||||
&*ptr
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn alloc_str(&self, s: &str) -> &str {
|
||||
unsafe {
|
||||
let ptr = self.current.borrow().alloc_str(s) as *const str;
|
||||
&*ptr
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn intern_string(&self, s: &str) -> u32 {
|
||||
let mut interner = self.string_interner.lock();
|
||||
interner.get_or_intern(s).to_usize() as u32
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_interned(&self, id: u32) -> Option<String> {
|
||||
let interner = self.string_interner.lock();
|
||||
let symbol = SymbolU32::try_from_usize(id as usize)?;
|
||||
interner.resolve(symbol)
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
|
||||
pub fn reset(&self) {
|
||||
let mut current = self.current.borrow_mut();
|
||||
current.reset();
|
||||
|
||||
let mut arenas = self.arenas.borrow_mut();
|
||||
for arena in arenas.iter_mut() {
|
||||
arena.reset();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_arena(&self) {
|
||||
let mut arenas = self.arenas.borrow_mut();
|
||||
let old = std::mem::replace(&mut *self.current.borrow_mut(),
|
||||
Bump::with_capacity(ARENA_SIZE));
|
||||
arenas.push(old);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ObjectPool<T> {
|
||||
pool: Vec<Box<T>>,
|
||||
factory: fn() -> T,
|
||||
}
|
||||
|
||||
impl<T> ObjectPool<T> {
|
||||
pub fn new(capacity: usize, factory: fn() -> T) -> Self {
|
||||
let mut pool = Vec::with_capacity(capacity);
|
||||
for _ in 0..capacity {
|
||||
pool.push(Box::new(factory()));
|
||||
}
|
||||
Self { pool, factory }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn acquire(&mut self) -> Box<T> {
|
||||
self.pool.pop().unwrap_or_else(|| Box::new((self.factory)()))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn release(&mut self, obj: Box<T>) {
|
||||
if self.pool.len() < POOL_SIZE {
|
||||
self.pool.push(obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C, align(64))]
|
||||
pub struct StackBuffer<const N: usize> {
|
||||
data: [MaybeUninit<u8>; N],
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl<const N: usize> StackBuffer<N> {
|
||||
#[inline(always)]
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
data: unsafe { MaybeUninit::uninit().assume_init() },
|
||||
len: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn push(&mut self, byte: u8) -> bool {
|
||||
if self.len < N {
|
||||
self.data[self.len] = MaybeUninit::new(byte);
|
||||
self.len += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
self.data.as_ptr() as *const u8,
|
||||
self.len
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn clear(&mut self) {
|
||||
self.len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_arena_allocator() {
|
||||
let arena = ArenaAllocator::new();
|
||||
let s1 = arena.alloc_str("hello");
|
||||
let s2 = arena.alloc_str("world");
|
||||
assert_eq!(s1, "hello");
|
||||
assert_eq!(s2, "world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_string_interning() {
|
||||
let arena = ArenaAllocator::new();
|
||||
let id1 = arena.intern_string("test");
|
||||
let id2 = arena.intern_string("test");
|
||||
assert_eq!(id1, id2);
|
||||
|
||||
let s = arena.get_interned(id1).unwrap();
|
||||
assert_eq!(s, "test");
|
||||
}
|
||||
}
|
||||
40
src/bin/crabrl_bench.rs
Normal file
40
src/bin/crabrl_bench.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
use crabrl::Parser;
|
||||
use std::env;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() != 2 {
|
||||
eprintln!("Usage: {} <xbrl_file>", args[0]);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let filepath = &args[1];
|
||||
let parser = Parser::new();
|
||||
|
||||
let start = Instant::now();
|
||||
match parser.parse_file(filepath) {
|
||||
Ok(doc) => {
|
||||
let elapsed = start.elapsed();
|
||||
let ms = elapsed.as_secs_f64() * 1000.0;
|
||||
|
||||
println!("crabrl found: {} facts, {} contexts, {} units (in {:.3}ms)",
|
||||
doc.facts.len(),
|
||||
doc.contexts.len(),
|
||||
doc.units.len(),
|
||||
ms);
|
||||
|
||||
// Additional stats
|
||||
println!("Facts: {}", doc.facts.len());
|
||||
println!("Contexts: {}", doc.contexts.len());
|
||||
println!("Units: {}", doc.units.len());
|
||||
println!("Tuples: {}", doc.tuples.len());
|
||||
println!("Footnotes: {}", doc.footnotes.len());
|
||||
println!("Time: {:.3}ms", ms);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error parsing file: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
47
src/cache.rs
Normal file
47
src/cache.rs
Normal file
@@ -0,0 +1,47 @@
|
||||
use dashmap::DashMap;
|
||||
use std::sync::Arc;
|
||||
use std::hash::Hash;
|
||||
|
||||
pub struct LockFreeCache<K, V> {
|
||||
map: Arc<DashMap<K, V>>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl<K, V> LockFreeCache<K, V>
|
||||
where
|
||||
K: Eq + Hash + Clone,
|
||||
V: Clone,
|
||||
{
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
map: Arc::new(DashMap::with_capacity(capacity)),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get(&self, key: &K) -> Option<V> {
|
||||
self.map.get(key).map(|v| v.clone())
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn insert(&self, key: K, value: V) {
|
||||
if self.map.len() >= self.capacity {
|
||||
if let Some(entry) = self.map.iter().next() {
|
||||
let k = entry.key().clone();
|
||||
drop(entry);
|
||||
self.map.remove(&k);
|
||||
}
|
||||
}
|
||||
self.map.insert(key, value);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn contains(&self, key: &K) -> bool {
|
||||
self.map.contains_key(key)
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
self.map.clear();
|
||||
}
|
||||
}
|
||||
21
src/instance.rs
Normal file
21
src/instance.rs
Normal file
@@ -0,0 +1,21 @@
|
||||
use crate::model::Document;
|
||||
use crate::Result;
|
||||
|
||||
pub struct InstanceValidator {
|
||||
strict: bool,
|
||||
}
|
||||
|
||||
impl InstanceValidator {
|
||||
pub fn new() -> Self {
|
||||
Self { strict: false }
|
||||
}
|
||||
|
||||
pub fn with_strict(mut self, strict: bool) -> Self {
|
||||
self.strict = strict;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn validate(&self, _document: &Document) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
438
src/linkbase.rs
Normal file
438
src/linkbase.rs
Normal file
@@ -0,0 +1,438 @@
|
||||
// Linkbase processing for XBRL
|
||||
use crate::{Error, Result, model::*};
|
||||
use compact_str::CompactString;
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
pub struct LinkbaseProcessor {
|
||||
presentation_links: HashMap<CompactString, Vec<PresentationLink>>,
|
||||
calculation_links: HashMap<CompactString, Vec<CalculationLink>>,
|
||||
definition_links: HashMap<CompactString, Vec<DefinitionLink>>,
|
||||
label_links: HashMap<CompactString, Vec<LabelLink>>,
|
||||
reference_links: HashMap<CompactString, Vec<ReferenceLink>>,
|
||||
}
|
||||
|
||||
impl LinkbaseProcessor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
presentation_links: HashMap::new(),
|
||||
calculation_links: HashMap::new(),
|
||||
definition_links: HashMap::new(),
|
||||
label_links: HashMap::new(),
|
||||
reference_links: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_linkbase<P: AsRef<Path>>(&mut self, path: P) -> Result<()> {
|
||||
let content = std::fs::read(path)?;
|
||||
self.parse_linkbase(&content)
|
||||
}
|
||||
|
||||
pub fn parse_linkbase(&mut self, data: &[u8]) -> Result<()> {
|
||||
// Skip BOM if present
|
||||
let data = if data.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
&data[3..]
|
||||
} else {
|
||||
data
|
||||
};
|
||||
|
||||
let text = std::str::from_utf8(data)
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in linkbase".to_string()))?;
|
||||
|
||||
// Detect linkbase type and parse accordingly
|
||||
if text.contains("presentationLink") {
|
||||
self.parse_presentation_linkbase(text)?;
|
||||
}
|
||||
if text.contains("calculationLink") {
|
||||
self.parse_calculation_linkbase(text)?;
|
||||
}
|
||||
if text.contains("definitionLink") {
|
||||
self.parse_definition_linkbase(text)?;
|
||||
}
|
||||
if text.contains("labelLink") {
|
||||
self.parse_label_linkbase(text)?;
|
||||
}
|
||||
if text.contains("referenceLink") {
|
||||
self.parse_reference_linkbase(text)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_presentation_linkbase(&mut self, text: &str) -> Result<()> {
|
||||
// Parse presentation arcs
|
||||
let mut pos = 0;
|
||||
while let Some(arc_start) = text[pos..].find("<link:presentationArc") {
|
||||
let arc_start = pos + arc_start;
|
||||
pos = arc_start + 1;
|
||||
|
||||
if let Some(arc_end) = text[arc_start..].find("/>") {
|
||||
let arc_text = &text[arc_start..arc_start + arc_end];
|
||||
|
||||
let mut link = PresentationLink {
|
||||
from: CompactString::new(""),
|
||||
to: CompactString::new(""),
|
||||
order: 1.0,
|
||||
priority: None,
|
||||
use_attribute: None,
|
||||
};
|
||||
|
||||
// Extract from
|
||||
if let Some(from_start) = arc_text.find("xlink:from=\"") {
|
||||
let from_start = from_start + 12;
|
||||
if let Some(from_end) = arc_text[from_start..].find('"') {
|
||||
link.from = CompactString::from(&arc_text[from_start..from_start + from_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract to
|
||||
if let Some(to_start) = arc_text.find("xlink:to=\"") {
|
||||
let to_start = to_start + 10;
|
||||
if let Some(to_end) = arc_text[to_start..].find('"') {
|
||||
link.to = CompactString::from(&arc_text[to_start..to_start + to_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract order
|
||||
if let Some(order_start) = arc_text.find("order=\"") {
|
||||
let order_start = order_start + 7;
|
||||
if let Some(order_end) = arc_text[order_start..].find('"') {
|
||||
if let Ok(order) = arc_text[order_start..order_start + order_end].parse() {
|
||||
link.order = order;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract priority
|
||||
if let Some(priority_start) = arc_text.find("priority=\"") {
|
||||
let priority_start = priority_start + 10;
|
||||
if let Some(priority_end) = arc_text[priority_start..].find('"') {
|
||||
if let Ok(priority) = arc_text[priority_start..priority_start + priority_end].parse() {
|
||||
link.priority = Some(priority);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract use
|
||||
if let Some(use_start) = arc_text.find("use=\"") {
|
||||
let use_start = use_start + 5;
|
||||
if let Some(use_end) = arc_text[use_start..].find('"') {
|
||||
link.use_attribute = Some(CompactString::from(&arc_text[use_start..use_start + use_end]));
|
||||
}
|
||||
}
|
||||
|
||||
self.presentation_links
|
||||
.entry(link.from.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(link);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_calculation_linkbase(&mut self, text: &str) -> Result<()> {
|
||||
// Parse calculation arcs
|
||||
let mut pos = 0;
|
||||
while let Some(arc_start) = text[pos..].find("<link:calculationArc") {
|
||||
let arc_start = pos + arc_start;
|
||||
pos = arc_start + 1;
|
||||
|
||||
if let Some(arc_end) = text[arc_start..].find("/>") {
|
||||
let arc_text = &text[arc_start..arc_start + arc_end];
|
||||
|
||||
let mut link = CalculationLink {
|
||||
from: CompactString::new(""),
|
||||
to: CompactString::new(""),
|
||||
weight: 1.0,
|
||||
order: 1.0,
|
||||
};
|
||||
|
||||
// Extract from
|
||||
if let Some(from_start) = arc_text.find("xlink:from=\"") {
|
||||
let from_start = from_start + 12;
|
||||
if let Some(from_end) = arc_text[from_start..].find('"') {
|
||||
link.from = CompactString::from(&arc_text[from_start..from_start + from_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract to
|
||||
if let Some(to_start) = arc_text.find("xlink:to=\"") {
|
||||
let to_start = to_start + 10;
|
||||
if let Some(to_end) = arc_text[to_start..].find('"') {
|
||||
link.to = CompactString::from(&arc_text[to_start..to_start + to_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract weight
|
||||
if let Some(weight_start) = arc_text.find("weight=\"") {
|
||||
let weight_start = weight_start + 8;
|
||||
if let Some(weight_end) = arc_text[weight_start..].find('"') {
|
||||
if let Ok(weight) = arc_text[weight_start..weight_start + weight_end].parse() {
|
||||
link.weight = weight;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract order
|
||||
if let Some(order_start) = arc_text.find("order=\"") {
|
||||
let order_start = order_start + 7;
|
||||
if let Some(order_end) = arc_text[order_start..].find('"') {
|
||||
if let Ok(order) = arc_text[order_start..order_start + order_end].parse() {
|
||||
link.order = order;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.calculation_links
|
||||
.entry(link.from.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(link);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_definition_linkbase(&mut self, text: &str) -> Result<()> {
|
||||
// Parse definition arcs
|
||||
let mut pos = 0;
|
||||
while let Some(arc_start) = text[pos..].find("<link:definitionArc") {
|
||||
let arc_start = pos + arc_start;
|
||||
pos = arc_start + 1;
|
||||
|
||||
if let Some(arc_end) = text[arc_start..].find("/>") {
|
||||
let arc_text = &text[arc_start..arc_start + arc_end];
|
||||
|
||||
let mut link = DefinitionLink {
|
||||
from: CompactString::new(""),
|
||||
to: CompactString::new(""),
|
||||
arcrole: CompactString::new(""),
|
||||
order: 1.0,
|
||||
};
|
||||
|
||||
// Extract from
|
||||
if let Some(from_start) = arc_text.find("xlink:from=\"") {
|
||||
let from_start = from_start + 12;
|
||||
if let Some(from_end) = arc_text[from_start..].find('"') {
|
||||
link.from = CompactString::from(&arc_text[from_start..from_start + from_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract to
|
||||
if let Some(to_start) = arc_text.find("xlink:to=\"") {
|
||||
let to_start = to_start + 10;
|
||||
if let Some(to_end) = arc_text[to_start..].find('"') {
|
||||
link.to = CompactString::from(&arc_text[to_start..to_start + to_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract arcrole
|
||||
if let Some(arcrole_start) = arc_text.find("xlink:arcrole=\"") {
|
||||
let arcrole_start = arcrole_start + 15;
|
||||
if let Some(arcrole_end) = arc_text[arcrole_start..].find('"') {
|
||||
link.arcrole = CompactString::from(&arc_text[arcrole_start..arcrole_start + arcrole_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract order
|
||||
if let Some(order_start) = arc_text.find("order=\"") {
|
||||
let order_start = order_start + 7;
|
||||
if let Some(order_end) = arc_text[order_start..].find('"') {
|
||||
if let Ok(order) = arc_text[order_start..order_start + order_end].parse() {
|
||||
link.order = order;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.definition_links
|
||||
.entry(link.from.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(link);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_label_linkbase(&mut self, text: &str) -> Result<()> {
|
||||
// Parse labels
|
||||
let mut pos = 0;
|
||||
while let Some(label_start) = text[pos..].find("<link:label") {
|
||||
let label_start = pos + label_start;
|
||||
pos = label_start + 1;
|
||||
|
||||
if let Some(label_end) = text[label_start..].find("</link:label>") {
|
||||
let label_text = &text[label_start..label_start + label_end];
|
||||
|
||||
let mut link = LabelLink {
|
||||
concept: CompactString::new(""),
|
||||
label: CompactString::new(""),
|
||||
role: CompactString::new(""),
|
||||
lang: CompactString::new("en"),
|
||||
};
|
||||
|
||||
// Extract label ID for concept mapping
|
||||
if let Some(id_start) = label_text.find("xlink:label=\"") {
|
||||
let id_start = id_start + 13;
|
||||
if let Some(id_end) = label_text[id_start..].find('"') {
|
||||
link.concept = CompactString::from(&label_text[id_start..id_start + id_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract role
|
||||
if let Some(role_start) = label_text.find("xlink:role=\"") {
|
||||
let role_start = role_start + 12;
|
||||
if let Some(role_end) = label_text[role_start..].find('"') {
|
||||
link.role = CompactString::from(&label_text[role_start..role_start + role_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract lang
|
||||
if let Some(lang_start) = label_text.find("xml:lang=\"") {
|
||||
let lang_start = lang_start + 10;
|
||||
if let Some(lang_end) = label_text[lang_start..].find('"') {
|
||||
link.lang = CompactString::from(&label_text[lang_start..lang_start + lang_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract label text content
|
||||
if let Some(content_start) = label_text.find('>') {
|
||||
let content = &label_text[content_start + 1..];
|
||||
link.label = CompactString::from(content.trim());
|
||||
}
|
||||
|
||||
self.label_links
|
||||
.entry(link.concept.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(link);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_reference_linkbase(&mut self, text: &str) -> Result<()> {
|
||||
// Parse references - simplified version
|
||||
let mut pos = 0;
|
||||
while let Some(ref_start) = text[pos..].find("<link:reference") {
|
||||
let ref_start = pos + ref_start;
|
||||
pos = ref_start + 1;
|
||||
|
||||
if let Some(ref_end) = text[ref_start..].find("</link:reference>") {
|
||||
let ref_text = &text[ref_start..ref_start + ref_end];
|
||||
|
||||
let mut reference = Reference {
|
||||
role: CompactString::new(""),
|
||||
parts: HashMap::new(),
|
||||
};
|
||||
|
||||
// Extract role
|
||||
if let Some(role_start) = ref_text.find("xlink:role=\"") {
|
||||
let role_start = role_start + 12;
|
||||
if let Some(role_end) = ref_text[role_start..].find('"') {
|
||||
reference.role = CompactString::from(&ref_text[role_start..role_start + role_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse reference parts (simplified)
|
||||
let parts = ["Name", "Number", "Section", "Subsection", "Paragraph", "Subparagraph", "Clause"];
|
||||
for part in &parts {
|
||||
let tag = format!("<link:{}", part);
|
||||
if let Some(part_start) = ref_text.find(&tag) {
|
||||
let part_start = part_start + tag.len();
|
||||
if let Some(content_start) = ref_text[part_start..].find('>') {
|
||||
let content_start = part_start + content_start + 1;
|
||||
if let Some(content_end) = ref_text[content_start..].find('<') {
|
||||
let content = &ref_text[content_start..content_start + content_end];
|
||||
reference.parts.insert(
|
||||
CompactString::from(*part),
|
||||
content.trim().to_string()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find concept this reference belongs to
|
||||
if let Some(label_start) = ref_text.find("xlink:label=\"") {
|
||||
let label_start = label_start + 13;
|
||||
if let Some(label_end) = ref_text[label_start..].find('"') {
|
||||
let concept = CompactString::from(&ref_text[label_start..label_start + label_end]);
|
||||
|
||||
let link = ReferenceLink {
|
||||
concept: concept.clone(),
|
||||
reference,
|
||||
};
|
||||
|
||||
self.reference_links
|
||||
.entry(concept)
|
||||
.or_insert_with(Vec::new)
|
||||
.push(link);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_presentation_tree(&self, root: &str) -> Vec<&PresentationLink> {
|
||||
self.presentation_links
|
||||
.get(root)
|
||||
.map(|links| {
|
||||
let mut sorted = links.iter().collect::<Vec<_>>();
|
||||
sorted.sort_by(|a, b| a.order.partial_cmp(&b.order).unwrap());
|
||||
sorted
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn calculate_total(&self, parent: &str, facts: &HashMap<String, f64>) -> f64 {
|
||||
if let Some(links) = self.calculation_links.get(parent) {
|
||||
links.iter()
|
||||
.map(|link| {
|
||||
facts.get(link.to.as_str())
|
||||
.map(|value| value * link.weight)
|
||||
.unwrap_or(0.0)
|
||||
})
|
||||
.sum()
|
||||
} else {
|
||||
facts.get(parent).copied().unwrap_or(0.0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_label(&self, concept: &str, role: &str, lang: &str) -> Option<&str> {
|
||||
self.label_links
|
||||
.get(concept)
|
||||
.and_then(|labels| {
|
||||
labels.iter()
|
||||
.find(|l| l.role == role && l.lang == lang)
|
||||
.or_else(|| labels.iter().find(|l| l.lang == lang))
|
||||
.or_else(|| labels.first())
|
||||
})
|
||||
.map(|l| l.label.as_str())
|
||||
}
|
||||
|
||||
pub fn validate_calculations(&self, facts: &HashMap<String, f64>) -> Vec<ValidationError> {
|
||||
let mut errors = Vec::new();
|
||||
|
||||
for (parent, links) in &self.calculation_links {
|
||||
let calculated = self.calculate_total(parent, facts);
|
||||
if let Some(&actual) = facts.get(parent.as_str()) {
|
||||
let diff = (calculated - actual).abs();
|
||||
let tolerance = 0.01; // Allow small rounding differences
|
||||
|
||||
if diff > tolerance {
|
||||
errors.push(ValidationError::CalculationInconsistency {
|
||||
concept: parent.to_string(),
|
||||
expected: calculated,
|
||||
actual,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errors
|
||||
}
|
||||
}
|
||||
1552
src/parser.rs
Normal file
1552
src/parser.rs
Normal file
File diff suppressed because it is too large
Load Diff
238
src/parser_base.rs
Normal file
238
src/parser_base.rs
Normal file
@@ -0,0 +1,238 @@
|
||||
// Base parsing methods for FullXbrlParser
|
||||
|
||||
impl<'a> FullXbrlParser<'a> {
|
||||
#[inline(always)]
|
||||
fn read_tag_name(&mut self) -> Result<&'a str> {
|
||||
let start = self.scanner.pos;
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b' ' || ch == b'>' || ch == b'/' || ch == b'\t' || ch == b'\n' || ch == b'\r' {
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
let end = self.scanner.pos;
|
||||
|
||||
if start == end {
|
||||
return Err(Error::Parse("Empty tag name".to_string()));
|
||||
}
|
||||
|
||||
std::str::from_utf8(&self.scanner.data[start..end])
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in tag name".to_string()))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn parse_attributes(&mut self) -> Result<Vec<(&'a str, &'a str)>> {
|
||||
let mut attrs = Vec::new();
|
||||
|
||||
loop {
|
||||
self.scanner.skip_whitespace();
|
||||
|
||||
match self.scanner.peek() {
|
||||
Some(b'>') => {
|
||||
// End of tag
|
||||
break;
|
||||
}
|
||||
Some(b'/') => {
|
||||
// Self-closing tag
|
||||
self.scanner.advance(1);
|
||||
if self.scanner.peek() == Some(b'>') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
None => return Err(Error::Parse("Unexpected EOF in attributes".to_string())),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let name_start = self.scanner.pos;
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'=' || ch == b' ' || ch == b'>' || ch == b'/' {
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
|
||||
if self.scanner.pos == name_start {
|
||||
break; // No more attributes
|
||||
}
|
||||
|
||||
let name = std::str::from_utf8(&self.scanner.data[name_start..self.scanner.pos])
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in attribute name".to_string()))?;
|
||||
|
||||
self.scanner.skip_whitespace();
|
||||
|
||||
if self.scanner.peek() != Some(b'=') {
|
||||
continue;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
|
||||
self.scanner.skip_whitespace();
|
||||
|
||||
let quote = self.scanner.peek()
|
||||
.ok_or_else(|| Error::Parse("Expected quote".to_string()))?;
|
||||
|
||||
if quote != b'"' && quote != b'\'' {
|
||||
return Err(Error::Parse("Expected quote in attribute".to_string()));
|
||||
}
|
||||
|
||||
self.scanner.advance(1);
|
||||
let value_start = self.scanner.pos;
|
||||
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == quote {
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
|
||||
let value = std::str::from_utf8(&self.scanner.data[value_start..self.scanner.pos])
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in attribute value".to_string()))?;
|
||||
|
||||
self.scanner.advance(1); // Skip closing quote
|
||||
|
||||
attrs.push((name, value));
|
||||
}
|
||||
|
||||
Ok(attrs)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn skip_to_tag_end(&mut self) -> Result<()> {
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'>' {
|
||||
self.scanner.advance(1);
|
||||
return Ok(());
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
Err(Error::Parse("Expected '>'".to_string()))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn read_text_content(&mut self) -> Result<&'a str> {
|
||||
let start = self.scanner.pos;
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'<' {
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
|
||||
let text = std::str::from_utf8(&self.scanner.data[start..self.scanner.pos])
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in text content".to_string()))?;
|
||||
|
||||
Ok(text.trim())
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn skip_element_from_tag(&mut self) -> Result<()> {
|
||||
// We've already read the tag name, now skip to end of opening tag
|
||||
self.skip_to_tag_end()?;
|
||||
|
||||
// Check if it was self-closing
|
||||
if self.scanner.pos >= 2 && self.scanner.data[self.scanner.pos - 2] == b'/' {
|
||||
return Ok(()); // Self-closing tag, we're done
|
||||
}
|
||||
|
||||
// Skip element content and find matching closing tag
|
||||
let mut depth = 1;
|
||||
|
||||
while depth > 0 && !self.scanner.is_eof() {
|
||||
// Find next tag
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'<' {
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
|
||||
if self.scanner.is_eof() {
|
||||
break;
|
||||
}
|
||||
|
||||
self.scanner.advance(1); // consume '<'
|
||||
|
||||
if self.scanner.peek() == Some(b'/') {
|
||||
depth -= 1;
|
||||
} else if self.scanner.peek() != Some(b'!') && self.scanner.peek() != Some(b'?') {
|
||||
// Check if it's a self-closing tag
|
||||
let mut is_self_closing = false;
|
||||
let _saved_pos = self.scanner.pos;
|
||||
|
||||
// Skip to end of tag to check
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'/' {
|
||||
if self.scanner.pos + 1 < self.scanner.data.len()
|
||||
&& self.scanner.data[self.scanner.pos + 1] == b'>' {
|
||||
is_self_closing = true;
|
||||
}
|
||||
}
|
||||
if ch == b'>' {
|
||||
self.scanner.advance(1);
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
|
||||
if !is_self_closing {
|
||||
depth += 1;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip to end of this tag
|
||||
while let Some(ch) = self.scanner.peek() {
|
||||
if ch == b'>' {
|
||||
self.scanner.advance(1);
|
||||
break;
|
||||
}
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn skip_processing_instruction(&mut self) -> Result<()> {
|
||||
// Skip until ?>
|
||||
while !self.scanner.is_eof() {
|
||||
if self.scanner.peek() == Some(b'?') {
|
||||
self.scanner.advance(1);
|
||||
if self.scanner.peek() == Some(b'>') {
|
||||
self.scanner.advance(1);
|
||||
return Ok(());
|
||||
}
|
||||
} else {
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
}
|
||||
Err(Error::Parse("Unclosed processing instruction".to_string()))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn skip_comment(&mut self) -> Result<()> {
|
||||
// Skip until -->
|
||||
while !self.scanner.is_eof() {
|
||||
if self.scanner.peek() == Some(b'-') {
|
||||
self.scanner.advance(1);
|
||||
if self.scanner.peek() == Some(b'-') {
|
||||
self.scanner.advance(1);
|
||||
if self.scanner.peek() == Some(b'>') {
|
||||
self.scanner.advance(1);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
self.scanner.advance(1);
|
||||
}
|
||||
}
|
||||
Err(Error::Parse("Unclosed comment".to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Parser {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
275
src/schema.rs
Normal file
275
src/schema.rs
Normal file
@@ -0,0 +1,275 @@
|
||||
// Schema loading and validation for XBRL
|
||||
use crate::{Error, Result, model::*};
|
||||
use compact_str::CompactString;
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
pub struct SchemaLoader {
|
||||
cache: HashMap<CompactString, Schema>,
|
||||
}
|
||||
|
||||
impl SchemaLoader {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
cache: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_schema<P: AsRef<Path>>(&mut self, path: P) -> Result<&Schema> {
|
||||
let path_str = path.as_ref().to_string_lossy();
|
||||
let key = CompactString::from(path_str.as_ref());
|
||||
|
||||
if self.cache.contains_key(&key) {
|
||||
return Ok(self.cache.get(&key).unwrap());
|
||||
}
|
||||
|
||||
let schema = self.parse_schema_file(path)?;
|
||||
self.cache.insert(key.clone(), schema);
|
||||
Ok(self.cache.get(&key).unwrap())
|
||||
}
|
||||
|
||||
fn parse_schema_file<P: AsRef<Path>>(&self, path: P) -> Result<Schema> {
|
||||
let content = std::fs::read(path)?;
|
||||
self.parse_schema_bytes(&content)
|
||||
}
|
||||
|
||||
fn parse_schema_bytes(&self, data: &[u8]) -> Result<Schema> {
|
||||
// Simple XML parsing for schema
|
||||
let mut schema = Schema {
|
||||
target_namespace: CompactString::new(""),
|
||||
elements: HashMap::new(),
|
||||
types: HashMap::new(),
|
||||
imports: Vec::new(),
|
||||
};
|
||||
|
||||
// Skip BOM if present
|
||||
let data = if data.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
&data[3..]
|
||||
} else {
|
||||
data
|
||||
};
|
||||
|
||||
let text = std::str::from_utf8(data)
|
||||
.map_err(|_| Error::Parse("Invalid UTF-8 in schema".to_string()))?;
|
||||
|
||||
// Extract target namespace
|
||||
if let Some(ns_start) = text.find("targetNamespace=\"") {
|
||||
let ns_start = ns_start + 17;
|
||||
if let Some(ns_end) = text[ns_start..].find('"') {
|
||||
schema.target_namespace = CompactString::from(&text[ns_start..ns_start + ns_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse elements
|
||||
let mut pos = 0;
|
||||
while let Some(elem_start) = text[pos..].find("<xs:element") {
|
||||
let elem_start = pos + elem_start;
|
||||
pos = elem_start + 1;
|
||||
|
||||
// Find element end
|
||||
let elem_end = if let Some(end) = text[elem_start..].find("/>") {
|
||||
elem_start + end + 2
|
||||
} else if let Some(end) = text[elem_start..].find("</xs:element>") {
|
||||
elem_start + end + 13
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let elem_text = &text[elem_start..elem_end];
|
||||
|
||||
// Extract element attributes
|
||||
let mut element = SchemaElement {
|
||||
name: CompactString::new(""),
|
||||
element_type: CompactString::new(""),
|
||||
substitution_group: None,
|
||||
period_type: None,
|
||||
balance: None,
|
||||
abstract_element: elem_text.contains("abstract=\"true\""),
|
||||
nillable: elem_text.contains("nillable=\"true\""),
|
||||
};
|
||||
|
||||
// Extract name
|
||||
if let Some(name_start) = elem_text.find("name=\"") {
|
||||
let name_start = name_start + 6;
|
||||
if let Some(name_end) = elem_text[name_start..].find('"') {
|
||||
element.name = CompactString::from(&elem_text[name_start..name_start + name_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract type
|
||||
if let Some(type_start) = elem_text.find("type=\"") {
|
||||
let type_start = type_start + 6;
|
||||
if let Some(type_end) = elem_text[type_start..].find('"') {
|
||||
element.element_type = CompactString::from(&elem_text[type_start..type_start + type_end]);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract substitutionGroup
|
||||
if let Some(sg_start) = elem_text.find("substitutionGroup=\"") {
|
||||
let sg_start = sg_start + 19;
|
||||
if let Some(sg_end) = elem_text[sg_start..].find('"') {
|
||||
element.substitution_group = Some(CompactString::from(&elem_text[sg_start..sg_start + sg_end]));
|
||||
}
|
||||
}
|
||||
|
||||
// Extract XBRL-specific attributes
|
||||
if let Some(pt_start) = elem_text.find("xbrli:periodType=\"") {
|
||||
let pt_start = pt_start + 18;
|
||||
if let Some(pt_end) = elem_text[pt_start..].find('"') {
|
||||
element.period_type = Some(CompactString::from(&elem_text[pt_start..pt_start + pt_end]));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(bal_start) = elem_text.find("xbrli:balance=\"") {
|
||||
let bal_start = bal_start + 15;
|
||||
if let Some(bal_end) = elem_text[bal_start..].find('"') {
|
||||
element.balance = Some(CompactString::from(&elem_text[bal_start..bal_start + bal_end]));
|
||||
}
|
||||
}
|
||||
|
||||
if !element.name.is_empty() {
|
||||
schema.elements.insert(element.name.clone(), element);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse imports
|
||||
pos = 0;
|
||||
while let Some(import_start) = text[pos..].find("<xs:import") {
|
||||
let import_start = pos + import_start;
|
||||
pos = import_start + 1;
|
||||
|
||||
if let Some(import_end) = text[import_start..].find("/>") {
|
||||
let import_text = &text[import_start..import_start + import_end];
|
||||
|
||||
let mut import = SchemaImport {
|
||||
namespace: CompactString::new(""),
|
||||
schema_location: CompactString::new(""),
|
||||
};
|
||||
|
||||
if let Some(ns_start) = import_text.find("namespace=\"") {
|
||||
let ns_start = ns_start + 11;
|
||||
if let Some(ns_end) = import_text[ns_start..].find('"') {
|
||||
import.namespace = CompactString::from(&import_text[ns_start..ns_start + ns_end]);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(loc_start) = import_text.find("schemaLocation=\"") {
|
||||
let loc_start = loc_start + 16;
|
||||
if let Some(loc_end) = import_text[loc_start..].find('"') {
|
||||
import.schema_location = CompactString::from(&import_text[loc_start..loc_start + loc_end]);
|
||||
}
|
||||
}
|
||||
|
||||
schema.imports.push(import);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(schema)
|
||||
}
|
||||
|
||||
pub fn validate_element(&self, name: &str, value: &str, schema: &Schema) -> Result<()> {
|
||||
if let Some(element) = schema.elements.get(name) {
|
||||
// Check if element is abstract
|
||||
if element.abstract_element {
|
||||
return Err(Error::Validation(format!("Element {} is abstract", name)));
|
||||
}
|
||||
|
||||
// Validate type
|
||||
if let Some(type_def) = schema.types.get(&element.element_type) {
|
||||
self.validate_type(value, type_def)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
} else {
|
||||
// Element not found in schema - might be from imported schema
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_type(&self, value: &str, type_def: &SchemaType) -> Result<()> {
|
||||
for restriction in &type_def.restrictions {
|
||||
match restriction {
|
||||
TypeRestriction::MinInclusive(min) => {
|
||||
if let (Ok(val), Ok(min_val)) = (value.parse::<f64>(), min.parse::<f64>()) {
|
||||
if val < min_val {
|
||||
return Err(Error::Validation(format!("Value {} is less than minimum {}", val, min_val)));
|
||||
}
|
||||
}
|
||||
}
|
||||
TypeRestriction::MaxInclusive(max) => {
|
||||
if let (Ok(val), Ok(max_val)) = (value.parse::<f64>(), max.parse::<f64>()) {
|
||||
if val > max_val {
|
||||
return Err(Error::Validation(format!("Value {} is greater than maximum {}", val, max_val)));
|
||||
}
|
||||
}
|
||||
}
|
||||
TypeRestriction::Pattern(pattern) => {
|
||||
// Simple pattern matching - could use regex for complex patterns
|
||||
if !value.contains(pattern) {
|
||||
return Err(Error::Validation(format!("Value {} doesn't match pattern {}", value, pattern)));
|
||||
}
|
||||
}
|
||||
TypeRestriction::MinLength(min) => {
|
||||
if value.len() < *min {
|
||||
return Err(Error::Validation(format!("Value length {} is less than minimum {}", value.len(), min)));
|
||||
}
|
||||
}
|
||||
TypeRestriction::MaxLength(max) => {
|
||||
if value.len() > *max {
|
||||
return Err(Error::Validation(format!("Value length {} is greater than maximum {}", value.len(), max)));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Schema validator for documents
|
||||
pub struct SchemaValidator {
|
||||
schemas: Vec<Schema>,
|
||||
}
|
||||
|
||||
impl SchemaValidator {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
schemas: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_schema(&mut self, schema: Schema) {
|
||||
self.schemas.push(schema);
|
||||
}
|
||||
|
||||
pub fn validate_document(&self, doc: &Document) -> Vec<ValidationError> {
|
||||
let mut errors = Vec::new();
|
||||
|
||||
// Validate facts against schemas
|
||||
for i in 0..doc.facts.len() {
|
||||
if let Some(_fact) = doc.facts.get(i) {
|
||||
// Would need to map fact concept_id back to concept name
|
||||
// and validate against schema
|
||||
// This is simplified for now
|
||||
}
|
||||
}
|
||||
|
||||
// Check for required elements
|
||||
for schema in &self.schemas {
|
||||
for (name, element) in &schema.elements {
|
||||
if !element.nillable && !element.abstract_element {
|
||||
// Check if this required element exists in document
|
||||
// This would require reverse mapping from concept names to facts
|
||||
let _found = false;
|
||||
// if !found {
|
||||
// errors.push(ValidationError::MissingRequiredElement {
|
||||
// element: name.to_string(),
|
||||
// });
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errors
|
||||
}
|
||||
}
|
||||
51
src/sec.rs
Normal file
51
src/sec.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
// SEC EDGAR XBRL filing support (local files only)
|
||||
use crate::{Parser, Document, Result};
|
||||
use std::path::Path;
|
||||
|
||||
pub struct SecFilingParser {
|
||||
parser: Parser,
|
||||
}
|
||||
|
||||
impl SecFilingParser {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
parser: Parser::new().with_validation(true),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_filing<P: AsRef<Path>>(&self, path: P) -> Result<Document> {
|
||||
self.parser.parse_file(path)
|
||||
}
|
||||
|
||||
pub fn with_validation(mut self, validate: bool) -> Self {
|
||||
self.parser = self.parser.with_validation(validate);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// Test utilities for SEC filings
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_local_sec_filing() {
|
||||
let parser = SecFilingParser::new();
|
||||
|
||||
// Test with local test files
|
||||
if std::path::Path::new("test_data/test_tiny.xbrl").exists() {
|
||||
match parser.parse_filing("test_data/test_tiny.xbrl") {
|
||||
Ok(doc) => {
|
||||
println!("Successfully parsed filing:");
|
||||
println!(" Facts: {}", doc.facts.len());
|
||||
println!(" Contexts: {}", doc.contexts.len());
|
||||
println!(" Units: {}", doc.units.len());
|
||||
assert!(doc.contexts.len() > 0, "Should have contexts");
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Failed to parse filing: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
208
src/simd.rs
Normal file
208
src/simd.rs
Normal file
@@ -0,0 +1,208 @@
|
||||
use memchr::{memchr, memchr2, memchr3};
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
const XML_TAG_START: u8 = b'<';
|
||||
const XML_TAG_END: u8 = b'>';
|
||||
const XML_SLASH: u8 = b'/';
|
||||
const XML_QUOTE: u8 = b'"';
|
||||
const XML_EQUALS: u8 = b'=';
|
||||
const XML_SPACE: u8 = b' ';
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_tag_start(haystack: &[u8]) -> Option<usize> {
|
||||
memchr(XML_TAG_START, haystack)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_tag_end(haystack: &[u8]) -> Option<usize> {
|
||||
memchr(XML_TAG_END, haystack)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_quote(haystack: &[u8]) -> Option<usize> {
|
||||
memchr(XML_QUOTE, haystack)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_any_delimiter(haystack: &[u8]) -> Option<usize> {
|
||||
memchr3(XML_TAG_START, XML_TAG_END, XML_QUOTE, haystack)
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
#[inline]
|
||||
pub unsafe fn find_pattern_avx2(haystack: &[u8], pattern: &[u8]) -> Option<usize> {
|
||||
if pattern.is_empty() || haystack.len() < pattern.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let first_byte = _mm256_set1_epi8(pattern[0] as i8);
|
||||
let mut i = 0;
|
||||
|
||||
while i + 32 <= haystack.len() {
|
||||
let chunk = _mm256_loadu_si256(haystack.as_ptr().add(i) as *const _);
|
||||
let cmp = _mm256_cmpeq_epi8(chunk, first_byte);
|
||||
let mask = _mm256_movemask_epi8(cmp);
|
||||
|
||||
if mask != 0 {
|
||||
for bit_pos in 0..32 {
|
||||
if (mask & (1 << bit_pos)) != 0 {
|
||||
let pos = i + bit_pos;
|
||||
if pos + pattern.len() <= haystack.len()
|
||||
&& &haystack[pos..pos + pattern.len()] == pattern {
|
||||
return Some(pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 32;
|
||||
}
|
||||
|
||||
while i < haystack.len() - pattern.len() + 1 {
|
||||
if &haystack[i..i + pattern.len()] == pattern {
|
||||
return Some(i);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
#[inline]
|
||||
pub unsafe fn skip_whitespace_avx2(data: &[u8], mut pos: usize) -> usize {
|
||||
let space = _mm256_set1_epi8(0x20);
|
||||
let tab = _mm256_set1_epi8(0x09);
|
||||
let newline = _mm256_set1_epi8(0x0A);
|
||||
let carriage = _mm256_set1_epi8(0x0D);
|
||||
|
||||
while pos + 32 <= data.len() {
|
||||
let chunk = _mm256_loadu_si256(data.as_ptr().add(pos) as *const _);
|
||||
|
||||
let is_space = _mm256_cmpeq_epi8(chunk, space);
|
||||
let is_tab = _mm256_cmpeq_epi8(chunk, tab);
|
||||
let is_newline = _mm256_cmpeq_epi8(chunk, newline);
|
||||
let is_carriage = _mm256_cmpeq_epi8(chunk, carriage);
|
||||
|
||||
let is_whitespace = _mm256_or_si256(
|
||||
_mm256_or_si256(is_space, is_tab),
|
||||
_mm256_or_si256(is_newline, is_carriage)
|
||||
);
|
||||
|
||||
let mask = _mm256_movemask_epi8(is_whitespace);
|
||||
|
||||
if mask != -1 {
|
||||
for i in 0..32 {
|
||||
if (mask & (1 << i)) == 0 {
|
||||
return pos + i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pos += 32;
|
||||
}
|
||||
|
||||
while pos < data.len() {
|
||||
match data[pos] {
|
||||
b' ' | b'\t' | b'\n' | b'\r' => pos += 1,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
|
||||
pos
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn skip_whitespace(data: &[u8], mut pos: usize) -> usize {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
if is_x86_feature_detected!("avx2") && data.len() - pos >= 32 {
|
||||
return unsafe { skip_whitespace_avx2(data, pos) };
|
||||
}
|
||||
}
|
||||
|
||||
while pos < data.len() {
|
||||
match data[pos] {
|
||||
b' ' | b'\t' | b'\n' | b'\r' => pos += 1,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
pos
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_pattern(haystack: &[u8], pattern: &[u8]) -> Option<usize> {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
if is_x86_feature_detected!("avx2") && haystack.len() >= 32 {
|
||||
return unsafe { find_pattern_avx2(haystack, pattern) };
|
||||
}
|
||||
}
|
||||
|
||||
haystack.windows(pattern.len())
|
||||
.position(|window| window == pattern)
|
||||
}
|
||||
|
||||
pub struct SimdScanner<'a> {
|
||||
pub data: &'a [u8],
|
||||
pub pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> SimdScanner<'a> {
|
||||
#[inline(always)]
|
||||
pub fn new(data: &'a [u8]) -> Self {
|
||||
Self { data, pos: 0 }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn skip_whitespace(&mut self) {
|
||||
self.pos = skip_whitespace(self.data, self.pos);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_next(&self, byte: u8) -> Option<usize> {
|
||||
memchr(byte, &self.data[self.pos..]).map(|i| self.pos + i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn find_pattern(&self, pattern: &[u8]) -> Option<usize> {
|
||||
find_pattern(&self.data[self.pos..], pattern).map(|i| self.pos + i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn advance(&mut self, n: usize) {
|
||||
self.pos = (self.pos + n).min(self.data.len());
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn peek(&self) -> Option<u8> {
|
||||
self.data.get(self.pos).copied()
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn remaining(&self) -> &'a [u8] {
|
||||
&self.data[self.pos..]
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn is_eof(&self) -> bool {
|
||||
self.pos >= self.data.len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_find_pattern() {
|
||||
let haystack = b"<xbrl:context id=\"c1\">";
|
||||
let pattern = b"context";
|
||||
assert_eq!(find_pattern(haystack, pattern), Some(6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skip_whitespace() {
|
||||
let data = b" \t\n\r<tag>";
|
||||
assert_eq!(skip_whitespace(data, 0), 6);
|
||||
}
|
||||
}
|
||||
49
src/taxonomy.rs
Normal file
49
src/taxonomy.rs
Normal file
@@ -0,0 +1,49 @@
|
||||
use crate::Result;
|
||||
use compact_str::CompactString;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub struct Taxonomy {
|
||||
pub schemas: Vec<Schema>,
|
||||
pub linkbases: Vec<Linkbase>,
|
||||
}
|
||||
|
||||
pub struct Schema {
|
||||
pub target_namespace: CompactString,
|
||||
pub elements: HashMap<CompactString, Element>,
|
||||
}
|
||||
|
||||
pub struct Element {
|
||||
pub name: CompactString,
|
||||
pub element_type: CompactString,
|
||||
pub substitution_group: Option<CompactString>,
|
||||
pub period_type: Option<CompactString>,
|
||||
}
|
||||
|
||||
pub struct Linkbase {
|
||||
pub role: CompactString,
|
||||
pub arcs: Vec<Arc>,
|
||||
}
|
||||
|
||||
pub struct Arc {
|
||||
pub from: CompactString,
|
||||
pub to: CompactString,
|
||||
pub order: f32,
|
||||
pub weight: f32,
|
||||
}
|
||||
|
||||
impl Taxonomy {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
schemas: Vec::new(),
|
||||
linkbases: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_schema(&mut self, _path: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn load_linkbase(&mut self, _path: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user