//! SQLite-backed persistence for research workspaces, notes, links, ghosts, and jobs.

use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};

use rusqlite::types::Value as SqlValue;
use rusqlite::{params, params_from_iter, Connection, OptionalExtension};

use crate::research::errors::{ResearchError, Result};
use crate::research::types::{
    AuditEvent, FinancialModel, GhostLifecycleState, GhostNote, GhostVisibilityState, JobKind,
    JobStatus, ModelCellAnnotation, ModelDataBinding, NoteLink, NoteType, PipelineJob,
    ResearchFile, ResearchNote, ResearchWorkspace, SourceRecord,
};

pub(crate) struct WorkspaceProjectionParts {
    pub workspace: ResearchWorkspace,
    pub notes: Vec<ResearchNote>,
    pub links: Vec<NoteLink>,
    pub ghosts: Vec<GhostNote>,
}

#[derive(Clone)]
pub struct ResearchRepository {
    db_path: PathBuf,
    connections: Arc<Mutex<Vec<Connection>>>,
}

impl ResearchRepository {
    pub fn new(db_path: PathBuf) -> Result<Self> {
        if let Some(parent) = db_path.parent() {
            std::fs::create_dir_all(parent)?;
        }

        let connection = open_connection(&db_path)?;
        let repository = Self {
            db_path,
            connections: Arc::new(Mutex::new(Vec::new())),
        };
        repository.initialize_schema(&connection)?;
        repository.store_connection(connection)?;
        Ok(repository)
    }

    pub async fn create_workspace(
        &self,
        workspace: ResearchWorkspace,
    ) -> Result<ResearchWorkspace> {
        let value = workspace.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO research_workspaces (
                    id, name, primary_ticker, stage, default_view, archived, created_at, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
                params![
                    value.id,
                    value.name,
                    value.primary_ticker,
                    serde_json::to_string(&value.stage)?,
                    serde_json::to_string(&value.default_view)?,
                    i64::from(value.archived),
                    value.created_at,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn save_workspace(&self, workspace: ResearchWorkspace) -> Result<ResearchWorkspace> {
        let value = workspace.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO research_workspaces (
                    id, name, primary_ticker, stage, default_view, archived, created_at, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
                ON CONFLICT(id) DO UPDATE SET
                    name = excluded.name,
                    primary_ticker = excluded.primary_ticker,
                    stage = excluded.stage,
                    default_view = excluded.default_view,
                    archived = excluded.archived,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.name,
                    value.primary_ticker,
                    serde_json::to_string(&value.stage)?,
                    serde_json::to_string(&value.default_view)?,
                    i64::from(value.archived),
                    value.created_at,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn list_workspaces(&self) -> Result<Vec<ResearchWorkspace>> {
        self.with_connection(|connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json
                 FROM research_workspaces
                 WHERE archived = 0
                 ORDER BY updated_at DESC",
            )?;
            let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn get_workspace(&self, workspace_id: &str) -> Result<ResearchWorkspace> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM research_workspaces WHERE id = ?1",
                    params![workspace_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| ResearchError::WorkspaceNotFound(workspace_id.clone()))?;
            Ok(serde_json::from_str(&json)?)
        })
        .await
    }

    pub async fn list_financial_models(&self, workspace_id: &str) -> Result<Vec<FinancialModel>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json
                 FROM financial_models
                 WHERE workspace_id = ?1
                 ORDER BY updated_at DESC",
            )?;
            let rows = statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn get_financial_model(&self, model_id: &str) -> Result<FinancialModel> {
        let model_id = model_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM financial_models WHERE id = ?1",
                    params![model_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| ResearchError::Validation("model not found".to_string()))?;
            Ok(serde_json::from_str(&json)?)
        })
        .await
    }

    pub async fn save_financial_model(&self, model: FinancialModel) -> Result<FinancialModel> {
        let value = model.clone();
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;
            transaction.execute(
                "INSERT INTO financial_models (
                    id, workspace_id, name, ticker, template_type, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
                ON CONFLICT(id) DO UPDATE SET
                    workspace_id = excluded.workspace_id,
                    name = excluded.name,
                    ticker = excluded.ticker,
                    template_type = excluded.template_type,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.workspace_id,
                    value.name,
                    value.ticker,
                    serde_json::to_string(&value.template_type)?,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            transaction.execute(
                "DELETE FROM model_cell_annotations WHERE model_id = ?1",
                params![value.id.clone()],
            )?;
            for annotation in &value.annotations {
                transaction.execute(
                    "INSERT INTO model_cell_annotations (
                        model_id, cell_id, sheet_id, address, evidence_status, updated_at, entity_json
                    ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
                    params![
                        &value.id,
                        &annotation.cell_id,
                        &annotation.sheet_id,
                        &annotation.address,
                        serde_json::to_string(&annotation.evidence_status)?,
                        &annotation.updated_at,
                        serde_json::to_string(annotation)?,
                    ],
                )?;
            }
            transaction.commit()?;
            Ok(value)
        })
        .await
    }

    pub async fn list_model_cell_annotations(
        &self,
        model_id: &str,
    ) -> Result<Vec<ModelCellAnnotation>> {
        let model_id = model_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json
                 FROM model_cell_annotations
                 WHERE model_id = ?1
                 ORDER BY sheet_id, address",
            )?;
            let rows = statement.query_map(params![model_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn list_model_data_bindings(&self, model_id: &str) -> Result<Vec<ModelDataBinding>> {
        let model_id = model_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json
                 FROM model_data_bindings
                 WHERE model_id = ?1
                 ORDER BY updated_at DESC",
            )?;
            let rows = statement.query_map(params![model_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn save_model_data_binding(
        &self,
        binding: ModelDataBinding,
    ) -> Result<ModelDataBinding> {
        let value = binding.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO model_data_bindings (
                    id, model_id, sheet_id, range_address, kind, source_key, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
                ON CONFLICT(id) DO UPDATE SET
                    model_id = excluded.model_id,
                    sheet_id = excluded.sheet_id,
                    range_address = excluded.range_address,
                    kind = excluded.kind,
                    source_key = excluded.source_key,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.model_id,
                    value.sheet_id,
                    value.range,
                    serde_json::to_string(&value.kind)?,
                    value.source_key,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(binding)
        })
        .await
    }

    pub async fn touch_workspace_updated_at(
        &self,
        workspace_id: &str,
        updated_at: String,
    ) -> Result<ResearchWorkspace> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM research_workspaces WHERE id = ?1",
                    params![workspace_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| ResearchError::WorkspaceNotFound(workspace_id.clone()))?;
            let mut workspace: ResearchWorkspace = serde_json::from_str(&json)?;
            workspace.updated_at = updated_at;

            connection.execute(
                "UPDATE research_workspaces
                 SET updated_at = ?2,
                     entity_json = ?3
                 WHERE id = ?1",
                params![
                    workspace.id,
                    workspace.updated_at,
                    serde_json::to_string(&workspace)?,
                ],
            )?;
            Ok(workspace)
        })
        .await
    }

    pub async fn create_note(&self, note: ResearchNote) -> Result<ResearchNote> {
        self.save_note(note).await
    }

    pub async fn save_note(&self, note: ResearchNote) -> Result<ResearchNote> {
        let value = note.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO research_notes (
                    id, workspace_id, note_type, ticker, source_id, archived, pinned, revision,
                    evidence_status, created_at, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)
                ON CONFLICT(id) DO UPDATE SET
                    workspace_id = excluded.workspace_id,
                    note_type = excluded.note_type,
                    ticker = excluded.ticker,
                    source_id = excluded.source_id,
                    archived = excluded.archived,
                    pinned = excluded.pinned,
                    revision = excluded.revision,
                    evidence_status = excluded.evidence_status,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.workspace_id,
                    serde_json::to_string(&value.note_type)?,
                    value.ticker,
                    value.source_id,
                    i64::from(value.archived),
                    i64::from(value.pinned),
                    i64::from(value.revision),
                    serde_json::to_string(&value.evidence_status)?,
                    value.created_at,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            connection.execute(
                "DELETE FROM research_fts WHERE note_id = ?1",
                params![value.id.clone()],
            )?;
            connection.execute(
                "INSERT INTO research_fts (note_id, title, cleaned_text, ai_annotation)
                 VALUES (?1, ?2, ?3, ?4)",
                params![
                    value.id,
                    value.title.clone().unwrap_or_default(),
                    value.cleaned_text,
                    value.ai_annotation.clone().unwrap_or_default(),
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn save_notes_batch(&self, notes: Vec<ResearchNote>) -> Result<()> {
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;

            for note in &notes {
                transaction.execute(
                    "INSERT INTO research_notes (
                        id, workspace_id, note_type, ticker, source_id, archived, pinned, revision,
                        evidence_status, created_at, updated_at, entity_json
                    ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)
                    ON CONFLICT(id) DO UPDATE SET
                        workspace_id = excluded.workspace_id,
                        note_type = excluded.note_type,
                        ticker = excluded.ticker,
                        source_id = excluded.source_id,
                        archived = excluded.archived,
                        pinned = excluded.pinned,
                        revision = excluded.revision,
                        evidence_status = excluded.evidence_status,
                        updated_at = excluded.updated_at,
                        entity_json = excluded.entity_json",
                    params![
                        &note.id,
                        &note.workspace_id,
                        serde_json::to_string(&note.note_type)?,
                        note.ticker.as_deref(),
                        note.source_id.as_deref(),
                        i64::from(note.archived),
                        i64::from(note.pinned),
                        i64::from(note.revision),
                        serde_json::to_string(&note.evidence_status)?,
                        &note.created_at,
                        &note.updated_at,
                        serde_json::to_string(note)?,
                    ],
                )?;
                transaction.execute(
                    "DELETE FROM research_fts WHERE note_id = ?1",
                    params![&note.id],
                )?;
                transaction.execute(
                    "INSERT INTO research_fts (note_id, title, cleaned_text, ai_annotation)
                     VALUES (?1, ?2, ?3, ?4)",
                    params![
                        &note.id,
                        note.title.clone().unwrap_or_default(),
                        &note.cleaned_text,
                        note.ai_annotation.clone().unwrap_or_default(),
                    ],
                )?;
            }

            transaction.commit()?;
            Ok(())
        })
        .await
    }

    pub async fn get_note(&self, note_id: &str) -> Result<ResearchNote> {
        let note_id = note_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM research_notes WHERE id = ?1",
                    params![note_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| ResearchError::NoteNotFound(note_id.clone()))?;
            Ok(serde_json::from_str(&json)?)
        })
        .await
    }

    pub async fn list_notes(
        &self,
        workspace_id: &str,
        include_archived: bool,
        note_type: Option<NoteType>,
    ) -> Result<Vec<ResearchNote>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let json_rows = if let Some(note_type) = note_type {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM research_notes
                     WHERE workspace_id = ?1 AND (?2 = 1 OR archived = 0) AND note_type = ?3
                     ORDER BY pinned DESC, updated_at DESC",
                )?;
                let rows = statement.query_map(
                    params![
                        workspace_id,
                        i64::from(include_archived),
                        serde_json::to_string(&note_type)?,
                    ],
                    |row| row.get::<_, String>(0),
                )?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            } else {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM research_notes
                     WHERE workspace_id = ?1 AND (?2 = 1 OR archived = 0)
                     ORDER BY pinned DESC, updated_at DESC",
                )?;
                let rows = statement
                    .query_map(params![workspace_id, i64::from(include_archived)], |row| {
                        row.get::<_, String>(0)
                    })?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            };

            json_rows
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn archive_note(&self, note_id: &str, archived: bool) -> Result<ResearchNote> {
        let mut note = self.get_note(note_id).await?;
        note.archived = archived;
        note.updated_at = crate::research::util::now_rfc3339();
        self.save_note(note).await
    }

    pub async fn find_source_reference_note(
        &self,
        workspace_id: &str,
        source_id: &str,
    ) -> Result<Option<ResearchNote>> {
        let workspace_id = workspace_id.to_string();
        let source_id = source_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json
                     FROM research_notes
                     WHERE workspace_id = ?1 AND source_id = ?2 AND note_type = ?3
                     LIMIT 1",
                    params![
                        workspace_id,
                        source_id,
                        serde_json::to_string(&NoteType::SourceReference)?,
                    ],
                    |row| row.get::<_, String>(0),
                )
                .optional()?;
            json.map(|value| serde_json::from_str(&value).map_err(ResearchError::from))
                .transpose()
        })
        .await
    }

    pub async fn search_notes(
        &self,
        workspace_id: &str,
        query: &str,
        include_archived: bool,
        note_type: Option<NoteType>,
        limit: Option<usize>,
    ) -> Result<Vec<ResearchNote>> {
        let workspace_id = workspace_id.to_string();
        let fts_expr = fts_query(query);
        let note_type_str = note_type.map(|nt| serde_json::to_string(&nt)).transpose()?;

        self.with_connection(move |connection| {
            let limit_val = limit.unwrap_or(50);

            let json_rows = if let Some(ref nt) = note_type_str {
                let mut stmt = connection.prepare(
                    "SELECT n.entity_json
                     FROM research_notes n
                     JOIN research_fts f ON f.note_id = n.id
                     WHERE n.workspace_id = ?1
                       AND (?2 = 1 OR n.archived = 0)
                       AND n.note_type = ?5
                       AND research_fts MATCH ?3
                     ORDER BY rank
                     LIMIT ?4",
                )?;
                let mapped: Vec<String> = stmt
                    .query_map(
                        params![
                            workspace_id,
                            i64::from(include_archived),
                            fts_expr,
                            limit_val,
                            nt
                        ],
                        |row| row.get::<_, String>(0),
                    )?
                    .filter_map(|r| r.ok())
                    .collect();
                mapped
            } else {
                let mut stmt = connection.prepare(
                    "SELECT n.entity_json
                     FROM research_notes n
                     JOIN research_fts f ON f.note_id = n.id
                     WHERE n.workspace_id = ?1
                       AND (?2 = 1 OR n.archived = 0)
                       AND research_fts MATCH ?3
                     ORDER BY rank
                     LIMIT ?4",
                )?;
                let mapped: Vec<String> = stmt
                    .query_map(
                        params![
                            workspace_id,
                            i64::from(include_archived),
                            fts_expr,
                            limit_val
                        ],
                        |row| row.get::<_, String>(0),
                    )?
                    .filter_map(|r| r.ok())
                    .collect();
                mapped
            };

            json_rows
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn save_source(&self, source: SourceRecord) -> Result<SourceRecord> {
        let value = source.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO source_records (
                    id, workspace_id, ticker, kind, published_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
                ON CONFLICT(id) DO UPDATE SET
                    workspace_id = excluded.workspace_id,
                    ticker = excluded.ticker,
                    kind = excluded.kind,
                    published_at = excluded.published_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.workspace_id,
                    value.ticker,
                    serde_json::to_string(&value.kind)?,
                    value.published_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn list_sources(&self, workspace_id: &str) -> Result<Vec<SourceRecord>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json FROM source_records WHERE workspace_id = ?1 ORDER BY published_at DESC, id DESC",
            )?;
            let rows = statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn list_sources_by_ids(&self, source_ids: &[String]) -> Result<Vec<SourceRecord>> {
        if source_ids.is_empty() {
            return Ok(Vec::new());
        }

        let mut seen = HashSet::new();
        let ids = source_ids
            .iter()
            .filter(|source_id| seen.insert((*source_id).clone()))
            .cloned()
            .collect::<Vec<_>>();
        self.with_connection(move |connection| {
            let placeholders = std::iter::repeat_n("?", ids.len())
                .collect::<Vec<_>>()
                .join(", ");
            let query =
                format!("SELECT id, entity_json FROM source_records WHERE id IN ({placeholders})");
            let query_params = ids.iter().cloned().map(SqlValue::Text).collect::<Vec<_>>();
            let mut statement = connection.prepare(&query)?;
            let rows = statement.query_map(params_from_iter(query_params.iter()), |row| {
                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
            })?;
            let rows = rows.collect::<std::result::Result<Vec<_>, _>>()?;
            let sources_by_id = rows
                .into_iter()
                .map(|(id, json)| {
                    serde_json::from_str::<SourceRecord>(&json)
                        .map(|source| (id, source))
                        .map_err(ResearchError::from)
                })
                .collect::<Result<HashMap<_, _>>>()?;

            Ok(ids
                .into_iter()
                .filter_map(|source_id| sources_by_id.get(&source_id).cloned())
                .collect())
        })
        .await
    }

    pub async fn find_source_by_checksum_or_accession(
        &self,
        workspace_id: &str,
        checksum: Option<&str>,
        filing_accession: Option<&str>,
    ) -> Result<Option<SourceRecord>> {
        let workspace_id = workspace_id.to_string();
        let checksum = checksum.map(ToOwned::to_owned);
        let filing_accession = filing_accession.map(ToOwned::to_owned);
        self.with_connection(move |connection| {
            let json = if let Some(checksum) = checksum {
                connection
                    .query_row(
                        "SELECT entity_json FROM source_records
                         WHERE workspace_id = ?1 AND json_extract(entity_json, '$.checksum') = ?2
                         LIMIT 1",
                        params![workspace_id, checksum],
                        |row| row.get::<_, String>(0),
                    )
                    .optional()?
            } else if let Some(filing_accession) = filing_accession {
                connection
                    .query_row(
                        "SELECT entity_json FROM source_records
                         WHERE workspace_id = ?1 AND json_extract(entity_json, '$.filingAccession') = ?2
                         LIMIT 1",
                        params![workspace_id, filing_accession],
                        |row| row.get::<_, String>(0),
                    )
                    .optional()?
            } else {
                None
            };

            json.map(|value| serde_json::from_str(&value).map_err(ResearchError::from))
                .transpose()
        })
        .await
    }

    pub async fn save_file(&self, file: ResearchFile) -> Result<ResearchFile> {
        let value = file.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO research_files (
                    id, workspace_id, checksum, original_name, created_at, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
                ON CONFLICT(id) DO UPDATE SET
                    workspace_id = excluded.workspace_id,
                    checksum = excluded.checksum,
                    original_name = excluded.original_name,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.workspace_id,
                    value.checksum,
                    value.original_name,
                    value.created_at,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn list_files(&self, workspace_id: &str) -> Result<Vec<ResearchFile>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json FROM research_files
                 WHERE workspace_id = ?1
                 ORDER BY created_at DESC, id DESC",
            )?;
            let rows = statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn list_files_by_ids(&self, file_ids: &[String]) -> Result<Vec<ResearchFile>> {
        if file_ids.is_empty() {
            return Ok(Vec::new());
        }

        let mut seen = HashSet::new();
        let ids = file_ids
            .iter()
            .filter(|file_id| seen.insert((*file_id).clone()))
            .cloned()
            .collect::<Vec<_>>();
        self.with_connection(move |connection| {
            let placeholders = std::iter::repeat_n("?", ids.len())
                .collect::<Vec<_>>()
                .join(", ");
            let query =
                format!("SELECT id, entity_json FROM research_files WHERE id IN ({placeholders})");
            let query_params = ids.iter().cloned().map(SqlValue::Text).collect::<Vec<_>>();
            let mut statement = connection.prepare(&query)?;
            let rows = statement.query_map(params_from_iter(query_params.iter()), |row| {
                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
            })?;
            let rows = rows.collect::<std::result::Result<Vec<_>, _>>()?;
            let files_by_id = rows
                .into_iter()
                .map(|(id, json)| {
                    serde_json::from_str::<ResearchFile>(&json)
                        .map(|file| (id, file))
                        .map_err(ResearchError::from)
                })
                .collect::<Result<HashMap<_, _>>>()?;

            Ok(ids
                .into_iter()
                .filter_map(|file_id| files_by_id.get(&file_id).cloned())
                .collect())
        })
        .await
    }

    pub async fn get_file(&self, file_id: &str) -> Result<ResearchFile> {
        let file_id = file_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM research_files WHERE id = ?1",
                    params![file_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| {
                    ResearchError::Validation(format!("research file not found: {file_id}"))
                })?;
            Ok(serde_json::from_str(&json)?)
        })
        .await
    }

    pub async fn find_file_by_checksum_and_name(
        &self,
        workspace_id: &str,
        checksum: &str,
        original_name: &str,
    ) -> Result<Option<ResearchFile>> {
        let workspace_id = workspace_id.to_string();
        let checksum = checksum.to_string();
        let original_name = original_name.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM research_files
                     WHERE workspace_id = ?1 AND checksum = ?2 AND original_name = ?3
                     LIMIT 1",
                    params![workspace_id, checksum, original_name],
                    |row| row.get::<_, String>(0),
                )
                .optional()?;
            json.map(|value| serde_json::from_str(&value).map_err(ResearchError::from))
                .transpose()
        })
        .await
    }

    pub async fn replace_links_for_workspace(
        &self,
        workspace_id: &str,
        links: Vec<NoteLink>,
    ) -> Result<Vec<NoteLink>> {
        let workspace_id = workspace_id.to_string();
        let values = links.clone();
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;
            transaction.execute("DELETE FROM note_links WHERE workspace_id = ?1", params![workspace_id])?;
            for link in &values {
                transaction.execute(
                    "INSERT INTO note_links (
                        id, workspace_id, from_note_id, to_note_id, link_type, stale, updated_at, entity_json
                    ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
                    params![
                        link.id,
                        link.workspace_id,
                        link.from_note_id,
                        link.to_note_id,
                        serde_json::to_string(&link.link_type)?,
                        i64::from(link.stale),
                        link.updated_at,
                        serde_json::to_string(link)?,
                    ],
                )?;
            }
            transaction.commit()?;
            Ok(values)
        })
        .await
    }

    pub async fn list_links(
        &self,
        workspace_id: &str,
        note_id: Option<&str>,
    ) -> Result<Vec<NoteLink>> {
        let workspace_id = workspace_id.to_string();
        let note_id = note_id.map(ToOwned::to_owned);
        self.with_connection(move |connection| {
            let json_rows = if let Some(note_id) = note_id {
                let mut statement = connection.prepare(
                    "SELECT entity_json
                     FROM note_links
                     WHERE workspace_id = ?1 AND (from_note_id = ?2 OR to_note_id = ?2)
                     ORDER BY updated_at DESC",
                )?;
                let rows = statement.query_map(params![workspace_id, note_id], |row| {
                    row.get::<_, String>(0)
                })?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            } else {
                let mut statement = connection.prepare(
                    "SELECT entity_json
                     FROM note_links
                     WHERE workspace_id = ?1
                     ORDER BY updated_at DESC",
                )?;
                let rows =
                    statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            };

            json_rows
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn replace_ghosts_for_workspace(
        &self,
        workspace_id: &str,
        ghosts: Vec<GhostNote>,
    ) -> Result<Vec<GhostNote>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;
            let existing = {
                let mut existing_statement = transaction
                    .prepare("SELECT id, entity_json FROM ghost_notes WHERE workspace_id = ?1")?;
                let existing_rows = existing_statement
                    .query_map(params![workspace_id.clone()], |row| {
                        Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
                    })?;
                existing_rows.collect::<std::result::Result<Vec<_>, _>>()?
            };
            transaction.execute(
                "DELETE FROM ghost_notes WHERE workspace_id = ?1",
                params![workspace_id],
            )?;

            let mut result = Vec::with_capacity(ghosts.len());
            for mut ghost in ghosts {
                if let Some((_, json)) = existing.iter().find(|(id, _)| *id == ghost.id) {
                    let prior: GhostNote = serde_json::from_str(json)?;
                    if matches!(
                        prior.state,
                        GhostLifecycleState::Accepted
                            | GhostLifecycleState::Dismissed
                            | GhostLifecycleState::Converted
                            | GhostLifecycleState::Ignored
                    ) {
                        ghost.state = prior.state;
                    }
                    if matches!(prior.visibility_state, GhostVisibilityState::Pinned) {
                        ghost.visibility_state = prior.visibility_state;
                    }
                    ghost.promoted_note_id = prior.promoted_note_id;
                }

                transaction.execute(
                    "INSERT INTO ghost_notes (
                        id, workspace_id, state, visibility_state, updated_at, entity_json
                    ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
                    params![
                        ghost.id,
                        ghost.workspace_id,
                        serde_json::to_string(&ghost.state)?,
                        serde_json::to_string(&ghost.visibility_state)?,
                        ghost.updated_at,
                        serde_json::to_string(&ghost)?,
                    ],
                )?;
                result.push(ghost);
            }

            transaction.commit()?;
            Ok(result)
        })
        .await
    }

    pub async fn get_ghost(&self, ghost_note_id: &str) -> Result<GhostNote> {
        let ghost_note_id = ghost_note_id.to_string();
        self.with_connection(move |connection| {
            let json = connection
                .query_row(
                    "SELECT entity_json FROM ghost_notes WHERE id = ?1",
                    params![ghost_note_id],
                    |row| row.get::<_, String>(0),
                )
                .optional()?
                .ok_or_else(|| ResearchError::GhostNoteNotFound(ghost_note_id.clone()))?;
            Ok(serde_json::from_str(&json)?)
        })
        .await
    }

    pub async fn save_ghost(&self, ghost: GhostNote) -> Result<GhostNote> {
        let value = ghost.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO ghost_notes (
                    id, workspace_id, state, visibility_state, updated_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
                ON CONFLICT(id) DO UPDATE SET
                    state = excluded.state,
                    visibility_state = excluded.visibility_state,
                    updated_at = excluded.updated_at,
                    entity_json = excluded.entity_json",
                params![
                    value.id,
                    value.workspace_id,
                    serde_json::to_string(&value.state)?,
                    serde_json::to_string(&value.visibility_state)?,
                    value.updated_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn list_ghosts(
        &self,
        workspace_id: &str,
        include_hidden: bool,
    ) -> Result<Vec<GhostNote>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json FROM ghost_notes
                 WHERE workspace_id = ?1
                 ORDER BY updated_at DESC",
            )?;
            let rows = statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str::<GhostNote>(&json).map_err(ResearchError::from))
                .collect::<Result<Vec<_>>>()
                .map(|ghosts| {
                    ghosts
                        .into_iter()
                        .filter(|ghost| {
                            include_hidden
                                || !matches!(ghost.visibility_state, GhostVisibilityState::Hidden)
                        })
                        .collect()
                })
        })
        .await
    }

    pub async fn load_workspace_projection_parts(
        &self,
        workspace_id: &str,
        include_hidden_ghosts: bool,
    ) -> Result<WorkspaceProjectionParts> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let workspace = {
                let json = connection
                    .query_row(
                        "SELECT entity_json FROM research_workspaces WHERE id = ?1",
                        params![workspace_id.clone()],
                        |row| row.get::<_, String>(0),
                    )
                    .optional()?
                    .ok_or_else(|| ResearchError::WorkspaceNotFound(workspace_id.clone()))?;
                serde_json::from_str(&json)?
            };

            let notes = {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM research_notes
                     WHERE workspace_id = ?1 AND archived = 0
                     ORDER BY pinned DESC, updated_at DESC",
                )?;
                let rows = statement
                    .query_map(params![workspace_id.clone()], |row| row.get::<_, String>(0))?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
                    .into_iter()
                    .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                    .collect::<Result<Vec<_>>>()?
            };

            let links = {
                let mut statement = connection.prepare(
                    "SELECT entity_json
                     FROM note_links
                     WHERE workspace_id = ?1
                     ORDER BY updated_at DESC",
                )?;
                let rows = statement
                    .query_map(params![workspace_id.clone()], |row| row.get::<_, String>(0))?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
                    .into_iter()
                    .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                    .collect::<Result<Vec<_>>>()?
            };

            let ghosts = {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM ghost_notes
                     WHERE workspace_id = ?1
                     ORDER BY updated_at DESC",
                )?;
                let rows =
                    statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
                    .into_iter()
                    .map(|json| {
                        serde_json::from_str::<GhostNote>(&json).map_err(ResearchError::from)
                    })
                    .collect::<Result<Vec<_>>>()?
                    .into_iter()
                    .filter(|ghost| {
                        include_hidden_ghosts
                            || !matches!(ghost.visibility_state, GhostVisibilityState::Hidden)
                    })
                    .collect()
            };

            Ok(WorkspaceProjectionParts {
                workspace,
                notes,
                links,
                ghosts,
            })
        })
        .await
    }

    pub async fn enqueue_jobs(&self, jobs: Vec<PipelineJob>) -> Result<Vec<PipelineJob>> {
        let values = jobs.clone();
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;
            for job in &values {
                transaction.execute(
                    "INSERT INTO pipeline_jobs (
                        id, workspace_id, entity_id, job_kind, status, next_attempt_at, updated_at, entity_json
                    ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
                    ON CONFLICT(id) DO UPDATE SET
                        status = excluded.status,
                        next_attempt_at = excluded.next_attempt_at,
                        updated_at = excluded.updated_at,
                        entity_json = excluded.entity_json",
                    params![
                        job.id,
                        job.workspace_id,
                        job.entity_id,
                        serde_json::to_string(&job.job_kind)?,
                        serde_json::to_string(&job.status)?,
                        job.next_attempt_at,
                        job.updated_at,
                        serde_json::to_string(job)?,
                    ],
                )?;
            }
            transaction.commit()?;
            Ok(values)
        })
        .await
    }

    pub async fn save_job(&self, job: PipelineJob) -> Result<PipelineJob> {
        self.enqueue_jobs(vec![job.clone()]).await?;
        Ok(job)
    }

    pub async fn claim_due_jobs(&self, limit: usize) -> Result<Vec<PipelineJob>> {
        let now = crate::research::util::now_rfc3339();
        self.with_connection(move |connection| {
            let transaction = connection.transaction()?;
            let mut statement = transaction.prepare(
                "SELECT entity_json
                 FROM pipeline_jobs
                 WHERE status IN (?1, ?2)
                   AND (next_attempt_at IS NULL OR next_attempt_at <= ?3)
                 ORDER BY updated_at ASC
                 LIMIT ?4",
            )?;
            let rows = statement.query_map(
                params![
                    serde_json::to_string(&JobStatus::Queued)?,
                    serde_json::to_string(&JobStatus::Failed)?,
                    now.clone(),
                    i64::try_from(limit)
                        .map_err(|error| ResearchError::Validation(error.to_string()))?,
                ],
                |row| row.get::<_, String>(0),
            )?;
            let mut jobs = rows
                .collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str::<PipelineJob>(&json).map_err(ResearchError::from))
                .collect::<Result<Vec<_>>>()?;

            drop(statement);

            for job in &mut jobs {
                job.status = JobStatus::Running;
                job.attempt_count += 1;
                job.updated_at = now.clone();
                transaction.execute(
                    "UPDATE pipeline_jobs
                     SET status = ?2,
                         updated_at = ?3,
                         entity_json = ?4
                    WHERE id = ?1",
                    params![
                        &job.id,
                        serde_json::to_string(&job.status)?,
                        &job.updated_at,
                        serde_json::to_string(job)?,
                    ],
                )?;
            }

            transaction.commit()?;
            Ok(jobs)
        })
        .await
    }

    pub async fn list_jobs(
        &self,
        workspace_id: &str,
        job_kind: Option<JobKind>,
    ) -> Result<Vec<PipelineJob>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let json_rows = if let Some(job_kind) = job_kind {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM pipeline_jobs
                     WHERE workspace_id = ?1 AND job_kind = ?2
                     ORDER BY updated_at DESC",
                )?;
                let rows = statement.query_map(
                    params![workspace_id, serde_json::to_string(&job_kind)?],
                    |row| row.get::<_, String>(0),
                )?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            } else {
                let mut statement = connection.prepare(
                    "SELECT entity_json FROM pipeline_jobs
                     WHERE workspace_id = ?1
                     ORDER BY updated_at DESC",
                )?;
                let rows =
                    statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
                rows.collect::<std::result::Result<Vec<_>, _>>()?
            };

            json_rows
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn retry_failed_jobs(
        &self,
        workspace_id: &str,
        job_kind: Option<JobKind>,
    ) -> Result<Vec<PipelineJob>> {
        let mut jobs = self.list_jobs(workspace_id, job_kind).await?;
        let mut retried = Vec::new();
        for job in &mut jobs {
            if matches!(job.status, JobStatus::Failed | JobStatus::Skipped) {
                job.status = JobStatus::Queued;
                job.next_attempt_at = None;
                job.last_error = None;
                job.updated_at = crate::research::util::now_rfc3339();
                retried.push(job.clone());
            }
        }
        self.enqueue_jobs(retried.clone()).await?;
        Ok(retried)
    }

    pub async fn append_audit_event(&self, event: AuditEvent) -> Result<AuditEvent> {
        let value = event.clone();
        self.with_connection(move |connection| {
            connection.execute(
                "INSERT INTO audit_events (
                    id, workspace_id, entity_id, entity_kind, created_at, entity_json
                ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
                params![
                    value.id,
                    value.workspace_id,
                    value.entity_id,
                    serde_json::to_string(&value.entity_kind)?,
                    value.created_at,
                    serde_json::to_string(&value)?,
                ],
            )?;
            Ok(value)
        })
        .await
    }

    pub async fn list_audit_events_for_entity(&self, entity_id: &str) -> Result<Vec<AuditEvent>> {
        let entity_id = entity_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json FROM audit_events WHERE entity_id = ?1 ORDER BY created_at ASC",
            )?;
            let rows = statement.query_map(params![entity_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    pub async fn list_audit_events_for_workspace(
        &self,
        workspace_id: &str,
    ) -> Result<Vec<AuditEvent>> {
        let workspace_id = workspace_id.to_string();
        self.with_connection(move |connection| {
            let mut statement = connection.prepare(
                "SELECT entity_json FROM audit_events WHERE workspace_id = ?1 ORDER BY created_at ASC",
            )?;
            let rows = statement.query_map(params![workspace_id], |row| row.get::<_, String>(0))?;
            rows.collect::<std::result::Result<Vec<_>, _>>()?
                .into_iter()
                .map(|json| serde_json::from_str(&json).map_err(ResearchError::from))
                .collect()
        })
        .await
    }

    fn initialize_schema(&self, connection: &Connection) -> Result<()> {
        connection.execute_batch(
            "PRAGMA foreign_keys = ON;
             PRAGMA journal_mode = WAL;
             PRAGMA synchronous = NORMAL;
             CREATE TABLE IF NOT EXISTS research_workspaces (
                 id TEXT PRIMARY KEY,
                 name TEXT NOT NULL,
                 primary_ticker TEXT NOT NULL,
                 stage TEXT NOT NULL,
                 default_view TEXT NOT NULL,
                 archived INTEGER NOT NULL DEFAULT 0,
                 created_at TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE TABLE IF NOT EXISTS research_notes (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 note_type TEXT NOT NULL,
                 ticker TEXT,
                 source_id TEXT,
                 archived INTEGER NOT NULL DEFAULT 0,
                 pinned INTEGER NOT NULL DEFAULT 0,
                 revision INTEGER NOT NULL,
                 evidence_status TEXT NOT NULL,
                 created_at TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS research_notes_workspace_archived_updated_idx
                 ON research_notes (workspace_id, archived, updated_at DESC);
             CREATE INDEX IF NOT EXISTS research_notes_workspace_type_archived_idx
                 ON research_notes (workspace_id, note_type, archived);
             CREATE INDEX IF NOT EXISTS research_notes_workspace_ticker_updated_idx
                 ON research_notes (workspace_id, ticker, updated_at DESC);
             CREATE INDEX IF NOT EXISTS research_notes_workspace_source_type_idx
                 ON research_notes (workspace_id, source_id, note_type);
             CREATE TABLE IF NOT EXISTS note_links (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 from_note_id TEXT NOT NULL,
                 to_note_id TEXT NOT NULL,
                 link_type TEXT NOT NULL,
                 stale INTEGER NOT NULL DEFAULT 0,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS note_links_workspace_from_type_idx
                 ON note_links (workspace_id, from_note_id, link_type);
             CREATE TABLE IF NOT EXISTS ghost_notes (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 state TEXT NOT NULL,
                 visibility_state TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS ghost_notes_workspace_state_visibility_idx
                 ON ghost_notes (workspace_id, state, visibility_state, updated_at DESC);
             CREATE TABLE IF NOT EXISTS source_records (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 ticker TEXT,
                 kind TEXT NOT NULL,
                 published_at TEXT,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS source_records_workspace_ticker_kind_published_idx
                 ON source_records (workspace_id, ticker, kind, published_at DESC);
             CREATE INDEX IF NOT EXISTS source_records_workspace_checksum_idx
                 ON source_records (workspace_id, json_extract(entity_json, '$.checksum'));
             CREATE INDEX IF NOT EXISTS source_records_workspace_accession_idx
                 ON source_records (workspace_id, json_extract(entity_json, '$.filingAccession'));
             CREATE TABLE IF NOT EXISTS research_files (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 checksum TEXT NOT NULL,
                 original_name TEXT NOT NULL,
                 created_at TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS research_files_workspace_created_idx
                 ON research_files (workspace_id, created_at DESC);
             CREATE INDEX IF NOT EXISTS research_files_workspace_checksum_idx
                 ON research_files (workspace_id, checksum);
             CREATE TABLE IF NOT EXISTS financial_models (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 name TEXT NOT NULL,
                 ticker TEXT,
                 template_type TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS financial_models_workspace_updated_idx
                 ON financial_models (workspace_id, updated_at DESC);
             CREATE TABLE IF NOT EXISTS model_cell_annotations (
                 model_id TEXT NOT NULL REFERENCES financial_models(id) ON DELETE CASCADE,
                 cell_id TEXT NOT NULL,
                 sheet_id TEXT NOT NULL,
                 address TEXT NOT NULL,
                 evidence_status TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL,
                 PRIMARY KEY (model_id, cell_id)
             );
             CREATE INDEX IF NOT EXISTS model_cell_annotations_model_address_idx
                 ON model_cell_annotations (model_id, sheet_id, address);
             CREATE TABLE IF NOT EXISTS model_data_bindings (
                 id TEXT PRIMARY KEY,
                 model_id TEXT NOT NULL REFERENCES financial_models(id) ON DELETE CASCADE,
                 sheet_id TEXT NOT NULL,
                 range_address TEXT NOT NULL,
                 kind TEXT NOT NULL,
                 source_key TEXT NOT NULL,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS model_data_bindings_model_idx
                 ON model_data_bindings (model_id);
             CREATE INDEX IF NOT EXISTS model_data_bindings_kind_source_idx
                 ON model_data_bindings (kind, source_key);
             CREATE TABLE IF NOT EXISTS source_excerpts (
                 id TEXT PRIMARY KEY,
                 source_id TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE TABLE IF NOT EXISTS pipeline_jobs (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 entity_id TEXT NOT NULL,
                 job_kind TEXT NOT NULL,
                 status TEXT NOT NULL,
                 next_attempt_at TEXT,
                 updated_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS pipeline_jobs_status_next_attempt_idx
                 ON pipeline_jobs (status, next_attempt_at);
             CREATE TABLE IF NOT EXISTS workspace_view_state (
                 workspace_id TEXT PRIMARY KEY,
                 entity_json TEXT NOT NULL
             );
             CREATE TABLE IF NOT EXISTS memo_exports (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE TABLE IF NOT EXISTS audit_events (
                 id TEXT PRIMARY KEY,
                 workspace_id TEXT NOT NULL REFERENCES research_workspaces(id) ON DELETE CASCADE,
                 entity_id TEXT NOT NULL,
                 entity_kind TEXT NOT NULL,
                 created_at TEXT NOT NULL,
                 entity_json TEXT NOT NULL
             );
             CREATE INDEX IF NOT EXISTS audit_events_entity_created_idx
                 ON audit_events (entity_id, created_at ASC);
             CREATE INDEX IF NOT EXISTS audit_events_workspace_created_idx
                 ON audit_events (workspace_id, created_at ASC);
             CREATE VIRTUAL TABLE IF NOT EXISTS research_fts USING fts5(note_id UNINDEXED, title, cleaned_text, ai_annotation);",
        )?;
        Ok(())
    }

    async fn with_connection<F, T>(&self, task: F) -> Result<T>
    where
        F: FnOnce(&mut Connection) -> Result<T> + Send + 'static,
        T: Send + 'static,
    {
        let db_path = self.db_path.clone();
        let connections = self.connections.clone();
        tokio::task::spawn_blocking(move || {
            let mut connection = take_connection(&connections, &db_path)?;
            let result = task(&mut connection);
            let store_result = store_connection(&connections, connection);
            match (result, store_result) {
                (Ok(value), Ok(())) => Ok(value),
                (Err(error), Ok(())) => Err(error),
                (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error),
            }
        })
        .await
        .map_err(|error| ResearchError::Join(error.to_string()))?
    }

    fn store_connection(&self, connection: Connection) -> Result<()> {
        store_connection(&self.connections, connection)
    }
}

fn open_connection(path: &Path) -> Result<Connection> {
    let connection = Connection::open(path)?;
    connection.execute_batch(
        "PRAGMA foreign_keys = ON;
         PRAGMA busy_timeout = 5000;
         PRAGMA journal_mode = WAL;
         PRAGMA synchronous = NORMAL;",
    )?;
    Ok(connection)
}

fn take_connection(pool: &Mutex<Vec<Connection>>, db_path: &Path) -> Result<Connection> {
    {
        let mut guard = pool.lock().map_err(|error| {
            ResearchError::Validation(format!("research connection pool poisoned: {error}"))
        })?;
        if let Some(connection) = guard.pop() {
            return Ok(connection);
        }
    }
    open_connection(db_path)
}

fn fts_query(input: &str) -> String {
    let tokens = input
        .split(|character: char| !character.is_ascii_alphanumeric())
        .filter(|value| !value.is_empty())
        .map(|value| format!("{value}*"))
        .collect::<Vec<_>>();

    if tokens.is_empty() {
        input.to_string()
    } else {
        tokens.join(" AND ")
    }
}

fn store_connection(pool: &Mutex<Vec<Connection>>, connection: Connection) -> Result<()> {
    let mut guard = pool.lock().map_err(|error| {
        ResearchError::Validation(format!("research connection pool poisoned: {error}"))
    })?;
    guard.push(connection);
    Ok(())
}

#[cfg(test)]
mod fts_tests {
    use super::*;

    #[test]
    fn fts_query_should_tokenize_and_append_wildcards() {
        assert_eq!(fts_query("AAPL earnings"), "AAPL* AND earnings*");
    }

    #[test]
    fn fts_query_should_handle_single_token() {
        assert_eq!(fts_query("thesis"), "thesis*");
    }

    #[test]
    fn fts_query_should_strip_non_alphanumeric() {
        assert_eq!(fts_query("NVDA/QCOM merger"), "NVDA* AND QCOM* AND merger*");
    }

    #[test]
    fn fts_query_should_handle_empty() {
        assert_eq!(fts_query(""), "");
    }
}

#[cfg(test)]
mod tests {
    use std::time::Duration;

    use rusqlite::params;
    use tempfile::tempdir;

    use super::{open_connection, ResearchRepository};
    use crate::research::types::{
        AnalystStatus, CaptureMethod, EvidenceBasis, EvidenceStatus, FinancialModel,
        GhostFreshnessRisk, GhostLifecycleState, GhostNote, GhostNoteClass, GhostRecommendedAction,
        GhostResolutionStatus, GhostStatus, GhostSupportStrength, GhostTargetClaimKind, GhostTone,
        GhostVisibilityState, LinkOrigin, LinkStrength, LinkType, ModelBindingRefreshPolicy,
        ModelCellAnnotation, ModelDataBinding, ModelDataBindingKind, ModelEvidenceStatus,
        ModelTemplateType, NoteLink, NotePriority, NoteProvenance, NoteType, ProvenanceActor,
        ResearchNote, ResearchStage, ResearchWorkspace, SourceKind, ThesisStatus, WorkspaceScope,
        WorkspaceViewKind,
    };

    fn workspace() -> ResearchWorkspace {
        let now = crate::research::util::now_rfc3339();
        ResearchWorkspace {
            id: "workspace-1".to_string(),
            name: "AAPL".to_string(),
            primary_ticker: "AAPL".to_string(),
            scope: WorkspaceScope::SingleCompany,
            stage: ResearchStage::Capture,
            default_view: WorkspaceViewKind::Canvas,
            pinned_note_ids: Vec::new(),
            archived: false,
            created_at: now.clone(),
            updated_at: now,
        }
    }

    fn note(id: &str) -> ResearchNote {
        let now = crate::research::util::now_rfc3339();
        ResearchNote {
            id: id.to_string(),
            workspace_id: "workspace-1".to_string(),
            company_id: None,
            ticker: Some("AAPL".to_string()),
            source_id: None,
            raw_text: format!("raw {id}"),
            cleaned_text: format!("clean {id}"),
            body_html: None,
            embeds: Vec::new(),
            file_ids: Vec::new(),
            title: Some(format!("Note {id}")),
            note_type: NoteType::Claim,
            subtype: None,
            analyst_status: AnalystStatus::Captured,
            ai_annotation: None,
            confidence: 0.7,
            evidence_status: EvidenceStatus::Unsourced,
            inferred_links: Vec::new(),
            ghost_status: GhostStatus::CandidateInput,
            thesis_status: ThesisStatus::None,
            created_at: now.clone(),
            updated_at: now.clone(),
            provenance: NoteProvenance {
                created_by: ProvenanceActor::Manual,
                capture_method: CaptureMethod::QuickEntry,
                source_kind: SourceKind::Manual,
                origin_note_id: None,
                origin_ghost_id: None,
                model_info: None,
                created_at: now,
                raw_input_hash: "hash".to_string(),
            },
            tags: Vec::new(),
            catalysts: Vec::new(),
            risks: Vec::new(),
            valuation_refs: Vec::new(),
            time_horizon: None,
            scenario: None,
            priority: NotePriority::Normal,
            pinned: false,
            archived: false,
            revision: 1,
            source_excerpt: None,
            last_enriched_at: None,
            last_linked_at: None,
            stale_reason: None,
            superseded_by_note_id: None,
        }
    }

    fn link() -> NoteLink {
        let now = crate::research::util::now_rfc3339();
        NoteLink {
            id: "link-1".to_string(),
            workspace_id: "workspace-1".to_string(),
            from_note_id: "note-1".to_string(),
            to_note_id: "note-2".to_string(),
            link_type: LinkType::Supports,
            directional: true,
            confidence: 0.8,
            strength: LinkStrength::Medium,
            evidence_basis: EvidenceBasis::Lexical,
            created_by: LinkOrigin::Heuristic,
            created_at: now.clone(),
            updated_at: now,
            source_revision_pair: (1, 1),
            stale: false,
            stale_reason: None,
        }
    }

    fn ghost() -> GhostNote {
        let now = crate::research::util::now_rfc3339();
        GhostNote {
            id: "ghost-1".to_string(),
            workspace_id: "workspace-1".to_string(),
            ghost_class: GhostNoteClass::CandidateThesis,
            headline: "Possible thesis".to_string(),
            body: "Evidence suggests a thesis.".to_string(),
            tone: GhostTone::Tentative,
            confidence: 0.75,
            visibility_state: GhostVisibilityState::Visible,
            state: GhostLifecycleState::Generated,
            supporting_note_ids: vec!["note-1".to_string()],
            contradicting_note_ids: Vec::new(),
            source_ids: Vec::new(),
            evidence_threshold_met: true,
            created_at: now.clone(),
            updated_at: now,
            superseded_by_ghost_id: None,
            promoted_note_id: None,
            memo_section_hint: None,
            target_note_id: Some("note-1".to_string()),
            parent_thesis_note_id: None,
            target_claim_text: "Evidence suggests a thesis.".to_string(),
            target_claim_kind: GhostTargetClaimKind::CoreThesis,
            why_it_matters: "This thesis candidate affects conviction.".to_string(),
            current_support_summary: "One supporting note.".to_string(),
            counter_case: "The evidence may not be decisive.".to_string(),
            recommended_action: GhostRecommendedAction::RewriteClaim,
            recommended_action_rationale: "Tighten the thesis claim.".to_string(),
            resolution_status: GhostResolutionStatus::Open,
            leverage_score: 0.75,
            support_strength: GhostSupportStrength::Weak,
            freshness_risk: GhostFreshnessRisk::Low,
            agent_reasoning_version: 1,
        }
    }

    #[tokio::test]
    async fn new_should_initialize_schema() {
        let dir = tempdir().unwrap();
        let repository = ResearchRepository::new(dir.path().join("research.sqlite")).unwrap();
        repository.create_workspace(workspace()).await.unwrap();

        let workspaces = repository.list_workspaces().await.unwrap();
        assert_eq!(workspaces.len(), 1);
    }

    #[tokio::test]
    async fn load_workspace_projection_parts_should_return_saved_read_model() {
        let dir = tempdir().unwrap();
        let repository = ResearchRepository::new(dir.path().join("research.sqlite")).unwrap();
        repository.create_workspace(workspace()).await.unwrap();
        repository.create_note(note("note-1")).await.unwrap();
        repository.create_note(note("note-2")).await.unwrap();
        repository
            .replace_links_for_workspace("workspace-1", vec![link()])
            .await
            .unwrap();
        repository
            .replace_ghosts_for_workspace("workspace-1", vec![ghost()])
            .await
            .unwrap();

        let parts = repository
            .load_workspace_projection_parts("workspace-1", false)
            .await
            .unwrap();

        assert_eq!(parts.workspace.id, "workspace-1");
        assert_eq!(parts.notes.len(), 2);
        assert_eq!(parts.links.len(), 1);
        assert_eq!(parts.ghosts.len(), 1);
    }

    #[tokio::test]
    async fn busy_timeout_should_allow_repository_write_to_wait_for_lock() {
        let dir = tempdir().unwrap();
        let db_path = dir.path().join("research.sqlite");
        let repository = ResearchRepository::new(db_path.clone()).unwrap();
        repository.create_workspace(workspace()).await.unwrap();

        let mut blocker = open_connection(&db_path).unwrap();
        let transaction = blocker.transaction().unwrap();
        transaction
            .execute(
                "UPDATE research_workspaces SET updated_at = updated_at WHERE id = ?1",
                params!["workspace-1"],
            )
            .unwrap();

        let repository_clone = repository.clone();
        let waiting_write = tokio::spawn(async move {
            repository_clone
                .touch_workspace_updated_at("workspace-1", crate::research::util::now_rfc3339())
                .await
        });

        tokio::time::sleep(Duration::from_millis(100)).await;
        transaction.commit().unwrap();

        let touched = waiting_write.await.unwrap().unwrap();
        assert_eq!(touched.id, "workspace-1");
    }

    #[tokio::test]
    async fn financial_model_persistence_replaces_annotations_and_saves_bindings() {
        let dir = tempdir().unwrap();
        let repository = ResearchRepository::new(dir.path().join("research.sqlite")).unwrap();
        repository.create_workspace(workspace()).await.unwrap();
        let now = crate::research::util::now_rfc3339();
        let mut model = FinancialModel {
            id: "model-1".to_string(),
            workspace_id: "workspace-1".to_string(),
            name: "Base DCF".to_string(),
            ticker: Some("AAPL".to_string()),
            template_type: ModelTemplateType::Dcf,
            scenario_group_id: None,
            workbook_json: serde_json::json!({
                "version": 1,
                "sheets": [{"id": "dcf", "name": "DCF", "cells": {"B4": {"value": 0.05}}}]
            }),
            annotations: vec![ModelCellAnnotation {
                cell_id: "cell-1".to_string(),
                sheet_id: "dcf".to_string(),
                address: "B4".to_string(),
                linked_note_ids: vec!["note-1".to_string()],
                evidence_status: ModelEvidenceStatus::SingleSource,
                confidence_score: 0.6,
                last_evidence_update: Some(now.clone()),
                updated_at: now.clone(),
            }],
            created_at: now.clone(),
            updated_at: now.clone(),
        };

        repository
            .save_financial_model(model.clone())
            .await
            .unwrap();
        let listed = repository
            .list_financial_models("workspace-1")
            .await
            .unwrap();
        assert_eq!(listed.len(), 1);
        assert_eq!(listed[0].annotations.len(), 1);

        model.annotations = vec![ModelCellAnnotation {
            cell_id: "cell-2".to_string(),
            sheet_id: "dcf".to_string(),
            address: "B5".to_string(),
            linked_note_ids: Vec::new(),
            evidence_status: ModelEvidenceStatus::Unsourced,
            confidence_score: 0.0,
            last_evidence_update: None,
            updated_at: now.clone(),
        }];
        repository.save_financial_model(model).await.unwrap();
        let annotations = repository
            .list_model_cell_annotations("model-1")
            .await
            .unwrap();
        assert_eq!(annotations.len(), 1);
        assert_eq!(annotations[0].cell_id, "cell-2");

        let binding = ModelDataBinding {
            id: "binding-1".to_string(),
            model_id: "model-1".to_string(),
            sheet_id: "dcf".to_string(),
            range: "B4:F4".to_string(),
            kind: ModelDataBindingKind::SecEdgar,
            source_key: "total_revenues".to_string(),
            refresh_policy: ModelBindingRefreshPolicy::Manual,
            last_refresh_at: None,
            created_at: now.clone(),
            updated_at: now,
        };
        repository.save_model_data_binding(binding).await.unwrap();
        let bindings = repository
            .list_model_data_bindings("model-1")
            .await
            .unwrap();
        assert_eq!(bindings.len(), 1);
        assert_eq!(bindings[0].source_key, "total_revenues");
    }
}
