import { createHash } from 'node:crypto';
import type { Database } from 'bun:sqlite';
import type {
  ResearchJournalEntry,
  SearchAnswerResponse,
  SearchCitation,
  SearchResult,
  SearchSource,
  TaskStageContext
} from '@/lib/types';
import { runAiAnalysis, runAiEmbeddings } from '@/lib/server/ai';
import { __dbInternals, getSqliteClient } from '@/lib/server/db';
import { fetchPrimaryFilingText } from '@/lib/server/sec';
import { getFilingByAccession, listFilingsRecords } from '@/lib/server/repos/filings';
import {
  getResearchJournalEntryRecord,
  listResearchJournalEntries,
  listResearchJournalEntriesForUser
} from '@/lib/server/repos/research-journal';

type SearchDocumentScope = 'global' | 'user';
type SearchDocumentSourceKind = 'filing_document' | 'filing_brief' | 'research_note';

type MaterializedSearchDocument = {
  sourceKind: SearchDocumentSourceKind;
  sourceRef: string;
  scope: SearchDocumentScope;
  userId: string | null;
  ticker: string | null;
  accessionNumber: string | null;
  filingDate: string | null;
  title: string | null;
  contentText: string;
  metadata: Record<string, unknown>;
};

type SearchChunkRecord = {
  chunkIndex: number;
  chunkText: string;
  charCount: number;
  startOffset: number;
  endOffset: number;
  headingPath: string | null;
  citationLabel: string;
};

type SearchDocumentRow = {
  id: number;
  source_kind: SearchDocumentSourceKind;
  source_ref: string;
  scope: SearchDocumentScope;
  user_id: string | null;
  ticker: string | null;
  accession_number: string | null;
  title: string | null;
  content_text: string;
  content_hash: string;
  metadata: Record<string, unknown> | null;
  index_status: 'pending' | 'indexed' | 'failed';
  indexed_at: string | null;
  last_error: string | null;
  created_at: string;
  updated_at: string;
};

type SearchChunkJoinRow = {
  chunk_id: number;
  document_id: number;
  chunk_text: string;
  heading_path: string | null;
  citation_label: string;
  source_kind: SearchDocumentSourceKind;
  source_ref: string;
  title: string | null;
  ticker: string | null;
  accession_number: string | null;
  metadata: Record<string, unknown> | null;
};

type DeleteSourceRef = {
  sourceKind: SearchDocumentSourceKind;
  sourceRef: string;
  scope: SearchDocumentScope;
  userId?: string | null;
};

type IndexSearchDocumentsInput = {
  userId: string;
  ticker?: string | null;
  accessionNumber?: string | null;
  journalEntryId?: number | null;
  sourceKinds?: SearchDocumentSourceKind[];
  deleteSourceRefs?: DeleteSourceRef[];
  onStage?: (
    stage: 'collect' | 'fetch' | 'chunk' | 'embed' | 'persist',
    detail: string,
    context?: TaskStageContext | null
  ) => Promise<void> | void;
};

type SearchInput = {
  userId: string;
  query: string;
  ticker?: string | null;
  sources?: SearchSource[];
  limit?: number;
};

type SearchMatch = {
  chunkId: number;
  vectorRank: number | null;
  lexicalRank: number | null;
  snippet: string | null;
};

const RRF_K = 60;
const SEARCH_RESULT_LIMIT_DEFAULT = 10;
const SEARCH_RESULT_LIMIT_MIN = 4;
const SEARCH_RESULT_LIMIT_MAX = 12;
const MAX_RESULTS_PER_DOCUMENT = 2;
const MAX_CONTEXT_RESULTS = 6;
const MAX_CONTEXT_CHARS = 8_000;

const SOURCE_KIND_BY_SEARCH_SOURCE: Record<SearchSource, SearchDocumentSourceKind> = {
  documents: 'filing_document',
  filings: 'filing_brief',
  research: 'research_note'
};

function escapeLike(value: string) {
  return value.replace(/[%_]/g, (match) => `\\${match}`);
}

function normalizeTicker(value: string | null | undefined) {
  const normalized = value?.trim().toUpperCase() ?? '';
  return normalized.length > 0 ? normalized : null;
}

function normalizeSearchSources(sources?: SearchSource[]) {
  const normalized = new Set<SearchSource>();

  for (const source of sources ?? ['documents', 'filings', 'research']) {
    if (source === 'documents' || source === 'filings' || source === 'research') {
      normalized.add(source);
    }
  }

  return normalized.size > 0
    ? [...normalized]
    : ['documents', 'filings', 'research'] as SearchSource[];
}

function clampLimit(limit?: number) {
  const value = Number.isFinite(limit) ? Number(limit) : SEARCH_RESULT_LIMIT_DEFAULT;
  const intValue = Math.trunc(value);
  return Math.min(Math.max(intValue, SEARCH_RESULT_LIMIT_MIN), SEARCH_RESULT_LIMIT_MAX);
}

function hashContent(content: string) {
  return createHash('sha256').update(content).digest('hex');
}

function stripMarkdown(markdown: string) {
  return markdown
    .replace(/```[\s\S]*?```/g, ' ')
    .replace(/`([^`]+)`/g, '$1')
    .replace(/!\[[^\]]*\]\([^)]+\)/g, ' ')
    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
    .replace(/^>\s?/gm, '')
    .replace(/^#{1,6}\s+/gm, '')
    .replace(/[*_~]/g, '')
    .replace(/^\s*[-+]\s+/gm, '')
    .replace(/\r/g, '\n')
    .replace(/[ \t]+\n/g, '\n')
    .replace(/\n{3,}/g, '\n\n')
    .trim();
}

function normalizeWhitespace(value: string) {
  return value
    .replace(/\r/g, '\n')
    .replace(/[ \t]+\n/g, '\n')
    .replace(/\n[ \t]+/g, '\n')
    .replace(/[ \t]{2,}/g, ' ')
    .replace(/\n{3,}/g, '\n\n')
    .trim();
}

function buildFilingBriefContent(input: {
  ticker: string;
  companyName: string;
  accessionNumber: string;
  filingDate: string;
  filingType: string;
  metrics: Record<string, number | null> | null;
  analysis: Record<string, unknown> | null;
}) {
  const extraction = input.analysis?.extraction;
  const extractionLines = extraction && typeof extraction === 'object'
    ? Object.entries(extraction as Record<string, unknown>)
      .map(([key, value]) => `${key}: ${Array.isArray(value) ? value.join(' | ') : String(value ?? '')}`)
      .filter((line) => !line.endsWith(': '))
    : [];

  const reportText = typeof input.analysis?.text === 'string'
    ? input.analysis.text
    : typeof input.analysis?.legacyInsights === 'string'
      ? input.analysis.legacyInsights
      : null;

  return normalizeWhitespace([
    `${input.companyName} (${input.ticker}) filing brief`,
    `Form: ${input.filingType}`,
    `Filed: ${input.filingDate}`,
    `Accession: ${input.accessionNumber}`,
    input.metrics ? `Key metrics: ${JSON.stringify(input.metrics)}` : null,
    reportText ? `AI summary:\n${reportText}` : null,
    extractionLines.length > 0 ? `Structured extraction:\n${extractionLines.join('\n')}` : null
  ].filter((entry): entry is string => Boolean(entry)).join('\n\n'));
}

function buildCitationLabel(document: MaterializedSearchDocument, chunkIndex: number) {
  if (document.sourceKind === 'research_note') {
    return `${document.ticker ?? 'Research'} journal note [${chunkIndex + 1}]`;
  }

  const parts = [
    document.ticker,
    document.accessionNumber,
    document.filingDate
  ].filter((entry): entry is string => Boolean(entry));

  return `${parts.join(' · ')} [${chunkIndex + 1}]`;
}

function inferHeadingPath(text: string, offset: number) {
  const windowStart = Math.max(0, offset - 600);
  const context = text.slice(windowStart, offset);
  const lines = context
    .split('\n')
    .map((line) => line.trim())
    .filter((line) => line.length > 0);

  for (let index = lines.length - 1; index >= 0; index -= 1) {
    const candidate = lines[index]!;
    const looksLikeHeading = candidate.length <= 100 && (
      /:$/.test(candidate)
      || /^[A-Z0-9][A-Z0-9\s/&,-]{4,}$/.test(candidate)
      || /^\d+(\.\d+)*\s+[A-Z]/.test(candidate)
    );

    if (looksLikeHeading) {
      return candidate;
    }
  }

  return null;
}

function chunkText(
  text: string,
  options: { targetChars: number; overlapChars: number; maxSingleChunk?: number },
  document: MaterializedSearchDocument
) {
  const normalized = normalizeWhitespace(text);
  if (!normalized) {
    return [] satisfies SearchChunkRecord[];
  }

  const maxSingleChunk = options.maxSingleChunk ?? 0;
  if (maxSingleChunk > 0 && normalized.length <= maxSingleChunk) {
    return [{
      chunkIndex: 0,
      chunkText: normalized,
      charCount: normalized.length,
      startOffset: 0,
      endOffset: normalized.length,
      headingPath: inferHeadingPath(normalized, 0),
      citationLabel: buildCitationLabel(document, 0)
    }];
  }

  const chunks: SearchChunkRecord[] = [];
  let start = 0;
  let chunkIndex = 0;

  while (start < normalized.length) {
    const tentativeEnd = Math.min(start + options.targetChars, normalized.length);
    let end = tentativeEnd;

    if (tentativeEnd < normalized.length) {
      const localWindow = normalized.slice(start, Math.min(normalized.length, tentativeEnd + 180));
      const paragraphBreak = localWindow.lastIndexOf('\n\n');
      const sentenceBreak = Math.max(localWindow.lastIndexOf('. '), localWindow.lastIndexOf('\n'));
      const boundary = Math.max(paragraphBreak, sentenceBreak);

      if (boundary >= options.targetChars * 0.55) {
        end = start + boundary + (paragraphBreak === boundary ? 0 : 1);
      }
    }

    const chunkTextValue = normalized.slice(start, end).trim();
    if (chunkTextValue) {
      chunks.push({
        chunkIndex,
        chunkText: chunkTextValue,
        charCount: chunkTextValue.length,
        startOffset: start,
        endOffset: end,
        headingPath: inferHeadingPath(normalized, start),
        citationLabel: buildCitationLabel(document, chunkIndex)
      });
      chunkIndex += 1;
    }

    if (end >= normalized.length) {
      break;
    }

    start = Math.max(end - options.overlapChars, start + 1);
  }

  return chunks;
}

function chunkDocument(document: MaterializedSearchDocument) {
  switch (document.sourceKind) {
    case 'filing_document':
      return chunkText(document.contentText, { targetChars: 1400, overlapChars: 200 }, document);
    case 'filing_brief':
      return chunkText(document.contentText, { targetChars: 1000, overlapChars: 100 }, document);
    case 'research_note':
      return chunkText(document.contentText, {
        targetChars: 1000,
        overlapChars: 100,
        maxSingleChunk: 2000
      }, document);
    default:
      return [];
  }
}

function mapSourceKindToSearchSource(sourceKind: SearchDocumentSourceKind): SearchSource {
  switch (sourceKind) {
    case 'filing_document':
      return 'documents';
    case 'filing_brief':
      return 'filings';
    case 'research_note':
      return 'research';
    default:
      return 'documents';
  }
}

function buildSearchHref(row: SearchChunkJoinRow) {
  if (row.source_kind === 'research_note') {
    return `/analysis?ticker=${encodeURIComponent(row.ticker ?? '')}&journalId=${encodeURIComponent(row.source_ref)}`;
  }

  const hasAnalysis = Boolean((row.metadata as { hasAnalysis?: unknown } | null)?.hasAnalysis);
  if (hasAnalysis && row.ticker && row.accession_number) {
    return `/analysis/reports/${encodeURIComponent(row.ticker)}/${encodeURIComponent(row.accession_number)}`;
  }

  if (row.ticker) {
    return `/filings?ticker=${encodeURIComponent(row.ticker)}`;
  }

  return '/filings';
}

function manualSnippet(text: string, query: string) {
  const tokens = query.toLowerCase().split(/\W+/).filter((token) => token.length > 1);
  const lower = text.toLowerCase();
  const matchIndex = tokens
    .map((token) => lower.indexOf(token))
    .find((index) => index >= 0) ?? 0;
  const start = Math.max(0, matchIndex - 90);
  const end = Math.min(text.length, start + 220);
  const prefix = start > 0 ? '... ' : '';
  const suffix = end < text.length ? ' ...' : '';
  return `${prefix}${text.slice(start, end).trim()}${suffix}`.trim();
}

function scoreSearchMatch(match: SearchMatch) {
  let score = 0;
  if (match.vectorRank !== null) {
    score += 1 / (RRF_K + match.vectorRank);
  }

  if (match.lexicalRank !== null) {
    score += 1 / (RRF_K + match.lexicalRank);
  }

  return score;
}

function toFtsQuery(query: string) {
  const tokens = query
    .trim()
    .split(/\W+/)
    .map((token) => token.trim().toLowerCase())
    .filter((token) => token.length > 1);

  if (tokens.length === 0) {
    return null;
  }

  return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ');
}

function createPlaceholders(length: number) {
  return new Array(length).fill('?').join(', ');
}

function queryOneDocument(
  client: Database,
  input: Pick<MaterializedSearchDocument, 'scope' | 'userId' | 'sourceKind' | 'sourceRef'>
) {
  return client
    .query(`
      SELECT *
      FROM search_document
      WHERE scope = ?
        AND ifnull(user_id, '') = ?
        AND source_kind = ?
        AND source_ref = ?
      LIMIT 1
    `)
    .get(
      input.scope,
      input.userId ?? '',
      input.sourceKind,
      input.sourceRef
    ) as SearchDocumentRow | null;
}

function listDocumentsForScope(client: Database, input: {
  scope: SearchDocumentScope;
  userId?: string | null;
  sourceKind: SearchDocumentSourceKind;
  ticker?: string | null;
}) {
  const conditions = [
    'scope = ?',
    "ifnull(user_id, '') = ?",
    'source_kind = ?'
  ];
  const values: Array<string | null> = [input.scope, input.userId ?? '', input.sourceKind];

  if (input.ticker) {
    conditions.push('ticker = ?');
    values.push(input.ticker);
  }

  return client
    .query(`
      SELECT *
      FROM search_document
      WHERE ${conditions.join(' AND ')}
    `)
    .all(...values) as SearchDocumentRow[];
}

function deleteDocumentCascade(client: Database, documentId: number) {
  const chunkRows = client
    .query('SELECT id FROM search_chunk WHERE document_id = ?')
    .all(documentId) as Array<{ id: number }>;

  for (const row of chunkRows) {
    client.query('DELETE FROM search_chunk_vec WHERE chunk_id = ?').run(row.id);
    client.query('DELETE FROM search_chunk_fts WHERE chunk_id = ?').run(row.id);
  }

  client.query('DELETE FROM search_chunk WHERE document_id = ?').run(documentId);
  client.query('DELETE FROM search_document WHERE id = ?').run(documentId);
}

function withTransaction<T>(client: Database, fn: () => T) {
  client.exec('BEGIN IMMEDIATE');

  try {
    const result = fn();
    client.exec('COMMIT');
    return result;
  } catch (error) {
    client.exec('ROLLBACK');
    throw error;
  }
}

async function collectFilingDocuments(ticker?: string | null, accessionNumber?: string | null) {
  const filings = accessionNumber
    ? await Promise.all([getFilingByAccession(accessionNumber)]).then((rows) => rows.filter(Boolean))
    : await listFilingsRecords({
      ticker: ticker ?? undefined,
      limit: 250
    });

  const documents: MaterializedSearchDocument[] = [];

  for (const filing of filings) {
    if (!filing) {
      continue;
    }

    const filingText = await fetchPrimaryFilingText({
      filingUrl: filing.filing_url,
      cik: filing.cik,
      accessionNumber: filing.accession_number,
      primaryDocument: filing.primary_document ?? null
    }).catch(() => null);

    if (!filingText?.text) {
      continue;
    }

    documents.push({
      sourceKind: 'filing_document',
      sourceRef: filing.accession_number,
      scope: 'global',
      userId: null,
      ticker: filing.ticker,
      accessionNumber: filing.accession_number,
      filingDate: filing.filing_date,
      title: `${filing.ticker} ${filing.filing_type} primary filing`,
      contentText: filingText.text,
      metadata: {
        filingType: filing.filing_type,
        filingDate: filing.filing_date,
        filingUrl: filing.filing_url,
        submissionUrl: filing.submission_url ?? null,
        primaryDocument: filing.primary_document ?? null,
        hasAnalysis: Boolean(filing.analysis?.text || filing.analysis?.legacyInsights)
      }
    });
  }

  return documents;
}

async function collectFilingBriefs(ticker?: string | null, accessionNumber?: string | null) {
  const filings = accessionNumber
    ? await Promise.all([getFilingByAccession(accessionNumber)]).then((rows) => rows.filter(Boolean))
    : await listFilingsRecords({
      ticker: ticker ?? undefined,
      limit: 250
    });

  return filings
    .filter((filing): filing is NonNullable<typeof filing> => Boolean(filing))
    .map((filing) => ({
      sourceKind: 'filing_brief' as const,
      sourceRef: filing.accession_number,
      scope: 'global' as const,
      userId: null,
      ticker: filing.ticker,
      accessionNumber: filing.accession_number,
      filingDate: filing.filing_date,
      title: `${filing.ticker} ${filing.filing_type} filing brief`,
      contentText: buildFilingBriefContent({
        ticker: filing.ticker,
        companyName: filing.company_name,
        accessionNumber: filing.accession_number,
        filingDate: filing.filing_date,
        filingType: filing.filing_type,
        metrics: filing.metrics,
        analysis: filing.analysis as Record<string, unknown> | null
      }),
      metadata: {
        filingType: filing.filing_type,
        filingDate: filing.filing_date,
        hasAnalysis: Boolean(filing.analysis?.text || filing.analysis?.legacyInsights)
      }
    }));
}

function materializeResearchNote(entry: ResearchJournalEntry): MaterializedSearchDocument | null {
  const text = normalizeWhitespace(stripMarkdown(entry.body_markdown));
  if (!text) {
    return null;
  }

  return {
    sourceKind: 'research_note',
    sourceRef: String(entry.id),
    scope: 'user',
    userId: entry.user_id,
    ticker: entry.ticker,
    accessionNumber: entry.accession_number,
    filingDate: null,
    title: entry.title ?? `${entry.ticker} research note`,
    contentText: text,
    metadata: {
      entryType: entry.entry_type,
      createdAt: entry.created_at,
      updatedAt: entry.updated_at
    }
  };
}

async function collectResearchNotes(userId: string, ticker?: string | null, journalEntryId?: number | null) {
  if (journalEntryId) {
    const entry = await getResearchJournalEntryRecord(userId, journalEntryId);
    const materialized = entry ? materializeResearchNote(entry) : null;
    return materialized ? [materialized] : [];
  }

  const entries = ticker
    ? await listResearchJournalEntries(userId, ticker, 250)
    : await listResearchJournalEntriesForUser(userId, 250);

  return entries
    .map(materializeResearchNote)
    .filter((entry): entry is MaterializedSearchDocument => Boolean(entry));
}

async function collectMaterializedDocuments(input: IndexSearchDocumentsInput) {
  const sourceKinds = input.sourceKinds ?? ['filing_document', 'filing_brief', 'research_note'];
  const documents: MaterializedSearchDocument[] = [];

  for (const sourceKind of sourceKinds) {
    if (sourceKind === 'filing_document') {
      documents.push(...await collectFilingDocuments(input.ticker ?? null, input.accessionNumber ?? null));
      continue;
    }

    if (sourceKind === 'filing_brief') {
      documents.push(...await collectFilingBriefs(input.ticker ?? null, input.accessionNumber ?? null));
      continue;
    }

    if (sourceKind === 'research_note') {
      documents.push(...await collectResearchNotes(
        input.userId,
        input.ticker ?? null,
        input.journalEntryId ?? null
      ));
    }
  }

  return documents;
}

function persistDocumentIndex(
  client: Database,
  document: MaterializedSearchDocument,
  chunks: SearchChunkRecord[],
  embeddings: number[][]
) {
  const now = new Date().toISOString();
  const contentHash = hashContent(document.contentText);
  const existing = queryOneDocument(client, document);

  if (existing && existing.content_hash === contentHash && existing.index_status === 'indexed') {
    return { indexed: false, skipped: true, chunkCount: 0 };
  }

  const documentId = withTransaction(client, () => {
    if (existing) {
      deleteDocumentCascade(client, existing.id);
    }

    const inserted = client
      .query(`
        INSERT INTO search_document (
          source_kind,
          source_ref,
          scope,
          user_id,
          ticker,
          accession_number,
          title,
          content_text,
          content_hash,
          metadata,
          index_status,
          indexed_at,
          last_error,
          created_at,
          updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'indexed', ?, NULL, ?, ?)
        RETURNING id
      `)
      .get(
        document.sourceKind,
        document.sourceRef,
        document.scope,
        document.userId,
        document.ticker,
        document.accessionNumber,
        document.title,
        document.contentText,
        contentHash,
        JSON.stringify(document.metadata),
        now,
        existing?.created_at ?? now,
        now
      ) as { id: number };

    for (let index = 0; index < chunks.length; index += 1) {
      const chunk = chunks[index]!;
      const embedding = embeddings[index]!;
      const insertedChunk = client
        .query(`
          INSERT INTO search_chunk (
            document_id,
            chunk_index,
            chunk_text,
            char_count,
            start_offset,
            end_offset,
            heading_path,
            citation_label,
            created_at
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
          RETURNING id
        `)
        .get(
          inserted.id,
          chunk.chunkIndex,
          chunk.chunkText,
          chunk.charCount,
          chunk.startOffset,
          chunk.endOffset,
          chunk.headingPath,
          chunk.citationLabel,
          now
        ) as { id: number };

      client
        .query(`
          INSERT INTO search_chunk_fts (
            rowid,
            chunk_text,
            citation_label,
            heading_path,
            chunk_id,
            document_id,
            chunk_index,
            scope,
            user_id,
            source_kind,
            ticker,
            accession_number,
            filing_date
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `)
        .run(
          insertedChunk.id,
          chunk.chunkText,
          chunk.citationLabel,
          chunk.headingPath,
          insertedChunk.id,
          inserted.id,
          chunk.chunkIndex,
          document.scope,
          document.userId,
          document.sourceKind,
          document.ticker,
          document.accessionNumber,
          document.filingDate
        );

      client
        .query(`
          INSERT INTO search_chunk_vec (
            chunk_id,
            embedding,
            scope,
            user_id,
            source_kind,
            ticker,
            accession_number,
            filing_date,
            document_id,
            chunk_index,
            citation_label
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `)
        .run(
          insertedChunk.id,
          JSON.stringify(embedding),
          document.scope,
          document.userId,
          document.sourceKind,
          document.ticker,
          document.accessionNumber,
          document.filingDate,
          inserted.id,
          chunk.chunkIndex,
          chunk.citationLabel
        );
    }

    return inserted.id;
  });

  return {
    indexed: true,
    skipped: false,
    chunkCount: chunks.length,
    documentId
  };
}

function deleteSourceRefs(client: Database, refs: DeleteSourceRef[]) {
  let deleted = 0;

  for (const ref of refs) {
    const row = queryOneDocument(client, {
      scope: ref.scope,
      userId: ref.userId ?? null,
      sourceKind: ref.sourceKind,
      sourceRef: ref.sourceRef
    });

    if (!row) {
      continue;
    }

    withTransaction(client, () => {
      deleteDocumentCascade(client, row.id);
    });
    deleted += 1;
  }

  return deleted;
}

export async function indexSearchDocuments(input: IndexSearchDocumentsInput) {
  const client = getSqliteClient();
  await input.onStage?.('collect', 'Collecting materialized search sources');
  const materialized = await collectMaterializedDocuments(input);
  const sourceKinds = input.sourceKinds ?? ['filing_document', 'filing_brief', 'research_note'];

  let indexed = 0;
  let skipped = 0;
  let deleted = 0;
  let chunksEmbedded = 0;
  const totalDocuments = materialized.length;

  const stageContext = (current: number, subject?: TaskStageContext['subject'] | null): TaskStageContext => ({
    progress: {
      current,
      total: totalDocuments || 1,
      unit: 'sources'
    },
    counters: {
      sourcesCollected: totalDocuments,
      indexed,
      skipped,
      deleted,
      chunksEmbedded
    },
    subject: subject ?? (input.ticker ? { ticker: input.ticker } : input.accessionNumber ? { accessionNumber: input.accessionNumber } : null)
  });

  if (input.deleteSourceRefs && input.deleteSourceRefs.length > 0) {
    deleted += deleteSourceRefs(client, input.deleteSourceRefs);
  }

  await input.onStage?.(
    'collect',
    `Collected ${materialized.length} source records for search indexing`,
    {
      counters: {
        sourcesCollected: materialized.length,
        deleted
      },
      subject: input.ticker ? { ticker: input.ticker } : input.accessionNumber ? { accessionNumber: input.accessionNumber } : null
    }
  );

  for (let index = 0; index < materialized.length; index += 1) {
    const document = materialized[index];

    await input.onStage?.(
      'fetch',
      `Preparing ${document.sourceKind} ${document.sourceRef}`,
      stageContext(index + 1, {
        ticker: document.ticker ?? undefined,
        accessionNumber: document.accessionNumber ?? undefined,
        label: document.sourceRef
      })
    );
    const chunks = chunkDocument(document);
    if (chunks.length === 0) {
      continue;
    }

    await input.onStage?.(
      'chunk',
      `Chunking ${document.sourceKind} ${document.sourceRef}`,
      stageContext(index + 1, {
        ticker: document.ticker ?? undefined,
        accessionNumber: document.accessionNumber ?? undefined,
        label: document.sourceRef
      })
    );
    await input.onStage?.(
      'embed',
      `Embedding ${chunks.length} chunks for ${document.sourceRef}`,
      {
        ...stageContext(index + 1, {
          ticker: document.ticker ?? undefined,
          accessionNumber: document.accessionNumber ?? undefined,
          label: document.sourceRef
        }),
        counters: {
          sourcesCollected: totalDocuments,
          indexed,
          skipped,
          deleted,
          chunksEmbedded
        }
      }
    );
    const embeddings = await runAiEmbeddings(chunks.map((chunk) => chunk.chunkText));
    await input.onStage?.(
      'persist',
      `Persisting indexed chunks for ${document.sourceRef}`,
      stageContext(index + 1, {
        ticker: document.ticker ?? undefined,
        accessionNumber: document.accessionNumber ?? undefined,
        label: document.sourceRef
      })
    );
    const result = persistDocumentIndex(client, document, chunks, embeddings);

    if (result.skipped) {
      skipped += 1;
      continue;
    }

    indexed += 1;
    chunksEmbedded += result.chunkCount;
  }

  if (input.ticker && !input.accessionNumber && !input.journalEntryId) {
    for (const sourceKind of sourceKinds) {
      const scope = sourceKind === 'research_note' ? 'user' : 'global';
      const expectedRefs = new Set(
        materialized
          .filter((document) => document.sourceKind === sourceKind)
          .map((document) => document.sourceRef)
      );
      const existingRows = listDocumentsForScope(client, {
        scope,
        userId: scope === 'user' ? input.userId : null,
        sourceKind,
        ticker: input.ticker
      });

      for (const row of existingRows) {
        if (expectedRefs.has(row.source_ref)) {
          continue;
        }

        withTransaction(client, () => {
          deleteDocumentCascade(client, row.id);
        });
        deleted += 1;
      }
    }
  }

  return {
    sourcesCollected: materialized.length,
    indexed,
    skipped,
    deleted,
    chunksEmbedded
  };
}

function vectorSearch(
  client: Database,
  input: {
    embedding: number[];
    limit: number;
    sourceKind: SearchDocumentSourceKind;
    scope: SearchDocumentScope;
    userId?: string | null;
    ticker?: string | null;
  }
) {
  if (!__dbInternals.isVectorExtensionLoaded(client)) {
    const conditions = [
      'scope = ?',
      'source_kind = ?'
    ];
    const values: Array<string | null> = [input.scope, input.sourceKind];

    if (input.scope === 'user') {
      conditions.push('user_id = ?');
      values.push(input.userId ?? null);
    }

    if (input.ticker) {
      conditions.push('ticker = ?');
      values.push(input.ticker);
    }

    const rows = client
      .query(`
        SELECT chunk_id, embedding
        FROM search_chunk_vec
        WHERE ${conditions.join(' AND ')}
      `)
      .all(...values) as Array<{ chunk_id: number; embedding: string }>;

    const queryNorm = Math.hypot(...input.embedding) || 1;
    return rows
      .map((row) => {
        const candidate = JSON.parse(row.embedding) as number[];
        const dot = candidate.reduce((sum, value, index) => sum + (value * (input.embedding[index] ?? 0)), 0);
        const candidateNorm = Math.hypot(...candidate) || 1;
        const cosineDistance = 1 - (dot / (candidateNorm * queryNorm));

        return {
          chunk_id: row.chunk_id,
          distance: cosineDistance
        };
      })
      .sort((left, right) => left.distance - right.distance)
      .slice(0, Math.max(input.limit, 4));
  }

  const conditions = [
    'embedding MATCH ?',
    'k = ?',
    'scope = ?',
    'source_kind = ?'
  ];
  const values: Array<string | number | null> = [
    JSON.stringify(input.embedding),
    Math.max(input.limit, 4),
    input.scope,
    input.sourceKind
  ];

  if (input.scope === 'user') {
    conditions.push('user_id = ?');
    values.push(input.userId ?? null);
  }

  if (input.ticker) {
    conditions.push('ticker = ?');
    values.push(input.ticker);
  }

  const rows = client
    .query(`
      SELECT chunk_id, distance
      FROM search_chunk_vec
      WHERE ${conditions.join(' AND ')}
      ORDER BY distance ASC
      LIMIT ?
    `)
    .all(...values, Math.max(input.limit, 4)) as Array<{ chunk_id: number; distance: number }>;

  return rows;
}

function lexicalSearch(
  client: Database,
  input: {
    ftsQuery: string;
    limit: number;
    sourceKind: SearchDocumentSourceKind;
    scope: SearchDocumentScope;
    userId?: string | null;
    ticker?: string | null;
  }
) {
  const conditions = [
    'search_chunk_fts MATCH ?',
    'source_kind = ?',
    'scope = ?'
  ];
  const values: Array<string | number | null> = [
    input.ftsQuery,
    input.sourceKind,
    input.scope
  ];

  if (input.scope === 'user') {
    conditions.push('user_id = ?');
    values.push(input.userId ?? null);
  }

  if (input.ticker) {
    conditions.push('ticker = ?');
    values.push(input.ticker);
  }

  const rows = client
    .query(`
      SELECT
        chunk_id,
        bm25(search_chunk_fts) AS bm25,
        snippet(search_chunk_fts, 0, '[', ']', ' ... ', 18) AS snippet
      FROM search_chunk_fts
      WHERE ${conditions.join(' AND ')}
      ORDER BY bm25(search_chunk_fts)
      LIMIT ?
    `)
    .all(...values, Math.max(input.limit, 4)) as Array<{
      chunk_id: number;
      bm25: number;
      snippet: string | null;
    }>;

  return rows;
}

function hydrateResults(
  client: Database,
  query: string,
  matches: Map<number, SearchMatch>,
  limit: number
) {
  const chunkIds = [...matches.keys()];
  if (chunkIds.length === 0) {
    return [] satisfies SearchResult[];
  }

  const placeholders = createPlaceholders(chunkIds.length);
  const rows = client
    .query(`
      SELECT
        c.id AS chunk_id,
        c.document_id,
        c.chunk_text,
        c.heading_path,
        c.citation_label,
        d.source_kind,
        d.source_ref,
        d.title,
        d.ticker,
        d.accession_number,
        d.metadata
      FROM search_chunk c
      INNER JOIN search_document d ON d.id = c.document_id
      WHERE c.id IN (${placeholders})
    `)
    .all(...chunkIds) as SearchChunkJoinRow[];

  const dedupePerDocument = new Map<number, number>();
  const enriched = rows
    .map((row) => {
      const match = matches.get(row.chunk_id);
      if (!match) {
        return null;
      }

      return {
        chunkId: row.chunk_id,
        documentId: row.document_id,
        source: mapSourceKindToSearchSource(row.source_kind),
        sourceKind: row.source_kind,
        sourceRef: row.source_ref,
        title: row.title,
        ticker: row.ticker,
        accessionNumber: row.accession_number,
        filingDate: typeof row.metadata?.filingDate === 'string' ? row.metadata.filingDate : null,
        citationLabel: row.citation_label,
        headingPath: row.heading_path,
        chunkText: row.chunk_text,
        snippet: match.snippet ?? manualSnippet(row.chunk_text, query),
        score: scoreSearchMatch(match),
        vectorRank: match.vectorRank,
        lexicalRank: match.lexicalRank,
        href: buildSearchHref(row)
      } satisfies SearchResult;
    })
    .filter((row): row is SearchResult => Boolean(row))
    .sort((left, right) => right.score - left.score);

  const results: SearchResult[] = [];

  for (const row of enriched) {
    const count = dedupePerDocument.get(row.documentId) ?? 0;
    if (count >= MAX_RESULTS_PER_DOCUMENT) {
      continue;
    }

    dedupePerDocument.set(row.documentId, count + 1);
    results.push(row);

    if (results.length >= limit) {
      break;
    }
  }

  return results;
}

export async function searchKnowledgeBase(input: SearchInput) {
  const normalizedQuery = input.query.trim();
  if (normalizedQuery.length < 2) {
    return [] satisfies SearchResult[];
  }

  const limit = clampLimit(input.limit);
  const normalizedTicker = normalizeTicker(input.ticker);
  const includedSources = normalizeSearchSources(input.sources);
  const client = getSqliteClient();
  const [queryEmbedding] = await runAiEmbeddings([normalizedQuery]);
  const ftsQuery = toFtsQuery(normalizedQuery);
  const matches = new Map<number, SearchMatch>();

  for (const source of includedSources) {
    const sourceKind = SOURCE_KIND_BY_SEARCH_SOURCE[source];
    const scope = sourceKind === 'research_note' ? 'user' : 'global';
    const vectorRows = vectorSearch(client, {
      embedding: queryEmbedding,
      limit: limit * 3,
      sourceKind,
      scope,
      userId: scope === 'user' ? input.userId : null,
      ticker: normalizedTicker
    });

    vectorRows.forEach((row, index) => {
      const existing = matches.get(row.chunk_id);
      matches.set(row.chunk_id, {
        chunkId: row.chunk_id,
        vectorRank: existing?.vectorRank ?? index + 1,
        lexicalRank: existing?.lexicalRank ?? null,
        snippet: existing?.snippet ?? null
      });
    });

    if (!ftsQuery) {
      continue;
    }

    const lexicalRows = lexicalSearch(client, {
      ftsQuery,
      limit: limit * 3,
      sourceKind,
      scope,
      userId: scope === 'user' ? input.userId : null,
      ticker: normalizedTicker
    });

    lexicalRows.forEach((row, index) => {
      const existing = matches.get(row.chunk_id);
      matches.set(row.chunk_id, {
        chunkId: row.chunk_id,
        vectorRank: existing?.vectorRank ?? null,
        lexicalRank: existing?.lexicalRank ?? index + 1,
        snippet: existing?.snippet ?? row.snippet ?? null
      });
    });
  }

  return hydrateResults(client, normalizedQuery, matches, limit);
}

function buildAnswerPrompt(query: string, evidence: SearchResult[]) {
  const evidenceText = evidence.map((result, index) => {
    const reference = index + 1;
    return [
      `[${reference}] ${result.citationLabel}`,
      `Source: ${result.title ?? result.sourceRef}`,
      `Ticker: ${result.ticker ?? 'n/a'}`,
      `Excerpt: ${result.chunkText}`
    ].join('\n');
  }).join('\n\n');

  return [
    'Answer the question using only the evidence below.',
    'Every factual claim must include at least one citation like [1] or [2].',
    'If the evidence is insufficient, respond with exactly INSUFFICIENT_EVIDENCE.',
    `Question: ${query}`,
    '',
    'Evidence:',
    evidenceText
  ].join('\n');
}

function finalizeAnswer(answer: string, evidence: SearchResult[]) {
  const trimmed = answer.trim();
  if (!trimmed || trimmed === 'INSUFFICIENT_EVIDENCE') {
    return {
      answer: 'Insufficient evidence to answer from the indexed sources.',
      citations: [] satisfies SearchCitation[]
    };
  }

  const matches = [...trimmed.matchAll(/\[(\d+)\]/g)];
  const seen = new Set<number>();
  const citations: SearchCitation[] = [];

  for (const match of matches) {
    const index = Number(match[1]);
    if (!Number.isInteger(index) || seen.has(index) || index < 1 || index > evidence.length) {
      continue;
    }

    seen.add(index);
    const result = evidence[index - 1]!;
    citations.push({
      index,
      label: result.citationLabel,
      chunkId: result.chunkId,
      href: result.href
    });
  }

  if (citations.length === 0) {
    return {
      answer: 'Insufficient evidence to answer from the indexed sources.',
      citations: [] satisfies SearchCitation[]
    };
  }

  return {
    answer: trimmed,
    citations
  };
}

export async function answerSearchQuery(input: SearchInput): Promise<SearchAnswerResponse> {
  const results = await searchKnowledgeBase({
    ...input,
    limit: clampLimit(input.limit)
  });

  if (results.length === 0) {
    return {
      answer: 'Insufficient evidence to answer from the indexed sources.',
      citations: [],
      results
    };
  }

  const evidence: SearchResult[] = [];
  let totalChars = 0;

  for (const result of results) {
    if (evidence.length >= MAX_CONTEXT_RESULTS) {
      break;
    }

    if (totalChars + result.chunkText.length > MAX_CONTEXT_CHARS && evidence.length > 0) {
      break;
    }

    evidence.push(result);
    totalChars += result.chunkText.length;
  }

  const response = await runAiAnalysis(
    buildAnswerPrompt(input.query, evidence),
    'Use neutral analyst prose. Do not use outside knowledge.',
    {
      workload: 'report'
    }
  );

  const finalized = finalizeAnswer(response.text, evidence);

  return {
    answer: finalized.answer,
    citations: finalized.citations,
    results
  };
}

export const __searchInternals = {
  buildCitationLabel,
  buildFilingBriefContent,
  chunkDocument,
  chunkText,
  deleteSourceRefs,
  finalizeAnswer,
  hashContent,
  hydrateResults,
  lexicalSearch,
  normalizeSearchSources,
  persistDocumentIndex,
  queryOneDocument,
  scoreSearchMatch,
  stripMarkdown,
  toFtsQuery,
  vectorSearch
};