Neon-Desk/lib/server/yahoo-company-profile.ts

import { normalizeTicker } from '@/lib/server/utils';

type FetchImpl = typeof fetch;

type CacheEntry<T> = {
  expiresAt: number;
  value: T;
};

type YahooQuoteSummaryPayload = {
  quoteSummary?: {
    result?: Array<{
      assetProfile?: {
        longBusinessSummary?: string;
      };
    }>;
  };
};

const YAHOO_COOKIE_URL = 'https://fc.yahoo.com';
const YAHOO_CRUMB_URL = 'https://query1.finance.yahoo.com/v1/test/getcrumb';
const YAHOO_QUOTE_SUMMARY_BASE = 'https://query1.finance.yahoo.com/v10/finance/quoteSummary';

const YAHOO_SESSION_TTL_MS = 1000 * 60 * 15;
const DESCRIPTION_CACHE_TTL_MS = 1000 * 60 * 60 * 6;
const DESCRIPTION_MAX_CHARS = 1_600;

let yahooSessionCache: CacheEntry<{ cookie: string; crumb: string }> | null = null;
const descriptionCache = new Map<string, CacheEntry<string | null>>();

function yahooUserAgent() {
  return 'Mozilla/5.0 (compatible; FiscalClone/3.0)';
}

function normalizeWhitespace(value: string) {
  return value
    .replace(/[ \t]+/g, ' ')
    .replace(/\n{3,}/g, '\n\n')
    .trim();
}

function clipAtSentenceBoundary(value: string, maxChars = DESCRIPTION_MAX_CHARS) {
  if (value.length <= maxChars) {
    return value;
  }

  const slice = value.slice(0, maxChars);
  const sentenceBoundary = Math.max(
    slice.lastIndexOf('. '),
    slice.lastIndexOf('! '),
    slice.lastIndexOf('? ')
  );

  if (sentenceBoundary > maxChars * 0.6) {
    return slice.slice(0, sentenceBoundary + 1).trim();
  }

  const wordBoundary = slice.lastIndexOf(' ');
  return (wordBoundary > maxChars * 0.7 ? slice.slice(0, wordBoundary) : slice).trim();
}

function normalizeDescription(value: unknown) {
  if (typeof value !== 'string') {
    return null;
  }

  const normalized = clipAtSentenceBoundary(normalizeWhitespace(value));
  return normalized.length > 0 ? normalized : null;
}

function readSetCookieHeader(headers: Headers) {
  const maybeHeaders = headers as Headers & { getSetCookie?: () => string[] };

  if (typeof maybeHeaders.getSetCookie === 'function') {
    const values = maybeHeaders.getSetCookie().filter((value) => value.trim().length > 0);
    if (values.length > 0) {
      return values;
    }
  }

  const single = headers.get('set-cookie');
  return single ? [single] : [];
}

function pickYahooSessionCookie(headers: Headers) {
  const cookies = readSetCookieHeader(headers);
  const match = cookies
    .map((value) => /^([^=;,\s]+)=([^;]+)/.exec(value))
    .find((entry) => entry && (entry[1] === 'A3' || entry[1] === 'A1'));

  return match ? `${match[1]}=${match[2]}` : null;
}

async function getYahooSession(fetchImpl: FetchImpl = fetch) {
  if (yahooSessionCache && yahooSessionCache.expiresAt > Date.now()) {
    return yahooSessionCache.value;
  }

  const cookieResponse = await fetchImpl(YAHOO_COOKIE_URL, {
    headers: {
      'User-Agent': yahooUserAgent(),
      Accept: '*/*'
    },
    cache: 'no-store'
  });

  const cookie = pickYahooSessionCookie(cookieResponse.headers);
  if (!cookie) {
    throw new Error(
      cookieResponse.ok
        ? 'Yahoo session cookie unavailable'
        : `Yahoo cookie request failed (${cookieResponse.status})`
    );
  }

  const crumbResponse = await fetchImpl(YAHOO_CRUMB_URL, {
    headers: {
      'User-Agent': yahooUserAgent(),
      Accept: 'text/plain',
      Cookie: cookie
    },
    cache: 'no-store'
  });

  if (!crumbResponse.ok) {
    throw new Error(`Yahoo crumb request failed (${crumbResponse.status})`);
  }

  const crumb = (await crumbResponse.text()).trim();
  if (!crumb || crumb.startsWith('{')) {
    throw new Error('Yahoo crumb unavailable');
  }

  const session = { cookie, crumb };
  yahooSessionCache = {
    value: session,
    expiresAt: Date.now() + YAHOO_SESSION_TTL_MS
  };

  return session;
}

export async function getYahooCompanyDescription(
  ticker: string,
  options?: { fetchImpl?: FetchImpl }
) {
  const normalizedTicker = normalizeTicker(ticker);
  if (!normalizedTicker) {
    return null;
  }

  const cached = descriptionCache.get(normalizedTicker);
  if (cached && cached.expiresAt > Date.now()) {
    return cached.value;
  }

  try {
    const session = await getYahooSession(options?.fetchImpl);
    const url = new URL(`${YAHOO_QUOTE_SUMMARY_BASE}/${normalizedTicker}`);
    url.searchParams.set('modules', 'assetProfile');
    url.searchParams.set('crumb', session.crumb);

    const response = await (options?.fetchImpl ?? fetch)(url, {
      headers: {
        'User-Agent': yahooUserAgent(),
        Accept: 'application/json',
        Cookie: session.cookie
      },
      cache: 'no-store'
    });

    if (!response.ok) {
      throw new Error(`Yahoo profile request failed (${response.status})`);
    }

    const payload = await response.json() as YahooQuoteSummaryPayload;
    const description = normalizeDescription(
      payload.quoteSummary?.result?.[0]?.assetProfile?.longBusinessSummary
    );

    descriptionCache.set(normalizedTicker, {
      value: description,
      expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS
    });

    return description;
  } catch {
    descriptionCache.set(normalizedTicker, {
      value: null,
      expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS
    });
    return null;
  }
}

export const __yahooCompanyProfileInternals = {
  clipAtSentenceBoundary,
  getYahooSession,
  normalizeDescription,
  pickYahooSessionCookie,
  resetCaches() {
    yahooSessionCache = null;
    descriptionCache.clear();
  }
};