Use Yahoo Finance company descriptions

lib/server/yahoo-company-profile.ts

@@ -0,0 +1,203 @@
+type FetchImpl = typeof fetch;
+
+type CacheEntry<T> = {
+  expiresAt: number;
+  value: T;
+};
+
+type YahooQuoteSummaryPayload = {
+  quoteSummary?: {
+    result?: Array<{
+      assetProfile?: {
+        longBusinessSummary?: string;
+      };
+    }>;
+  };
+};
+
+const YAHOO_COOKIE_URL = 'https://fc.yahoo.com';
+const YAHOO_CRUMB_URL = 'https://query1.finance.yahoo.com/v1/test/getcrumb';
+const YAHOO_QUOTE_SUMMARY_BASE = 'https://query1.finance.yahoo.com/v10/finance/quoteSummary';
+
+const YAHOO_SESSION_TTL_MS = 1000 * 60 * 15;
+const DESCRIPTION_CACHE_TTL_MS = 1000 * 60 * 60 * 6;
+const DESCRIPTION_MAX_CHARS = 1_600;
+
+let yahooSessionCache: CacheEntry<{ cookie: string; crumb: string }> | null = null;
+const descriptionCache = new Map<string, CacheEntry<string | null>>();
+
+function yahooUserAgent() {
+  return 'Mozilla/5.0 (compatible; FiscalClone/3.0)';
+}
+
+function normalizeWhitespace(value: string) {
+  return value
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function clipAtSentenceBoundary(value: string, maxChars = DESCRIPTION_MAX_CHARS) {
+  if (value.length <= maxChars) {
+    return value;
+  }
+
+  const slice = value.slice(0, maxChars);
+  const sentenceBoundary = Math.max(
+    slice.lastIndexOf('. '),
+    slice.lastIndexOf('! '),
+    slice.lastIndexOf('? ')
+  );
+
+  if (sentenceBoundary > maxChars * 0.6) {
+    return slice.slice(0, sentenceBoundary + 1).trim();
+  }
+
+  const wordBoundary = slice.lastIndexOf(' ');
+  return (wordBoundary > maxChars * 0.7 ? slice.slice(0, wordBoundary) : slice).trim();
+}
+
+function normalizeDescription(value: unknown) {
+  if (typeof value !== 'string') {
+    return null;
+  }
+
+  const normalized = clipAtSentenceBoundary(normalizeWhitespace(value));
+  return normalized.length > 0 ? normalized : null;
+}
+
+function readSetCookieHeader(headers: Headers) {
+  const maybeHeaders = headers as Headers & { getSetCookie?: () => string[] };
+
+  if (typeof maybeHeaders.getSetCookie === 'function') {
+    const values = maybeHeaders.getSetCookie().filter((value) => value.trim().length > 0);
+    if (values.length > 0) {
+      return values;
+    }
+  }
+
+  const single = headers.get('set-cookie');
+  return single ? [single] : [];
+}
+
+function pickYahooSessionCookie(headers: Headers) {
+  const cookies = readSetCookieHeader(headers);
+  const match = cookies
+    .map((value) => /^([^=;,\s]+)=([^;]+)/.exec(value))
+    .find((entry) => entry && (entry[1] === 'A3' || entry[1] === 'A1'));
+
+  return match ? `${match[1]}=${match[2]}` : null;
+}
+
+async function getYahooSession(fetchImpl: FetchImpl = fetch) {
+  if (yahooSessionCache && yahooSessionCache.expiresAt > Date.now()) {
+    return yahooSessionCache.value;
+  }
+
+  const cookieResponse = await fetchImpl(YAHOO_COOKIE_URL, {
+    headers: {
+      'User-Agent': yahooUserAgent(),
+      Accept: '*/*'
+    },
+    cache: 'no-store'
+  });
+
+  const cookie = pickYahooSessionCookie(cookieResponse.headers);
+  if (!cookie) {
+    throw new Error(
+      cookieResponse.ok
+        ? 'Yahoo session cookie unavailable'
+        : `Yahoo cookie request failed (${cookieResponse.status})`
+    );
+  }
+
+  const crumbResponse = await fetchImpl(YAHOO_CRUMB_URL, {
+    headers: {
+      'User-Agent': yahooUserAgent(),
+      Accept: 'text/plain',
+      Cookie: cookie
+    },
+    cache: 'no-store'
+  });
+
+  if (!crumbResponse.ok) {
+    throw new Error(`Yahoo crumb request failed (${crumbResponse.status})`);
+  }
+
+  const crumb = (await crumbResponse.text()).trim();
+  if (!crumb || crumb.startsWith('{')) {
+    throw new Error('Yahoo crumb unavailable');
+  }
+
+  const session = { cookie, crumb };
+  yahooSessionCache = {
+    value: session,
+    expiresAt: Date.now() + YAHOO_SESSION_TTL_MS
+  };
+
+  return session;
+}
+
+export async function getYahooCompanyDescription(
+  ticker: string,
+  options?: { fetchImpl?: FetchImpl }
+) {
+  const normalizedTicker = ticker.trim().toUpperCase();
+  if (!normalizedTicker) {
+    return null;
+  }
+
+  const cached = descriptionCache.get(normalizedTicker);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.value;
+  }
+
+  try {
+    const session = await getYahooSession(options?.fetchImpl);
+    const url = new URL(`${YAHOO_QUOTE_SUMMARY_BASE}/${normalizedTicker}`);
+    url.searchParams.set('modules', 'assetProfile');
+    url.searchParams.set('crumb', session.crumb);
+
+    const response = await (options?.fetchImpl ?? fetch)(url, {
+      headers: {
+        'User-Agent': yahooUserAgent(),
+        Accept: 'application/json',
+        Cookie: session.cookie
+      },
+      cache: 'no-store'
+    });
+
+    if (!response.ok) {
+      throw new Error(`Yahoo profile request failed (${response.status})`);
+    }
+
+    const payload = await response.json() as YahooQuoteSummaryPayload;
+    const description = normalizeDescription(
+      payload.quoteSummary?.result?.[0]?.assetProfile?.longBusinessSummary
+    );
+
+    descriptionCache.set(normalizedTicker, {
+      value: description,
+      expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS
+    });
+
+    return description;
+  } catch {
+    descriptionCache.set(normalizedTicker, {
+      value: null,
+      expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS
+    });
+    return null;
+  }
+}
+
+export const __yahooCompanyProfileInternals = {
+  clipAtSentenceBoundary,
+  getYahooSession,
+  normalizeDescription,
+  pickYahooSessionCookie,
+  resetCaches() {
+    yahooSessionCache = null;
+    descriptionCache.clear();
+  }
+};