type FetchImpl = typeof fetch; type CacheEntry = { expiresAt: number; value: T; }; type YahooQuoteSummaryPayload = { quoteSummary?: { result?: Array<{ assetProfile?: { longBusinessSummary?: string; }; }>; }; }; const YAHOO_COOKIE_URL = 'https://fc.yahoo.com'; const YAHOO_CRUMB_URL = 'https://query1.finance.yahoo.com/v1/test/getcrumb'; const YAHOO_QUOTE_SUMMARY_BASE = 'https://query1.finance.yahoo.com/v10/finance/quoteSummary'; const YAHOO_SESSION_TTL_MS = 1000 * 60 * 15; const DESCRIPTION_CACHE_TTL_MS = 1000 * 60 * 60 * 6; const DESCRIPTION_MAX_CHARS = 1_600; let yahooSessionCache: CacheEntry<{ cookie: string; crumb: string }> | null = null; const descriptionCache = new Map>(); function yahooUserAgent() { return 'Mozilla/5.0 (compatible; FiscalClone/3.0)'; } function normalizeWhitespace(value: string) { return value .replace(/[ \t]+/g, ' ') .replace(/\n{3,}/g, '\n\n') .trim(); } function clipAtSentenceBoundary(value: string, maxChars = DESCRIPTION_MAX_CHARS) { if (value.length <= maxChars) { return value; } const slice = value.slice(0, maxChars); const sentenceBoundary = Math.max( slice.lastIndexOf('. '), slice.lastIndexOf('! '), slice.lastIndexOf('? ') ); if (sentenceBoundary > maxChars * 0.6) { return slice.slice(0, sentenceBoundary + 1).trim(); } const wordBoundary = slice.lastIndexOf(' '); return (wordBoundary > maxChars * 0.7 ? slice.slice(0, wordBoundary) : slice).trim(); } function normalizeDescription(value: unknown) { if (typeof value !== 'string') { return null; } const normalized = clipAtSentenceBoundary(normalizeWhitespace(value)); return normalized.length > 0 ? normalized : null; } function readSetCookieHeader(headers: Headers) { const maybeHeaders = headers as Headers & { getSetCookie?: () => string[] }; if (typeof maybeHeaders.getSetCookie === 'function') { const values = maybeHeaders.getSetCookie().filter((value) => value.trim().length > 0); if (values.length > 0) { return values; } } const single = headers.get('set-cookie'); return single ? [single] : []; } function pickYahooSessionCookie(headers: Headers) { const cookies = readSetCookieHeader(headers); const match = cookies .map((value) => /^([^=;,\s]+)=([^;]+)/.exec(value)) .find((entry) => entry && (entry[1] === 'A3' || entry[1] === 'A1')); return match ? `${match[1]}=${match[2]}` : null; } async function getYahooSession(fetchImpl: FetchImpl = fetch) { if (yahooSessionCache && yahooSessionCache.expiresAt > Date.now()) { return yahooSessionCache.value; } const cookieResponse = await fetchImpl(YAHOO_COOKIE_URL, { headers: { 'User-Agent': yahooUserAgent(), Accept: '*/*' }, cache: 'no-store' }); const cookie = pickYahooSessionCookie(cookieResponse.headers); if (!cookie) { throw new Error( cookieResponse.ok ? 'Yahoo session cookie unavailable' : `Yahoo cookie request failed (${cookieResponse.status})` ); } const crumbResponse = await fetchImpl(YAHOO_CRUMB_URL, { headers: { 'User-Agent': yahooUserAgent(), Accept: 'text/plain', Cookie: cookie }, cache: 'no-store' }); if (!crumbResponse.ok) { throw new Error(`Yahoo crumb request failed (${crumbResponse.status})`); } const crumb = (await crumbResponse.text()).trim(); if (!crumb || crumb.startsWith('{')) { throw new Error('Yahoo crumb unavailable'); } const session = { cookie, crumb }; yahooSessionCache = { value: session, expiresAt: Date.now() + YAHOO_SESSION_TTL_MS }; return session; } export async function getYahooCompanyDescription( ticker: string, options?: { fetchImpl?: FetchImpl } ) { const normalizedTicker = ticker.trim().toUpperCase(); if (!normalizedTicker) { return null; } const cached = descriptionCache.get(normalizedTicker); if (cached && cached.expiresAt > Date.now()) { return cached.value; } try { const session = await getYahooSession(options?.fetchImpl); const url = new URL(`${YAHOO_QUOTE_SUMMARY_BASE}/${normalizedTicker}`); url.searchParams.set('modules', 'assetProfile'); url.searchParams.set('crumb', session.crumb); const response = await (options?.fetchImpl ?? fetch)(url, { headers: { 'User-Agent': yahooUserAgent(), Accept: 'application/json', Cookie: session.cookie }, cache: 'no-store' }); if (!response.ok) { throw new Error(`Yahoo profile request failed (${response.status})`); } const payload = await response.json() as YahooQuoteSummaryPayload; const description = normalizeDescription( payload.quoteSummary?.result?.[0]?.assetProfile?.longBusinessSummary ); descriptionCache.set(normalizedTicker, { value: description, expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS }); return description; } catch { descriptionCache.set(normalizedTicker, { value: null, expiresAt: Date.now() + DESCRIPTION_CACHE_TTL_MS }); return null; } } export const __yahooCompanyProfileInternals = { clipAtSentenceBoundary, getYahooSession, normalizeDescription, pickYahooSessionCookie, resetCaches() { yahooSessionCache = null; descriptionCache.clear(); } };