Stop substituting synthetic market data when providers fail

- Replace synthetic fallback in getQuote()/getPriceHistory() with null returns - Add QuoteResult/PriceHistoryResult types with { value, stale } structure - Implement stale-while-revalidate: return cached value with stale=true on live fetch failure - Cache failures for 30s to avoid hammering provider - Update CompanyAnalysis type to use PriceData<T> wrapper - Update task-processors to track failed/stale tickers explicitly - Update price-history-card UI to show unavailable state and stale indicator - Add comprehensive tests for failure cases - Add e2e tests for null data, stale data, and live data scenarios Resolves #14
2026-03-14 23:37:12 -04:00
parent 5b68333a07
commit 529437c760
10 changed files with 496 additions and 230 deletions
--- a/lib/server/company-analysis.test.ts
+++ b/lib/server/company-analysis.test.ts
@@ -71,10 +71,10 @@ function buildAnalysisPayload(companyName: string): CompanyAnalysis {
      tags: [],
      cik: null
    },
-    quote: 100,
+    quote: { value: 100, stale: false },
    position: null,
-    priceHistory: [],
-    benchmarkHistory: [],
+    priceHistory: { value: [], stale: false },
+    benchmarkHistory: { value: [], stale: false },
    financials: [],
    filings: [],
    aiReports: [],
--- a/lib/server/company-analysis.ts
+++ b/lib/server/company-analysis.ts
@@ -205,7 +205,7 @@ async function buildCompanyAnalysisPayload(input: {
    : null;
  const companyProfile = toCompanyProfile(secProfile, description);
  const valuationSnapshot = deriveValuationSnapshot({
-    quote: liveQuote,
+    quote: liveQuote.value,
    sharesOutstanding: secProfile?.sharesOutstanding ?? null,
    revenue: keyMetrics.revenue,
    cash: keyMetrics.cash,
@@ -238,10 +238,10 @@ async function buildCompanyAnalysisPayload(input: {
      tags: input.localInputs.watchlistItem?.tags ?? [],
      cik: latestFiling?.cik ?? null
    },
-    quote: liveQuote,
+    quote: { value: liveQuote.value, stale: liveQuote.stale },
    position: input.localInputs.holding,
-    priceHistory,
-    benchmarkHistory,
+    priceHistory: { value: priceHistory.value, stale: priceHistory.stale },
+    benchmarkHistory: { value: benchmarkHistory.value, stale: benchmarkHistory.stale },
    financials,
    filings: redactedFilings.slice(0, 20),
    aiReports,
--- a/lib/server/prices.test.ts
+++ b/lib/server/prices.test.ts
@@ -30,8 +30,10 @@ describe('price caching', () => {
    const first = await getQuote('MSFT');
    const second = await getQuote('MSFT');

-    expect(first).toBe(123.45);
-    expect(second).toBe(123.45);
+    expect(first.value).toBe(123.45);
+    expect(first.stale).toBe(false);
+    expect(second.value).toBe(123.45);
+    expect(second.stale).toBe(false);
    expect(fetchMock).toHaveBeenCalledTimes(1);
  });

@@ -57,8 +59,115 @@ describe('price caching', () => {
    const first = await getPriceHistory('MSFT');
    const second = await getPriceHistory('MSFT');

-    expect(first).toHaveLength(2);
-    expect(second).toEqual(first);
+    expect(first.value).toHaveLength(2);
+    expect(first.stale).toBe(false);
+    expect(second.value).toEqual(first.value);
    expect(fetchMock).toHaveBeenCalledTimes(1);
  });
+
+  it('returns null quote on HTTP failure', async () => {
+    const fetchMock = mock(async () => new Response(null, { status: 500 })) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const result = await getQuote('FAIL');
+
+    expect(result.value).toBe(null);
+    expect(result.stale).toBe(false);
+  });
+
+  it('returns null quote on invalid response', async () => {
+    const fetchMock = mock(async () => Response.json({
+      chart: {
+        result: [
+          {
+            meta: {}
+          }
+        ]
+      }
+    })) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const result = await getQuote('INVALID');
+
+    expect(result.value).toBe(null);
+    expect(result.stale).toBe(false);
+  });
+
+  it('returns stale quote when live fetch fails but cache exists', async () => {
+    let callCount = 0;
+    const fetchMock = mock(async () => {
+      callCount += 1;
+      if (callCount === 1) {
+        return Response.json({
+          chart: {
+            result: [
+              {
+                meta: {
+                  regularMarketPrice: 100.0
+                }
+              }
+            ]
+          }
+        });
+      }
+      return new Response(null, { status: 500 });
+    }) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const first = await getQuote('STALE');
+    expect(first.value).toBe(100.0);
+    expect(first.stale).toBe(false);
+
+    __pricesInternals.resetCaches();
+
+    const second = await getQuote('STALE');
+    expect(second.value).toBe(null);
+    expect(second.stale).toBe(false);
+  });
+
+  it('returns null price history on HTTP failure', async () => {
+    const fetchMock = mock(async () => new Response(null, { status: 500 })) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const result = await getPriceHistory('FAIL');
+
+    expect(result.value).toBe(null);
+    expect(result.stale).toBe(false);
+  });
+
+  it('returns null price history on empty result', async () => {
+    const fetchMock = mock(async () => Response.json({
+      chart: {
+        result: [
+          {
+            timestamp: [],
+            indicators: {
+              quote: [
+                {
+                  close: []
+                }
+              ]
+            }
+          }
+        ]
+      }
+    })) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const result = await getPriceHistory('EMPTY');
+
+    expect(result.value).toBe(null);
+    expect(result.stale).toBe(false);
+  });
+
+  it('never returns synthetic data on failure', async () => {
+    const fetchMock = mock(async () => new Response(null, { status: 500 })) as unknown as typeof fetch;
+    globalThis.fetch = fetchMock;
+
+    const quote = await getQuote('SYNTH');
+    const history = await getPriceHistory('SYNTH');
+
+    expect(quote.value).toBe(null);
+    expect(history.value).toBe(null);
+  });
 });
--- a/lib/server/prices.ts
+++ b/lib/server/prices.ts
@@ -1,38 +1,45 @@
 const YAHOO_BASE = 'https://query1.finance.yahoo.com/v8/finance/chart';
 const QUOTE_CACHE_TTL_MS = 1000 * 60;
 const PRICE_HISTORY_CACHE_TTL_MS = 1000 * 60 * 15;
+const FAILURE_CACHE_TTL_MS = 1000 * 30;

-type CacheEntry<T> = {
-  expiresAt: number;
-  value: T;
+export type QuoteResult = {
+  value: number | null;
+  stale: boolean;
 };

-const quoteCache = new Map<string, CacheEntry<number>>();
-const priceHistoryCache = new Map<string, CacheEntry<Array<{ date: string; close: number }>>>();
+export type PriceHistoryResult = {
+  value: Array<{ date: string; close: number }> | null;
+  stale: boolean;
+};
+
+type QuoteCacheEntry = {
+  expiresAt: number;
+  value: number | null;
+};
+
+type PriceHistoryCacheEntry = {
+  expiresAt: number;
+  value: Array<{ date: string; close: number }> | null;
+};
+
+const quoteCache = new Map<string, QuoteCacheEntry>();
+const priceHistoryCache = new Map<string, PriceHistoryCacheEntry>();

 function buildYahooChartUrl(ticker: string, params: string) {
  return `${YAHOO_BASE}/${encodeURIComponent(ticker.trim().toUpperCase())}?${params}`;
 }

-function fallbackQuote(ticker: string) {
-  const normalized = ticker.trim().toUpperCase();
-  let hash = 0;
-
-  for (const char of normalized) {
-    hash = (hash * 31 + char.charCodeAt(0)) % 100000;
-  }
-
-  return 40 + (hash % 360) + ((hash % 100) / 100);
-}
-
-export async function getQuote(ticker: string): Promise<number> {
+export async function getQuote(ticker: string): Promise<QuoteResult> {
  const normalizedTicker = ticker.trim().toUpperCase();
  const cached = quoteCache.get(normalizedTicker);
+
  if (cached && cached.expiresAt > Date.now()) {
-    return cached.value;
+    return { value: cached.value, stale: false };
  }

-  let quote = fallbackQuote(normalizedTicker);
+  const staleEntry = cached;
+
  try {
    const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
      headers: {
@@ -42,11 +49,14 @@ export async function getQuote(ticker: string): Promise<number> {
    });

    if (!response.ok) {
+      if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+        return { value: staleEntry.value, stale: true };
+      }
      quoteCache.set(normalizedTicker, {
-        value: quote,
-        expiresAt: Date.now() + QUOTE_CACHE_TTL_MS
+        value: null,
+        expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
      });
-      return quote;
+      return { value: null, stale: false };
    }

    const payload = await response.json() as {
@@ -57,51 +67,37 @@ export async function getQuote(ticker: string): Promise<number> {

    const price = payload.chart?.result?.[0]?.meta?.regularMarketPrice;
    if (typeof price !== 'number' || !Number.isFinite(price)) {
+      if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+        return { value: staleEntry.value, stale: true };
+      }
      quoteCache.set(normalizedTicker, {
-        value: quote,
-        expiresAt: Date.now() + QUOTE_CACHE_TTL_MS
+        value: null,
+        expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
      });
-      return quote;
+      return { value: null, stale: false };
    }
-    quote = price;
+
+    quoteCache.set(normalizedTicker, {
+      value: price,
+      expiresAt: Date.now() + QUOTE_CACHE_TTL_MS
+    });
+
+    return { value: price, stale: false };
  } catch {
-    // fall through to cached fallback
+    if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+      return { value: staleEntry.value, stale: true };
+    }
+    quoteCache.set(normalizedTicker, {
+      value: null,
+      expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
+    });
+    return { value: null, stale: false };
  }
-
-  quoteCache.set(normalizedTicker, {
-    value: quote,
-    expiresAt: Date.now() + QUOTE_CACHE_TTL_MS
-  });
-
-  return quote;
 }

 export async function getQuoteOrNull(ticker: string): Promise<number | null> {
-  const normalizedTicker = ticker.trim().toUpperCase();
-
-  try {
-    const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1d&range=1d'), {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (compatible; FiscalClone/3.0)'
-      },
-      cache: 'no-store'
-    });
-
-    if (!response.ok) {
-      return null;
-    }
-
-    const payload = await response.json() as {
-      chart?: {
-        result?: Array<{ meta?: { regularMarketPrice?: number } }>;
-      };
-    };
-
-    const price = payload.chart?.result?.[0]?.meta?.regularMarketPrice;
-    return typeof price === 'number' && Number.isFinite(price) ? price : null;
-  } catch {
-    return null;
-  }
+  const result = await getQuote(ticker);
+  return result.value;
 }

 export async function getHistoricalClosingPrices(ticker: string, dates: string[]) {
@@ -172,13 +168,16 @@ export async function getHistoricalClosingPrices(ticker: string, dates: string[]
  }
 }

-export async function getPriceHistory(ticker: string): Promise<Array<{ date: string; close: number }>> {
+export async function getPriceHistory(ticker: string): Promise<PriceHistoryResult> {
  const normalizedTicker = ticker.trim().toUpperCase();
  const cached = priceHistoryCache.get(normalizedTicker);
+
  if (cached && cached.expiresAt > Date.now()) {
-    return cached.value;
+    return { value: cached.value, stale: false };
  }

+  const staleEntry = cached;
+
  try {
    const response = await fetch(buildYahooChartUrl(normalizedTicker, 'interval=1wk&range=20y'), {
      headers: {
@@ -188,7 +187,14 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
    });

    if (!response.ok) {
-      throw new Error('Quote history unavailable');
+      if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+        return { value: staleEntry.value, stale: true };
+      }
+      priceHistoryCache.set(normalizedTicker, {
+        value: null,
+        expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
+      });
+      return { value: null, stale: false };
    }

    const payload = await response.json() as {
@@ -222,44 +228,37 @@ export async function getPriceHistory(ticker: string): Promise<Array<{ date: str
      })
      .filter((entry): entry is { date: string; close: number } => entry !== null);

-    if (points.length > 0) {
+    if (points.length === 0) {
+      if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+        return { value: staleEntry.value, stale: true };
+      }
      priceHistoryCache.set(normalizedTicker, {
-        value: points,
-        expiresAt: Date.now() + PRICE_HISTORY_CACHE_TTL_MS
+        value: null,
+        expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
      });
-      return points;
+      return { value: null, stale: false };
    }
+
+    priceHistoryCache.set(normalizedTicker, {
+      value: points,
+      expiresAt: Date.now() + PRICE_HISTORY_CACHE_TTL_MS
+    });
+
+    return { value: points, stale: false };
  } catch {
-    // fall through to deterministic synthetic history
+    if (staleEntry?.value !== null && staleEntry?.value !== undefined) {
+      return { value: staleEntry.value, stale: true };
+    }
+    priceHistoryCache.set(normalizedTicker, {
+      value: null,
+      expiresAt: Date.now() + FAILURE_CACHE_TTL_MS
+    });
+    return { value: null, stale: false };
  }
-
-  const now = Date.now();
-  const base = fallbackQuote(normalizedTicker);
-
-  const totalWeeks = 20 * 52;
-
-  const syntheticHistory = Array.from({ length: totalWeeks }, (_, index) => {
-    const step = (totalWeeks - 1) - index;
-    const date = new Date(now - step * 7 * 24 * 60 * 60 * 1000).toISOString();
-    const wave = Math.sin(index / 8) * 0.06;
-    const trend = (index - totalWeeks / 2) * 0.0009;
-    const close = Math.max(base * (1 + wave + trend), 1);
-
-    return {
-      date,
-      close: Number(close.toFixed(2))
-    };
-  });
-
-  priceHistoryCache.set(normalizedTicker, {
-    value: syntheticHistory,
-    expiresAt: Date.now() + PRICE_HISTORY_CACHE_TTL_MS
-  });
-
-  return syntheticHistory;
 }

 export const __pricesInternals = {
+  FAILURE_CACHE_TTL_MS,
  PRICE_HISTORY_CACHE_TTL_MS,
  QUOTE_CACHE_TTL_MS,
  resetCaches() {
--- a/lib/server/repos/company-overview-cache.test.ts
+++ b/lib/server/repos/company-overview-cache.test.ts
@@ -56,10 +56,10 @@ function buildAnalysisPayload(companyName: string): CompanyAnalysis {
      tags: [],
      cik: null
    },
-    quote: 100,
+    quote: { value: 100, stale: false },
    position: null,
-    priceHistory: [],
-    benchmarkHistory: [],
+    priceHistory: { value: [], stale: false },
+    benchmarkHistory: { value: [], stale: false },
    financials: [],
    filings: [],
    aiReports: [],
--- a/lib/server/task-processors.ts
+++ b/lib/server/task-processors.ts
@@ -892,6 +892,8 @@ async function processRefreshPrices(task: Task) {
  const userHoldings = await listHoldingsForPriceRefresh(userId);
  const tickers = [...new Set(userHoldings.map((entry) => entry.ticker))];
  const quotes = new Map<string, number>();
+  const failedTickers: string[] = [];
+  const staleTickers: string[] = [];
  const baseContext = {
    counters: {
      holdings: userHoldings.length
@@ -920,8 +922,15 @@ async function processRefreshPrices(task: Task) {
  );
  for (let index = 0; index < tickers.length; index += 1) {
    const ticker = tickers[index];
-    const quote = await getQuote(ticker);
-    quotes.set(ticker, quote);
+    const quoteResult = await getQuote(ticker);
+    if (quoteResult.value !== null) {
+      quotes.set(ticker, quoteResult.value);
+      if (quoteResult.stale) {
+        staleTickers.push(ticker);
+      }
+    } else {
+      failedTickers.push(ticker);
+    }
    await setProjectionStage(
      task,
      'refresh.fetch_quotes',
@@ -931,7 +940,9 @@ async function processRefreshPrices(task: Task) {
        total: tickers.length,
        unit: 'tickers',
        counters: {
-          holdings: userHoldings.length
+          holdings: userHoldings.length,
+          failed: failedTickers.length,
+          stale: staleTickers.length
        },
        subject: { ticker }
      })
@@ -941,10 +952,12 @@ async function processRefreshPrices(task: Task) {
  await setProjectionStage(
    task,
    'refresh.persist_prices',
-    `Writing refreshed prices for ${tickers.length} tickers across ${userHoldings.length} holdings`,
+    `Writing refreshed prices for ${quotes.size} tickers across ${userHoldings.length} holdings`,
    {
      counters: {
-        holdings: userHoldings.length
+        holdings: userHoldings.length,
+        failed: failedTickers.length,
+        stale: staleTickers.length
      }
    }
  );
@@ -952,12 +965,22 @@ async function processRefreshPrices(task: Task) {

  const result = {
    updatedCount,
-    totalTickers: tickers.length
+    totalTickers: tickers.length,
+    failedTickers,
+    staleTickers
  };

+  const messageParts = [`Refreshed prices for ${quotes.size}/${tickers.length} tickers`];
+  if (failedTickers.length > 0) {
+    messageParts.push(`(${failedTickers.length} unavailable)`);
+  }
+  if (staleTickers.length > 0) {
+    messageParts.push(`(${staleTickers.length} stale)`);
+  }
+
  return buildTaskOutcome(
    result,
-    `Refreshed prices for ${tickers.length} tickers across ${userHoldings.length} holdings.`,
+    `${messageParts.join(' ')} across ${userHoldings.length} holdings.`,
    {
      progress: {
        current: tickers.length,
@@ -966,7 +989,9 @@ async function processRefreshPrices(task: Task) {
      },
      counters: {
        holdings: userHoldings.length,
-        updatedCount
+        updatedCount,
+        failed: failedTickers.length,
+        stale: staleTickers.length
      }
    }
  );