From 57c4d3f0dfdcb9ad050e4e10373881d5a88eb232 Mon Sep 17 00:00:00 2001
From: DJP <DJP>
Date: Tue, 7 Apr 2026 11:53:05 -0400
Subject: [PATCH] Fix Apify budget: run scrapers sequentially instead of
 parallel

Promise.all() launched all platform scrapers simultaneously, so multiple
expensive runs started before any costs were tracked. Budget check only
saw totals after each run finished, allowing $7+ overspend on a $5 limit.

Now Stage 3 and Stage 5 run each scraper sequentially so the budget
gate can cut off between calls.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../stages/stage3-discovery-scrape.ts              | 14 ++++++++------
 .../stages/stage5-enrichment-scrape.ts             | 12 +++++-------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/agents/social-listening/stages/stage3-discovery-scrape.ts b/agents/social-listening/stages/stage3-discovery-scrape.ts
index be6dc56..c38db1e 100644
--- a/agents/social-listening/stages/stage3-discovery-scrape.ts
+++ b/agents/social-listening/stages/stage3-discovery-scrape.ts
@@ -188,20 +188,22 @@ export async function runStage3(brief: ClientBrief): Promise<StageResult<Discove
   const start = Date.now();
   console.log('[Stage 3] Starting discovery scrape...');
 
-  const platformScrapers: Promise<{ platform: Platform; videos: Video[] }>[] = [];
+  // Run platforms sequentially so Apify budget check works between calls
+  const results: { platform: Platform; videos: Video[] }[] = [];
 
   if (brief.platforms.includes('tiktok')) {
-    platformScrapers.push(scrapeTikTok(brief).then(videos => ({ platform: 'tiktok' as Platform, videos })));
+    const videos = await scrapeTikTok(brief);
+    results.push({ platform: 'tiktok', videos });
   }
   if (brief.platforms.includes('instagram')) {
-    platformScrapers.push(scrapeInstagram(brief).then(videos => ({ platform: 'instagram' as Platform, videos })));
+    const videos = await scrapeInstagram(brief);
+    results.push({ platform: 'instagram', videos });
   }
   if (brief.platforms.includes('youtube')) {
-    platformScrapers.push(scrapeYouTube(brief).then(videos => ({ platform: 'youtube' as Platform, videos })));
+    const videos = await scrapeYouTube(brief);
+    results.push({ platform: 'youtube', videos });
   }
 
-  const results = await Promise.all(platformScrapers);
-
   let allVideos: Video[] = [];
   const byPlatform: Record<Platform, Video[]> = { tiktok: [], instagram: [], youtube: [] };
 
diff --git a/agents/social-listening/stages/stage5-enrichment-scrape.ts b/agents/social-listening/stages/stage5-enrichment-scrape.ts
index a14af1f..2fdb91f 100644
--- a/agents/social-listening/stages/stage5-enrichment-scrape.ts
+++ b/agents/social-listening/stages/stage5-enrichment-scrape.ts
@@ -105,13 +105,11 @@ export async function runStage5(
   const instagramUrls = selection.videos.filter(v => v.platform === 'instagram').map(v => v.url);
   const youtubeUrls = selection.videos.filter(v => v.platform === 'youtube').map(v => v.url);
 
-  // Run all fetches in parallel
-  const [tiktokTranscripts, instagramTranscripts, youtubeTranscripts, tiktokComments] = await Promise.all([
-    fetchTikTokTranscripts(tiktokUrls),
-    fetchInstagramTranscripts(instagramUrls),
-    fetchYouTubeTranscripts(youtubeUrls),
-    fetchTikTokComments(tiktokUrls),
-  ]);
+  // Run fetches sequentially so Apify budget check works between calls
+  const tiktokTranscripts = await fetchTikTokTranscripts(tiktokUrls);
+  const instagramTranscripts = await fetchInstagramTranscripts(instagramUrls);
+  const youtubeTranscripts = await fetchYouTubeTranscripts(youtubeUrls);
+  const tiktokComments = await fetchTikTokComments(tiktokUrls);
 
   // Merge all transcript maps
   const allTranscripts = new Map<string, string>();