Oliver/social-reporting-tool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix Apify budget: run scrapers sequentially instead of parallel

Browse source 
Promise.all() launched all platform scrapers simultaneously, so multiple
expensive runs started before any costs were tracked. Budget check only
saw totals after each run finished, allowing $7+ overspend on a $5 limit.

Now Stage 3 and Stage 5 run each scraper sequentially so the budget
gate can cut off between calls.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
DJP 2026-04-07 11:53:05 -04:00
parent 247da45297
commit 57c4d3f0df
2 changed files with 13 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
agents/social-listening/stages/stage3-discovery-scrape.ts
View file

@ -188,20 +188,22 @@ export async function runStage3(brief: ClientBrief): Promise<StageResult<Discove
const start = Date.now();
console.log('[Stage 3] Starting discovery scrape...');
const platformScrapers: Promise<{ platform: Platform; videos: Video[] }>[] = [];
// Run platforms sequentially so Apify budget check works between calls
const results: { platform: Platform; videos: Video[] }[] = [];
if (brief.platforms.includes('tiktok')) {
platformScrapers.push(scrapeTikTok(brief).then(videos => ({ platform: 'tiktok' as Platform, videos })));
const videos = await scrapeTikTok(brief);
results.push({ platform: 'tiktok', videos });
}
if (brief.platforms.includes('instagram')) {
platformScrapers.push(scrapeInstagram(brief).then(videos => ({ platform: 'instagram' as Platform, videos })));
const videos = await scrapeInstagram(brief);
results.push({ platform: 'instagram', videos });
}
if (brief.platforms.includes('youtube')) {
platformScrapers.push(scrapeYouTube(brief).then(videos => ({ platform: 'youtube' as Platform, videos })));
const videos = await scrapeYouTube(brief);
results.push({ platform: 'youtube', videos });
}
const results = await Promise.all(platformScrapers);
let allVideos: Video[] = [];
const byPlatform: Record<Platform, Video[]> = { tiktok: [], instagram: [], youtube: [] };

12
agents/social-listening/stages/stage5-enrichment-scrape.ts
View file

@ -105,13 +105,11 @@ export async function runStage5(
const instagramUrls = selection.videos.filter(v => v.platform === 'instagram').map(v => v.url);
const youtubeUrls = selection.videos.filter(v => v.platform === 'youtube').map(v => v.url);
// Run all fetches in parallel
const [tiktokTranscripts, instagramTranscripts, youtubeTranscripts, tiktokComments] = await Promise.all([
fetchTikTokTranscripts(tiktokUrls),
fetchInstagramTranscripts(instagramUrls),
fetchYouTubeTranscripts(youtubeUrls),
fetchTikTokComments(tiktokUrls),
]);
// Run fetches sequentially so Apify budget check works between calls
const tiktokTranscripts = await fetchTikTokTranscripts(tiktokUrls);
const instagramTranscripts = await fetchInstagramTranscripts(instagramUrls);
const youtubeTranscripts = await fetchYouTubeTranscripts(youtubeUrls);
const tiktokComments = await fetchTikTokComments(tiktokUrls);
// Merge all transcript maps
const allTranscripts = new Map<string, string>();