From 568cf1d40db2a107e5bd41cbd83d50a6efa32389 Mon Sep 17 00:00:00 2001 From: DJP Date: Mon, 13 Apr 2026 10:36:30 -0400 Subject: [PATCH] Add per-brief Apify budget with platform splitting - Add apifyBudget field to ClientBrief (default $10) - Budget split: 70% discovery (evenly across platforms), 30% enrichment - Per-platform soft cap prevents one platform hogging the budget - Budget input field added to both frontend and dashboard forms - Saved briefs preserve budget setting - Fix Claude Vision 5MB limit: filter oversized thumbnails before batching - Fix Docker: ensure node user can write to volume-mounted dirs Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 6 +++--- agents/social-listening/apify.ts | 9 +++++++- agents/social-listening/dashboard/index.html | 9 ++++++++ agents/social-listening/pipeline-v2.ts | 4 ++-- .../social-listening/stages/stage1-brief.ts | 1 + .../stages/stage3-discovery-scrape.ts | 21 ++++++++++++++++++- .../social-listening/stages/stage8-report.ts | 11 +++++++++- agents/social-listening/types-v2.ts | 1 + frontend/index.html | 9 ++++++++ 9 files changed, 63 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85f6b3b..e2e22b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,10 @@ RUN npm ci --production COPY tsconfig.json ./ COPY agents/ ./agents/ -# Output directory -RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs && \ - chown -R node:node agents/social-listening/outputs agents/social-listening/briefs +# Output and briefs directories +RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs +# Run as node user (uid 1000) — host volume dirs must be writable by uid 1000 USER node EXPOSE 3456 diff --git a/agents/social-listening/apify.ts b/agents/social-listening/apify.ts index c2fd746..c0c5f6e 100644 --- a/agents/social-listening/apify.ts +++ b/agents/social-listening/apify.ts @@ -66,16 +66,23 @@ export function isTestMode(): boolean { return IS_TEST; } // ─── Budget tracking ─── let _runningApifyCost = 0; let _apifyCostLimit = APIFY_COST_LIMIT; +let _softCap: number | null = null; // per-platform soft cap export function resetApifyCost(limit?: number): void { _runningApifyCost = 0; - if (limit !== undefined) _apifyCostLimit = limit; + _softCap = null; + if (limit !== undefined && limit > 0) _apifyCostLimit = limit; } export function getApifyCost(): number { return _runningApifyCost; } export function getApifyCostLimit(): number { return _apifyCostLimit; } +/** Set a soft cap for the current platform/phase. Calls exceeding this are skipped. */ +export function setSoftCap(cap: number | null): void { _softCap = cap; } +export function getSoftCap(): number | null { return _softCap; } + function isBudgetExceeded(): boolean { + if (_softCap !== null && _runningApifyCost >= _softCap) return true; return _runningApifyCost >= _apifyCostLimit; } diff --git a/agents/social-listening/dashboard/index.html b/agents/social-listening/dashboard/index.html index 9175e71..e04fe74 100644 --- a/agents/social-listening/dashboard/index.html +++ b/agents/social-listening/dashboard/index.html @@ -129,6 +129,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
+

Budget

+
+
Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).
@@ -228,6 +231,7 @@ function buildBriefFromForm() { youtube: splitVal('inf-youtube'), }, dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined, + apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10, }; } @@ -245,6 +249,7 @@ function populateForm(brief) { if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', '); if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', '); } + if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget; } // ─── Save/load briefs to server ─── @@ -402,6 +407,9 @@ function startPipeline() { const now = new Date(); const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); + const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10; + apifyBudgetLimit = budgetVal; + const brief = { clientName: document.getElementById('clientName').value, category: document.getElementById('category').value, @@ -416,6 +424,7 @@ function startPipeline() { dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : { from: ago.toISOString(), to: now.toISOString() }, + apifyBudget: budgetVal, }; eventSource = new EventSource('/events'); diff --git a/agents/social-listening/pipeline-v2.ts b/agents/social-listening/pipeline-v2.ts index eb2cb62..1691f3b 100644 --- a/agents/social-listening/pipeline-v2.ts +++ b/agents/social-listening/pipeline-v2.ts @@ -49,8 +49,8 @@ export async function runPipeline( let runId = 0; let runningTotal = 0; - // Reset Apify budget tracker for this run - resetApifyCost(); + // Reset Apify budget tracker for this run (brief budget overrides env default) + resetApifyCost(rawBrief.apifyBudget); console.log(`[PIPELINE] Apify budget: $${getApifyCostLimit().toFixed(2)}`); try { diff --git a/agents/social-listening/stages/stage1-brief.ts b/agents/social-listening/stages/stage1-brief.ts index 8715fc5..a076d15 100644 --- a/agents/social-listening/stages/stage1-brief.ts +++ b/agents/social-listening/stages/stage1-brief.ts @@ -48,6 +48,7 @@ export function runStage1(raw: Partial): StageResult { platforms: raw.platforms!, influencers: raw.influencers!, dateRange: raw.dateRange!, + apifyBudget: raw.apifyBudget && raw.apifyBudget > 0 ? raw.apifyBudget : undefined, }; console.log(`[Stage 1] Brief validated — ${brief.clientName} / ${brief.category}`); diff --git a/agents/social-listening/stages/stage3-discovery-scrape.ts b/agents/social-listening/stages/stage3-discovery-scrape.ts index 3cf6aaf..c36e510 100644 --- a/agents/social-listening/stages/stage3-discovery-scrape.ts +++ b/agents/social-listening/stages/stage3-discovery-scrape.ts @@ -1,6 +1,6 @@ // ─── Stage 3: Discovery Scrape (First Apify Run) ─── import { ClientBrief, DiscoveryData, Video, Platform, StageResult, RawTikTokItem, RawInstagramItem, RawYouTubeItem } from '../types-v2.js'; -import { runActor, ACTORS, getLimits } from '../apify.js'; +import { runActor, ACTORS, getLimits, getApifyCost, getApifyCostLimit, setSoftCap } from '../apify.js'; // ─── Normalization ─── @@ -196,22 +196,41 @@ export async function runStage3(brief: ClientBrief): Promise = { tiktok: [], instagram: [], youtube: [] }; diff --git a/agents/social-listening/stages/stage8-report.ts b/agents/social-listening/stages/stage8-report.ts index ad3261a..3a611c6 100644 --- a/agents/social-listening/stages/stage8-report.ts +++ b/agents/social-listening/stages/stage8-report.ts @@ -24,8 +24,17 @@ async function analyseVisualLanguage( // Build lookup: url -> video info const videoLookup = new Map(enrichment.videos.map(v => [v.url, v])); + // Filter out oversized images (Claude Vision limit: 5MB per image) + const MAX_B64_SIZE = 5 * 1024 * 1024 * 0.95; // 95% of 5MB to account for encoding overhead + const validEntries = entries.filter(([_, b64]) => { + const dataStart = b64.indexOf(','); + const dataSize = dataStart > 0 ? (b64.length - dataStart - 1) * 0.75 : b64.length * 0.75; // base64 → bytes + return dataSize < MAX_B64_SIZE; + }); + console.log(`[Stage 8] ${validEntries.length} of ${entries.length} thumbnails under 5MB limit`); + // Take top 50, split into 5 batches of 10 - const top50 = entries.slice(0, 50); + const top50 = validEntries.slice(0, 50); const batchSize = 10; const batchResults: string[] = []; diff --git a/agents/social-listening/types-v2.ts b/agents/social-listening/types-v2.ts index 2f92f4f..e7d2086 100644 --- a/agents/social-listening/types-v2.ts +++ b/agents/social-listening/types-v2.ts @@ -15,6 +15,7 @@ export interface ClientBrief { from: string; to: string; }; + apifyBudget?: number; } export type Platform = 'tiktok' | 'instagram' | 'youtube'; diff --git a/frontend/index.html b/frontend/index.html index 0d02d78..ee9778d 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -114,6 +114,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
+

Budget

+
+
Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).
@@ -231,6 +234,7 @@ function buildBriefFromForm() { youtube: splitVal('inf-youtube'), }, dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined, + apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10, }; } @@ -248,6 +252,7 @@ function populateForm(brief) { if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', '); if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', '); } + if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget; } // ─── Save/load briefs to server ─── @@ -402,6 +407,9 @@ function startPipeline() { const now = new Date(); const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); + const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10; + apifyBudgetLimit = budgetVal; + const brief = { clientName: document.getElementById('clientName').value, category: document.getElementById('category').value, @@ -416,6 +424,7 @@ function startPipeline() { dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : { from: ago.toISOString(), to: now.toISOString() }, + apifyBudget: budgetVal, }; const sseUrl = (SSE_BASE || API) + '/events';