Add per-brief Apify budget with platform splitting

- Add apifyBudget field to ClientBrief (default $10)
- Budget split: 70% discovery (evenly across platforms), 30% enrichment
- Per-platform soft cap prevents one platform hogging the budget
- Budget input field added to both frontend and dashboard forms
- Saved briefs preserve budget setting
- Fix Claude Vision 5MB limit: filter oversized thumbnails before batching
- Fix Docker: ensure node user can write to volume-mounted dirs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Dockerfile

@@ -8,10 +8,10 @@ RUN npm ci --production
 COPY tsconfig.json ./
 COPY agents/ ./agents/
 
-# Output directory
-RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs && \
-    chown -R node:node agents/social-listening/outputs agents/social-listening/briefs
+# Output and briefs directories
+RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs
 
+# Run as node user (uid 1000) — host volume dirs must be writable by uid 1000
 USER node
 
 EXPOSE 3456

agents/social-listening/apify.ts

@@ -66,16 +66,23 @@ export function isTestMode(): boolean { return IS_TEST; }
 // ─── Budget tracking ───
 let _runningApifyCost = 0;
 let _apifyCostLimit = APIFY_COST_LIMIT;
+let _softCap: number | null = null; // per-platform soft cap
 
 export function resetApifyCost(limit?: number): void {
   _runningApifyCost = 0;
-  if (limit !== undefined) _apifyCostLimit = limit;
+  _softCap = null;
+  if (limit !== undefined && limit > 0) _apifyCostLimit = limit;
 }
 
 export function getApifyCost(): number { return _runningApifyCost; }
 export function getApifyCostLimit(): number { return _apifyCostLimit; }
 
+/** Set a soft cap for the current platform/phase. Calls exceeding this are skipped. */
+export function setSoftCap(cap: number | null): void { _softCap = cap; }
+export function getSoftCap(): number | null { return _softCap; }
+
 function isBudgetExceeded(): boolean {
+  if (_softCap !== null && _runningApifyCost >= _softCap) return true;
   return _runningApifyCost >= _apifyCostLimit;
 }
 

agents/social-listening/dashboard/index.html

@@ -129,6 +129,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
 <div class="field"><label>TikTok handles</label><input id="inf-tiktok" placeholder="@hm, @hmusa"></div>
 <div class="field"><label>Instagram handles</label><input id="inf-instagram" placeholder="hm, hmusa"></div>
 <div class="field"><label>YouTube handles</label><input id="inf-youtube" placeholder="@hm"></div>
+<h2 style="margin-top:24px">Budget</h2>
+<div class="field"><label>Apify Budget ($)</label><input id="apifyBudget" type="number" min="1" max="50" step="1" value="10" placeholder="10" style="max-width:120px"></div>
+<div style="font-size:11px;color:#666;margin-top:-12px;margin-bottom:8px">Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).</div>
 </div>
 
 <button class="run" id="runBtn" onclick="startPipeline()">Run Pipeline</button>
@@ -228,6 +231,7 @@ function buildBriefFromForm() {
       youtube: splitVal('inf-youtube'),
     },
     dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined,
+    apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10,
   };
 }
 
@@ -245,6 +249,7 @@ function populateForm(brief) {
     if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', ');
     if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', ');
   }
+  if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget;
 }
 
 // ─── Save/load briefs to server ───
@@ -402,6 +407,9 @@ function startPipeline() {
   const now = new Date();
   const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
 
+  const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10;
+  apifyBudgetLimit = budgetVal;
+
   const brief = {
     clientName: document.getElementById('clientName').value,
     category: document.getElementById('category').value,
@@ -416,6 +424,7 @@ function startPipeline() {
     dateRange: (loadedBrief && loadedBrief.dateRange)
       ? loadedBrief.dateRange
       : { from: ago.toISOString(), to: now.toISOString() },
+    apifyBudget: budgetVal,
   };
 
   eventSource = new EventSource('/events');

agents/social-listening/pipeline-v2.ts

@@ -49,8 +49,8 @@ export async function runPipeline(
   let runId = 0;
   let runningTotal = 0;
 
-  // Reset Apify budget tracker for this run
-  resetApifyCost();
+  // Reset Apify budget tracker for this run (brief budget overrides env default)
+  resetApifyCost(rawBrief.apifyBudget);
   console.log(`[PIPELINE] Apify budget: $${getApifyCostLimit().toFixed(2)}`);
 
   try {

agents/social-listening/stages/stage1-brief.ts

@@ -48,6 +48,7 @@ export function runStage1(raw: Partial<ClientBrief>): StageResult<ClientBrief> {
     platforms: raw.platforms!,
     influencers: raw.influencers!,
     dateRange: raw.dateRange!,
+    apifyBudget: raw.apifyBudget && raw.apifyBudget > 0 ? raw.apifyBudget : undefined,
   };
 
   console.log(`[Stage 1] Brief validated — ${brief.clientName} / ${brief.category}`);

agents/social-listening/stages/stage3-discovery-scrape.ts

@@ -1,6 +1,6 @@
 // ─── Stage 3: Discovery Scrape (First Apify Run) ───
 import { ClientBrief, DiscoveryData, Video, Platform, StageResult, RawTikTokItem, RawInstagramItem, RawYouTubeItem } from '../types-v2.js';
-import { runActor, ACTORS, getLimits } from '../apify.js';
+import { runActor, ACTORS, getLimits, getApifyCost, getApifyCostLimit, setSoftCap } from '../apify.js';
 
 // ─── Normalization ───
 
@@ -196,22 +196,41 @@ export async function runStage3(brief: ClientBrief): Promise<StageResult<Discove
   const start = Date.now();
   console.log('[Stage 3] Starting discovery scrape...');
 
+  // Budget splitting: reserve 30% for enrichment (stage 5), split rest across platforms
+  const totalBudget = getApifyCostLimit();
+  const discoveryBudget = totalBudget * 0.7;
+  const platformCount = brief.platforms.length;
+  const perPlatformBudget = discoveryBudget / platformCount;
+  console.log(`[Stage 3] Budget: $${totalBudget.toFixed(2)} total → $${discoveryBudget.toFixed(2)} discovery ($${perPlatformBudget.toFixed(2)}/platform), $${(totalBudget * 0.3).toFixed(2)} reserved for enrichment`);
+
   // Run platforms sequentially so Apify budget check works between calls
   const results: { platform: Platform; videos: Video[] }[] = [];
 
   if (brief.platforms.includes('tiktok')) {
+    const cap = getApifyCost() + perPlatformBudget;
+    setSoftCap(cap);
+    console.log(`[Stage 3] TikTok soft cap: $${cap.toFixed(2)}`);
     const videos = await scrapeTikTok(brief);
     results.push({ platform: 'tiktok', videos });
   }
   if (brief.platforms.includes('instagram')) {
+    const cap = getApifyCost() + perPlatformBudget;
+    setSoftCap(cap);
+    console.log(`[Stage 3] Instagram soft cap: $${cap.toFixed(2)}`);
     const videos = await scrapeInstagram(brief);
     results.push({ platform: 'instagram', videos });
   }
   if (brief.platforms.includes('youtube')) {
+    const cap = getApifyCost() + perPlatformBudget;
+    setSoftCap(cap);
+    console.log(`[Stage 3] YouTube soft cap: $${cap.toFixed(2)}`);
     const videos = await scrapeYouTube(brief);
     results.push({ platform: 'youtube', videos });
   }
 
+  // Remove soft cap for enrichment stage
+  setSoftCap(null);
+
   let allVideos: Video[] = [];
   const byPlatform: Record<Platform, Video[]> = { tiktok: [], instagram: [], youtube: [] };
 

agents/social-listening/stages/stage8-report.ts

@@ -24,8 +24,17 @@ async function analyseVisualLanguage(
   // Build lookup: url -> video info
   const videoLookup = new Map(enrichment.videos.map(v => [v.url, v]));
 
+  // Filter out oversized images (Claude Vision limit: 5MB per image)
+  const MAX_B64_SIZE = 5 * 1024 * 1024 * 0.95; // 95% of 5MB to account for encoding overhead
+  const validEntries = entries.filter(([_, b64]) => {
+    const dataStart = b64.indexOf(',');
+    const dataSize = dataStart > 0 ? (b64.length - dataStart - 1) * 0.75 : b64.length * 0.75; // base64 → bytes
+    return dataSize < MAX_B64_SIZE;
+  });
+  console.log(`[Stage 8] ${validEntries.length} of ${entries.length} thumbnails under 5MB limit`);
+
   // Take top 50, split into 5 batches of 10
-  const top50 = entries.slice(0, 50);
+  const top50 = validEntries.slice(0, 50);
   const batchSize = 10;
   const batchResults: string[] = [];
 

agents/social-listening/types-v2.ts

@@ -15,6 +15,7 @@ export interface ClientBrief {
     from: string;
     to: string;
   };
+  apifyBudget?: number;
 }
 
 export type Platform = 'tiktok' | 'instagram' | 'youtube';

frontend/index.html

@@ -114,6 +114,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
 <div class="field"><label>TikTok handles</label><input id="inf-tiktok" placeholder="@hm, @hmusa"></div>
 <div class="field"><label>Instagram handles</label><input id="inf-instagram" placeholder="hm, hmusa"></div>
 <div class="field"><label>YouTube handles</label><input id="inf-youtube" placeholder="@hm"></div>
+<h2 style="margin-top:24px">Budget</h2>
+<div class="field"><label>Apify Budget ($)</label><input id="apifyBudget" type="number" min="1" max="50" step="1" value="10" placeholder="10" style="max-width:120px"></div>
+<div style="font-size:11px;color:#666;margin-top:-12px;margin-bottom:8px">Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).</div>
 </div>
 
 <button class="run" id="runBtn" onclick="startPipeline()">Run Pipeline</button>
@@ -231,6 +234,7 @@ function buildBriefFromForm() {
       youtube: splitVal('inf-youtube'),
     },
     dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined,
+    apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10,
   };
 }
 
@@ -248,6 +252,7 @@ function populateForm(brief) {
     if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', ');
     if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', ');
   }
+  if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget;
 }
 
 // ─── Save/load briefs to server ───
@@ -402,6 +407,9 @@ function startPipeline() {
   const now = new Date();
   const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
 
+  const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10;
+  apifyBudgetLimit = budgetVal;
+
   const brief = {
     clientName: document.getElementById('clientName').value,
     category: document.getElementById('category').value,
@@ -416,6 +424,7 @@ function startPipeline() {
     dateRange: (loadedBrief && loadedBrief.dateRange)
       ? loadedBrief.dateRange
       : { from: ago.toISOString(), to: now.toISOString() },
+    apifyBudget: budgetVal,
   };
 
   const sseUrl = (SSE_BASE || API) + '/events';