diff --git a/Dockerfile b/Dockerfile
index 85f6b3b..e2e22b3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,10 +8,10 @@ RUN npm ci --production
COPY tsconfig.json ./
COPY agents/ ./agents/
-# Output directory
-RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs && \
- chown -R node:node agents/social-listening/outputs agents/social-listening/briefs
+# Output and briefs directories
+RUN mkdir -p agents/social-listening/outputs agents/social-listening/briefs
+# Run as node user (uid 1000) — host volume dirs must be writable by uid 1000
USER node
EXPOSE 3456
diff --git a/agents/social-listening/apify.ts b/agents/social-listening/apify.ts
index c2fd746..c0c5f6e 100644
--- a/agents/social-listening/apify.ts
+++ b/agents/social-listening/apify.ts
@@ -66,16 +66,23 @@ export function isTestMode(): boolean { return IS_TEST; }
// ─── Budget tracking ───
let _runningApifyCost = 0;
let _apifyCostLimit = APIFY_COST_LIMIT;
+let _softCap: number | null = null; // per-platform soft cap
export function resetApifyCost(limit?: number): void {
_runningApifyCost = 0;
- if (limit !== undefined) _apifyCostLimit = limit;
+ _softCap = null;
+ if (limit !== undefined && limit > 0) _apifyCostLimit = limit;
}
export function getApifyCost(): number { return _runningApifyCost; }
export function getApifyCostLimit(): number { return _apifyCostLimit; }
+/** Set a soft cap for the current platform/phase. Calls exceeding this are skipped. */
+export function setSoftCap(cap: number | null): void { _softCap = cap; }
+export function getSoftCap(): number | null { return _softCap; }
+
function isBudgetExceeded(): boolean {
+ if (_softCap !== null && _runningApifyCost >= _softCap) return true;
return _runningApifyCost >= _apifyCostLimit;
}
diff --git a/agents/social-listening/dashboard/index.html b/agents/social-listening/dashboard/index.html
index 9175e71..e04fe74 100644
--- a/agents/social-listening/dashboard/index.html
+++ b/agents/social-listening/dashboard/index.html
@@ -129,6 +129,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
+Budget
+
+Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).
@@ -228,6 +231,7 @@ function buildBriefFromForm() {
youtube: splitVal('inf-youtube'),
},
dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined,
+ apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10,
};
}
@@ -245,6 +249,7 @@ function populateForm(brief) {
if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', ');
if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', ');
}
+ if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget;
}
// ─── Save/load briefs to server ───
@@ -402,6 +407,9 @@ function startPipeline() {
const now = new Date();
const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+ const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10;
+ apifyBudgetLimit = budgetVal;
+
const brief = {
clientName: document.getElementById('clientName').value,
category: document.getElementById('category').value,
@@ -416,6 +424,7 @@ function startPipeline() {
dateRange: (loadedBrief && loadedBrief.dateRange)
? loadedBrief.dateRange
: { from: ago.toISOString(), to: now.toISOString() },
+ apifyBudget: budgetVal,
};
eventSource = new EventSource('/events');
diff --git a/agents/social-listening/pipeline-v2.ts b/agents/social-listening/pipeline-v2.ts
index eb2cb62..1691f3b 100644
--- a/agents/social-listening/pipeline-v2.ts
+++ b/agents/social-listening/pipeline-v2.ts
@@ -49,8 +49,8 @@ export async function runPipeline(
let runId = 0;
let runningTotal = 0;
- // Reset Apify budget tracker for this run
- resetApifyCost();
+ // Reset Apify budget tracker for this run (brief budget overrides env default)
+ resetApifyCost(rawBrief.apifyBudget);
console.log(`[PIPELINE] Apify budget: $${getApifyCostLimit().toFixed(2)}`);
try {
diff --git a/agents/social-listening/stages/stage1-brief.ts b/agents/social-listening/stages/stage1-brief.ts
index 8715fc5..a076d15 100644
--- a/agents/social-listening/stages/stage1-brief.ts
+++ b/agents/social-listening/stages/stage1-brief.ts
@@ -48,6 +48,7 @@ export function runStage1(raw: Partial): StageResult {
platforms: raw.platforms!,
influencers: raw.influencers!,
dateRange: raw.dateRange!,
+ apifyBudget: raw.apifyBudget && raw.apifyBudget > 0 ? raw.apifyBudget : undefined,
};
console.log(`[Stage 1] Brief validated — ${brief.clientName} / ${brief.category}`);
diff --git a/agents/social-listening/stages/stage3-discovery-scrape.ts b/agents/social-listening/stages/stage3-discovery-scrape.ts
index 3cf6aaf..c36e510 100644
--- a/agents/social-listening/stages/stage3-discovery-scrape.ts
+++ b/agents/social-listening/stages/stage3-discovery-scrape.ts
@@ -1,6 +1,6 @@
// ─── Stage 3: Discovery Scrape (First Apify Run) ───
import { ClientBrief, DiscoveryData, Video, Platform, StageResult, RawTikTokItem, RawInstagramItem, RawYouTubeItem } from '../types-v2.js';
-import { runActor, ACTORS, getLimits } from '../apify.js';
+import { runActor, ACTORS, getLimits, getApifyCost, getApifyCostLimit, setSoftCap } from '../apify.js';
// ─── Normalization ───
@@ -196,22 +196,41 @@ export async function runStage3(brief: ClientBrief): Promise = { tiktok: [], instagram: [], youtube: [] };
diff --git a/agents/social-listening/stages/stage8-report.ts b/agents/social-listening/stages/stage8-report.ts
index ad3261a..3a611c6 100644
--- a/agents/social-listening/stages/stage8-report.ts
+++ b/agents/social-listening/stages/stage8-report.ts
@@ -24,8 +24,17 @@ async function analyseVisualLanguage(
// Build lookup: url -> video info
const videoLookup = new Map(enrichment.videos.map(v => [v.url, v]));
+ // Filter out oversized images (Claude Vision limit: 5MB per image)
+ const MAX_B64_SIZE = 5 * 1024 * 1024 * 0.95; // 95% of 5MB to account for encoding overhead
+ const validEntries = entries.filter(([_, b64]) => {
+ const dataStart = b64.indexOf(',');
+ const dataSize = dataStart > 0 ? (b64.length - dataStart - 1) * 0.75 : b64.length * 0.75; // base64 → bytes
+ return dataSize < MAX_B64_SIZE;
+ });
+ console.log(`[Stage 8] ${validEntries.length} of ${entries.length} thumbnails under 5MB limit`);
+
// Take top 50, split into 5 batches of 10
- const top50 = entries.slice(0, 50);
+ const top50 = validEntries.slice(0, 50);
const batchSize = 10;
const batchResults: string[] = [];
diff --git a/agents/social-listening/types-v2.ts b/agents/social-listening/types-v2.ts
index 2f92f4f..e7d2086 100644
--- a/agents/social-listening/types-v2.ts
+++ b/agents/social-listening/types-v2.ts
@@ -15,6 +15,7 @@ export interface ClientBrief {
from: string;
to: string;
};
+ apifyBudget?: number;
}
export type Platform = 'tiktok' | 'instagram' | 'youtube';
diff --git a/frontend/index.html b/frontend/index.html
index 0d02d78..ee9778d 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -114,6 +114,9 @@ button.run:disabled { background: #333; color: #666; cursor: not-allowed; }
+Budget
+
+Split evenly across platforms. 70% discovery, 30% enrichment (transcripts + comments).
@@ -231,6 +234,7 @@ function buildBriefFromForm() {
youtube: splitVal('inf-youtube'),
},
dateRange: (loadedBrief && loadedBrief.dateRange) ? loadedBrief.dateRange : undefined,
+ apifyBudget: parseFloat(document.getElementById('apifyBudget').value) || 10,
};
}
@@ -248,6 +252,7 @@ function populateForm(brief) {
if (brief.influencers.instagram) document.getElementById('inf-instagram').value = brief.influencers.instagram.join(', ');
if (brief.influencers.youtube) document.getElementById('inf-youtube').value = brief.influencers.youtube.join(', ');
}
+ if (brief.apifyBudget) document.getElementById('apifyBudget').value = brief.apifyBudget;
}
// ─── Save/load briefs to server ───
@@ -402,6 +407,9 @@ function startPipeline() {
const now = new Date();
const ago = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+ const budgetVal = parseFloat(document.getElementById('apifyBudget').value) || 10;
+ apifyBudgetLimit = budgetVal;
+
const brief = {
clientName: document.getElementById('clientName').value,
category: document.getElementById('category').value,
@@ -416,6 +424,7 @@ function startPipeline() {
dateRange: (loadedBrief && loadedBrief.dateRange)
? loadedBrief.dateRange
: { from: ago.toISOString(), to: now.toISOString() },
+ apifyBudget: budgetVal,
};
const sseUrl = (SSE_BASE || API) + '/events';