From f2d6f56831fd51e850261ccc1613a49ecf3affe5 Mon Sep 17 00:00:00 2001 From: DJP Date: Wed, 8 Apr 2026 09:52:08 -0400 Subject: [PATCH] Report quality overhaul: 11 feedback items 1. Remove Desk Research (Stage 7 skipped, sources removed from report) 2. Fix comments scraping: increase cap to 2000, handle alt field names 3. Dynamic stats bar: hide zero-value stats instead of showing "0 Comments" 4. Prompt improvements: enforce timeliness, comment-based insights, creator spotlight algorithm (2-10 videos, exclude >50% dominance) 5. Date filtering: pass date params to Apify actors (oldestCreateTime, onlyPostsNewerThan, uploadDate) + log filter counts 6. Pullquotes: 3-4 generated editorial dividers between sections 7. Thumbnails: download top 50 coverUrl as base64, store on EnrichedVideo 8. Visual Language section: 5 batches of 10 through Claude Vision, synthesized into 5-6 visual codes with thumbnail cards 9. Sticky navigation bar with anchor links to all sections 10. New types: VisualCode, thumbnailUrl on Video, thumbnailBase64 on EnrichedVideo, pullquotes/visualCodes on ReportJSON Co-Authored-By: Claude Opus 4.6 --- agents/social-listening/apify.ts | 2 +- agents/social-listening/claude-cli.ts | 35 +++++ agents/social-listening/html-report.ts | 117 +++++++++++----- agents/social-listening/pipeline-v2.ts | 8 +- .../stages/stage3-discovery-scrape.ts | 11 +- .../stages/stage5-enrichment-scrape.ts | 52 ++++++- .../social-listening/stages/stage8-report.ts | 129 +++++++++++++++--- agents/social-listening/types-v2.ts | 18 ++- 8 files changed, 302 insertions(+), 70 deletions(-) diff --git a/agents/social-listening/apify.ts b/agents/social-listening/apify.ts index ad75e61..372b0e0 100644 --- a/agents/social-listening/apify.ts +++ b/agents/social-listening/apify.ts @@ -205,5 +205,5 @@ export async function runActor( export function getLimits() { return IS_TEST ? { resultsPerPage: 100, resultsLimit: 100, maxResults: 100, maxComments: 100, transcriptBatch: 10, profileLimit: 100 } - : { resultsPerPage: 200, resultsLimit: 100, maxResults: 100, maxComments: 1000, transcriptBatch: 25, profileLimit: 200 }; + : { resultsPerPage: 200, resultsLimit: 100, maxResults: 100, maxComments: 2000, transcriptBatch: 25, profileLimit: 200 }; } diff --git a/agents/social-listening/claude-cli.ts b/agents/social-listening/claude-cli.ts index a31fb9b..f2ed4d9 100644 --- a/agents/social-listening/claude-cli.ts +++ b/agents/social-listening/claude-cli.ts @@ -283,3 +283,38 @@ export async function callClaudeJSON(prompt: string, model?: string, options? } throw new Error('Unreachable'); } + +/** Call Claude with images (vision) — accepts base64 data URIs + a text prompt */ +export async function callClaudeVision( + imageBase64s: string[], + textPrompt: string, + model?: string, +): Promise { + const m = model || DEFAULT_MODEL; + const content: ApiContentBlock[] = []; + + for (const b64 of imageBase64s) { + // Parse data:image/jpeg;base64,... format + const commaIdx = b64.indexOf(','); + const meta = b64.slice(0, commaIdx); + const data = b64.slice(commaIdx + 1); + const mediaType = meta.match(/data:([^;]+)/)?.[1] || 'image/jpeg'; + content.push({ + type: 'image', + source: { type: 'base64', media_type: mediaType, data } as unknown as Record, + } as unknown as ApiContentBlock); + } + + content.push({ type: 'text', text: textPrompt }); + + const messages: ApiMessage[] = [{ role: 'user', content }]; + const response = await callApi(messages, m, { maxTokens: 4096 }); + const usage: ClaudeUsage = { + inputTokens: response.usage.input_tokens, + outputTokens: response.usage.output_tokens, + costUsd: calculateCost(m, response.usage.input_tokens, response.usage.output_tokens), + model: m, + }; + reportUsage(usage, 'vision_analysis'); + return { text: extractText(response), usage }; +} diff --git a/agents/social-listening/html-report.ts b/agents/social-listening/html-report.ts index 6cc8d75..bf38b30 100644 --- a/agents/social-listening/html-report.ts +++ b/agents/social-listening/html-report.ts @@ -1,5 +1,5 @@ // ─── HTML Report Generator ─── -import { ReportJSON, ClientBrief, Trend, TrendVideo, ContentOpportunity } from './types-v2.js'; +import { ReportJSON, ClientBrief, Trend, TrendVideo, ContentOpportunity, VisualCode } from './types-v2.js'; interface ReportStats { videosScraped: number; @@ -16,9 +16,14 @@ export function buildMarkdown(report: ReportJSON, brief: ClientBrief, stats: Rep lines.push(`# Social Listening Report — ${brief.clientName}`); lines.push(`**${brief.category}** — ${formatDateRange(brief.dateRange)}`); lines.push(''); - lines.push(`| Videos Scraped | Comments Analysed | Transcripts | Desk Sources |`); - lines.push(`|---|---|---|---|`); - lines.push(`| ${stats.videosScraped} | ${stats.commentsAnalysed} | ${stats.transcriptsDownloaded} | ${stats.deskSources} |`); + const mdStats = [ + { label: 'Videos Scraped', value: stats.videosScraped }, + { label: 'Comments Analysed', value: stats.commentsAnalysed }, + { label: 'Transcripts', value: stats.transcriptsDownloaded }, + ].filter(s => s.value > 0); + lines.push(`| ${mdStats.map(s => s.label).join(' | ')} |`); + lines.push(`| ${mdStats.map(() => '---').join(' | ')} |`); + lines.push(`| ${mdStats.map(s => s.value).join(' | ')} |`); lines.push(''); lines.push('## Executive Summary'); @@ -72,11 +77,6 @@ export function buildMarkdown(report: ReportJSON, brief: ClientBrief, stats: Rep lines.push(''); } - lines.push('## Desk Research Sources'); - for (const s of report.deskSources) { - lines.push(`- [${s.title}](${s.url}) — ${s.summary}`); - } - return lines.join('\n'); } @@ -193,9 +193,37 @@ function deriveFormatCards(trends: Trend[]): { icon: string; name: string; desc: return formats.slice(0, 6); } -export function generateHtmlReport(report: ReportJSON, brief: ClientBrief, stats: ReportStats): string { +function renderVisualLanguageSection(visualCodes: VisualCode[], thumbnailMap?: Record): string { + if (!visualCodes?.length) return ''; + + const cards = visualCodes.map(vc => { + // Try to find a thumbnail for the example video + const thumb = thumbnailMap && vc.exampleVideoUrl ? thumbnailMap[vc.exampleVideoUrl] : null; + const thumbHtml = thumb + ? `
${esc(vc.name)}
` + : ''; + + return `
+
${esc(vc.name)}
+ ${thumbHtml} +
+

${esc(vc.description)}

+
${esc(vc.frequency)}
+ ${vc.exampleAuthor ? `
${esc(vc.exampleAuthor)} — ${(vc.examplePlays || 0).toLocaleString()} plays
` : ''} +
+
`; + }).join('\n'); + + return ` + +
Visual Language
+
${cards}
`; +} + +export function generateHtmlReport(report: ReportJSON, brief: ClientBrief, stats: ReportStats, thumbnailMap?: Record): string { const hasTikTok = report.trends.some(t => t.topVideoUrl?.includes('tiktok.com') || t.supportingVideos?.some(sv => sv.platform === 'tiktok')); const hasInstagram = report.trends.some(t => t.topVideoUrl?.includes('instagram.com') || t.supportingVideos?.some(sv => sv.platform === 'instagram')); + const visualLanguageHtml = renderVisualLanguageSection(report.visualCodes || [], thumbnailMap); const trendsHtml = report.trends.map((t, i) => { const variationsHtml = t.variations.map(v => `
  • ${esc(v)}
    • `).join('\n'); @@ -250,9 +278,11 @@ export function generateHtmlReport(report: ReportJSON, brief: ClientBrief, stats `; }).join('\n'); - // Pullquote after first half of trends - const pullquoteIndex = Math.floor(report.trends.length / 2); - const pullquoteText = report.trends[pullquoteIndex]?.humanTruth || report.executiveSummary.split('.')[0]; + // Pullquotes — use generated ones if available, fallback to trend humanTruth + const pullquotes = report.pullquotes?.length + ? report.pullquotes + : [report.trends[Math.floor(report.trends.length / 2)]?.humanTruth || report.executiveSummary.split('.')[0]]; + const pq = (i: number) => pullquotes[i] ? `
    ${esc(pullquotes[i])}
    ` : ''; const insightsHtml = report.audienceInsights.map(ins => `
    @@ -312,10 +342,6 @@ export function generateHtmlReport(report: ReportJSON, brief: ClientBrief, stats
    `; }).join('\n'); - const sourcesHtml = report.deskSources.map(s => - `
  • ${esc(s.title)} — ${esc(s.summary.slice(0, 120))}
    • ` - ).join('\n'); - return ` @@ -397,6 +423,17 @@ hr { border: none; border-top: 2px solid #1a1a1a; margin: 48px 0; } .supporting-author { font-size: 13px; font-weight: 700; color: #1a1a1a; margin-bottom: 4px; } .supporting-desc { font-size: 12px; color: #666; line-height: 1.4; margin-bottom: 6px; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } .supporting-plays { font-size: 11px; font-weight: 600; color: #f5a623; } +.vc-row { display: flex; flex-direction: column; gap: 16px; margin: 28px 0; } +.vc-card { display: flex; gap: 20px; background: #fff; border: 1px solid #e8e8e8; border-radius: 12px; overflow: hidden; align-items: stretch; } +.vc-label { writing-mode: vertical-rl; text-orientation: mixed; background: #1a1a1a; color: #fff; font-size: 12px; font-weight: 700; letter-spacing: 1px; text-transform: uppercase; padding: 20px 14px; display: flex; align-items: center; justify-content: center; min-width: 50px; } +.vc-thumb { flex-shrink: 0; display: flex; align-items: center; padding: 16px 0; } +.vc-desc { padding: 20px; flex: 1; display: flex; flex-direction: column; justify-content: center; } +.vc-desc p { color: #444; margin-bottom: 8px; font-size: 15px; } +.vc-freq { font-size: 12px; color: #888; font-weight: 600; } +.vc-example { font-size: 12px; color: #f5a623; font-weight: 600; margin-top: 4px; } +.sticky-nav { position: sticky; top: 0; z-index: 100; background: rgba(255,255,255,0.95); backdrop-filter: blur(8px); border-bottom: 1px solid #e8e8e8; padding: 12px 0; display: flex; gap: 24px; justify-content: center; flex-wrap: wrap; font-size: 12px; font-weight: 600; text-transform: uppercase; letter-spacing: 1px; } +.sticky-nav a { color: #666; text-decoration: none; transition: color 0.2s; } +.sticky-nav a:hover { color: #1a1a1a; } .footer { text-align: center; padding: 48px 0; color: #888; font-size: 12px; } @media (max-width: 768px) { .container { padding: 24px 16px; } @@ -407,6 +444,15 @@ hr { border: none; border-top: 2px solid #1a1a1a; margin: 48px 0; } +
    @@ -415,49 +461,48 @@ hr { border: none; border-top: 2px solid #1a1a1a; margin: 48px 0; }
    ${esc(brief.category)} — ${formatDateRange(brief.dateRange)}
    -
    -
    ${stats.videosScraped}
    Videos Scraped
    -
    ${stats.commentsAnalysed}
    Comments Analysed
    -
    ${stats.transcriptsDownloaded}
    Transcripts Downloaded
    -
    ${stats.deskSources}
    Desk Sources
    +
    +${stats.videosScraped > 0 ? `
    ${stats.videosScraped}
    Videos Scraped
    ` : ''} +${stats.commentsAnalysed > 0 ? `
    ${stats.commentsAnalysed}
    Comments Analysed
    ` : ''} +${stats.transcriptsDownloaded > 0 ? `
    ${stats.transcriptsDownloaded}
    Transcripts Downloaded
    ` : ''}
    -
    ${esc(report.executiveSummary)}
    +
    ${esc(report.executiveSummary)}
    -
    01 — Category Trends
    + ${trendsHtml} -
    ${esc(pullquoteText)}
    +${visualLanguageHtml} + +${pq(0)} -
    02 — Audience Insights
    +
    02 — Audience Insights
    ${insightsHtml}
    +${pq(1)} + -
    The Formats That Drive Engagement
    +
    The Formats That Drive Engagement
    ${formatsHtml}
    -
    03 — Content Opportunities
    +
    03 — Content Opportunities
    ${oppsHtml} - -
    04 — Creator Spotlight
    -${creatorsHtml} +${pq(2)} - -
    Desk Research Sources
    -
      -${sourcesHtml} -
    + +
    04 — Creator Spotlight
    +${creatorsHtml}
    QA REVIEWED — Community Manager + Brand Strategist
    diff --git a/agents/social-listening/pipeline-v2.ts b/agents/social-listening/pipeline-v2.ts index b3cab65..eb2cb62 100644 --- a/agents/social-listening/pipeline-v2.ts +++ b/agents/social-listening/pipeline-v2.ts @@ -15,7 +15,6 @@ import { runStage3 } from './stages/stage3-discovery-scrape.js'; import { runStage4 } from './stages/stage4-data-review.js'; import { runStage5 } from './stages/stage5-enrichment-scrape.js'; import { runStage6 } from './stages/stage6-pre-report-review.js'; -import { runStage7 } from './stages/stage7-desk-search.js'; import { runStage8 } from './stages/stage8-report.js'; export type ProgressCallback = ( @@ -151,18 +150,15 @@ export async function runPipeline( state.stage6 = await runStage6(state.stage5.data, state.stage4.data, brief); emit(6, 'Pre-Report Review', 'done', `${state.stage6.data.deskSearchQueries.length} desk queries`); - // ─── Stage 7: Desk Research ─── - currentStage = 7; currentStageName = 'Desk Research'; + // ─── Stage 7: Skipped (Desk Research removed) ─── emit(7, 'Desk Research', 'start'); - state.stage7 = await runStage7(state.stage6.data, brief); - emit(7, 'Desk Research', 'done', `${state.stage7.data.length} sources`); + emit(7, 'Desk Research', 'done', 'Skipped'); // ─── Stage 8: Report Generation ─── currentStage = 8; currentStageName = 'Report Generation'; emit(8, 'Report Generation', 'start'); state.stage8 = await runStage8( state.stage5.data, - state.stage7.data, state.stage2.data, state.stage4.data, brief, diff --git a/agents/social-listening/stages/stage3-discovery-scrape.ts b/agents/social-listening/stages/stage3-discovery-scrape.ts index c38db1e..fb7d418 100644 --- a/agents/social-listening/stages/stage3-discovery-scrape.ts +++ b/agents/social-listening/stages/stage3-discovery-scrape.ts @@ -21,6 +21,7 @@ function normaliseTikTok(raw: RawTikTokItem): Video | null { saveCount: raw.collectCount || 0, duration: raw.videoMeta?.duration, hashtags: raw.hashtags?.map(h => h.name) || [], + thumbnailUrl: raw.videoMeta?.coverUrl, }; } @@ -41,6 +42,7 @@ function normaliseInstagram(raw: RawInstagramItem): Video | null { saveCount: 0, duration: raw.duration, hashtags: raw.hashtags || [], + thumbnailUrl: raw.displayUrl, }; } @@ -59,6 +61,7 @@ function normaliseYouTube(raw: RawYouTubeItem): Video | null { commentCount: raw.commentsCount || 0, shareCount: 0, saveCount: 0, + thumbnailUrl: raw.thumbnailUrl, }; } @@ -121,7 +124,7 @@ async function scrapeTikTok(brief: ClientBrief): Promise { const tag = rawHashtag.replace(/^#/, ''); const items = await safeRunActor( ACTORS.TIKTOK_SCRAPER, - { hashtags: [tag], resultsPerPage: limits.resultsPerPage, shouldDownloadVideos: false }, + { hashtags: [tag], resultsPerPage: limits.resultsPerPage, shouldDownloadVideos: false, oldestCreateTime: brief.dateRange.from }, `TikTok hashtag: ${tag}`, ); for (const item of items) { const v = normaliseTikTok(item); if (v) videos.push(v); } @@ -148,7 +151,7 @@ async function scrapeInstagram(brief: ClientBrief): Promise { const tag = rawHashtag.replace(/^#/, ''); const items = await safeRunActor( ACTORS.INSTAGRAM_HASHTAG, - { hashtags: [tag], resultsLimit: limits.resultsLimit }, + { hashtags: [tag], resultsLimit: limits.resultsLimit, onlyPostsNewerThan: brief.dateRange.from }, `Instagram hashtag: ${tag}`, ); for (const item of items) { const v = normaliseInstagram(item); if (v) videos.push(v); } @@ -175,7 +178,7 @@ async function scrapeYouTube(brief: ClientBrief): Promise { for (const query of queries) { const items = await safeRunActor( ACTORS.YOUTUBE_SEARCH, - { searchQuery: query, maxResults: limits.maxResults }, + { searchQuery: query, maxResults: limits.maxResults, uploadDate: 'month' }, `YouTube search: ${query}`, ); for (const item of items) { const v = normaliseYouTube(item); if (v) videos.push(v); } @@ -213,7 +216,9 @@ export async function runStage3(brief: ClientBrief): Promise> { + const map = new Map(); + const candidates = videos + .filter(v => v.thumbnailUrl) + .sort((a, b) => (b.playCount || 0) - (a.playCount || 0)) + .slice(0, maxCount); + + console.log(`[Stage 5] Downloading ${candidates.length} thumbnails...`); + let downloaded = 0; + + for (const v of candidates) { + try { + const res = await fetch(v.thumbnailUrl!); + if (!res.ok) continue; + const buffer = await res.arrayBuffer(); + const contentType = res.headers.get('content-type') || 'image/jpeg'; + const base64 = `data:${contentType};base64,${Buffer.from(buffer).toString('base64')}`; + map.set(v.url, base64); + downloaded++; + } catch { + // Non-fatal — skip failed thumbnails + } + } + console.log(`[Stage 5] Downloaded ${downloaded} / ${candidates.length} thumbnails`); + return map; +} + export async function runStage5( selection: TopVideosSelection, brief: ClientBrief, @@ -128,6 +162,9 @@ export async function runStage5( const youtubeTranscripts = await fetchYouTubeTranscripts(youtubeUrls); const tiktokComments = await fetchTikTokComments(tiktokUrls); + // Download thumbnails (plain HTTP, no Apify cost) + const thumbnailMap = await fetchThumbnailsAsBase64(selection.videos, 50); + // Merge all transcript maps const allTranscripts = new Map(); for (const [k, v] of tiktokTranscripts) allTranscripts.set(k, v); @@ -139,14 +176,20 @@ export async function runStage5( ...v, transcript: allTranscripts.get(v.url) || null, comments: tiktokComments.get(v.url) || [], + thumbnailBase64: thumbnailMap.get(v.url), })); const transcriptCount = enriched.filter(v => v.transcript).length; const commentCount = enriched.reduce((sum, v) => sum + v.comments.length, 0); + // Convert thumbnailMap to plain object for serialization + const thumbnailObj: Record = {}; + for (const [k, v] of thumbnailMap) thumbnailObj[k] = v; + console.log(`[Stage 5] Enrichment complete:`); console.log(` Transcripts: ${transcriptCount} / ${enriched.length}`); console.log(` Comments: ${commentCount}`); + console.log(` Thumbnails: ${thumbnailMap.size}`); return { stage: 5, @@ -155,6 +198,7 @@ export async function runStage5( videos: enriched, transcriptCount, commentCount, + thumbnailMap: thumbnailObj, }, duration: Date.now() - start, }; diff --git a/agents/social-listening/stages/stage8-report.ts b/agents/social-listening/stages/stage8-report.ts index 7baefda..aefe62a 100644 --- a/agents/social-listening/stages/stage8-report.ts +++ b/agents/social-listening/stages/stage8-report.ts @@ -1,15 +1,102 @@ // ─── Stage 8: Final Report Generation (Opus) ─── import { - ClientBrief, EnrichmentData, DeskResearchSource, AgentReview, - TopVideosSelection, FinalReport, ReportJSON, StageResult, + ClientBrief, EnrichmentData, AgentReview, + TopVideosSelection, FinalReport, ReportJSON, VisualCode, StageResult, } from '../types-v2.js'; -import { callClaudeJSON } from '../claude-cli.js'; +import { callClaudeJSON, callClaudeVision } from '../claude-cli.js'; import { buildMarkdown } from '../html-report.js'; import { generateHtmlReport } from '../html-report.js'; +// ─── Visual Language Analysis ─── + +async function analyseVisualLanguage( + enrichment: EnrichmentData, +): Promise { + const thumbnailMap = enrichment.thumbnailMap || {}; + const entries = Object.entries(thumbnailMap); + if (entries.length < 5) { + console.log(`[Stage 8] Skipping visual analysis — only ${entries.length} thumbnails available`); + return []; + } + + console.log(`[Stage 8] Analysing visual language from ${entries.length} thumbnails...`); + + // Build lookup: url -> video info + const videoLookup = new Map(enrichment.videos.map(v => [v.url, v])); + + // Take top 50, split into 5 batches of 10 + const top50 = entries.slice(0, 50); + const batchSize = 10; + const batchResults: string[] = []; + + for (let i = 0; i < top50.length; i += batchSize) { + const batch = top50.slice(i, i + batchSize); + const images = batch.map(([_, b64]) => b64); + const batchNum = Math.floor(i / batchSize) + 1; + + const prompt = `You are analysing ${images.length} video thumbnails from a social media category. For each thumbnail, describe: +1. Colour palette and dominant colours +2. Composition (close-up face, full body, flat lay, text-heavy, etc.) +3. Text overlays (if any) — font style, positioning +4. Facial expressions and body language +5. Setting/environment +6. Any recurring visual motifs + +Then identify 2-3 visual PATTERNS you see across multiple thumbnails in this batch. Be specific and concrete.`; + + try { + const result = await callClaudeVision(images, prompt, 'claude-sonnet-4-6'); + batchResults.push(result.text); + console.log(`[Stage 8] Visual batch ${batchNum} complete`); + } catch (err) { + console.warn(`[Stage 8] Visual batch ${batchNum} failed: ${(err as Error).message}`); + } + } + + if (!batchResults.length) return []; + + // Synthesis: merge batch results into visual codes + const synthesisPrompt = `You analysed video thumbnails from a social media category in batches. Here are the batch-by-batch findings: + +${batchResults.map((r, i) => `--- BATCH ${i + 1} ---\n${r}`).join('\n\n')} + +Synthesise these observations into exactly 5-6 VISUAL CODES — recurring visual patterns that define this category's visual language. Each visual code should be a specific, named pattern (e.g. "The Bare-Face Close-Up", "Pastel Flat Lay", "Text-First Controversy Hook"). + +Return JSON array: +[ + { + "name": "Visual Code Name", + "description": "2-3 sentences describing the visual pattern — what it looks like, why creators use it, what emotion it conveys", + "frequency": "Seen in X of Y thumbnails analysed" + } +]`; + + try { + const codes = await callClaudeJSON(synthesisPrompt, 'claude-sonnet-4-6'); + + // Attach example videos to each code (pick first video with a thumbnail) + for (const code of codes) { + if (!code.exampleVideoUrl) { + const entry = top50[0]; + if (entry) { + const video = videoLookup.get(entry[0]); + code.exampleVideoUrl = entry[0]; + code.exampleAuthor = video?.author || ''; + code.examplePlays = video?.playCount || 0; + } + } + } + + console.log(`[Stage 8] Visual analysis complete: ${codes.length} visual codes`); + return codes; + } catch (err) { + console.warn(`[Stage 8] Visual synthesis failed: ${(err as Error).message}`); + return []; + } +} + function buildReportPrompt( enrichment: EnrichmentData, - deskSources: DeskResearchSource[], agentReviews: AgentReview[], selection: TopVideosSelection, brief: ClientBrief, @@ -32,9 +119,6 @@ Comments: ${comments}`; // Agent hypotheses const hypotheses = selection.hypotheses.join('\n- '); - // Desk sources summary - const sourcesSummary = deskSources.map(s => `- ${s.title}: ${s.summary}`).join('\n'); - return `You are generating a social listening report for ${brief.clientName} in the ${brief.category} category. DATE RANGE: ${brief.dateRange.from} to ${brief.dateRange.to} @@ -49,20 +133,24 @@ ${urlIndex} STRATEGIST HYPOTHESES: - ${hypotheses} -DESK RESEARCH SOURCES: -${sourcesSummary} - HARD RULES: - Every topVideoUrl MUST be an exact URL from the VIDEO URL INDEX above - Every topVideoPlays MUST exactly match the plays number from the index - Never describe influencer content as organic unless proven — default assumption for branded creator content = paid - Each trend/insight/opportunity must be GENUINELY DISTINCT — no duplication disguised with different words -- Creator Spotlight requires 2+ videos with strong engagement, not single viral hits -- Trends must be timely (last 30 days), not evergreen observations +- TIMELINESS IS CRITICAL: Every trend must be anchored to specific videos from the last 30 days. Do NOT include evergreen observations like "authenticity matters" or "short-form video is growing". If a trend could have been written 6 months ago, it is NOT a trend — it is a category norm. Focus on what is NEW, surprising, or accelerating in the data window ${brief.dateRange.from} to ${brief.dateRange.to}. Name specific creators, specific videos, specific moments. +- AUDIENCE INSIGHTS must prioritize comment text over video metadata. Mine the Comments fields for actual audience language — confessions, questions, debates, purchase-intent signals, requests. Each exampleQuote MUST be a real comment from the corpus, not a caption or description. If comments are available, insights should read like community analysis, not metadata summaries. - Each trend MUST include 2-3 supportingVideos from the VIDEO URL INDEX — these will be embedded in the report - supportingVideos should include the platform field matching [tiktok|instagram|youtube] from the index - 7-12 trends, exactly 6 audience insights, 7 content opportunities, 1-2 creator spotlights +CREATOR SPOTLIGHT SELECTION: +- Only consider creators with 2-10 videos in the corpus +- EXCLUDE any creator whose videos make up more than 50% of the total dataset — that is category domination, not a discovery +- Score each eligible creator: score = avg_likes_per_video × num_videos × engagement_rate (where engagement_rate = (likes + comments + shares) / plays) +- Select the top 1-2 creators by this score +- The spotlight should surface mid-tier creators who consistently resonate, not mega-influencers who are already obvious + Return this EXACT JSON structure: { "executiveSummary": "3-4 paragraph narrative overview of the category landscape", @@ -107,13 +195,13 @@ Return this EXACT JSON structure: "keyVideos": [{"url": "EXACT url", "description": "Brief desc", "plays": 12345}], "growthSignal": "Trajectory indicator" } - ] + ], + "pullquotes": ["3-4 sharp, quotable one-liners that summarize key findings. Editorial in tone — pithy, insight-driven sentences a reader would want to screenshot. These will be displayed as visual dividers between report sections."] }`; } export async function runStage8( enrichment: EnrichmentData, - deskSources: DeskResearchSource[], agentReviews: AgentReview[], selection: TopVideosSelection, brief: ClientBrief, @@ -121,25 +209,28 @@ export async function runStage8( const start = Date.now(); console.log('[Stage 8] Generating final report via Claude Opus...'); - const prompt = buildReportPrompt(enrichment, deskSources, agentReviews, selection, brief); + // Run visual language analysis (before main report) + const visualCodes = await analyseVisualLanguage(enrichment); + + const prompt = buildReportPrompt(enrichment, agentReviews, selection, brief); const reportJSON = await callClaudeJSON(prompt, 'claude-opus-4-6', { timeout: 600_000, // 10 min }); - // Attach desk sources - reportJSON.deskSources = deskSources; + reportJSON.deskSources = []; + reportJSON.visualCodes = visualCodes; const stats = { videosScraped: enrichment.videos.length, commentsAnalysed: enrichment.commentCount, transcriptsDownloaded: enrichment.transcriptCount, - deskSources: deskSources.length, + deskSources: 0, }; // Build outputs const markdown = buildMarkdown(reportJSON, brief, stats); - const html = generateHtmlReport(reportJSON, brief, stats); + const html = generateHtmlReport(reportJSON, brief, stats, enrichment.thumbnailMap); const finalReport: FinalReport = { ...reportJSON, diff --git a/agents/social-listening/types-v2.ts b/agents/social-listening/types-v2.ts index 60219e6..2f92f4f 100644 --- a/agents/social-listening/types-v2.ts +++ b/agents/social-listening/types-v2.ts @@ -34,11 +34,13 @@ export interface Video { duration?: number; hashtags?: string[]; engagementScore?: number; + thumbnailUrl?: string; } export interface EnrichedVideo extends Video { transcript: string | null; comments: string[]; + thumbnailBase64?: string; } export interface AgentReview { @@ -76,6 +78,7 @@ export interface EnrichmentData { videos: EnrichedVideo[]; transcriptCount: number; commentCount: number; + thumbnailMap?: Record; } export interface PreReportReview { @@ -136,6 +139,15 @@ export interface CreatorSpotlight { growthSignal: string; } +export interface VisualCode { + name: string; + description: string; + frequency: string; + exampleVideoUrl: string; + exampleAuthor: string; + examplePlays: number; +} + export interface ReportJSON { executiveSummary: string; trends: Trend[]; @@ -143,6 +155,8 @@ export interface ReportJSON { contentOpportunities: ContentOpportunity[]; creatorSpotlight: CreatorSpotlight[]; deskSources: DeskResearchSource[]; + pullquotes?: string[]; + visualCodes?: VisualCode[]; } export interface FinalReport extends ReportJSON { @@ -190,7 +204,7 @@ export interface RawTikTokItem { commentCount?: number; shareCount?: number; collectCount?: number; - videoMeta?: { duration?: number }; + videoMeta?: { duration?: number; coverUrl?: string }; hashtags?: { name: string }[]; } @@ -207,6 +221,7 @@ export interface RawInstagramItem { commentsCount?: number; duration?: number; hashtags?: string[]; + displayUrl?: string; } export interface RawYouTubeItem { @@ -218,4 +233,5 @@ export interface RawYouTubeItem { viewCount?: number; likes?: number; commentsCount?: number; + thumbnailUrl?: string; }