added full persona profile export in bulk actions menu (CSV, JSON and Markdown formats)

This commit is contained in:
michael 2025-09-10 22:38:35 -05:00
parent 4165677451
commit c7f868e5b1
10 changed files with 1319 additions and 12 deletions

View file

@ -22,7 +22,8 @@
"Bash(pip uninstall:*)",
"Bash(pip install:*)",
"mcp__gpt5-bridge__call_gpt5",
"WebSearch"
"WebSearch",
"Bash(pip show:*)"
],
"deny": []
},

View file

@ -8,9 +8,10 @@ Provides jwt_required decorator and token management functions.
import os
import jwt
import functools
import json
from datetime import datetime, timedelta
from typing import Optional, Dict, Any
from quart import request, g, current_app, jsonify
from quart import request, g, current_app, jsonify, Response
# JWT Configuration - ensure compatibility with Flask-JWT-Extended
JWT_SECRET_KEY = os.environ.get('SECRET_KEY', 'your-secret-key-for-sessions-and-tokens')
@ -123,8 +124,11 @@ def jwt_required(optional: bool = False):
return result
else:
# Token required but not provided
from quart import make_response
return make_response(jsonify({'error': 'Missing authorization token'}), 401)
return Response(
json.dumps({'error': 'Missing authorization token'}),
status=401,
mimetype="application/json"
)
# Validate token
try:
@ -171,8 +175,11 @@ def jwt_required(optional: bool = False):
return result
else:
# Invalid token and required
from quart import make_response
return make_response(jsonify({'error': f'Invalid token: {str(e)}'}), 401)
return Response(
json.dumps({'error': f'Invalid token: {str(e)}'}),
status=401,
mimetype="application/json"
)
except Exception as e:
current_app.logger.error(f"JWT validation error: {e}")
@ -191,8 +198,11 @@ def jwt_required(optional: bool = False):
else:
return result
else:
from quart import make_response
return make_response(jsonify({'error': 'Authentication error'}), 500)
return Response(
json.dumps({'error': 'Authentication error'}),
status=500,
mimetype="application/json"
)
return wrapper
return decorator

View file

@ -1,12 +1,21 @@
from quart import Blueprint, request, jsonify
from quart import Blueprint, request, jsonify, send_file, Response
from app.auth.quart_jwt import jwt_required, get_jwt_identity
from app.models.persona import Persona
import json
from app.services.persona_export_service import PersonaExportService
from app.services.bulk_persona_export_service import BulkPersonaExportService
from app.services.persona_modification_service import PersonaModificationService, PersonaModificationError
from app.services.task_manager import CancellableTask
from bson import ObjectId
import datetime
import asyncio
import os
import uuid
# Helper function for safe JSON responses (avoids async issues)
def json_response(payload: dict, status: int = 200) -> Response:
"""Create a JSON response without async complications."""
return Response(json.dumps(payload), status=status, mimetype="application/json")
# Helper function to make MongoDB documents JSON serializable
def make_serializable(obj):
@ -315,4 +324,121 @@ async def export_persona_profile(persona_id):
except Exception as e:
print(f"Error in export_persona_profile: {e}")
return jsonify({"error": f"Failed to export persona profile: {str(e)}"}), 500
return jsonify({"error": f"Failed to export persona profile: {str(e)}"}), 500
@personas_bp.route('/bulk-export', methods=['POST'])
@jwt_required()
async def bulk_export_personas():
"""
Export multiple personas in bulk to specified format (markdown, JSON, CSV).
Request body should include:
- persona_ids: List of persona IDs to export
- export_format: Format for export ('markdown', 'json', 'csv')
"""
try:
# Check if get_jwt_identity is async and handle accordingly
user_id = get_jwt_identity()
if hasattr(user_id, "__await__"): # cheap awaitable check
user_id = await user_id
request_data = await request.get_json()
if not request_data:
return json_response({"error": "No request data provided"}, 400)
persona_ids = request_data.get('persona_ids', [])
if not persona_ids or not isinstance(persona_ids, list):
return json_response({"error": "persona_ids must be a non-empty list"}, 400)
export_format = request_data.get('export_format', 'markdown')
if export_format not in ['markdown', 'json', 'csv']:
return json_response({"error": "export_format must be 'markdown', 'json', or 'csv'"}, 400)
print(f"🚀 Backend: Starting bulk export for {len(persona_ids)} personas (format: {export_format})")
# Initialize bulk export service
bulk_export_service = BulkPersonaExportService()
# Start export asynchronously (returns task_id immediately)
success, result_message, task_id = await bulk_export_service.export_personas_bulk(
persona_ids=persona_ids,
user_id=user_id,
export_format=export_format
)
if success:
# Since export is instant, directly serve the file instead of returning path
try:
file_path = result_message
if os.path.exists(file_path):
filename = os.path.basename(file_path)
print(f"📥 Direct serving: {filename} ({os.path.getsize(file_path)} bytes)")
return await send_file(
file_path,
as_attachment=True
)
else:
return json_response({"error": "Export file not found"}, 500)
except Exception as file_error:
print(f"Error serving export file: {file_error}")
return json_response({"error": f"Failed to serve file: {str(file_error)}"}, 500)
else:
return json_response({
"error": result_message,
"task_id": task_id
}, 400)
except Exception as e:
print(f"Error in bulk_export_personas: {e}")
return json_response({"error": f"Failed to start bulk export: {str(e)}"}, 500)
@personas_bp.route('/download/<path:file_path>', methods=['GET'])
@jwt_required()
async def download_export_file(file_path):
"""
Download an exported file by its file path.
This endpoint serves files from the temp directory.
"""
try:
user_id = get_jwt_identity()
# Security: Only allow files from temp directory
temp_dir = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"temp"
)
# Build full file path
full_file_path = os.path.join(temp_dir, file_path)
# Security check: ensure path is within temp directory
temp_dir_real = os.path.realpath(temp_dir)
full_file_path_real = os.path.realpath(full_file_path)
if not full_file_path_real.startswith(temp_dir_real):
print(f"⚠️ Security: Attempted access outside temp directory: {file_path}")
return jsonify({"error": "File not found"}), 404
# Check if file exists
if not os.path.exists(full_file_path):
print(f"📁 File not found: {full_file_path}")
return jsonify({"error": "File not found or expired"}), 404
filename = os.path.basename(full_file_path)
print(f"📥 Serving download: {filename} to user {user_id}")
# Use Quart's send_file with correct parameters for v0.20.0
return await send_file(
full_file_path,
as_attachment=True,
download_name=filename # Quart 0.20+ uses download_name, not attachment_filename
)
except Exception as e:
print(f"Error in download_export_file: {e}")
return jsonify({"error": f"Failed to download file: {str(e)}"}), 500

View file

@ -0,0 +1,751 @@
"""
Bulk Persona Export Service
Handles bulk export of persona profiles to various formats (markdown, JSON, CSV)
with real-time progress tracking via WebSocket events.
"""
import os
import json
import logging
import zipfile
import tempfile
import uuid
import asyncio
from typing import Dict, List, Any, Optional, Tuple
from datetime import datetime
# Removed PersonaExportService dependency - using direct conversion
from app.models.persona import Persona
from app.websocket_manager_async import get_async_websocket_manager
from app.services.task_manager import CancellableTask
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class BulkPersonaExportService:
"""Service for bulk exporting persona profiles with progress tracking."""
def __init__(self):
"""Initialize the bulk persona export service."""
self.websocket_manager = get_async_websocket_manager()
def _create_temp_directory(self) -> str:
"""Create a temporary directory for export files."""
temp_dir = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"temp"
)
os.makedirs(temp_dir, exist_ok=True)
# Create unique subdirectory for this export
export_id = str(uuid.uuid4())
export_dir = os.path.join(temp_dir, f"export_{export_id}")
os.makedirs(export_dir, exist_ok=True)
return export_dir
def _sanitize_filename(self, filename: str) -> str:
"""Sanitize filename for safe file system use."""
# Remove or replace invalid characters
invalid_chars = '<>:"/\\|?*'
for char in invalid_chars:
filename = filename.replace(char, '_')
# Limit length and ensure it's not empty
filename = filename[:200].strip()
if not filename:
filename = "persona"
return filename
async def _emit_progress(self, user_id: str, task_id: str, progress: int,
current_item: str, completed_count: int, total_count: int,
current_persona_name: Optional[str] = None):
"""Emit progress update via WebSocket."""
try:
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'bulk_export_progress',
{
'task_id': task_id,
'task_type': 'bulk_persona_export',
'progress': progress,
'current_item': current_item,
'completed_count': completed_count,
'total_count': total_count,
'current_persona_name': current_persona_name
}
)
logger.debug(f"Emitted progress: {progress}% - {current_item}")
else:
logger.warning("WebSocket manager not available for progress updates")
except Exception as e:
logger.error(f"Failed to emit progress update: {e}")
def _create_markdown_table(self, data: List[tuple], headers: List[str] = None) -> List[str]:
"""Create a markdown table from data tuples."""
if not data:
return []
# Use default headers if none provided
if not headers:
headers = ["Field", "Value"]
# Create table
table_lines = []
# Headers
header_line = "| " + " | ".join(headers) + " |"
separator_line = "|" + "|".join(["-" * (len(h) + 2) for h in headers]) + "|"
table_lines.append(header_line)
table_lines.append(separator_line)
# Data rows
for row in data:
# Ensure all values are strings and escape pipe characters
escaped_row = []
for value in row:
str_value = str(value) if value is not None else ""
# Escape pipe characters in cell content
str_value = str_value.replace("|", "\\|")
# Replace newlines with spaces for table formatting
str_value = str_value.replace("\n", " ").replace("\r", " ")
escaped_row.append(str_value)
row_line = "| " + " | ".join(escaped_row) + " |"
table_lines.append(row_line)
return table_lines
def _create_comprehensive_markdown(self, persona_data: Dict[str, Any]) -> str:
"""Create comprehensive markdown from persona data with all fields included."""
try:
name = persona_data.get('name', 'Unknown Persona')
# Build comprehensive markdown from all available data
markdown_parts = [f"# {name} - Complete Persona Profile\n"]
# AI-Generated Summary (if available) - put at top as overview
if 'aiSynthesizedBio' in persona_data and persona_data['aiSynthesizedBio']:
markdown_parts.append("## Overview")
markdown_parts.append(persona_data['aiSynthesizedBio'])
markdown_parts.append("")
# Core Demographics Section - Table Format
demo_fields = [
('age', 'Age'), ('gender', 'Gender'), ('occupation', 'Occupation'),
('education', 'Education'), ('location', 'Location'), ('ethnicity', 'Ethnicity'),
('householdIncome', 'Household Income'), ('householdComposition', 'Household Composition'),
('socialGrade', 'Social Grade')
]
demographics_data = []
for field, label in demo_fields:
if field in persona_data and persona_data[field]:
demographics_data.append((label, persona_data[field]))
if demographics_data:
markdown_parts.append("## Demographics")
table_lines = self._create_markdown_table(demographics_data, ["Attribute", "Value"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Behavioral Profile & Preferences Section - Table Format
behavioral_fields = [
('techSavviness', 'Tech Savviness'), ('personality', 'Personality'),
('brandLoyalty', 'Brand Loyalty'), ('priceConsciousness', 'Price Consciousness'),
('environmentalConcern', 'Environmental Concern'), ('interests', 'Interests'),
('shoppingHabits', 'Shopping Habits'), ('mediaConsumption', 'Media Consumption'),
('deviceUsage', 'Device Usage'), ('brandPreferences', 'Brand Preferences'),
('hasPurchasingPower', 'Has Purchasing Power'), ('hasChildren', 'Has Children')
]
behavioral_data = []
for field, label in behavioral_fields:
if field in persona_data and persona_data[field] is not None:
value = persona_data[field]
if isinstance(value, bool):
value = "Yes" if value else "No"
behavioral_data.append((label, value))
if behavioral_data:
markdown_parts.append("## Behavioral Profile & Preferences")
table_lines = self._create_markdown_table(behavioral_data, ["Attribute", "Value"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Goals, Motivations & Aspirations
goal_sections = [
("Goals", "goals"), ("Motivations", "motivations"),
("Frustrations", "frustrations"), ("Fears", "fears"),
("Scenarios", "scenarios")
]
for section_name, field in goal_sections:
if field in persona_data and persona_data[field]:
items = persona_data[field]
if isinstance(items, list) and items:
markdown_parts.append(f"## {section_name}")
for item in items:
if item and item.strip():
markdown_parts.append(f"- {item}")
markdown_parts.append("")
# Think, Feel, Do Psychology Framework - 3-Column Table
if 'thinkFeelDo' in persona_data and persona_data['thinkFeelDo']:
tfd = persona_data['thinkFeelDo']
markdown_parts.append("## Psychological Profile - Think, Feel, Do")
# Get the lists for each category
thinks = tfd.get('thinks', []) if isinstance(tfd.get('thinks'), list) else []
feels = tfd.get('feels', []) if isinstance(tfd.get('feels'), list) else []
does = tfd.get('does', []) if isinstance(tfd.get('does'), list) else []
# Create table data by combining the three lists
max_items = max(len(thinks), len(feels), len(does))
if max_items > 0:
tfd_data = []
for i in range(max_items):
think_item = thinks[i] if i < len(thinks) else ""
feel_item = feels[i] if i < len(feels) else ""
do_item = does[i] if i < len(does) else ""
# Only add row if at least one cell has content
if think_item or feel_item or do_item:
tfd_data.append((think_item, feel_item, do_item))
if tfd_data:
table_lines = self._create_markdown_table(tfd_data, ["Thinks", "Feels", "Does"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# OCEAN Personality Traits (Big Five) - Enhanced Table Format
if 'oceanTraits' in persona_data and persona_data['oceanTraits']:
ocean = persona_data['oceanTraits']
markdown_parts.append("## OCEAN Personality Traits (Big Five)")
trait_descriptions = {
'openness': 'Openness to Experience',
'conscientiousness': 'Conscientiousness',
'extraversion': 'Extraversion',
'agreeableness': 'Agreeableness',
'neuroticism': 'Neuroticism'
}
def get_level_description(score):
"""Get descriptive level for OCEAN trait score."""
if score < 0.3:
return "Low"
elif score < 0.7:
return "Moderate"
else:
return "High"
ocean_data = []
for trait, score in ocean.items():
if score is not None:
trait_name = trait_descriptions.get(trait, trait.title())
# Handle different score formats - scores are already decimal (0.0-1.0)
if isinstance(score, (int, float)):
# If score is already 0-1 range, multiply by 100; if it's 0-100 range, use as is
if score <= 1.0:
percentage = f"{round(float(score) * 100)}%"
else:
percentage = f"{round(float(score))}%"
level = get_level_description(float(score) if score <= 1.0 else float(score) / 100)
else:
percentage = f"{score}%"
level = "N/A"
ocean_data.append((trait_name, percentage, level))
if ocean_data:
table_lines = self._create_markdown_table(ocean_data, ["Trait", "Score", "Level"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Top Personality Traits
if 'topPersonalityTraits' in persona_data and persona_data['topPersonalityTraits']:
traits = persona_data['topPersonalityTraits']
if isinstance(traits, list) and traits:
markdown_parts.append("## Top Personality Traits")
markdown_parts.append(", ".join([trait for trait in traits if trait]))
markdown_parts.append("")
# Qualitative Attributes
if 'qualitativeAttributes' in persona_data and persona_data['qualitativeAttributes']:
attrs = persona_data['qualitativeAttributes']
if isinstance(attrs, list) and attrs:
markdown_parts.append("## Key Qualitative Attributes")
markdown_parts.append(", ".join([attr for attr in attrs if attr]))
markdown_parts.append("")
# Lifestyle & Consumer Behavior - Table Format
lifestyle_fields = [
('coreValues', 'Core Values'), ('lifestyleChoices', 'Lifestyle Choices'),
('socialActivities', 'Social Activities'), ('categoryKnowledge', 'Category Knowledge'),
('paymentMethods', 'Payment Methods'), ('purchaseBehaviour', 'Purchase Behavior'),
('decisionInfluences', 'Decision Influences'), ('painPoints', 'Pain Points'),
('journeyContext', 'Journey Context')
]
lifestyle_data = []
for field, label in lifestyle_fields:
if field in persona_data and persona_data[field]:
lifestyle_data.append((label, persona_data[field]))
if lifestyle_data:
markdown_parts.append("## Lifestyle & Consumer Behavior")
table_lines = self._create_markdown_table(lifestyle_data, ["Attribute", "Value"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Generation Context & Research - Table Format
context_data = []
if 'audience_brief' in persona_data and persona_data['audience_brief']:
context_data.append(("Audience Brief", persona_data['audience_brief']))
if 'research_objective' in persona_data and persona_data['research_objective']:
context_data.append(("Research Objective", persona_data['research_objective']))
if context_data:
markdown_parts.append("## Generation Context")
table_lines = self._create_markdown_table(context_data, ["Type", "Description"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Additional Data Fields (catch any remaining fields) - Table Format
processed_fields = {
'name', 'aiSynthesizedBio', 'age', 'gender', 'occupation', 'education', 'location',
'ethnicity', 'householdIncome', 'householdComposition', 'socialGrade', 'techSavviness',
'personality', 'brandLoyalty', 'priceConsciousness', 'environmentalConcern', 'interests',
'shoppingHabits', 'mediaConsumption', 'deviceUsage', 'brandPreferences', 'hasPurchasingPower',
'hasChildren', 'goals', 'motivations', 'frustrations', 'fears', 'scenarios', 'thinkFeelDo',
'oceanTraits', 'topPersonalityTraits', 'qualitativeAttributes', 'coreValues', 'lifestyleChoices',
'socialActivities', 'categoryKnowledge', 'paymentMethods', 'purchaseBehaviour', 'decisionInfluences',
'painPoints', 'journeyContext', 'audience_brief', 'research_objective', '_id', 'created_at',
'created_by', 'updated_at', 'folder_ids'
}
additional_data = []
for key, value in persona_data.items():
if key not in processed_fields and value is not None:
if isinstance(value, list):
if value: # Non-empty list
formatted_value = ", ".join([str(v) for v in value if v])
if formatted_value:
additional_data.append((key.replace('_', ' ').title(), formatted_value))
elif isinstance(value, dict):
if value: # Non-empty dict
formatted_dict = ", ".join([f"{k}: {v}" for k, v in value.items() if v is not None])
if formatted_dict:
additional_data.append((key.replace('_', ' ').title(), formatted_dict))
else:
if str(value).strip(): # Non-empty value
additional_data.append((key.replace('_', ' ').title(), str(value)))
if additional_data:
markdown_parts.append("## Additional Data")
table_lines = self._create_markdown_table(additional_data, ["Attribute", "Value"])
markdown_parts.extend(table_lines)
markdown_parts.append("")
# Metadata - Table Format
metadata_data = []
if 'created_at' in persona_data and persona_data['created_at']:
metadata_data.append(("Created", persona_data['created_at']))
if 'updated_at' in persona_data and persona_data['updated_at']:
metadata_data.append(("Last Updated", persona_data['updated_at']))
if 'created_by' in persona_data and persona_data['created_by']:
metadata_data.append(("Created By", persona_data['created_by']))
if 'folder_ids' in persona_data and persona_data['folder_ids']:
folder_count = len(persona_data['folder_ids'])
metadata_data.append(("Folder Assignments", f"{folder_count} folder(s)"))
if metadata_data:
markdown_parts.append("## Metadata")
table_lines = self._create_markdown_table(metadata_data, ["Field", "Value"])
markdown_parts.extend(table_lines)
return "\n".join(markdown_parts)
except Exception as e:
logger.error(f"Failed to create comprehensive markdown: {e}")
return f"# {persona_data.get('name', 'Unknown Persona')}\n\nError generating profile.\n\n```json\n{json.dumps(persona_data, indent=2, default=str)}\n```"
async def export_personas_bulk(
self,
persona_ids: List[str],
user_id: str,
export_format: str = 'markdown'
) -> Tuple[bool, str, Optional[str]]:
"""
Export multiple personas to specified format with progress tracking.
Args:
persona_ids: List of persona IDs to export
user_id: ID of user requesting export
export_format: Format for export ('markdown', 'json', 'csv')
Returns:
Tuple of (success, file_path_or_error_message, task_id)
"""
task_id = str(uuid.uuid4())
export_dir = None
try:
async with CancellableTask("bulk_persona_export", user_id, {"export_format": export_format}) as registered_task_id:
task_id = registered_task_id or task_id
logger.info(f"Starting bulk export for {len(persona_ids)} personas (user: {user_id}, format: {export_format})")
# Create temp directory
export_dir = self._create_temp_directory()
# Emit initial progress
await self._emit_progress(
user_id, task_id, 0,
"Initializing export...", 0, len(persona_ids)
)
# Fetch all personas
await self._emit_progress(
user_id, task_id, 5,
"Fetching persona data...", 0, len(persona_ids)
)
personas = []
for persona_id in persona_ids:
persona = await Persona.find_by_id(persona_id)
if persona:
personas.append(persona)
else:
logger.warning(f"Persona not found: {persona_id}")
if not personas:
await self._emit_progress(
user_id, task_id, 100,
"No valid personas found", 0, len(persona_ids)
)
return False, "No valid personas found for export", task_id
# Process personas based on format
if export_format == 'markdown':
return await self._export_as_markdown_zip(
personas, user_id, task_id, export_dir
)
elif export_format == 'json':
return await self._export_as_json_zip(
personas, user_id, task_id, export_dir
)
elif export_format == 'csv':
return await self._export_as_csv_zip(
personas, user_id, task_id, export_dir
)
else:
return False, f"Unsupported export format: {export_format}", task_id
except asyncio.CancelledError:
logger.info(f"Bulk export cancelled by user: {user_id}")
if export_dir and os.path.exists(export_dir):
import shutil
shutil.rmtree(export_dir, ignore_errors=True)
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'task_cancelled',
{
'task_id': task_id,
'message': 'Export cancelled successfully'
}
)
return False, "Export cancelled by user", task_id
except Exception as e:
logger.error(f"Bulk export error: {e}")
if export_dir and os.path.exists(export_dir):
import shutil
shutil.rmtree(export_dir, ignore_errors=True)
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'task_failed',
{
'task_id': task_id,
'message': f'Export failed: {str(e)}'
}
)
return False, f"Export failed: {str(e)}", task_id
async def _export_as_markdown_zip(
self,
personas: List[Dict[str, Any]],
user_id: str,
task_id: str,
export_dir: str
) -> Tuple[bool, str, str]:
"""Export personas as markdown files in a ZIP archive."""
try:
zip_path = os.path.join(export_dir, f"persona_profiles_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.zip")
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
total_personas = len(personas)
for i, persona in enumerate(personas):
# Check for cancellation
current_task = asyncio.current_task()
if current_task and current_task.cancelled():
raise asyncio.CancelledError("Task was cancelled")
persona_name = persona.get('name', f'Persona_{i+1}')
# Update progress
progress = int(10 + (i / total_personas) * 80) # 10-90%
await self._emit_progress(
user_id, task_id, progress,
f"Processing persona {i+1} of {total_personas}",
i, total_personas, persona_name
)
# Make persona data serializable and convert directly to markdown
from app.routes.personas import make_serializable
serializable_persona = make_serializable(persona)
# Generate comprehensive markdown directly from persona data
markdown_content = self._create_comprehensive_markdown(serializable_persona)
# Create safe filename
safe_name = self._sanitize_filename(persona_name)
filename = f"{safe_name}.md"
# Add to ZIP
zipf.writestr(filename, markdown_content.encode('utf-8'))
logger.info(f"Added {filename} to ZIP ({len(markdown_content)} chars)")
# Final progress update
await self._emit_progress(
user_id, task_id, 95,
"Finalizing ZIP file...", total_personas, total_personas
)
# Verify ZIP was created
if not os.path.exists(zip_path):
return False, "Failed to create ZIP file", task_id
file_size = os.path.getsize(zip_path)
logger.info(f"Created ZIP file: {zip_path} ({file_size} bytes)")
# Success notification
await self._emit_progress(
user_id, task_id, 100,
f"Export completed! {total_personas} personas exported.",
total_personas, total_personas
)
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'task_completed',
{
'task_id': task_id,
'message': f'Successfully exported {total_personas} persona profiles',
'file_path': zip_path,
'file_size': file_size
}
)
return True, zip_path, task_id
except Exception as e:
logger.error(f"Markdown ZIP export error: {e}")
return False, f"Markdown export failed: {str(e)}", task_id
async def _export_as_json_zip(
self,
personas: List[Dict[str, Any]],
user_id: str,
task_id: str,
export_dir: str
) -> Tuple[bool, str, str]:
"""Export personas as JSON files in a ZIP archive."""
try:
zip_path = os.path.join(export_dir, f"persona_data_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.zip")
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
total_personas = len(personas)
for i, persona in enumerate(personas):
# Check for cancellation
current_task = asyncio.current_task()
if current_task and current_task.cancelled():
raise asyncio.CancelledError("Task was cancelled")
persona_name = persona.get('name', f'Persona_{i+1}')
# Update progress
progress = int(10 + (i / total_personas) * 80) # 10-90%
await self._emit_progress(
user_id, task_id, progress,
f"Processing persona {i+1} of {total_personas}",
i, total_personas, persona_name
)
# Make persona data serializable and convert to JSON
from app.routes.personas import make_serializable
serializable_persona = make_serializable(persona)
json_content = json.dumps(serializable_persona, indent=2, ensure_ascii=False, default=str)
# Create safe filename
safe_name = self._sanitize_filename(persona_name)
filename = f"{safe_name}.json"
# Add to ZIP
zipf.writestr(filename, json_content.encode('utf-8'))
# Final steps
await self._emit_progress(
user_id, task_id, 100,
f"Export completed! {total_personas} personas exported.",
total_personas, total_personas
)
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'task_completed',
{
'task_id': task_id,
'message': f'Successfully exported {total_personas} persona JSON files',
'file_path': zip_path,
'file_size': os.path.getsize(zip_path)
}
)
return True, zip_path, task_id
except Exception as e:
logger.error(f"JSON ZIP export error: {e}")
return False, f"JSON export failed: {str(e)}", task_id
async def _export_as_csv_zip(
self,
personas: List[Dict[str, Any]],
user_id: str,
task_id: str,
export_dir: str
) -> Tuple[bool, str, str]:
"""Export personas as individual CSV files in a ZIP archive."""
try:
import csv
import io
zip_path = os.path.join(export_dir, f"persona_csvs_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.zip")
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
total_personas = len(personas)
for i, persona in enumerate(personas):
# Check for cancellation
current_task = asyncio.current_task()
if current_task and current_task.cancelled():
raise asyncio.CancelledError("Task was cancelled")
persona_name = persona.get('name', f'Persona_{i+1}')
# Update progress
progress = int(10 + (i / total_personas) * 80) # 10-90%
await self._emit_progress(
user_id, task_id, progress,
f"Processing persona {i+1} of {total_personas}",
i, total_personas, persona_name
)
# Make persona data serializable and flatten for CSV
from app.routes.personas import make_serializable
serializable_persona = make_serializable(persona)
# Flatten nested objects and arrays for CSV format
flat_persona = {}
for key, value in serializable_persona.items():
if isinstance(value, dict):
# Flatten nested objects: {"oceanTraits": {"openness": 0.7}} -> {"oceanTraits_openness": 0.7}
for subkey, subvalue in value.items():
flat_persona[f"{key}_{subkey}"] = subvalue
elif isinstance(value, list):
# Convert arrays to semicolon-separated strings
if value:
if isinstance(value[0], str):
flat_persona[key] = "; ".join(str(v) for v in value if v)
else:
flat_persona[key] = json.dumps(value)
else:
flat_persona[key] = ""
else:
flat_persona[key] = value if value is not None else ""
# Create CSV content using built-in csv module
output = io.StringIO()
if flat_persona: # Only proceed if we have data
fieldnames = list(flat_persona.keys())
writer = csv.DictWriter(output, fieldnames=fieldnames)
writer.writeheader()
writer.writerow(flat_persona)
csv_content = output.getvalue()
output.close()
# Create safe filename
safe_name = self._sanitize_filename(persona_name)
filename = f"{safe_name}.csv"
# Add to ZIP
zipf.writestr(filename, csv_content.encode('utf-8'))
logger.info(f"Added {filename} to ZIP ({len(csv_content)} chars)")
# Final progress update
await self._emit_progress(
user_id, task_id, 95,
"Finalizing ZIP file...", total_personas, total_personas
)
# Verify ZIP was created
if not os.path.exists(zip_path):
return False, "Failed to create ZIP file", task_id
file_size = os.path.getsize(zip_path)
logger.info(f"Created CSV ZIP file: {zip_path} ({file_size} bytes)")
# Success notification
await self._emit_progress(
user_id, task_id, 100,
f"Export completed! {total_personas} personas exported.",
total_personas, total_personas
)
if self.websocket_manager:
await self.websocket_manager.emit_to_user(
user_id,
'task_completed',
{
'task_id': task_id,
'message': f'Successfully exported {total_personas} persona CSV files',
'file_path': zip_path,
'file_size': file_size
}
)
return True, zip_path, task_id
except Exception as e:
logger.error(f"CSV ZIP export error: {e}")
return False, f"CSV export failed: {str(e)}", task_id

4
dist/index.html vendored
View file

@ -7,8 +7,8 @@
<meta name="description" content="Lovable Generated Project" />
<meta name="author" content="Lovable" />
<meta property="og:image" content="/og-image.png" />
<script type="module" crossorigin src="/semblance/assets/index-CBPpFZFw.js"></script>
<link rel="stylesheet" crossorigin href="/semblance/assets/index-BAs_xR_U.css">
<script type="module" crossorigin src="/semblance/assets/index-iDGrOQEx.js"></script>
<link rel="stylesheet" crossorigin href="/semblance/assets/index-sPfhNyTb.css">
</head>
<body>

View file

@ -0,0 +1,298 @@
import React, { useState, useEffect, useRef } from 'react';
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog';
import { Progress } from '@/components/ui/progress';
import { Button } from '@/components/ui/button';
import { X, Download, CheckCircle, AlertCircle } from 'lucide-react';
import { cn } from '@/lib/utils';
import { toast } from 'sonner';
interface BulkExportProgressData {
task_id: string;
task_type: 'bulk_persona_export';
progress: number;
current_item: string;
completed_count: number;
total_count: number;
current_persona_name?: string;
}
interface BulkExportProgressModalProps {
isOpen: boolean;
onClose: () => void;
taskId: string | null;
exportFormat: 'markdown' | 'json' | 'csv';
personaCount: number;
onCancel?: () => Promise<boolean>;
onDownload?: (filePath: string) => void;
}
export const BulkExportProgressModal: React.FC<BulkExportProgressModalProps> = ({
isOpen,
onClose,
taskId,
exportFormat,
personaCount,
onCancel,
onDownload
}) => {
const [progress, setProgress] = useState(0);
const [currentItem, setCurrentItem] = useState('Initializing export...');
const [completedCount, setCompletedCount] = useState(0);
const [currentPersonaName, setCurrentPersonaName] = useState<string | null>(null);
const [isComplete, setIsComplete] = useState(false);
const [hasError, setHasError] = useState(false);
const [isCancelling, setIsCancelling] = useState(false);
const [errorMessage, setErrorMessage] = useState<string | null>(null);
const [downloadFilePath, setDownloadFilePath] = useState<string | null>(null);
const stateRef = useRef({ taskId, isComplete, hasError });
stateRef.current = { taskId, isComplete, hasError };
// Reset state when modal opens
useEffect(() => {
if (isOpen && taskId) {
setProgress(0);
setCurrentItem('Initializing export...');
setCompletedCount(0);
setCurrentPersonaName(null);
setIsComplete(false);
setHasError(false);
setIsCancelling(false);
setErrorMessage(null);
setDownloadFilePath(null);
}
}, [isOpen, taskId]);
// Set up WebSocket event listeners for bulk export progress
useEffect(() => {
const handleBulkExportProgress = (event: CustomEvent) => {
const data: BulkExportProgressData = event.detail;
// Only handle events for our task
if (data.task_id !== stateRef.current.taskId) {
return;
}
setProgress(data.progress);
setCurrentItem(data.current_item);
setCompletedCount(data.completed_count);
setCurrentPersonaName(data.current_persona_name || null);
};
const handleTaskCompleted = (event: CustomEvent) => {
const data = event.detail;
if (data.task_id !== stateRef.current.taskId) {
return;
}
setIsComplete(true);
setProgress(100);
setCurrentItem('Export completed successfully!');
// Store the file path for download
if (data.file_path) {
setDownloadFilePath(data.file_path);
}
toast.success(`Successfully exported ${personaCount} persona profiles!`);
};
const handleTaskFailed = (event: CustomEvent) => {
const data = event.detail;
if (data.task_id !== stateRef.current.taskId) {
return;
}
setHasError(true);
setErrorMessage(data.message || 'Export failed');
setCurrentItem('Export failed');
toast.error('Export failed: ' + (data.message || 'Unknown error'));
};
const handleTaskCancelled = (event: CustomEvent) => {
const data = event.detail;
if (data.task_id !== stateRef.current.taskId) {
return;
}
setIsCancelling(false);
setCurrentItem('Export cancelled');
toast.success('Export cancelled successfully');
// Close modal after short delay
setTimeout(() => {
onClose();
}, 1000);
};
// Register window event listeners
window.addEventListener('ws:bulk_export_progress', handleBulkExportProgress as EventListener);
window.addEventListener('ws:task_completed', handleTaskCompleted as EventListener);
window.addEventListener('ws:task_failed', handleTaskFailed as EventListener);
window.addEventListener('ws:task_cancelled', handleTaskCancelled as EventListener);
// Cleanup listeners
return () => {
window.removeEventListener('ws:bulk_export_progress', handleBulkExportProgress as EventListener);
window.removeEventListener('ws:task_completed', handleTaskCompleted as EventListener);
window.removeEventListener('ws:task_failed', handleTaskFailed as EventListener);
window.removeEventListener('ws:task_cancelled', handleTaskCancelled as EventListener);
};
}, []); // Only set up once
const handleCancel = async () => {
if (!taskId || isCancelling) return;
setIsCancelling(true);
setCurrentItem('Cancelling export...');
const success = onCancel ? await onCancel() : false;
if (!success) {
setIsCancelling(false);
}
};
const handleDownload = () => {
if (downloadFilePath && onDownload) {
onDownload(downloadFilePath);
}
};
const handleClose = () => {
if (isComplete || hasError) {
onClose();
}
};
const getFormatDisplay = () => {
switch (exportFormat) {
case 'markdown': return 'Markdown';
case 'json': return 'JSON';
case 'csv': return 'CSV';
default: return 'Files';
}
};
const getProgressColor = () => {
if (hasError) return 'bg-red-500';
if (isComplete) return 'bg-green-500';
if (isCancelling) return 'bg-orange-500';
return 'bg-blue-500';
};
return (
<Dialog open={isOpen} onOpenChange={handleClose}>
<DialogContent className="sm:max-w-md">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
{hasError ? (
<AlertCircle className="h-5 w-5 text-red-500" />
) : isComplete ? (
<CheckCircle className="h-5 w-5 text-green-500" />
) : (
<Download className="h-5 w-5 text-blue-500" />
)}
Bulk Export - {getFormatDisplay()}
</DialogTitle>
<DialogDescription>
Exporting {personaCount} persona{personaCount !== 1 ? 's' : ''} to {getFormatDisplay().toLowerCase()} format
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
{/* Progress Bar */}
<div className="space-y-2">
<Progress
value={progress}
className={cn(
"w-full transition-all duration-200",
hasError && "opacity-75",
isCancelling && "opacity-60"
)}
/>
<div className="flex justify-between text-sm text-muted-foreground">
<span>{Math.round(progress)}%</span>
<span>{completedCount}/{personaCount}</span>
</div>
</div>
{/* Current Status */}
<div className="space-y-1">
<div className="text-sm font-medium">
{currentItem}
</div>
{currentPersonaName && !isComplete && !hasError && (
<div className="text-sm text-muted-foreground">
Processing: {currentPersonaName}
</div>
)}
</div>
{/* Error Message */}
{hasError && errorMessage && (
<div className="p-3 bg-red-50 border border-red-200 rounded-md">
<div className="text-sm text-red-800">
{errorMessage}
</div>
</div>
)}
{/* Action Buttons */}
<div className="flex justify-between">
<div>
{/* Cancel Button */}
{!isComplete && !hasError && taskId && (
<Button
variant="outline"
onClick={handleCancel}
disabled={isCancelling}
className="text-muted-foreground hover:text-destructive hover:border-destructive"
>
{isCancelling ? 'Cancelling...' : 'Cancel'}
</Button>
)}
</div>
<div className="flex gap-2">
{/* Download Button */}
{isComplete && downloadFilePath && (
<Button
onClick={handleDownload}
className="bg-green-600 hover:bg-green-700"
>
<Download className="h-4 w-4 mr-2" />
Download
</Button>
)}
{/* Close Button */}
{(isComplete || hasError) && (
<Button
variant="outline"
onClick={handleClose}
>
Close
</Button>
)}
</div>
</div>
{/* Success Message */}
{isComplete && (
<div className="p-3 bg-green-50 border border-green-200 rounded-md">
<div className="text-sm text-green-800">
Export completed successfully! Your {getFormatDisplay().toLowerCase()} files are ready for download.
</div>
</div>
)}
</div>
</DialogContent>
</Dialog>
);
};
export default BulkExportProgressModal;

View file

@ -193,6 +193,15 @@ export const personasApi = {
exportProfile: (id: string, options?: { llm_model?: string; temperature?: number }) =>
api.post(`/personas/${id}/export-profile`, options || {}, {
timeout: 300000 // 5 minutes for profile export
}),
// Bulk export personas to specified format
bulkExportPersonas: (data: {
persona_ids: string[];
export_format: 'markdown' | 'json' | 'csv';
}) =>
api.post('/personas/bulk-export', data, {
timeout: 600000 // 10 minutes for bulk export
})
};

View file

@ -91,6 +91,7 @@ const SyntheticUsers = () => {
// WebSocket and cancellable generation for summary generation
const socket = getSocket();
const [summaryGenerationState, summaryGenerationControls] = useCancellableGeneration('persona summary generation', socket);
// Bulk export no longer needs cancellable generation - it's instant
const [searchTerm, setSearchTerm] = useState('');
const [selectedUser, setSelectedUser] = useState<Persona | null>(null);
const [selectedFolder, setSelectedFolder] = useState<string>(DEFAULT_FOLDER_ID);
@ -143,6 +144,8 @@ const SyntheticUsers = () => {
const [downloadLlmModalOpen, setDownloadLlmModalOpen] = useState(false);
const [selectedDownloadLlmModel, setSelectedDownloadLlmModel] = useState<string>('gemini-2.5-pro');
// Bulk export no longer needs state - direct download
// Handle summary generation progress completion
const handleSummaryProgressComplete = () => {
summaryGenerationControls.resetGeneration();
@ -1103,6 +1106,71 @@ const SyntheticUsers = () => {
setIsLoading(false);
}
};
// Bulk export functions
const handleBulkExport = async (format: 'markdown' | 'json' | 'csv') => {
const selectedIds = Array.from(selectedPersonas);
if (selectedIds.length === 0) {
toastService.error("Please select personas to export");
return;
}
// Show loading toast
toastService.info(`Exporting ${selectedIds.length} persona${selectedIds.length !== 1 ? 's' : ''} to ${format.toUpperCase()}...`);
try {
// Get JWT token for the request
const token = localStorage.getItem('auth_token');
const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || '/semblance_back/api';
// Make direct fetch request since response will be a file
const response = await fetch(`${API_BASE_URL}/personas/bulk-export`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`,
},
body: JSON.stringify({
persona_ids: selectedIds,
export_format: format
})
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.error || 'Export failed');
}
// The response should be the file blob directly
const blob = await response.blob();
// Create download link
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = `persona_profiles_${format}_${new Date().toISOString().slice(0,19).replace(/:/g,'-')}.zip`;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
// Clean up
URL.revokeObjectURL(url);
toastService.success(`${format.toUpperCase()} export completed!`, {
description: `Successfully exported ${selectedIds.length} persona profile${selectedIds.length !== 1 ? 's' : ''}`
});
} catch (error) {
console.error("Error during bulk export:", error);
toastService.error("Export failed", {
description: error.message || 'Unknown error occurred'
});
}
};
// Removed separate download function - now using direct file response
return (
<div className="min-h-screen bg-slate-50">
@ -1260,6 +1328,39 @@ const SyntheticUsers = () => {
Remove from {folders.find(f => f._id === selectedFolder)?.name || 'folder'}
</DropdownMenuItem>
)}
<DropdownMenuItem
className="flex items-center gap-2 cursor-pointer"
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
handleBulkExport('markdown');
}}
>
<Download className="h-4 w-4" />
Download Full Persona Profiles (Markdown)
</DropdownMenuItem>
<DropdownMenuItem
className="flex items-center gap-2 cursor-pointer"
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
handleBulkExport('json');
}}
>
<Download className="h-4 w-4" />
Download Full Persona Profiles (JSON)
</DropdownMenuItem>
<DropdownMenuItem
className="flex items-center gap-2 cursor-pointer"
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
handleBulkExport('csv');
}}
>
<Download className="h-4 w-4" />
Download Full Persona Profiles (CSV)
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)}

View file

@ -128,6 +128,11 @@ export function initSocket(getToken: () => string): Socket {
window.dispatchEvent(new CustomEvent("ws:task_failed", { detail: payload }));
break;
case 'bulk_export_progress':
console.log('🔧 [GPT-5] *** ROUTING bulk_export_progress from onAny ***');
window.dispatchEvent(new CustomEvent("ws:bulk_export_progress", { detail: payload }));
break;
case 'error':
console.error('🔧 [GPT-5] *** ROUTING error from onAny ***', payload);
break;
@ -281,6 +286,11 @@ function bindCoreListeners() {
console.error('🔧 [GPT-5] auth_error:', payload);
window.dispatchEvent(new CustomEvent("ws:auth_error", { detail: payload }));
};
const onBulkExportProgress = (payload: any) => {
console.log('🔧 [GPT-5] bulk_export_progress:', payload);
window.dispatchEvent(new CustomEvent("ws:bulk_export_progress", { detail: payload }));
};
// Bind all listeners
console.log('🔧 [GPT-5] BINDING specific listeners to socket');
@ -293,6 +303,7 @@ function bindCoreListeners() {
socket.on("focus_group_update", onFG);
socket.on("mode_event_update", onModeEvent);
socket.on("auth_error", onAuthError);
socket.on("bulk_export_progress", onBulkExportProgress);
console.log('🔧 [GPT-5] BOUND specific listeners to socket');
// GPT-5 DEBUG: Verify listeners are actually attached