From 21544ab52922cc79aae9bebc74d1aa648f19191a Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 19 Nov 2025 15:36:43 -0600 Subject: [PATCH] Implement CSV import functionality --- main.py | 181 +++++++++++++++++++++++- templates/nav.html | 334 ++++++++++++++++++++++++++------------------- 2 files changed, 368 insertions(+), 147 deletions(-) diff --git a/main.py b/main.py index 5e45005..e0b259c 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,12 @@ -from fastapi import FastAPI, Depends, HTTPException, Request, Form, Query +from fastapi import FastAPI, Depends, HTTPException, Request, Form, Query, File, UploadFile from fastapi.security import OAuth2PasswordBearer from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse, StreamingResponse from starlette.middleware.sessions import SessionMiddleware from typing import List, Optional import crud -import models +import models import auth import config import msal_auth @@ -15,6 +15,9 @@ import os import re from dotenv import load_dotenv from fastapi import Header +import csv +import io +import json load_dotenv() @@ -964,12 +967,182 @@ async def update_user(email: str, user_update: models.UserUpdate, current_user: @app.get("/api/admin/agents", response_model=List[models.AiAgentResponse]) async def get_all_agents_admin(current_user: dict = Depends(require_admin)): - + agents = await crud.get_all_agents() return [ create_agent_response(agent) for agent in agents ] +@app.get("/api/admin/agents/export/csv") +async def export_agents_csv(current_user: dict = Depends(require_admin)): + """Export all agent data to CSV (excluding usage data)""" + + # Fetch all agents from database + agents = await crud.get_all_agents() + + # Create CSV in memory + output = io.StringIO() + + # Define CSV columns (all fields except usage-related ones) + fieldnames = [ + "agent_id", + "agent_name", + "agent_tool", + "agent_description", + "agent_purpose", + "agent_version", + "agent_status", + "agent_location", + "agent_department", + "agent_contact_person", + "agent_created_at", + "agent_updated_at", + "agent_tags", + "agent_metadata", + "agent_userbase", + "agent_capabilities", + "url", + "quality_audit_status", + "quality_audit_updated_by", + "quality_audit_updated_at", + "quality_audit_updated_by_name", + "risk_factor", + "last_edited_by", + "created_by" + ] + + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + + # Write agent data + for agent in agents: + row = { + "agent_id": str(agent["_id"]), + "agent_name": agent.get("agent_name", ""), + "agent_tool": agent.get("agent_tool", ""), + "agent_description": agent.get("agent_description", ""), + "agent_purpose": agent.get("agent_purpose", ""), + "agent_version": agent.get("agent_version", ""), + "agent_status": agent.get("agent_status", ""), + "agent_location": agent.get("agent_location", ""), + "agent_department": agent.get("agent_department", ""), + "agent_contact_person": agent.get("agent_contact_person", ""), + "agent_created_at": agent["created_at"].isoformat() if agent.get("created_at") else "", + "agent_updated_at": agent["updated_at"].isoformat() if agent.get("updated_at") else "", + # Convert lists to pipe-separated strings for CSV compatibility + "agent_tags": "|".join(agent.get("agent_tags", [])) if agent.get("agent_tags") else "", + "agent_userbase": "|".join(agent.get("agent_userbase", [])) if agent.get("agent_userbase") else "", + "agent_capabilities": "|".join(agent.get("agent_capabilities", [])) if agent.get("agent_capabilities") else "", + # Convert metadata dict to JSON string + "agent_metadata": json.dumps(sanitize_metadata(agent.get("agent_metadata"))) if agent.get("agent_metadata") else "", + "url": agent.get("url", ""), + "quality_audit_status": str(agent.get("quality_audit_status", False)), + "quality_audit_updated_by": agent.get("quality_audit_updated_by", ""), + "quality_audit_updated_at": agent.get("quality_audit_updated_at", ""), + "quality_audit_updated_by_name": agent.get("quality_audit_updated_by_name", ""), + "risk_factor": str(agent.get("risk_factor", "")) if agent.get("risk_factor") is not None else "", + "last_edited_by": agent.get("last_edited_by", ""), + "created_by": agent.get("created_by", "") + } + writer.writerow(row) + + # Get CSV content + csv_content = output.getvalue() + output.close() + + # Return as downloadable file + return StreamingResponse( + iter([csv_content]), + media_type="text/csv", + headers={ + "Content-Disposition": f"attachment; filename=agents_export_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.csv" + } + ) + +@app.post("/api/admin/agents/import/csv") +async def import_agents_csv( + request: Request, + file: UploadFile = File(...), + current_user: dict = Depends(require_admin) +): + """Import agents from CSV file""" + if not file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="File must be a CSV") + + try: + contents = await file.read() + decoded = contents.decode('utf-8') + csv_reader = csv.DictReader(io.StringIO(decoded)) + + success_count = 0 + skipped_count = 0 + error_count = 0 + errors = [] + + user_id = str(current_user["_id"]) + + for row_num, row in enumerate(csv_reader, start=1): + try: + agent_name = row.get("agent_name") + if not agent_name: + continue + + # Check for duplicates (skip if exists) + existing = await crud.get_agent_by_name(agent_name) + if existing and existing["created_by"] == user_id: + skipped_count += 1 + continue + + # Parse fields + agent_data = { + "agent_name": agent_name, + "agent_tool": row.get("agent_tool", ""), + "agent_description": row.get("agent_description"), + "agent_purpose": row.get("agent_purpose"), + "agent_version": row.get("agent_version"), + "agent_status": row.get("agent_status") or "Development", + "agent_location": row.get("agent_location"), + "agent_department": row.get("agent_department"), + "agent_contact_person": row.get("agent_contact_person"), + "url": row.get("url"), + "quality_audit_status": row.get("quality_audit_status", "False").lower() == "true", + "risk_factor": int(row.get("risk_factor")) if row.get("risk_factor") else None + } + + # Handle lists (pipe separated) + if row.get("agent_tags"): + agent_data["agent_tags"] = [t.strip() for t in row.get("agent_tags").split("|") if t.strip()] + if row.get("agent_userbase"): + agent_data["agent_userbase"] = [u.strip() for u in row.get("agent_userbase").split("|") if u.strip()] + if row.get("agent_capabilities"): + agent_data["agent_capabilities"] = [c.strip() for c in row.get("agent_capabilities").split("|") if c.strip()] + + # Handle metadata (JSON string) + if row.get("agent_metadata"): + try: + agent_data["agent_metadata"] = json.loads(row.get("agent_metadata")) + except: + pass # Ignore invalid metadata JSON + + # Create agent + await crud.create_agent(agent_data, user_id) + success_count += 1 + + except Exception as e: + error_count += 1 + errors.append(f"Row {row_num}: {str(e)}") + + return JSONResponse({ + "success": True, + "imported": success_count, + "skipped": skipped_count, + "errors": error_count, + "error_details": errors[:10] # Limit error details + }) + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Import failed: {str(e)}") + # Agent Collector API Endpoints (for compatibility with agent_collector app) @app.post("/agents") async def create_agent_collector( diff --git a/templates/nav.html b/templates/nav.html index 91cbf84..15865ae 100644 --- a/templates/nav.html +++ b/templates/nav.html @@ -4,11 +4,12 @@ AgentHub - - - + @@ -101,90 +116,123 @@ + async function uploadCsv(input) { + if (!input.files || !input.files[0]) return; + + const file = input.files[0]; + const formData = new FormData(); + formData.append('file', file); + + try { + const response = await fetch('{{ base_path }}/api/admin/agents/import/csv', { + method: 'POST', + body: formData + }); + + const result = await response.json(); + + if (response.ok && result.success) { + let msg = `Import complete!\nImported: ${result.imported}\nSkipped: ${result.skipped}\nErrors: ${result.errors}`; + if (result.errors > 0) { + msg += `\n\nFirst few errors:\n${result.error_details.join('\n')}`; + } + alert(msg); + window.location.reload(); + } else { + alert('Import failed: ' + (result.detail || 'Unknown error')); + } + } catch (error) { + console.error('Error uploading CSV:', error); + alert('Error uploading CSV: ' + error.message); + } + + // Reset input + input.value = ''; + } + \ No newline at end of file