Implement CSV import functionality

This commit is contained in:
michael 2025-11-19 15:36:43 -06:00
parent d160b1bc90
commit 21544ab529
2 changed files with 368 additions and 147 deletions

181
main.py
View file

@ -1,12 +1,12 @@
from fastapi import FastAPI, Depends, HTTPException, Request, Form, Query
from fastapi import FastAPI, Depends, HTTPException, Request, Form, Query, File, UploadFile
from fastapi.security import OAuth2PasswordBearer
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse, StreamingResponse
from starlette.middleware.sessions import SessionMiddleware
from typing import List, Optional
import crud
import models
import models
import auth
import config
import msal_auth
@ -15,6 +15,9 @@ import os
import re
from dotenv import load_dotenv
from fastapi import Header
import csv
import io
import json
load_dotenv()
@ -964,12 +967,182 @@ async def update_user(email: str, user_update: models.UserUpdate, current_user:
@app.get("/api/admin/agents", response_model=List[models.AiAgentResponse])
async def get_all_agents_admin(current_user: dict = Depends(require_admin)):
agents = await crud.get_all_agents()
return [
create_agent_response(agent) for agent in agents
]
@app.get("/api/admin/agents/export/csv")
async def export_agents_csv(current_user: dict = Depends(require_admin)):
"""Export all agent data to CSV (excluding usage data)"""
# Fetch all agents from database
agents = await crud.get_all_agents()
# Create CSV in memory
output = io.StringIO()
# Define CSV columns (all fields except usage-related ones)
fieldnames = [
"agent_id",
"agent_name",
"agent_tool",
"agent_description",
"agent_purpose",
"agent_version",
"agent_status",
"agent_location",
"agent_department",
"agent_contact_person",
"agent_created_at",
"agent_updated_at",
"agent_tags",
"agent_metadata",
"agent_userbase",
"agent_capabilities",
"url",
"quality_audit_status",
"quality_audit_updated_by",
"quality_audit_updated_at",
"quality_audit_updated_by_name",
"risk_factor",
"last_edited_by",
"created_by"
]
writer = csv.DictWriter(output, fieldnames=fieldnames)
writer.writeheader()
# Write agent data
for agent in agents:
row = {
"agent_id": str(agent["_id"]),
"agent_name": agent.get("agent_name", ""),
"agent_tool": agent.get("agent_tool", ""),
"agent_description": agent.get("agent_description", ""),
"agent_purpose": agent.get("agent_purpose", ""),
"agent_version": agent.get("agent_version", ""),
"agent_status": agent.get("agent_status", ""),
"agent_location": agent.get("agent_location", ""),
"agent_department": agent.get("agent_department", ""),
"agent_contact_person": agent.get("agent_contact_person", ""),
"agent_created_at": agent["created_at"].isoformat() if agent.get("created_at") else "",
"agent_updated_at": agent["updated_at"].isoformat() if agent.get("updated_at") else "",
# Convert lists to pipe-separated strings for CSV compatibility
"agent_tags": "|".join(agent.get("agent_tags", [])) if agent.get("agent_tags") else "",
"agent_userbase": "|".join(agent.get("agent_userbase", [])) if agent.get("agent_userbase") else "",
"agent_capabilities": "|".join(agent.get("agent_capabilities", [])) if agent.get("agent_capabilities") else "",
# Convert metadata dict to JSON string
"agent_metadata": json.dumps(sanitize_metadata(agent.get("agent_metadata"))) if agent.get("agent_metadata") else "",
"url": agent.get("url", ""),
"quality_audit_status": str(agent.get("quality_audit_status", False)),
"quality_audit_updated_by": agent.get("quality_audit_updated_by", ""),
"quality_audit_updated_at": agent.get("quality_audit_updated_at", ""),
"quality_audit_updated_by_name": agent.get("quality_audit_updated_by_name", ""),
"risk_factor": str(agent.get("risk_factor", "")) if agent.get("risk_factor") is not None else "",
"last_edited_by": agent.get("last_edited_by", ""),
"created_by": agent.get("created_by", "")
}
writer.writerow(row)
# Get CSV content
csv_content = output.getvalue()
output.close()
# Return as downloadable file
return StreamingResponse(
iter([csv_content]),
media_type="text/csv",
headers={
"Content-Disposition": f"attachment; filename=agents_export_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.csv"
}
)
@app.post("/api/admin/agents/import/csv")
async def import_agents_csv(
request: Request,
file: UploadFile = File(...),
current_user: dict = Depends(require_admin)
):
"""Import agents from CSV file"""
if not file.filename.endswith('.csv'):
raise HTTPException(status_code=400, detail="File must be a CSV")
try:
contents = await file.read()
decoded = contents.decode('utf-8')
csv_reader = csv.DictReader(io.StringIO(decoded))
success_count = 0
skipped_count = 0
error_count = 0
errors = []
user_id = str(current_user["_id"])
for row_num, row in enumerate(csv_reader, start=1):
try:
agent_name = row.get("agent_name")
if not agent_name:
continue
# Check for duplicates (skip if exists)
existing = await crud.get_agent_by_name(agent_name)
if existing and existing["created_by"] == user_id:
skipped_count += 1
continue
# Parse fields
agent_data = {
"agent_name": agent_name,
"agent_tool": row.get("agent_tool", ""),
"agent_description": row.get("agent_description"),
"agent_purpose": row.get("agent_purpose"),
"agent_version": row.get("agent_version"),
"agent_status": row.get("agent_status") or "Development",
"agent_location": row.get("agent_location"),
"agent_department": row.get("agent_department"),
"agent_contact_person": row.get("agent_contact_person"),
"url": row.get("url"),
"quality_audit_status": row.get("quality_audit_status", "False").lower() == "true",
"risk_factor": int(row.get("risk_factor")) if row.get("risk_factor") else None
}
# Handle lists (pipe separated)
if row.get("agent_tags"):
agent_data["agent_tags"] = [t.strip() for t in row.get("agent_tags").split("|") if t.strip()]
if row.get("agent_userbase"):
agent_data["agent_userbase"] = [u.strip() for u in row.get("agent_userbase").split("|") if u.strip()]
if row.get("agent_capabilities"):
agent_data["agent_capabilities"] = [c.strip() for c in row.get("agent_capabilities").split("|") if c.strip()]
# Handle metadata (JSON string)
if row.get("agent_metadata"):
try:
agent_data["agent_metadata"] = json.loads(row.get("agent_metadata"))
except:
pass # Ignore invalid metadata JSON
# Create agent
await crud.create_agent(agent_data, user_id)
success_count += 1
except Exception as e:
error_count += 1
errors.append(f"Row {row_num}: {str(e)}")
return JSONResponse({
"success": True,
"imported": success_count,
"skipped": skipped_count,
"errors": error_count,
"error_details": errors[:10] # Limit error details
})
except Exception as e:
raise HTTPException(status_code=500, detail=f"Import failed: {str(e)}")
# Agent Collector API Endpoints (for compatibility with agent_collector app)
@app.post("/agents")
async def create_agent_collector(

View file

@ -4,11 +4,12 @@
<i class="fas fa-users-cog me-2"></i>
<span>AgentHub</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNavAltMarkup" aria-controls="navbarNavAltMarkup" aria-expanded="false" aria-label="Toggle navigation">
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNavAltMarkup"
aria-controls="navbarNavAltMarkup" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarNavAltMarkup">
<div class="navbar-nav me-auto">
{% if current_user %}
@ -17,83 +18,97 @@
</a>
{% endif %}
{% if current_user %}
{% if not current_user.is_admin %}
<a class="nav-link" href="{{ base_path }}/agent-management">
<i class="fas fa-globe me-1"></i>All Agents
</a>
<a class="nav-link" href="{{ base_path }}/agent-management?view=my">
<i class="fas fa-robot me-1"></i>My Agents
</a>
{% endif %}
{% if current_user.is_admin %}
<a class="nav-link" href="{{ base_path }}/admin">
<i class="fas fa-tachometer-alt me-1"></i>Admin
</a>
{% endif %}
{% if not current_user.is_admin %}
<a class="nav-link" href="{{ base_path }}/agent-management">
<i class="fas fa-globe me-1"></i>All Agents
</a>
<a class="nav-link" href="{{ base_path }}/agent-management?view=my">
<i class="fas fa-robot me-1"></i>My Agents
</a>
{% endif %}
{% if current_user.is_admin %}
<a class="nav-link" href="{{ base_path }}/admin">
<i class="fas fa-tachometer-alt me-1"></i>Admin
</a>
<a class="nav-link" href="{{ base_path }}/api/admin/agents/export/csv" download>
<i class="fas fa-download me-1"></i>Export CSV
</a>
<a class="nav-link" href="#" onclick="document.getElementById('csv-import-input').click(); return false;">
<i class="fas fa-file-upload me-1"></i>Import CSV
</a>
<input type="file" id="csv-import-input" accept=".csv" style="display: none;" onchange="uploadCsv(this)">
{% endif %}
{% endif %}
</div>
<div class="navbar-nav">
{% if current_user %}
<a class="nav-link" href="{{ base_path }}/search">
<i class="fas fa-search me-1"></i>Search
<a class="nav-link" href="{{ base_path }}/search">
<i class="fas fa-search me-1"></i>Search
</a>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle d-flex align-items-center" href="#" role="button"
data-bs-toggle="dropdown">
<div class="user-avatar me-2">
{{ current_user.full_name[0] if current_user.full_name else current_user.email[0] }}
</div>
{{ current_user.full_name or current_user.email.split('@')[0] }}
</a>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle d-flex align-items-center" href="#" role="button" data-bs-toggle="dropdown">
<div class="user-avatar me-2">
{{ current_user.full_name[0] if current_user.full_name else current_user.email[0] }}
</div>
{{ current_user.full_name or current_user.email.split('@')[0] }}
</a>
<ul class="dropdown-menu dropdown-menu-end">
<li><h6 class="dropdown-header">{{ current_user.email }}</h6></li>
<li><hr class="dropdown-divider"></li>
<li>
<a class="dropdown-item" href="{{ base_path }}/profile">
<i class="fas fa-user me-2"></i>My Profile
</a>
</li>
<li>
<a class="dropdown-item" href="{{ base_path }}/agent-register">
<i class="fas fa-plus-circle me-2"></i>Add Agent
</a>
</li>
<li>
<a class="dropdown-item" href="{{ base_path }}/agent-management">
<i class="fas fa-list me-2"></i>My Agents
</a>
</li>
{% if current_user.is_admin %}
<li>
<a class="dropdown-item" href="{{ base_path }}/admin">
<i class="fas fa-cog me-2"></i>Admin Panel
</a>
</li>
{% endif %}
{% if current_user.get('actual_is_admin') or current_user.is_admin %}
<li>
<a class="dropdown-item" href="#" onclick="toggleAdminView(event)">
<i class="fas fa-exchange-alt me-2"></i>
{% if current_user.get('actual_is_admin') %}
Switch to Admin View
{% else %}
Switch to User View
{% endif %}
</a>
</li>
{% endif %}
<li><hr class="dropdown-divider"></li>
<li>
<a class="dropdown-item text-danger" href="{{ base_path }}/logout">
<i class="fas fa-sign-out-alt me-2"></i>Logout
</a>
</li>
</ul>
</div>
<ul class="dropdown-menu dropdown-menu-end">
<li>
<h6 class="dropdown-header">{{ current_user.email }}</h6>
</li>
<li>
<hr class="dropdown-divider">
</li>
<li>
<a class="dropdown-item" href="{{ base_path }}/profile">
<i class="fas fa-user me-2"></i>My Profile
</a>
</li>
<li>
<a class="dropdown-item" href="{{ base_path }}/agent-register">
<i class="fas fa-plus-circle me-2"></i>Add Agent
</a>
</li>
<li>
<a class="dropdown-item" href="{{ base_path }}/agent-management">
<i class="fas fa-list me-2"></i>My Agents
</a>
</li>
{% if current_user.is_admin %}
<li>
<a class="dropdown-item" href="{{ base_path }}/admin">
<i class="fas fa-cog me-2"></i>Admin Panel
</a>
</li>
{% endif %}
{% if current_user.get('actual_is_admin') or current_user.is_admin %}
<li>
<a class="dropdown-item" href="#" onclick="toggleAdminView(event)">
<i class="fas fa-exchange-alt me-2"></i>
{% if current_user.get('actual_is_admin') %}
Switch to Admin View
{% else %}
Switch to User View
{% endif %}
</a>
</li>
{% endif %}
<li>
<hr class="dropdown-divider">
</li>
<li>
<a class="dropdown-item text-danger" href="{{ base_path }}/logout">
<i class="fas fa-sign-out-alt me-2"></i>Logout
</a>
</li>
</ul>
</div>
{% else %}
<a class="nav-link" href="{{ base_path }}/login">
<i class="fas fa-sign-in-alt me-1"></i>Login
</a>
<a class="nav-link" href="{{ base_path }}/login">
<i class="fas fa-sign-in-alt me-1"></i>Login
</a>
{% endif %}
</div>
</div>
@ -101,90 +116,123 @@
</nav>
<style>
.user-avatar {
width: 28px;
height: 28px;
border-radius: 50%;
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.75rem;
}
.user-avatar {
width: 28px;
height: 28px;
border-radius: 50%;
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 0.75rem;
}
.dropdown-menu {
border-radius: 12px;
border: none;
box-shadow: var(--shadow-lg);
padding: 0.5rem 0;
margin-top: 0.5rem;
z-index: 1050 !important;
position: absolute !important;
}
.dropdown-menu {
border-radius: 12px;
border: none;
box-shadow: var(--shadow-lg);
padding: 0.5rem 0;
margin-top: 0.5rem;
z-index: 1050 !important;
position: absolute !important;
}
.dropdown-item {
padding: 0.5rem 1rem;
transition: all 0.3s ease;
border-radius: 8px;
margin: 0 0.5rem;
}
.dropdown-item {
padding: 0.5rem 1rem;
transition: all 0.3s ease;
border-radius: 8px;
margin: 0 0.5rem;
}
.dropdown-item:hover {
background-color: var(--bg-light);
transform: translateX(4px);
}
.dropdown-item:hover {
background-color: var(--bg-light);
transform: translateX(4px);
}
.navbar-toggler {
border: none;
padding: 4px 8px;
}
.navbar-toggler {
border: none;
padding: 4px 8px;
}
.navbar-toggler:focus {
box-shadow: none;
}
.navbar-toggler:focus {
box-shadow: none;
}
.navbar {
position: relative;
z-index: 1100 !important;
}
.navbar {
position: relative;
z-index: 1100 !important;
}
.nav-item.dropdown {
position: relative;
z-index: 1100;
}
.nav-item.dropdown {
position: relative;
z-index: 1100;
}
</style>
<script>
async function toggleAdminView(event) {
event.preventDefault();
async function toggleAdminView(event) {
event.preventDefault();
try {
const response = await fetch('{{ base_path }}/api/admin/toggle-view', {
method: 'POST',
credentials: 'include',
headers: {
'Content-Type': 'application/json'
}
});
try {
const response = await fetch('{{ base_path }}/api/admin/toggle-view', {
method: 'POST',
credentials: 'include',
headers: {
'Content-Type': 'application/json'
}
});
if (response.ok) {
const data = await response.json();
if (response.ok) {
const data = await response.json();
// Redirect based on new view mode
if (data.view_mode === 'user') {
window.location.href = '{{ base_path }}/agent-management';
// Redirect based on new view mode
if (data.view_mode === 'user') {
window.location.href = '{{ base_path }}/agent-management';
} else {
window.location.href = '{{ base_path }}/admin';
}
} else {
window.location.href = '{{ base_path }}/admin';
const error = await response.json();
alert('Failed to toggle view: ' + (error.detail || 'Unknown error'));
}
} else {
const error = await response.json();
alert('Failed to toggle view: ' + (error.detail || 'Unknown error'));
} catch (error) {
console.error('Error toggling admin view:', error);
alert('Failed to toggle view. Please try again.');
}
} catch (error) {
console.error('Error toggling admin view:', error);
alert('Failed to toggle view. Please try again.');
}
}
</script>
async function uploadCsv(input) {
if (!input.files || !input.files[0]) return;
const file = input.files[0];
const formData = new FormData();
formData.append('file', file);
try {
const response = await fetch('{{ base_path }}/api/admin/agents/import/csv', {
method: 'POST',
body: formData
});
const result = await response.json();
if (response.ok && result.success) {
let msg = `Import complete!\nImported: ${result.imported}\nSkipped: ${result.skipped}\nErrors: ${result.errors}`;
if (result.errors > 0) {
msg += `\n\nFirst few errors:\n${result.error_details.join('\n')}`;
}
alert(msg);
window.location.reload();
} else {
alert('Import failed: ' + (result.detail || 'Unknown error'));
}
} catch (error) {
console.error('Error uploading CSV:', error);
alert('Error uploading CSV: ' + error.message);
}
// Reset input
input.value = '';
}
</script>