Fix 504 timeouts on upload/match and broken exports
- Move AI parsing and matching into BackgroundTasks so both endpoints return immediately instead of blocking until Claude finishes (~60s+) - Frontend now polls project status after upload/match POST returns, keeping the spinner/progress UI working as before - Replace <a href> export links with programmatic Axios downloads to fix missing /gsb base path and missing auth token (401 in production) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Vadym Samoilenko
parent
b5a21764d8
commit
9596f4231e
2 changed files with 185 additions and 100 deletions
|
|
@ -2,11 +2,11 @@
|
|||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, UploadFile, File
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.database import get_db, async_session
|
||||
from app.models.gmal import GmalAsset
|
||||
from app.models.project import Project, ClientAsset, Match, ProjectStatus, MatchConfidence
|
||||
from app.schemas.project import ClientAssetOut, ClientAssetUpdate, MatchOut, MatchSelectRequest, ManualMatchRequest
|
||||
|
|
@ -17,23 +17,82 @@ router = APIRouter()
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _background_parse(project_id: int, filename: str, text: str, metadata: dict):
|
||||
"""Run AI parsing and save results in the background (own DB session)."""
|
||||
async with async_session() as db:
|
||||
try:
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
return
|
||||
|
||||
# Stage 3: AI parsing
|
||||
try:
|
||||
extracted, usage_info = parse_text_with_ai(text)
|
||||
except Exception as e:
|
||||
logger.error(f"AI parsing failed for project {project_id}: {e}")
|
||||
project.status = ProjectStatus.DRAFT
|
||||
project.parse_stage = f"AI parsing failed: {str(e)}"
|
||||
await db.commit()
|
||||
return
|
||||
|
||||
# Save AI costs
|
||||
project.ai_input_tokens = (project.ai_input_tokens or 0) + usage_info.get("input_tokens", 0)
|
||||
project.ai_output_tokens = (project.ai_output_tokens or 0) + usage_info.get("output_tokens", 0)
|
||||
project.ai_cost_usd = float(project.ai_cost_usd or 0) + usage_info.get("cost_usd", 0)
|
||||
project.ai_call_count = (project.ai_call_count or 0) + 1
|
||||
|
||||
# Stage 4: Saving results
|
||||
project.parse_stage = f"AI found {len(extracted)} assets. Saving..."
|
||||
await db.commit()
|
||||
|
||||
# Clear existing client assets
|
||||
existing = await db.execute(
|
||||
select(ClientAsset).where(ClientAsset.project_id == project_id)
|
||||
)
|
||||
for ca in existing.scalars().all():
|
||||
await db.delete(ca)
|
||||
|
||||
# Create client asset records
|
||||
assets = []
|
||||
for idx, item in enumerate(extracted):
|
||||
ca = ClientAsset(
|
||||
project_id=project_id,
|
||||
raw_name=item.get("name", "Unknown"),
|
||||
raw_description=item.get("description", ""),
|
||||
volume=item.get("volume", 1),
|
||||
sort_order=idx + 1,
|
||||
)
|
||||
db.add(ca)
|
||||
assets.append(ca)
|
||||
|
||||
project.status = ProjectStatus.REVIEW
|
||||
project.parse_stage = f"Done! {len(assets)} assets extracted."
|
||||
await db.commit()
|
||||
logger.info(f"Background parse complete for project {project_id}: {len(assets)} assets")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Background parse error for project {project_id}: {e}")
|
||||
|
||||
|
||||
@router.post("/{project_id}/upload")
|
||||
async def upload_client_document(
|
||||
project_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Upload a client document and extract assets using AI."""
|
||||
project = await _get_project(project_id, db)
|
||||
|
||||
# Stage 1: Uploading
|
||||
# Stage 1: Read file
|
||||
content = await file.read()
|
||||
project.source_filename = file.filename
|
||||
project.status = ProjectStatus.PARSING
|
||||
project.parse_stage = f"Uploading {file.filename}..."
|
||||
await db.commit()
|
||||
|
||||
# Stage 2: Extracting text
|
||||
# Stage 2: Extract text (fast, synchronous)
|
||||
project.parse_stage = "Extracting text from document..."
|
||||
await db.commit()
|
||||
|
||||
|
|
@ -49,57 +108,12 @@ async def upload_client_document(
|
|||
project.parse_stage = f"Extracted {metadata['char_count']:,} characters{sheets_info}. Sending to AI..."
|
||||
await db.commit()
|
||||
|
||||
# Stage 3: AI parsing
|
||||
try:
|
||||
extracted, usage_info = parse_text_with_ai(text)
|
||||
except Exception as e:
|
||||
logger.error(f"AI parsing failed: {e}")
|
||||
project.status = ProjectStatus.DRAFT
|
||||
project.parse_stage = None
|
||||
await db.commit()
|
||||
raise HTTPException(status_code=400, detail=f"Failed to parse document: {str(e)}")
|
||||
|
||||
# Save AI costs
|
||||
project.ai_input_tokens = (project.ai_input_tokens or 0) + usage_info.get("input_tokens", 0)
|
||||
project.ai_output_tokens = (project.ai_output_tokens or 0) + usage_info.get("output_tokens", 0)
|
||||
project.ai_cost_usd = float(project.ai_cost_usd or 0) + usage_info.get("cost_usd", 0)
|
||||
project.ai_call_count = (project.ai_call_count or 0) + 1
|
||||
|
||||
# Stage 4: Saving results
|
||||
project.parse_stage = f"AI found {len(extracted)} assets. Saving..."
|
||||
await db.commit()
|
||||
|
||||
# Clear existing client assets
|
||||
existing = await db.execute(
|
||||
select(ClientAsset).where(ClientAsset.project_id == project_id)
|
||||
)
|
||||
for ca in existing.scalars().all():
|
||||
await db.delete(ca)
|
||||
|
||||
# Create client asset records
|
||||
assets = []
|
||||
for idx, item in enumerate(extracted):
|
||||
ca = ClientAsset(
|
||||
project_id=project_id,
|
||||
raw_name=item.get("name", "Unknown"),
|
||||
raw_description=item.get("description", ""),
|
||||
volume=item.get("volume", 1),
|
||||
sort_order=idx + 1,
|
||||
)
|
||||
db.add(ca)
|
||||
assets.append(ca)
|
||||
|
||||
project.status = ProjectStatus.REVIEW
|
||||
project.parse_stage = f"Done! {len(assets)} assets extracted."
|
||||
await db.commit()
|
||||
# Stage 3+4: AI parsing runs in background — return 202 immediately
|
||||
background_tasks.add_task(_background_parse, project_id, file.filename, text, metadata)
|
||||
|
||||
return {
|
||||
"message": f"Extracted {len(assets)} assets from {file.filename}",
|
||||
"asset_count": len(assets),
|
||||
"assets": [
|
||||
{"name": a.raw_name, "description": a.raw_description, "volume": a.volume}
|
||||
for a in assets
|
||||
],
|
||||
"message": f"Document received. AI parsing started for {file.filename}.",
|
||||
"status": "parsing",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -139,8 +153,48 @@ async def update_client_asset(
|
|||
return ca
|
||||
|
||||
|
||||
async def _background_match(project_id: int, asset_snapshots: list):
|
||||
"""Run AI matching in the background (own DB session)."""
|
||||
async with async_session() as db:
|
||||
try:
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
return
|
||||
|
||||
# Reconstruct ClientAsset-like objects from snapshots for match_client_assets
|
||||
ca_result = await db.execute(
|
||||
select(ClientAsset).where(ClientAsset.id.in_([s["id"] for s in asset_snapshots]))
|
||||
.order_by(ClientAsset.sort_order)
|
||||
)
|
||||
client_assets = ca_result.scalars().all()
|
||||
|
||||
matches = await match_client_assets(db, project_id, client_assets)
|
||||
|
||||
await db.refresh(project)
|
||||
project.status = ProjectStatus.REVIEW
|
||||
await db.commit()
|
||||
logger.info(f"Background match complete for project {project_id}: {len(matches)} matches")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Background match error for project {project_id}: {e}")
|
||||
try:
|
||||
async with async_session() as db2:
|
||||
result = await db2.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if project:
|
||||
project.status = ProjectStatus.REVIEW
|
||||
await db2.commit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.post("/{project_id}/match")
|
||||
async def run_matching(project_id: int, db: AsyncSession = Depends(get_db)):
|
||||
async def run_matching(
|
||||
project_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Trigger AI matching for all client assets in this project."""
|
||||
project = await _get_project(project_id, db)
|
||||
|
||||
|
|
@ -153,6 +207,9 @@ async def run_matching(project_id: int, db: AsyncSession = Depends(get_db)):
|
|||
if not client_assets:
|
||||
raise HTTPException(status_code=400, detail="No client assets to match. Upload a document first.")
|
||||
|
||||
# Snapshot IDs before clearing (ORM objects expire after commit)
|
||||
asset_snapshots = [{"id": ca.id} for ca in client_assets]
|
||||
|
||||
# Clear existing matches
|
||||
for ca in client_assets:
|
||||
matches_result = await db.execute(select(Match).where(Match.client_asset_id == ca.id))
|
||||
|
|
@ -162,17 +219,12 @@ async def run_matching(project_id: int, db: AsyncSession = Depends(get_db)):
|
|||
project.status = ProjectStatus.MATCHING
|
||||
await db.commit()
|
||||
|
||||
# Run matching (batched, parallel, commits per batch)
|
||||
matches = await match_client_assets(db, project_id, client_assets)
|
||||
|
||||
# Refresh project and set final status
|
||||
await db.refresh(project)
|
||||
project.status = ProjectStatus.REVIEW
|
||||
await db.commit()
|
||||
# Run matching in background — return 202 immediately
|
||||
background_tasks.add_task(_background_match, project_id, asset_snapshots)
|
||||
|
||||
return {
|
||||
"message": f"Matched {len(client_assets)} client assets",
|
||||
"total_matches": len(matches),
|
||||
"message": f"Matching started for {len(client_assets)} client assets.",
|
||||
"status": "matching",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -91,53 +91,64 @@ export default function ProjectView() {
|
|||
setUploading(true);
|
||||
setUploadStage(`Uploading ${file.name}...`);
|
||||
|
||||
// Poll project status for stage updates
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
await api.post(`/projects/${id}/upload`, form);
|
||||
} catch (err: any) {
|
||||
alert(`Upload failed: ${err.response?.data?.detail || err.message}`);
|
||||
setUploading(false);
|
||||
setUploadStage('');
|
||||
return;
|
||||
}
|
||||
|
||||
// Poll until background parsing completes (status leaves 'parsing')
|
||||
const pollInterval = setInterval(async () => {
|
||||
try {
|
||||
const res = await api.get(`/projects/${id}`);
|
||||
if (res.data.parse_stage) {
|
||||
setUploadStage(res.data.parse_stage);
|
||||
}
|
||||
if (res.data.status !== 'parsing') {
|
||||
clearInterval(pollInterval);
|
||||
setUploading(false);
|
||||
setUploadStage('');
|
||||
await loadProject();
|
||||
setTab('matches');
|
||||
}
|
||||
} catch {}
|
||||
}, 1500);
|
||||
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
await api.post(`/projects/${id}/upload`, form);
|
||||
await loadProject();
|
||||
setTab('matches');
|
||||
} catch (err: any) {
|
||||
alert(`Upload failed: ${err.response?.data?.detail || err.message}`);
|
||||
} finally {
|
||||
clearInterval(pollInterval);
|
||||
setUploading(false);
|
||||
setUploadStage('');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleMatch() {
|
||||
setMatching(true);
|
||||
// Start polling for matches while the request runs
|
||||
const pollInterval = setInterval(async () => {
|
||||
try {
|
||||
const matchRes = await api.get(`/projects/${id}/matches`);
|
||||
setMatches(matchRes.data);
|
||||
} catch {}
|
||||
}, 3000);
|
||||
|
||||
try {
|
||||
await api.post(`/projects/${id}/match`);
|
||||
await loadProject();
|
||||
} catch (err: any) {
|
||||
if (!err.message?.includes('cancel')) {
|
||||
alert(`Matching failed: ${err.response?.data?.detail || err.message}`);
|
||||
}
|
||||
await loadProject();
|
||||
} finally {
|
||||
clearInterval(pollInterval);
|
||||
setMatching(false);
|
||||
await loadProject();
|
||||
return;
|
||||
}
|
||||
|
||||
// Poll until background matching completes (status leaves 'matching')
|
||||
const pollInterval = setInterval(async () => {
|
||||
try {
|
||||
const [matchRes, projRes] = await Promise.all([
|
||||
api.get(`/projects/${id}/matches`),
|
||||
api.get(`/projects/${id}`),
|
||||
]);
|
||||
setMatches(matchRes.data);
|
||||
if (projRes.data.status !== 'matching') {
|
||||
clearInterval(pollInterval);
|
||||
setMatching(false);
|
||||
await loadProject();
|
||||
}
|
||||
} catch {}
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
async function handleCancelMatch() {
|
||||
|
|
@ -184,10 +195,32 @@ export default function ProjectView() {
|
|||
});
|
||||
}
|
||||
|
||||
function getExcelExportUrl() {
|
||||
async function downloadFile(url: string, filename: string) {
|
||||
try {
|
||||
const response = await api.get(url, { responseType: 'blob' });
|
||||
const blob = new Blob([response.data], { type: response.headers['content-type'] });
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
} catch (err: any) {
|
||||
alert(`Export failed: ${err.response?.data?.detail || err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function handleExcelExport() {
|
||||
const levels = Array.from(selectedEfficiencyLevels).sort().join(',');
|
||||
const base = `/api/projects/${id}/ratecard/export/excel`;
|
||||
return levels ? `${base}?efficiency_levels=${levels}` : base;
|
||||
const base = `/projects/${id}/ratecard/export/excel`;
|
||||
const url = levels ? `${base}?efficiency_levels=${levels}` : base;
|
||||
downloadFile(url, `${project?.name || 'ratecard'}.xlsx`);
|
||||
}
|
||||
|
||||
function handlePdfExport() {
|
||||
downloadFile(`/projects/${id}/ratecard/export/pdf`, `${project?.name || 'caveats'}_caveats.pdf`);
|
||||
}
|
||||
|
||||
async function handleDelete() {
|
||||
|
|
@ -455,12 +488,12 @@ export default function ProjectView() {
|
|||
<span className="rc-assets">{ratecard.total_assets} assets</span>
|
||||
</div>
|
||||
<div className="rc-exports">
|
||||
<a href={getExcelExportUrl()} className="btn btn-secondary">
|
||||
<button onClick={handleExcelExport} className="btn btn-secondary">
|
||||
Export Excel
|
||||
</a>
|
||||
<a href={`/api/projects/${id}/ratecard/export/pdf`} className="btn btn-secondary">
|
||||
</button>
|
||||
<button onClick={handlePdfExport} className="btn btn-secondary">
|
||||
Export PDF Caveats
|
||||
</a>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -575,12 +608,12 @@ export default function ProjectView() {
|
|||
<div className="rc-header" style={{ marginBottom: 16 }}>
|
||||
<div />
|
||||
<div style={{ display: 'flex', gap: 8 }}>
|
||||
<a href={getExcelExportUrl()} className="btn btn-secondary">
|
||||
<button onClick={handleExcelExport} className="btn btn-secondary">
|
||||
Export Excel {selectedEfficiencyLevels.size > 0 ? `(+${selectedEfficiencyLevels.size} AI tabs)` : ''}
|
||||
</a>
|
||||
<a href={`/api/projects/${id}/ratecard/export/pdf`} className="btn btn-secondary">
|
||||
</button>
|
||||
<button onClick={handlePdfExport} className="btn btn-secondary">
|
||||
Export PDF Caveats
|
||||
</a>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue