Add admin analytics and update OpenAI integration
Backend changes: - Add admin analytics endpoints for daily usage per user - Add GET /tokens/daily-users endpoint with date/user breakdown - Update OpenAI SDK from 1.58.1 to 2.6.1 - Switch from Assistants API to Responses API with file_search tool - Implement strict RAG-only system instructions - Add citation validation to prevent hallucinations - Add get_daily_usage_by_user repository method - Add DailyUserUsage schema for admin analytics Frontend changes: - Implement comprehensive admin usage dashboard - Add overall system statistics (users, conversations, messages, tokens, cost) - Add daily usage table with per-user breakdown - Add chat state clearing on logout and user change for isolation - Center welcome message and input field in chat interface - Add admin-specific styling for usage analytics tables - Fix useCallback dependencies to prevent infinite loops Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
65aa0ae340
commit
2e6597ee08
12 changed files with 843 additions and 157 deletions
|
|
@ -5,12 +5,13 @@ API endpoints for token usage analytics
|
|||
"""
|
||||
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from typing import List
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.services.chat_service import ChatService
|
||||
from app.schemas.token import TokenUsageSummary
|
||||
from app.schemas.token import TokenUsageSummary, UserTokenStats, DailyUserUsage
|
||||
from app.core.middleware import get_current_active_user
|
||||
from app.models.user import User
|
||||
|
||||
|
|
@ -49,3 +50,75 @@ async def get_token_usage(
|
|||
)
|
||||
|
||||
return TokenUsageSummary(**summary)
|
||||
|
||||
|
||||
@router.get("/users", response_model=List[UserTokenStats])
|
||||
async def get_all_users_usage(
|
||||
days: int = Query(30, ge=1, le=365, description="Number of days to retrieve"),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get token usage for all users (admin only)
|
||||
|
||||
Args:
|
||||
days: Number of days to include in summary
|
||||
current_user: Current authenticated user (must be admin)
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of UserTokenStats with usage per user
|
||||
|
||||
Raises:
|
||||
HTTPException: If user is not admin
|
||||
"""
|
||||
# Check if user is admin
|
||||
if current_user.role not in ['admin', 'superadmin']:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Admin access required"
|
||||
)
|
||||
|
||||
chat_service = ChatService(db)
|
||||
|
||||
users_stats = await chat_service.get_all_users_token_usage(days=days)
|
||||
|
||||
logger.info(f"Admin {current_user.id} retrieved usage for {len(users_stats)} users")
|
||||
|
||||
return [UserTokenStats(**stats) for stats in users_stats]
|
||||
|
||||
|
||||
@router.get("/daily-users", response_model=List[DailyUserUsage])
|
||||
async def get_daily_usage_by_user(
|
||||
days: int = Query(30, ge=1, le=365, description="Number of days to retrieve"),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Get daily token usage breakdown by user (admin only)
|
||||
|
||||
Args:
|
||||
days: Number of days to include
|
||||
current_user: Current authenticated user (must be admin)
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of DailyUserUsage with usage per user per day
|
||||
|
||||
Raises:
|
||||
HTTPException: If user is not admin
|
||||
"""
|
||||
# Check if user is admin
|
||||
if current_user.role not in ['admin', 'superadmin']:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Admin access required"
|
||||
)
|
||||
|
||||
chat_service = ChatService(db)
|
||||
|
||||
daily_stats = await chat_service.get_daily_usage_by_user(days=days)
|
||||
|
||||
logger.info(f"Admin {current_user.id} retrieved daily usage: {len(daily_stats)} records")
|
||||
|
||||
return [DailyUserUsage(**stats) for stats in daily_stats]
|
||||
|
|
|
|||
|
|
@ -181,3 +181,135 @@ class TokenUsageRepository(BaseRepository[TokenUsage]):
|
|||
}
|
||||
for row in result
|
||||
]
|
||||
|
||||
async def get_all_users_stats(
|
||||
self,
|
||||
days: Optional[int] = None
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Get token usage statistics for all users (admin only)
|
||||
|
||||
Args:
|
||||
days: Optional filter for last N days
|
||||
|
||||
Returns:
|
||||
List of dicts with per-user statistics
|
||||
"""
|
||||
from app.models.user import User
|
||||
from app.models.conversation import Conversation
|
||||
from app.models.message import Message
|
||||
|
||||
# Build base query with user join
|
||||
query = select(
|
||||
User.id.label("user_id"),
|
||||
User.email.label("user_email"),
|
||||
User.display_name.label("user_name"),
|
||||
func.count(func.distinct(TokenUsage.conversation_id)).label("conversation_count"),
|
||||
func.count(TokenUsage.id).label("message_count"),
|
||||
func.sum(TokenUsage.total_tokens).label("total_tokens"),
|
||||
func.sum(TokenUsage.prompt_tokens).label("prompt_tokens"),
|
||||
func.sum(TokenUsage.completion_tokens).label("completion_tokens"),
|
||||
func.sum(TokenUsage.cached_tokens).label("cached_tokens"),
|
||||
func.sum(TokenUsage.cost_usd).label("total_cost_usd"),
|
||||
func.max(TokenUsage.created_at).label("last_activity")
|
||||
).select_from(User).join(
|
||||
TokenUsage,
|
||||
TokenUsage.user_id == User.id,
|
||||
isouter=False
|
||||
)
|
||||
|
||||
if days:
|
||||
since = datetime.utcnow() - timedelta(days=days)
|
||||
query = query.where(TokenUsage.created_at >= since)
|
||||
|
||||
query = query.group_by(User.id, User.email, User.display_name)
|
||||
query = query.order_by(func.sum(TokenUsage.total_tokens).desc())
|
||||
|
||||
result = await self.session.execute(query)
|
||||
|
||||
users_stats = []
|
||||
for row in result:
|
||||
total_tokens = int(row.total_tokens or 0)
|
||||
message_count = int(row.message_count or 0)
|
||||
avg_tokens = total_tokens / message_count if message_count > 0 else 0
|
||||
|
||||
users_stats.append({
|
||||
"user_id": str(row.user_id),
|
||||
"user_email": row.user_email,
|
||||
"user_name": row.user_name,
|
||||
"total_tokens": total_tokens,
|
||||
"prompt_tokens": int(row.prompt_tokens or 0),
|
||||
"completion_tokens": int(row.completion_tokens or 0),
|
||||
"cached_tokens": int(row.cached_tokens or 0),
|
||||
"total_cost_usd": float(row.total_cost_usd or 0.0),
|
||||
"message_count": message_count,
|
||||
"conversation_count": int(row.conversation_count or 0),
|
||||
"avg_tokens_per_message": round(avg_tokens, 1),
|
||||
"last_activity": row.last_activity.isoformat() if row.last_activity else None
|
||||
})
|
||||
|
||||
return users_stats
|
||||
|
||||
async def get_daily_usage_by_user(
|
||||
self,
|
||||
days: Optional[int] = None
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Get daily token usage breakdown by user (admin only)
|
||||
|
||||
Args:
|
||||
days: Optional filter for last N days
|
||||
|
||||
Returns:
|
||||
List of dicts with daily usage per user
|
||||
"""
|
||||
from app.models.user import User
|
||||
|
||||
# Build query grouped by date and user
|
||||
query = select(
|
||||
func.date(TokenUsage.created_at).label("date"),
|
||||
User.id.label("user_id"),
|
||||
User.email.label("user_email"),
|
||||
User.display_name.label("user_name"),
|
||||
func.count(TokenUsage.id).label("message_count"),
|
||||
func.sum(TokenUsage.total_tokens).label("total_tokens"),
|
||||
func.sum(TokenUsage.prompt_tokens).label("prompt_tokens"),
|
||||
func.sum(TokenUsage.completion_tokens).label("completion_tokens"),
|
||||
func.sum(TokenUsage.cached_tokens).label("cached_tokens"),
|
||||
func.sum(TokenUsage.cost_usd).label("total_cost_usd")
|
||||
).select_from(User).join(
|
||||
TokenUsage,
|
||||
TokenUsage.user_id == User.id,
|
||||
isouter=False
|
||||
)
|
||||
|
||||
if days:
|
||||
since = datetime.utcnow() - timedelta(days=days)
|
||||
query = query.where(TokenUsage.created_at >= since)
|
||||
|
||||
query = query.group_by(
|
||||
func.date(TokenUsage.created_at),
|
||||
User.id,
|
||||
User.email,
|
||||
User.display_name
|
||||
)
|
||||
query = query.order_by(func.date(TokenUsage.created_at).desc())
|
||||
|
||||
result = await self.session.execute(query)
|
||||
|
||||
daily_stats = []
|
||||
for row in result:
|
||||
daily_stats.append({
|
||||
"date": row.date.isoformat() if row.date else None,
|
||||
"user_id": str(row.user_id),
|
||||
"user_email": row.user_email,
|
||||
"user_name": row.user_name,
|
||||
"total_tokens": int(row.total_tokens or 0),
|
||||
"prompt_tokens": int(row.prompt_tokens or 0),
|
||||
"completion_tokens": int(row.completion_tokens or 0),
|
||||
"cached_tokens": int(row.cached_tokens or 0),
|
||||
"total_cost_usd": float(row.total_cost_usd or 0.0),
|
||||
"message_count": int(row.message_count or 0)
|
||||
})
|
||||
|
||||
return daily_stats
|
||||
|
|
|
|||
|
|
@ -21,3 +21,33 @@ class DailyUsage(BaseModel):
|
|||
date: str
|
||||
tokens: int
|
||||
cost: float
|
||||
|
||||
|
||||
class UserTokenStats(BaseModel):
|
||||
"""Token usage statistics per user (for admin)"""
|
||||
user_id: str
|
||||
user_email: str
|
||||
user_name: str
|
||||
total_tokens: int
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
cached_tokens: int
|
||||
total_cost_usd: float
|
||||
message_count: int
|
||||
conversation_count: int
|
||||
avg_tokens_per_message: float
|
||||
last_activity: Optional[str] = None
|
||||
|
||||
|
||||
class DailyUserUsage(BaseModel):
|
||||
"""Daily token usage per user (for admin)"""
|
||||
date: str
|
||||
user_id: str
|
||||
user_email: str
|
||||
user_name: str
|
||||
total_tokens: int
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
cached_tokens: int
|
||||
total_cost_usd: float
|
||||
message_count: int
|
||||
|
|
|
|||
|
|
@ -380,6 +380,36 @@ class ChatService:
|
|||
"period_days": days
|
||||
}
|
||||
|
||||
async def get_all_users_token_usage(
|
||||
self,
|
||||
days: Optional[int] = None
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Get token usage for all users (admin only)
|
||||
|
||||
Args:
|
||||
days: Optional filter for last N days
|
||||
|
||||
Returns:
|
||||
List of dicts with per-user statistics
|
||||
"""
|
||||
return await self.token_repo.get_all_users_stats(days)
|
||||
|
||||
async def get_daily_usage_by_user(
|
||||
self,
|
||||
days: Optional[int] = None
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Get daily token usage breakdown by user (admin only)
|
||||
|
||||
Args:
|
||||
days: Optional filter for last N days
|
||||
|
||||
Returns:
|
||||
List of dicts with daily usage per user
|
||||
"""
|
||||
return await self.token_repo.get_daily_usage_by_user(days)
|
||||
|
||||
def _estimate_tokens(self, text: str) -> int:
|
||||
"""
|
||||
Estimate token count for text (rough approximation)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
OpenAI Assistants API Service with RAG
|
||||
OpenAI Responses API Service with RAG
|
||||
|
||||
Integrates with OpenAI Assistants API using file_search tool for strict RAG-only responses.
|
||||
Integrates with OpenAI Responses API using file_search tool for strict RAG-only responses.
|
||||
Uses Vector Store to retrieve relevant documents before generating responses.
|
||||
"""
|
||||
|
||||
|
|
@ -17,13 +17,13 @@ settings = get_settings()
|
|||
|
||||
class OpenAIService:
|
||||
"""
|
||||
Service for OpenAI Assistants API integration with file_search tool.
|
||||
Service for OpenAI Responses API integration with file_search tool.
|
||||
|
||||
Key features:
|
||||
- Uses Assistants API with file_search tool for RAG
|
||||
- Uses Responses API with file_search tool for RAG
|
||||
- Connects to Vector Store for document retrieval
|
||||
- Enforces strict RAG-only responses
|
||||
- Creates threads for multi-turn conversations
|
||||
- Uses previous_response_id for multi-turn conversations
|
||||
"""
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
|
|
@ -38,7 +38,6 @@ class OpenAIService:
|
|||
self.async_client = AsyncOpenAI(api_key=self.api_key)
|
||||
self.model = settings.OPENAI_MODEL
|
||||
self.vector_store_id = settings.OPENAI_VECTOR_STORE_ID
|
||||
self.assistant_id = None # Will be created on first use
|
||||
|
||||
def get_system_instructions(self) -> str:
|
||||
"""
|
||||
|
|
@ -81,57 +80,21 @@ RESPONSE FORMAT:
|
|||
|
||||
Remember: When in doubt, DON'T answer. Redirect to HR or explain your scope."""
|
||||
|
||||
async def _get_or_create_assistant(self) -> str:
|
||||
"""
|
||||
Get existing assistant or create new one with file_search tool.
|
||||
|
||||
Returns:
|
||||
Assistant ID
|
||||
"""
|
||||
if self.assistant_id:
|
||||
return self.assistant_id
|
||||
|
||||
try:
|
||||
# Create assistant with file_search tool
|
||||
assistant = await self.async_client.beta.assistants.create(
|
||||
name="Seapac Ops Bot",
|
||||
instructions=self.get_system_instructions(),
|
||||
model=self.model,
|
||||
tools=[
|
||||
{
|
||||
"type": "file_search"
|
||||
}
|
||||
],
|
||||
tool_resources={
|
||||
"file_search": {
|
||||
"vector_store_ids": [self.vector_store_id]
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
self.assistant_id = assistant.id
|
||||
logger.info(f"Created assistant: {self.assistant_id}")
|
||||
return self.assistant_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create assistant: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
async def generate_response(
|
||||
self,
|
||||
user_message: str,
|
||||
previous_response_id: Optional[str] = None
|
||||
) -> Dict:
|
||||
"""
|
||||
Generate bot response using Assistants API with file_search.
|
||||
Generate bot response using Responses API with file_search.
|
||||
|
||||
Args:
|
||||
user_message: User's input message
|
||||
previous_response_id: Thread ID from previous conversation (for multi-turn)
|
||||
previous_response_id: Response ID from previous turn (for multi-turn)
|
||||
|
||||
Returns:
|
||||
Dict containing:
|
||||
- response_id: Thread ID for next turn
|
||||
- response_id: Response ID for next turn
|
||||
- content: Assistant's response text
|
||||
- usage: Token usage statistics
|
||||
- file_search_results: List of retrieved documents
|
||||
|
|
@ -142,93 +105,173 @@ Remember: When in doubt, DON'T answer. Redirect to HR or explain your scope."""
|
|||
Exception: If API call fails
|
||||
"""
|
||||
try:
|
||||
# Get or create assistant
|
||||
assistant_id = await self._get_or_create_assistant()
|
||||
# Build request payload
|
||||
request_payload = {
|
||||
"model": self.model,
|
||||
"input": user_message,
|
||||
"instructions": self.get_system_instructions(),
|
||||
"tools": [
|
||||
{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": [self.vector_store_id],
|
||||
"max_num_results": 20
|
||||
}
|
||||
],
|
||||
"store": True, # Store for conversation history
|
||||
}
|
||||
|
||||
# Create or use existing thread
|
||||
# Add previous response for multi-turn conversation
|
||||
if previous_response_id:
|
||||
thread_id = previous_response_id
|
||||
logger.info(f"Using existing thread: {thread_id}")
|
||||
request_payload["previous_response_id"] = previous_response_id
|
||||
logger.info(f"Using previous_response_id: {previous_response_id}")
|
||||
else:
|
||||
thread = await self.async_client.beta.threads.create()
|
||||
thread_id = thread.id
|
||||
logger.info(f"Created new thread: {thread_id}")
|
||||
logger.info("Creating new conversation")
|
||||
|
||||
# Add user message to thread
|
||||
await self.async_client.beta.threads.messages.create(
|
||||
thread_id=thread_id,
|
||||
role="user",
|
||||
content=user_message
|
||||
)
|
||||
# Call Responses API
|
||||
response = await self.async_client.responses.create(**request_payload)
|
||||
|
||||
# Run assistant
|
||||
run = await self.async_client.beta.threads.runs.create(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id
|
||||
)
|
||||
# Parse response
|
||||
parsed = self._parse_response(response)
|
||||
|
||||
# Wait for completion
|
||||
while run.status in ["queued", "in_progress"]:
|
||||
import asyncio
|
||||
await asyncio.sleep(0.5)
|
||||
run = await self.async_client.beta.threads.runs.retrieve(
|
||||
thread_id=thread_id,
|
||||
run_id=run.id
|
||||
)
|
||||
|
||||
if run.status != "completed":
|
||||
logger.error(f"Run failed with status: {run.status}")
|
||||
raise Exception(f"Assistant run failed: {run.status}")
|
||||
|
||||
# Get messages
|
||||
messages = await self.async_client.beta.threads.messages.list(
|
||||
thread_id=thread_id,
|
||||
limit=1,
|
||||
order="desc"
|
||||
)
|
||||
|
||||
assistant_message = messages.data[0]
|
||||
content = assistant_message.content[0].text.value
|
||||
|
||||
# Get usage from run
|
||||
usage = {
|
||||
"prompt_tokens": run.usage.prompt_tokens if run.usage else 0,
|
||||
"completion_tokens": run.usage.completion_tokens if run.usage else 0,
|
||||
"total_tokens": run.usage.total_tokens if run.usage else 0,
|
||||
"cached_tokens": 0
|
||||
}
|
||||
|
||||
# Parse file citations if present
|
||||
file_search_results = []
|
||||
if assistant_message.content[0].text.annotations:
|
||||
for annotation in assistant_message.content[0].text.annotations:
|
||||
if annotation.type == "file_citation":
|
||||
file_search_results.append({
|
||||
"file_id": annotation.file_citation.file_id,
|
||||
"quote": annotation.text
|
||||
})
|
||||
|
||||
result = {
|
||||
"response_id": thread_id,
|
||||
"content": content,
|
||||
"usage": usage,
|
||||
"file_search_results": file_search_results,
|
||||
"has_citations": len(file_search_results) > 0 or self._check_valid_response(content),
|
||||
"status": "completed"
|
||||
}
|
||||
# Validate RAG usage
|
||||
self._validate_rag_usage(parsed)
|
||||
|
||||
logger.info(
|
||||
f"Generated response in thread {thread_id}: "
|
||||
f"{usage['total_tokens']} tokens, "
|
||||
f"{len(file_search_results)} citations"
|
||||
f"Generated response {parsed['response_id']}: "
|
||||
f"{parsed['usage']['total_tokens']} tokens, "
|
||||
f"{len(parsed['file_search_results'])} search results"
|
||||
)
|
||||
|
||||
return result
|
||||
return parsed
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OpenAI Assistants API call failed: {e}", exc_info=True)
|
||||
logger.error(f"OpenAI Responses API call failed: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
def _parse_response(self, response) -> Dict:
|
||||
"""
|
||||
Parse Responses API response.
|
||||
|
||||
Args:
|
||||
response: Raw response from Responses API
|
||||
|
||||
Returns:
|
||||
Parsed response dictionary
|
||||
"""
|
||||
response_id = response.id
|
||||
status = getattr(response, 'status', 'completed')
|
||||
usage = getattr(response, 'usage', None)
|
||||
|
||||
# Extract assistant message and file search results
|
||||
assistant_message = None
|
||||
file_search_results = []
|
||||
|
||||
for output_item in response.output:
|
||||
# Get assistant message
|
||||
if output_item.type == "message" and output_item.role == "assistant":
|
||||
for content_block in output_item.content:
|
||||
if content_block.type == "output_text":
|
||||
assistant_message = content_block.text
|
||||
|
||||
# Get file search results
|
||||
if output_item.type == "file_search_call":
|
||||
if hasattr(output_item, 'results') and output_item.results:
|
||||
file_search_results = self._format_search_results(
|
||||
output_item.results
|
||||
)
|
||||
|
||||
return {
|
||||
"response_id": response_id,
|
||||
"content": assistant_message or "",
|
||||
"usage": {
|
||||
"prompt_tokens": usage.input_tokens if usage else 0,
|
||||
"completion_tokens": usage.output_tokens if usage else 0,
|
||||
"total_tokens": usage.total_tokens if usage else 0,
|
||||
"cached_tokens": getattr(usage, 'cached_tokens', 0) if usage else 0
|
||||
},
|
||||
"file_search_results": file_search_results,
|
||||
"has_citations": self._check_citations(assistant_message, file_search_results),
|
||||
"status": status
|
||||
}
|
||||
|
||||
def _format_search_results(self, results: List) -> List[Dict]:
|
||||
"""
|
||||
Format file search results for storage/display.
|
||||
|
||||
Args:
|
||||
results: Raw search results from file_search_call
|
||||
|
||||
Returns:
|
||||
Formatted search results list
|
||||
"""
|
||||
formatted = []
|
||||
|
||||
for result in results:
|
||||
formatted.append({
|
||||
"file_id": getattr(result, "file_id", None),
|
||||
"filename": getattr(result, "filename", "Unknown"),
|
||||
"content_snippet": getattr(result, "content", "")[:200],
|
||||
"score": getattr(result, "score", 0.0)
|
||||
})
|
||||
|
||||
return formatted
|
||||
|
||||
def _check_citations(self, message: Optional[str], search_results: List[Dict]) -> bool:
|
||||
"""
|
||||
Check if response includes citations or is a valid "no info" response.
|
||||
|
||||
Args:
|
||||
message: Response content
|
||||
search_results: File search results
|
||||
|
||||
Returns:
|
||||
True if response is valid (has citations or properly says no info)
|
||||
"""
|
||||
if not message:
|
||||
return False
|
||||
|
||||
# If we have search results, the response should reference them
|
||||
if len(search_results) > 0:
|
||||
citation_keywords = [
|
||||
"according to",
|
||||
"source:",
|
||||
"document",
|
||||
"as stated in",
|
||||
"refers to",
|
||||
"based on"
|
||||
]
|
||||
return any(kw in message.lower() for kw in citation_keywords)
|
||||
|
||||
# If no search results, check for valid "no info" response
|
||||
return self._check_valid_response(message)
|
||||
|
||||
def _validate_rag_usage(self, parsed_response: Dict) -> None:
|
||||
"""
|
||||
Validate that response uses RAG properly.
|
||||
|
||||
Args:
|
||||
parsed_response: Parsed response dictionary
|
||||
"""
|
||||
content = parsed_response["content"]
|
||||
has_citations = parsed_response["has_citations"]
|
||||
has_search_results = len(parsed_response["file_search_results"]) > 0
|
||||
|
||||
# Check for potential hallucination
|
||||
is_no_info_response = self._check_valid_response(content)
|
||||
|
||||
if not has_citations and not is_no_info_response:
|
||||
logger.warning(
|
||||
f"POTENTIAL HALLUCINATION: Response without citations\n"
|
||||
f"Response ID: {parsed_response['response_id']}\n"
|
||||
f"Response: {content[:100]}...\n"
|
||||
f"Has search results: {has_search_results}"
|
||||
)
|
||||
|
||||
# Add disclaimer (modify content in parsed_response)
|
||||
parsed_response["content"] += (
|
||||
"\n\n⚠️ Note: This response may not be fully verified against documents."
|
||||
)
|
||||
|
||||
def _check_valid_response(self, content: str) -> bool:
|
||||
"""
|
||||
Check if response is valid (either has info or properly says no info).
|
||||
|
|
@ -260,37 +303,89 @@ Remember: When in doubt, DON'T answer. Redirect to HR or explain your scope."""
|
|||
on_chunk_callback: Optional[Callable] = None
|
||||
) -> AsyncIterator[Dict]:
|
||||
"""
|
||||
Stream response from Assistants API.
|
||||
|
||||
Note: Assistants API streaming is complex, using regular response for now.
|
||||
Stream response from Responses API.
|
||||
|
||||
Args:
|
||||
user_message: User's input message
|
||||
previous_response_id: Thread ID from previous turn
|
||||
previous_response_id: Response ID from previous turn
|
||||
on_chunk_callback: Optional callback for each text chunk
|
||||
|
||||
Yields:
|
||||
Dicts containing text chunks and metadata
|
||||
"""
|
||||
# For now, use non-streaming and yield complete response
|
||||
# TODO: Implement proper streaming with Assistants API
|
||||
result = await self.generate_response(user_message, previous_response_id)
|
||||
try:
|
||||
# Build request payload
|
||||
request_payload = {
|
||||
"model": self.model,
|
||||
"input": user_message,
|
||||
"instructions": self.get_system_instructions(),
|
||||
"tools": [
|
||||
{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": [self.vector_store_id],
|
||||
"max_num_results": 20
|
||||
}
|
||||
],
|
||||
"stream": True, # Enable streaming
|
||||
}
|
||||
|
||||
# Yield as single chunk
|
||||
yield {
|
||||
"type": "chunk",
|
||||
"content": result["content"]
|
||||
}
|
||||
if previous_response_id:
|
||||
request_payload["previous_response_id"] = previous_response_id
|
||||
|
||||
yield {
|
||||
"type": "complete",
|
||||
"response_id": result["response_id"],
|
||||
"content": result["content"]
|
||||
}
|
||||
# Stream response
|
||||
full_message = ""
|
||||
response_id = None
|
||||
usage_data = None
|
||||
|
||||
stream = await self.async_client.responses.create(**request_payload)
|
||||
|
||||
async for chunk in stream:
|
||||
if chunk.id:
|
||||
response_id = chunk.id
|
||||
|
||||
# Extract delta text
|
||||
if hasattr(chunk, 'output'):
|
||||
for output_item in chunk.output:
|
||||
if output_item.type == "message":
|
||||
for content_block in output_item.content:
|
||||
if content_block.type == "output_text":
|
||||
delta_text = content_block.text
|
||||
full_message += delta_text
|
||||
|
||||
# Yield chunk
|
||||
yield {
|
||||
"type": "chunk",
|
||||
"content": delta_text
|
||||
}
|
||||
|
||||
# Call callback for real-time updates (WebSocket)
|
||||
if on_chunk_callback:
|
||||
await on_chunk_callback(delta_text)
|
||||
|
||||
# Capture usage data
|
||||
if hasattr(chunk, 'usage') and chunk.usage:
|
||||
usage_data = chunk.usage
|
||||
|
||||
# Yield completion
|
||||
yield {
|
||||
"type": "complete",
|
||||
"response_id": response_id,
|
||||
"content": full_message,
|
||||
"usage": {
|
||||
"prompt_tokens": usage_data.input_tokens if usage_data else 0,
|
||||
"completion_tokens": usage_data.output_tokens if usage_data else 0,
|
||||
"total_tokens": usage_data.total_tokens if usage_data else 0,
|
||||
"cached_tokens": getattr(usage_data, 'cached_tokens', 0) if usage_data else 0
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OpenAI Responses API streaming failed: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
async def test_connection(self) -> bool:
|
||||
"""
|
||||
Test connection to OpenAI API and Vector Store.
|
||||
Test connection to OpenAI Responses API and Vector Store.
|
||||
|
||||
Returns:
|
||||
True if connection successful
|
||||
|
|
@ -299,15 +394,20 @@ Remember: When in doubt, DON'T answer. Redirect to HR or explain your scope."""
|
|||
Exception: If connection fails
|
||||
"""
|
||||
try:
|
||||
# Test simple API call
|
||||
# Test simple API call with Vector Store
|
||||
response = await self.generate_response(
|
||||
"Hello, can you help me?",
|
||||
"Hello, can you help me with Oliver Agency APAC operations?",
|
||||
previous_response_id=None
|
||||
)
|
||||
|
||||
logger.info(f"OpenAI connection test successful: {response['response_id']}")
|
||||
logger.info(
|
||||
f"OpenAI Responses API connection test successful\n"
|
||||
f"Response ID: {response['response_id']}\n"
|
||||
f"Has search results: {len(response['file_search_results']) > 0}\n"
|
||||
f"Response: {response['content'][:100]}..."
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OpenAI connection test failed: {e}", exc_info=True)
|
||||
logger.error(f"OpenAI Responses API connection test failed: {e}", exc_info=True)
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ pyjwt==2.8.0
|
|||
httpx==0.26.0
|
||||
|
||||
# OpenAI
|
||||
openai==1.58.1
|
||||
openai==2.6.1
|
||||
|
||||
# Redis
|
||||
redis==5.0.1
|
||||
|
|
|
|||
|
|
@ -15,18 +15,36 @@ import AdminPanel from './components/AdminPanel';
|
|||
|
||||
const AppContent: React.FC = () => {
|
||||
const { isAuthenticated, isLoading, user, logout, login } = useAuth();
|
||||
const { loadConversations } = useChat();
|
||||
const { loadConversations, clearState } = useChat();
|
||||
const [activeView, setActiveView] = useState<'chat' | 'usage' | 'admin'>('chat');
|
||||
const [sidebarOpen, setSidebarOpen] = useState(true);
|
||||
const [prevUserId, setPrevUserId] = useState<string | null>(null);
|
||||
|
||||
const isAdmin = user?.role === 'admin' || user?.role === 'superadmin';
|
||||
|
||||
// Clear chat state when user logs out
|
||||
useEffect(() => {
|
||||
if (isAuthenticated) {
|
||||
// Load conversations when user is authenticated
|
||||
loadConversations();
|
||||
if (!isAuthenticated) {
|
||||
clearState();
|
||||
setPrevUserId(null);
|
||||
}
|
||||
}, [isAuthenticated, loadConversations]);
|
||||
}, [isAuthenticated, clearState]);
|
||||
|
||||
// Load conversations when user is authenticated
|
||||
// Clear and reload if user changed (different account logged in)
|
||||
useEffect(() => {
|
||||
if (isAuthenticated && user) {
|
||||
// If user changed, clear old state first
|
||||
if (prevUserId && prevUserId !== user.id) {
|
||||
console.log('User changed, clearing old chat state');
|
||||
clearState();
|
||||
}
|
||||
|
||||
// Load conversations for current user
|
||||
loadConversations();
|
||||
setPrevUserId(user.id);
|
||||
}
|
||||
}, [isAuthenticated, user, loadConversations, clearState, prevUserId]);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
|
|
@ -85,7 +103,13 @@ const AppContent: React.FC = () => {
|
|||
)}
|
||||
<div className="user-info">
|
||||
<span>👤 {user?.display_name} <span className="user-role">({user?.role})</span></span>
|
||||
<button onClick={logout} className="btn-logout">
|
||||
<button
|
||||
onClick={() => {
|
||||
clearState();
|
||||
logout();
|
||||
}}
|
||||
className="btn-logout"
|
||||
>
|
||||
Logout
|
||||
</button>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@
|
|||
* Token Usage Dashboard Component
|
||||
*
|
||||
* Displays token usage statistics and analytics
|
||||
* Different views for regular users vs admins
|
||||
*/
|
||||
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
import { tokenAPI } from '../services/api';
|
||||
import { useAuth } from '../context/AuthContext';
|
||||
|
||||
interface DailyUsage {
|
||||
date: string;
|
||||
|
|
@ -20,29 +22,82 @@ interface UsageSummary {
|
|||
period_days: number;
|
||||
}
|
||||
|
||||
interface DailyUserUsage {
|
||||
date: string;
|
||||
user_id: string;
|
||||
user_email: string;
|
||||
user_name: string;
|
||||
total_tokens: number;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
cached_tokens: number;
|
||||
total_cost_usd: number;
|
||||
message_count: number;
|
||||
}
|
||||
|
||||
interface UserTokenStats {
|
||||
user_id: string;
|
||||
user_email: string;
|
||||
user_name: string;
|
||||
total_tokens: number;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
cached_tokens: number;
|
||||
total_cost_usd: number;
|
||||
message_count: number;
|
||||
conversation_count: number;
|
||||
avg_tokens_per_message: number;
|
||||
last_activity: string | null;
|
||||
}
|
||||
|
||||
const TokenUsageDashboard: React.FC = () => {
|
||||
const { user } = useAuth();
|
||||
const [usage, setUsage] = useState<UsageSummary | null>(null);
|
||||
const [dailyUserUsage, setDailyUserUsage] = useState<DailyUserUsage[]>([]);
|
||||
const [usersStats, setUsersStats] = useState<UserTokenStats[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [days, setDays] = useState(30);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
loadUsage();
|
||||
}, [days]);
|
||||
const isAdmin = user?.role === 'admin' || user?.role === 'superadmin';
|
||||
|
||||
const loadUsage = async () => {
|
||||
const loadUsage = useCallback(async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
const response = await tokenAPI.getUsage(days);
|
||||
setUsage(response.data);
|
||||
if (isAdmin) {
|
||||
// Load admin analytics
|
||||
const [usageResponse, dailyResponse, statsResponse] = await Promise.all([
|
||||
tokenAPI.getUsage(days),
|
||||
tokenAPI.getDailyUsageByUser(days),
|
||||
tokenAPI.getUsersUsage(days)
|
||||
]);
|
||||
|
||||
setUsage(usageResponse.data);
|
||||
setDailyUserUsage(dailyResponse.data);
|
||||
setUsersStats(statsResponse.data);
|
||||
} else {
|
||||
// Load regular user usage
|
||||
const response = await tokenAPI.getUsage(days);
|
||||
setUsage(response.data);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error('Failed to load token usage:', err);
|
||||
setError('Failed to load token usage');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, [days, isAdmin]);
|
||||
|
||||
useEffect(() => {
|
||||
loadUsage();
|
||||
}, [loadUsage]);
|
||||
|
||||
const handleViewUserDetails = (userId: string) => {
|
||||
// TODO: Navigate to user details page (to be implemented)
|
||||
console.log('View details for user:', userId);
|
||||
alert('User details view coming soon!');
|
||||
};
|
||||
|
||||
if (isLoading) {
|
||||
|
|
@ -61,7 +116,119 @@ const TokenUsageDashboard: React.FC = () => {
|
|||
);
|
||||
}
|
||||
|
||||
// Get last 7 days for chart
|
||||
// Admin view
|
||||
if (isAdmin) {
|
||||
// Calculate overall statistics
|
||||
const totalSystemTokens = usersStats.reduce((sum, u) => sum + u.total_tokens, 0);
|
||||
const totalSystemCost = usersStats.reduce((sum, u) => sum + u.total_cost_usd, 0);
|
||||
const totalUsers = usersStats.length;
|
||||
const totalConversations = usersStats.reduce((sum, u) => sum + u.conversation_count, 0);
|
||||
const totalMessages = usersStats.reduce((sum, u) => sum + u.message_count, 0);
|
||||
|
||||
return (
|
||||
<div className="token-dashboard">
|
||||
<div className="token-dashboard-header">
|
||||
<h2>📊 Usage Analytics (Admin)</h2>
|
||||
<div className="token-dashboard-period">
|
||||
<button
|
||||
className={days === 7 ? 'active' : ''}
|
||||
onClick={() => setDays(7)}
|
||||
>
|
||||
7 Days
|
||||
</button>
|
||||
<button
|
||||
className={days === 30 ? 'active' : ''}
|
||||
onClick={() => setDays(30)}
|
||||
>
|
||||
30 Days
|
||||
</button>
|
||||
<button
|
||||
className={days === 90 ? 'active' : ''}
|
||||
onClick={() => setDays(90)}
|
||||
>
|
||||
90 Days
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Overall System Statistics */}
|
||||
<div className="admin-stats-section">
|
||||
<h3>Overall System Statistics</h3>
|
||||
<div className="admin-stats-grid">
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Total Users</div>
|
||||
<div className="admin-stat-value">{totalUsers}</div>
|
||||
</div>
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Total Conversations</div>
|
||||
<div className="admin-stat-value">{totalConversations.toLocaleString()}</div>
|
||||
</div>
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Total Messages</div>
|
||||
<div className="admin-stat-value">{totalMessages.toLocaleString()}</div>
|
||||
</div>
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Total Tokens Used</div>
|
||||
<div className="admin-stat-value">{totalSystemTokens.toLocaleString()}</div>
|
||||
</div>
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Total Cost</div>
|
||||
<div className="admin-stat-value">${totalSystemCost.toFixed(4)}</div>
|
||||
</div>
|
||||
<div className="admin-stat-card">
|
||||
<div className="admin-stat-label">Avg Cost per User</div>
|
||||
<div className="admin-stat-value">
|
||||
${totalUsers > 0 ? (totalSystemCost / totalUsers).toFixed(4) : '0.0000'}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Daily Usage by User Table */}
|
||||
<div className="admin-usage-table-section">
|
||||
<h3>Daily Usage by User</h3>
|
||||
<div className="admin-usage-table-wrapper">
|
||||
<table className="admin-usage-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Date</th>
|
||||
<th>User Name</th>
|
||||
<th>Email</th>
|
||||
<th>Total Tokens</th>
|
||||
<th>Total Cost</th>
|
||||
<th>Messages</th>
|
||||
<th>Details</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{dailyUserUsage.map((record, idx) => (
|
||||
<tr key={`${record.date}-${record.user_id}-${idx}`}>
|
||||
<td>{new Date(record.date).toLocaleDateString()}</td>
|
||||
<td>{record.user_name}</td>
|
||||
<td>{record.user_email}</td>
|
||||
<td>{record.total_tokens.toLocaleString()}</td>
|
||||
<td>${record.total_cost_usd.toFixed(4)}</td>
|
||||
<td>{record.message_count}</td>
|
||||
<td>
|
||||
<button
|
||||
className="btn-details"
|
||||
onClick={() => handleViewUserDetails(record.user_id)}
|
||||
title="View user details"
|
||||
>
|
||||
👁️ View
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Regular user view
|
||||
const recentData = usage.daily_breakdown.slice(-7);
|
||||
const maxTokens = Math.max(...recentData.map((d) => d.tokens), 1);
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ interface ChatContextType {
|
|||
updateConversationTitle: (id: string, title: string) => Promise<void>;
|
||||
deleteConversation: (id: string) => Promise<void>;
|
||||
archiveConversation: (id: string) => Promise<void>;
|
||||
clearState: () => void;
|
||||
}
|
||||
|
||||
const ChatContext = createContext<ChatContextType | undefined>(undefined);
|
||||
|
|
@ -211,6 +212,14 @@ export const ChatProvider: React.FC<{ children: ReactNode }> = ({ children }) =>
|
|||
}
|
||||
}, [currentConversation]);
|
||||
|
||||
const clearState = useCallback(() => {
|
||||
setConversations([]);
|
||||
setCurrentConversation(null);
|
||||
setMessages([]);
|
||||
setError(null);
|
||||
console.log('Chat state cleared');
|
||||
}, []);
|
||||
|
||||
const value: ChatContextType = {
|
||||
conversations,
|
||||
currentConversation,
|
||||
|
|
@ -225,6 +234,7 @@ export const ChatProvider: React.FC<{ children: ReactNode }> = ({ children }) =>
|
|||
updateConversationTitle,
|
||||
deleteConversation,
|
||||
archiveConversation,
|
||||
clearState,
|
||||
};
|
||||
|
||||
return <ChatContext.Provider value={value}>{children}</ChatContext.Provider>;
|
||||
|
|
|
|||
|
|
@ -125,6 +125,12 @@ export const messageAPI = {
|
|||
export const tokenAPI = {
|
||||
getUsage: (days: number = 30) =>
|
||||
apiClient.get('/tokens/usage', { params: { days } }),
|
||||
|
||||
getUsersUsage: (days: number = 30) =>
|
||||
apiClient.get('/tokens/users', { params: { days } }),
|
||||
|
||||
getDailyUsageByUser: (days: number = 30) =>
|
||||
apiClient.get('/tokens/daily-users', { params: { days } }),
|
||||
};
|
||||
|
||||
export const adminAPI = {
|
||||
|
|
|
|||
|
|
@ -215,3 +215,106 @@
|
|||
font-weight: 700;
|
||||
color: var(--primary-gold);
|
||||
}
|
||||
|
||||
/* Admin Usage Analytics */
|
||||
.admin-stats-section {
|
||||
margin-bottom: 3rem;
|
||||
}
|
||||
|
||||
.admin-stats-section h3 {
|
||||
margin: 0 0 1.5rem 0;
|
||||
font-size: 1.25rem;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.admin-usage-table-section {
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.admin-usage-table-section h3 {
|
||||
margin: 0 0 1.5rem 0;
|
||||
font-size: 1.25rem;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.admin-usage-table-wrapper {
|
||||
overflow-x: auto;
|
||||
background: var(--white);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--radius-md);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.admin-usage-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
min-width: 800px;
|
||||
}
|
||||
|
||||
.admin-usage-table thead {
|
||||
background: var(--bg-secondary);
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.admin-usage-table th {
|
||||
padding: 1rem;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
border-bottom: 2px solid var(--border-color);
|
||||
font-size: 0.9rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.admin-usage-table td {
|
||||
padding: 0.875rem 1rem;
|
||||
border-bottom: 1px solid var(--border-color);
|
||||
color: var(--text-primary);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.admin-usage-table tbody tr:hover {
|
||||
background: var(--bg-hover);
|
||||
}
|
||||
|
||||
.admin-usage-table tbody tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.btn-details {
|
||||
padding: 0.4rem 0.875rem;
|
||||
background: var(--primary-gold);
|
||||
border: none;
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--white);
|
||||
font-size: 0.875rem;
|
||||
cursor: pointer;
|
||||
transition: all var(--transition-normal);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.btn-details:hover {
|
||||
background: var(--primary-gold-dark);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
/* Empty state for admin tables */
|
||||
.admin-empty-state {
|
||||
text-align: center;
|
||||
padding: 3rem 2rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.admin-empty-state h4 {
|
||||
margin: 0 0 0.5rem 0;
|
||||
font-size: 1.25rem;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.admin-empty-state p {
|
||||
margin: 0;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -196,11 +196,18 @@ body {
|
|||
background: linear-gradient(180deg, var(--light-bg) 0%, var(--light-bg-gradient) 100%);
|
||||
}
|
||||
|
||||
/* Center welcome message vertically */
|
||||
.chat-body:has(.welcome-message) {
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
/* ========== WELCOME MESSAGE ========== */
|
||||
.welcome-message {
|
||||
text-align: center;
|
||||
padding: var(--spacing-2xl);
|
||||
color: var(--text-muted);
|
||||
max-width: 700px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.welcome-message h2 {
|
||||
|
|
@ -222,6 +229,10 @@ body {
|
|||
display: flex;
|
||||
gap: var(--spacing-md);
|
||||
align-items: flex-end;
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.input-wrapper {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue