+ + {/* Display image thumbnails if available */} + {images.length > 0 && ( +

Relevant Document Screenshots:

+ {images.map((image, idx) => { + // Handle both string and object formats for backward compatibility + const filename = typeof image === 'string' ? image : image.filename; + const metadata = typeof image === 'string' ? null : image; + + return ( +

openImage(image)} + > +

+ {metadata && ( +

+ pg. {metadata.page} +

+ )} +

+ ); + })} +

+ )} + +

+ {sources.length > 0 && ( + + + + + + + +

Sources Used:

+ {source.tool && ( +
+ Tool: {source.tool} +
+ )} +
{source.text}
+

+ + + + + + )} + + {reasoningSteps.length > 0 && ( + + + + + + + +

Reasoning Steps:

+
+ {step.type}: +
+
{step.content}
+ {step.thought && step.thought !== step.content && ( +
+ Thought: {step.thought} +
+ )} +

+ + + + + + )} +

+ + {/* Image viewer modal */} + +

+ ); + }; + + const handleSubmit = async () => { + if (!inputMessage.trim() || isProcessing) return; + + const currentMessage = inputMessage; + setMessages(prev => [...prev, { role: 'user', content: currentMessage }]); + setInputMessage(''); + setIsProcessing(true); + + // ENHANCED DEBUGGING: Log the current state of sessionId and related variables + console.log('SUBMIT DEBUG DATA:', { + sessionId, + sessionIdType: typeof sessionId, + sessionIdLength: sessionId ? sessionId.length : 0, + hasActiveConversation: !!activeConversation, + activeConversationId: activeConversation ? activeConversation.id : null, + activeConversationSessionId: activeConversation ? activeConversation.session_id : null, + localStorageSessionId: localStorage.getItem('chatSessionId') + }); + + // Check if this is a blank chat with no active conversation - create one if needed + if (!activeConversation) { + logToConsole('info', 'No active conversation found when submitting message, creating a new one', { sessionId }); + try { + await handleCreateNewConversation(); + } catch (err) { + console.error('Error creating new conversation before sending message:', err); + // Continue with the chat request anyway, as the backend might handle this + } + } + + // ENHANCEMENT: Try to ensure we have a valid sessionId + const effectiveSessionId = sessionId || + (activeConversation ? activeConversation.session_id : null) || + localStorage.getItem('chatSessionId'); + + if (!effectiveSessionId) { + console.error('No valid sessionId found from any source'); + setError('Session ID is missing. Please refresh the page.'); + setIsProcessing(false); + return; + } + + try { + // Log the sessionId to help debug + console.log('Sending chat request with sessionId:', effectiveSessionId); + + const fetchOptions = getFetchDefaults(); + const response = await fetchWithTimeout(`${BACKEND_URL}/chat`, { + ...fetchOptions, + method: 'POST', + body: JSON.stringify({ + message: currentMessage, + sessionId: effectiveSessionId + }) + }); + + if (!response.ok) { + const errorData = await response.json(); + // If the server says chat is not initialized, reset UI to show upload screen + if (errorData.error === 'Chat system not initialized') { + setIsInitialized(false); + throw new Error('Chat system not initialized. Please upload files first.'); + } + throw new Error(errorData.error || 'Failed to get response'); + } + + const responseData = await response.json(); + + let assistantMessage; + if (responseData.status === 'success' && responseData.data) { + assistantMessage = { + role: 'assistant', + content: responseData.data.response || '', + sources: responseData.data.sources || [], + reasoning: responseData.data.reasoning || [], + images: responseData.data.images || [] + }; + } else if (responseData.result) { + assistantMessage = { + role: 'assistant', + content: responseData.result.response || '', + sources: responseData.result.sources || [], + reasoning: responseData.result.reasoning || [], + images: responseData.result.images || [] + }; + } else { + assistantMessage = { + role: 'assistant', + content: responseData.response || '', + sources: responseData.sources || [], + reasoning: responseData.reasoning || [], + images: responseData.images || [] + }; + } + + console.log('Processed assistant message:', assistantMessage); + setMessages(prev => [...prev, assistantMessage]); + setError(null); + + // Update the conversations list to get the AI-generated title + // Add a longer delay to allow backend to generate title using OpenAI + setTimeout(() => { + // Need a longer delay for OpenAI to generate the title + updateConversationsList(); + + // Check again after a bit longer in case the first attempt was too soon + setTimeout(() => { + updateConversationsList(); + }, 3000); // Check again after 3 more seconds + }, 2000); // Initial 2 second delay + + } catch (e) { + console.error('Error in chat:', e); + setError('Failed to process response. Please try again.'); + } finally { + setIsProcessing(false); + } + }; + +// Helper function to slice file into chunks and upload them +const uploadFileInChunks = async (file, fileType, sessionId, controller) => { + const chunkSize = 1024 * 1024; // 1MB chunks + const totalChunks = Math.ceil(file.size / chunkSize); + + // Step 1: Initialize the chunked upload + const fetchOptions = getFetchDefaults(); + const initResponse = await fetch(`${BACKEND_URL}/init-chunked-upload`, { + method: 'POST', + headers: { + ...fetchOptions.headers, + 'Content-Type': 'application/json' + }, + signal: controller.signal, + body: JSON.stringify({ + fileName: file.name, + fileType: fileType, // 'brief' or 'supporting' + fileSize: file.size, + mimeType: file.type, + totalChunks, + sessionId + }) + }); + + if (!initResponse.ok) { + const error = await initResponse.json(); + throw new Error(error.message || 'Failed to initialize chunked upload'); + } + + const { uploadId } = await initResponse.json(); + + // Step 2: Upload each chunk + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const start = chunkIndex * chunkSize; + const end = Math.min(start + chunkSize, file.size); + const chunk = file.slice(start, end); + + const chunkFormData = new FormData(); + chunkFormData.append('chunk', chunk); + chunkFormData.append('uploadId', uploadId); + chunkFormData.append('chunkIndex', chunkIndex); + + const fetchOptions = getFetchDefaults(); + const chunkResponse = await fetch(`${BACKEND_URL}/upload-chunk`, { + method: 'POST', + body: chunkFormData, + signal: controller.signal, + headers: { + ...fetchOptions.headers, + 'X-Requested-With': 'XMLHttpRequest' + } + }); + + if (!chunkResponse.ok) { + throw new Error(`Failed to upload chunk ${chunkIndex}`); + } + } + + // Step 3: Finalize the upload + const finalizeOptions = getFetchDefaults(); + const finalizeResponse = await fetch(`${BACKEND_URL}/finalize-upload`, { + method: 'POST', + headers: { + ...finalizeOptions.headers, + 'Content-Type': 'application/json' + }, + signal: controller.signal, + body: JSON.stringify({ uploadId }) + }); + + if (!finalizeResponse.ok) { + throw new Error('Failed to finalize upload'); + } + + return { fileName: file.name, uploadId }; +}; + +// Main function to initialize chat +const initializeChat = async () => { + if (briefFiles.length === 0) { + setError('Please upload at least one brief file.'); + return; + } + + logToConsole('info', 'Starting chat initialization', { + briefFilesCount: briefFiles.length, + supportingFilesCount: supportingFiles.length + }); + + setIsProcessing(true); + + // Create an AbortController for proper timeout handling + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(); + logToConsole('error', 'Request timed out after 10 minutes'); + }, 600000); // 10 minute timeout + + try { + // Check for large files that might need chunked upload + const largeFiles = [...briefFiles, ...supportingFiles].filter(file => file.size > 1024 * 1024); + if (largeFiles.length > 0) { + logToConsole('info', 'Large files detected, using chunked upload approach', { + largeFiles: largeFiles.map(f => ({ name: f.name, size: f.size })) + }); + } + + // Upload all brief files + const briefUploads = []; + for (const file of briefFiles) { + if (file.size > 1024 * 1024) { // If file is larger than 1MB, use chunked upload + const result = await uploadFileInChunks(file, 'brief', sessionId, controller); + briefUploads.push(result); + } else { + // For small files, use regular upload + const smallFormData = new FormData(); + smallFormData.append('file', file); + smallFormData.append('fileType', 'brief'); + smallFormData.append('sessionId', sessionId); + + const fetchOptions = getFetchDefaults(); + const response = await fetch(`${BACKEND_URL}/upload-small-file`, { + method: 'POST', + body: smallFormData, + signal: controller.signal, + headers: { + ...fetchOptions.headers, + 'X-Requested-With': 'XMLHttpRequest' + } + }); + + if (!response.ok) { + throw new Error(`Failed to upload small file: ${file.name}`); + } + + const result = await response.json(); + briefUploads.push(result); + } + } + + // Upload all supporting files (similar approach) + const supportingUploads = []; + for (const file of supportingFiles) { + if (file.size > 1024 * 1024) { + const result = await uploadFileInChunks(file, 'supporting', sessionId, controller); + supportingUploads.push(result); + } else { + // Small file upload code similar to brief files + const smallFormData = new FormData(); + smallFormData.append('file', file); + smallFormData.append('fileType', 'supporting'); + smallFormData.append('sessionId', sessionId); + + const fetchOptions = getFetchDefaults(); + const response = await fetch(`${BACKEND_URL}/upload-small-file`, { + method: 'POST', + body: smallFormData, + signal: controller.signal, + headers: { + ...fetchOptions.headers, + 'X-Requested-With': 'XMLHttpRequest' + } + }); + + if (!response.ok) { + throw new Error(`Failed to upload small file: ${file.name}`); + } + + const result = await response.json(); + supportingUploads.push(result); + } + } + + // Now that all files are uploaded, initialize the chat + const finalizeData = { + sessionId, + briefFiles: briefUploads, + supportingFiles: supportingUploads + }; + + const finalizeOptions = getFetchDefaults(); + const finalizeResponse = await fetch(`${BACKEND_URL}/initialize-from-uploads`, { + method: 'POST', + headers: { + ...finalizeOptions.headers, + 'Content-Type': 'application/json', + 'X-Requested-With': 'XMLHttpRequest' + }, + body: JSON.stringify(finalizeData), + signal: controller.signal + }); + + // Clear the timeout + clearTimeout(timeoutId); + + if (!finalizeResponse.ok) { + const errorData = await finalizeResponse.json(); + throw new Error(errorData.error || `Server responded with ${finalizeResponse.status}`); + } + + const data = await finalizeResponse.json(); + + logToConsole('info', 'Chat initialization successful'); + setIsInitialized(true); + setMessages([{ + role: 'assistant', + content: 'Chat initialized! How can I help you?' + }]); + setError(null); + + } catch (err) { + // Clear the timeout if there's an error + clearTimeout(timeoutId); + + // Enhanced error logging + const errorDetails = { + message: err.message, + name: err.name, + stack: err.stack, + isAbortError: err.name === 'AbortError' + }; + + logToConsole('error', 'Initialization error', errorDetails); + console.error('Initialization error:', err); + + // Set appropriate error message for user + if (err.name === 'AbortError') { + setError('Request timed out. The file(s) may be too large.'); + } else { + setError(err.message || 'Failed to initialize chat'); + } + + setIsInitialized(false); + } finally { + setIsProcessing(false); + } +}; + + return ( + +

+ {/* Conversation Sidebar - Fixed to left side */} +

Conversations

+ +

+ {loadingConversations ? ( +

+ +

+ ) : conversations.length > 0 ? ( + conversations.map(convo => ( +

handleLoadConversation(convo)} + > +

{convo.title || "New conversation"}

+ {new Date(convo.last_updated).toLocaleDateString()} +

+ +

+ )) + ) : ( +

+ No conversations yet +

+ )} +

+ +

+ {getCurrentUser()} + +

+ + {!autoWidth && ( + + )} +

+ + {/* Resize handle */} +

+ + {/* Header with HP branding - completely rebuilt without any borders */} +

+ {/* Theme toggle in top right corner */} +

+ +

+ {/* HP Logo */} +

+ HP +

Marketing Bot

+ + {/* Main Chat Area - Centered with left margin to account for sidebar */} +

+ {isCheckingStatus ? ( +

+ +

Connecting to HP Marketing Materials Chatbot...

+ ) : !isInitialized ? ( +

HP Marketing Materials Chatbot

+ The system is currently initializing. Please try again later. +

+ +

+ ) : ( + <> +

+ {activeConversation?.title || "HP Marketing Materials Chatbot"} +

+ +

+ Ask questions about HP's marketing materials, branding guidelines, campaign strategies, and creative processes. +

+ +

+ {messages.map((message, index) => ( + + ))} +

+ +

+ setInputMessage(e.target.value)} + onKeyPress={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); // Prevent any default behavior + if (!isProcessing && inputMessage.trim()) { + handleSubmit(); + } + } + }} + placeholder="Type your message..." + className="flex-1 p-2 border dark:border-gray-600 dark:bg-gray-700 dark:text-white rounded focus:outline-none focus:ring-2 focus:ring-blue-500" + disabled={isProcessing} + /> + +

+ + )} + + {error && !isCheckingStatus && ( + + {error} + + )} +

+ + ); +} \ No newline at end of file diff --git a/chat-interface/src/App_11-19.jsx b/chat-interface/src/App_11-19.jsx new file mode 100644 index 0000000..6b8d72e --- /dev/null +++ b/chat-interface/src/App_11-19.jsx @@ -0,0 +1,691 @@ +import { useState, useRef, useEffect } from 'react'; +import { Send, Upload, Loader2, X, FileText, Info, FileDown } from 'lucide-react'; +import { Alert, AlertDescription } from "./components/ui/alert"; +import * as Tooltip from '@radix-ui/react-tooltip'; + +const BACKEND_URL = 'http://localhost:6173'; + +const fetchDefaults = { + mode: 'cors', + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + 'Accept': '*/*', + 'Access-Control-Allow-Origin': '*', + } +}; + +export default function ChatInterface() { + const [messages, setMessages] = useState([]); + const [inputMessage, setInputMessage] = useState(''); + const [isProcessing, setIsProcessing] = useState(false); + const [briefFiles, setBriefFiles] = useState([]); + const [supportingFiles, setSupportingFiles] = useState([]); + const [isInitialized, setIsInitialized] = useState(false); + const [error, setError] = useState(null); + + const messagesEndRef = useRef(null); + const fileInputBriefRef = useRef(null); + const fileInputSupportingRef = useRef(null); + + useEffect(() => { + // Enable CORS for all fetch requests + fetch.defaults = fetchDefaults; + }, []); + + const logToConsole = (type, message, data = null) => { + const timestamp = new Date().toISOString(); + const logMessage = { + timestamp, + type, + message, + data + }; + console.log(JSON.stringify(logMessage, null, 2)); + }; + + const scrollToBottom = () => { + messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); + }; + + useEffect(() => { + scrollToBottom(); + }, [messages]); + + const handleFilesChange = (e, type) => { + const files = Array.from(e.target.files); + logToConsole('info', `Handling ${type} files change`, { + fileCount: files.length, + fileDetails: files.map(f => ({ + name: f.name, + type: f.type, + size: f.size + })) + }); + + if (type === 'brief') { + setBriefFiles(prev => [...prev, ...files]); + } else { + setSupportingFiles(prev => [...prev, ...files]); + } + }; + + const removeFile = (fileName, type) => { + logToConsole('info', `Removing file`, { + fileName, + type + }); + + if (type === 'brief') { + setBriefFiles(prev => prev.filter(file => file.name !== fileName)); + } else { + setSupportingFiles(prev => prev.filter(file => file.name !== fileName)); + } + }; + + const MessageBubble = ({ message }) => { + // Helper function to safely extract source content + const formatSources = (sources) => { + if (!sources || !Array.isArray(sources)) return []; + + return sources.map((source) => { + // Handle the nested structure from the backend + if (source && typeof source === 'object') { + // Check for the different possible source structures + if (source.content) { + return { + text: typeof source.content === 'string' + ? source.content + : JSON.stringify(source.content), + tool: source.tool_name || '' + }; + } + // Fallback for other source structures + return { + text: JSON.stringify(source), + tool: '' + }; + } + // Handle string sources + return { + text: String(source), + tool: '' + }; + }).filter(source => source.text); // Remove empty sources + }; + + // Helper function to format reasoning steps + const formatReasoning = (reasoning) => { + if (!reasoning || !Array.isArray(reasoning)) return []; + + return reasoning.map((step, index) => { + // Handle the new reasoning step structure + const stepType = step.type || 'thought'; + let content = ''; + + if (step.action) { + content = `${step.action}${step.action_input ? `: ${step.action_input}` : ''}`; + } else if (step.observation) { + content = step.observation; + } else if (step.response) { + content = step.response; + } else if (step.thought) { + content = step.thought; + } + + return { + id: index, + type: stepType, + content: content, + thought: step.thought + }; + }).filter(step => step.content); + }; + + const sources = formatSources(message.sources); + const reasoningSteps = formatReasoning(message.reasoning); + + const handleDownloadBrief = async () => { + // First, send the message to generate the brief + const briefRequestMessage = "write a comprehensive, detailed, and organized marketing brief based on the entire history of this conversation"; + setMessages(prev => [...prev, { role: 'user', content: briefRequestMessage }]); + setIsProcessing(true); + + try { + // First request to generate the brief content + const response = await fetch(`${BACKEND_URL}/chat`, { + ...fetchDefaults, + method: 'POST', + body: JSON.stringify({ message: briefRequestMessage }) + }); + + if (!response.ok) { + throw new Error('Failed to generate brief'); + } + + const responseData = await response.json(); + + // Add the assistant's response to the chat + setMessages(prev => [...prev, { + role: 'assistant', + content: responseData.data.response, + sources: responseData.data.sources, + reasoning: responseData.data.reasoning + }]); + + // Then make a second request to download the document + const downloadResponse = await fetch(`${BACKEND_URL}/download-brief`, { + ...fetchDefaults, + method: 'POST', + body: JSON.stringify({ brief_content: responseData.data.response }) + }); + + if (!downloadResponse.ok) { + throw new Error('Failed to download brief'); + } + + // Get the blob from the response + const blob = await downloadResponse.blob(); + + // Create a download link and trigger it + const downloadUrl = window.URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = downloadUrl; + link.download = 'marketing_brief.docx'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + window.URL.revokeObjectURL(downloadUrl); + + } catch (e) { + console.error('Error in download brief:', e); + setError('Failed to download brief. Please try again.'); + } finally { + setIsProcessing(false); + } + }; + + + return ( + +

+ {!isInitialized ? ( + // ... (initialization UI remains the same) + ) : ( + <> +

+ {messages.map((message, index) => ( + + ))} +

+ +

+ setInputMessage(e.target.value)} + onKeyPress={(e) => e.key === 'Enter' && handleSubmit()} + placeholder="Type your message..." + className="flex-1 p-2 border rounded focus:outline-none focus:ring-2 focus:ring-blue-500" + disabled={isProcessing} + /> + + +

+ + )} + + {error && ( + + {error} + + )} +

+ + ); + } + + + // return ( + //

+ //

{message.content}

+ + //

+ // {sources.length > 0 && ( + // + // + // + // + // + // + // + //

+ //

Sources Used:

+ //

+ // {source.tool && ( + //
+ // Tool: {source.tool} + //
+ // )} + //
{source.text}
+ //

+ //

+ // + // + // + // + // + // )} + + // {reasoningSteps.length > 0 && ( + // + // + // + // + // + // + // + //

+ //

Reasoning Steps:

+ //

+ //
+ // {step.type}: + //
+ //
{step.content}
+ // {step.thought && step.thought !== step.content && ( + //
+ // Thought: {step.thought} + //
+ // )} + //

+ //

+ // + // + // + // + // + // )} + //

+ //

+ // ); + // }; + + + + const handleSubmit = async () => { + if (!inputMessage.trim()) return; + + const currentMessage = inputMessage; + setMessages(prev => [...prev, { role: 'user', content: currentMessage }]); + setInputMessage(''); + setIsProcessing(true); + + try { + const response = await fetch(`${BACKEND_URL}/chat`, { + ...fetchDefaults, + method: 'POST', + body: JSON.stringify({ message: currentMessage }) + }); + + if (!response.ok) { + throw new Error('Failed to get response'); + } + + const responseData = await response.json(); + + // Log the full response data for debugging + console.log('Full response data:', responseData); + + // Extract the relevant data based on the response structure + let assistantMessage; + if (responseData.status === 'success' && responseData.data) { + // New response structure + assistantMessage = { + role: 'assistant', + content: responseData.data.response || '', + sources: responseData.data.sources || [], + reasoning: responseData.data.reasoning || [] + }; + } else if (responseData.result) { + // Alternative response structure + assistantMessage = { + role: 'assistant', + content: responseData.result.response || '', + sources: responseData.result.sources || [], + reasoning: responseData.result.reasoning || [] + }; + } else { + // Fallback for direct response structure + assistantMessage = { + role: 'assistant', + content: responseData.response || '', + sources: responseData.sources || [], + reasoning: responseData.reasoning || [] + }; + } + + // Log the processed message for debugging + console.log('Processed assistant message:', assistantMessage); + + setMessages(prev => [...prev, assistantMessage]); + setError(null); + + } catch (e) { + console.error('Error in chat:', e); + setError('Failed to process response. Please try again.'); + } finally { + setIsProcessing(false); + } + }; + + + + const initializeChat = async () => { + if (briefFiles.length === 0) { + setError('Please upload at least one brief file.'); + return; + } + + logToConsole('info', 'Starting chat initialization', { + briefFilesCount: briefFiles.length, + supportingFilesCount: supportingFiles.length, + briefFiles: briefFiles.map(f => ({ + name: f.name, + type: f.type, + size: f.size + })), + supportingFiles: supportingFiles.map(f => ({ + name: f.name, + type: f.type, + size: f.size + })) + }); + + setIsProcessing(true); + const formData = new FormData(); + + briefFiles.forEach(file => { + formData.append('brief', file); + logToConsole('debug', 'Appending brief file to FormData', { + fileName: file.name, + fileType: file.type, + fileSize: file.size + }); + }); + + supportingFiles.forEach(file => { + formData.append('supporting', file); + logToConsole('debug', 'Appending supporting file to FormData', { + fileName: file.name, + fileType: file.type, + fileSize: file.size + }); + }); + + try { + logToConsole('info', 'Sending initialization request to backend'); + + const response = await fetch(`${BACKEND_URL}/initialize`, { + method: 'POST', + mode: 'cors', + credentials: 'include', + body: formData, + // headers: { + // 'Accept': '*/*', + // 'Access-Control-Allow-Origin': '*', + // } + }); + + logToConsole('debug', 'Received response from backend', { + status: response.status, + statusText: response.statusText, + headers: Object.fromEntries(response.headers.entries()) + }); + + const textResponse = await response.text(); + logToConsole('debug', 'Response text received', { textResponse }); + + let data; + try { + data = JSON.parse(textResponse); + logToConsole('debug', 'Successfully parsed response JSON', { data }); + } catch (e) { + logToConsole('error', 'Failed to parse response JSON', { + error: e.message, + textResponse + }); + throw new Error('Invalid response format from server'); + } + + if (!response.ok) { + logToConsole('error', 'Backend returned error response', { + status: response.status, + error: data.error + }); + throw new Error(data.error || 'Failed to initialize chat'); + } + + setIsInitialized(true); + setMessages([{ + role: 'assistant', + content: 'Chat initialized! How can I help you?' + }]); + + setError(null); + logToConsole('info', 'Chat initialization completed successfully'); + + } catch (err) { + logToConsole('error', 'Chat initialization failed', { + error: err.message, + stack: err.stack + }); + + let errorMessage = 'Failed to initialize chat. '; + if (err.message) { + errorMessage += err.message; + } else { + errorMessage += 'Please try again.'; + } + + setError(errorMessage); + setIsInitialized(false); + } finally { + setIsProcessing(false); + } + }; + + return ( + +

+ {!isInitialized ? ( +

Upload Files to Start Chat

+ +

Brief Files (Required)

+ handleFilesChange(e, 'brief')} + multiple + accept=".pdf,.doc,.docx,.txt,.xls,.xlsx,.ppt,.pptx,.eml" + className="hidden" + /> + +

+ {briefFiles.map((file) => ( +

+ + {file.name} + +

+ ))} +

+ +

Supporting Files (Optional)

+ handleFilesChange(e, 'supporting')} + multiple + accept=".pdf,.doc,.docx,.txt,.xls,.xlsx,.ppt,.pptx,.eml" + className="hidden" + /> + +

+ {supportingFiles.map((file) => ( +

+ + {file.name} + +

+ ))} +

+ + +

+ ) : ( + <> +

+ {messages.map((message, index) => ( + + ))} +

+ +

+ + )} + + {error && ( + + {error} + + )} +

+ + ); +} diff --git a/chat-interface/src/assets/react.svg b/chat-interface/src/assets/react.svg new file mode 100644 index 0000000..6c87de9 --- /dev/null +++ b/chat-interface/src/assets/react.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/chat-interface/src/auth.js b/chat-interface/src/auth.js new file mode 100644 index 0000000..5caa704 --- /dev/null +++ b/chat-interface/src/auth.js @@ -0,0 +1,119 @@ +// MSAL Authentication Configuration +const msalConfig = { + auth: { + clientId: "9079054c-9620-4757-a256-23413042f1ef", + authority: "https://login.microsoftonline.com/e519c2e6-bc6d-4fdf-8d9c-923c2f002385", + redirectUri: "https://ai-sandbox.oliver.solutions/format" + }, + cache: { + cacheLocation: "sessionStorage", + storeAuthStateInCookie: true, + } +}; + +const loginRequest = { + scopes: ["user.read"] +}; + +// Initialize MSAL object +let myMSALObj = null; +let thisUser = null; + +// Initialize MSAL when loaded +export function initializeMSAL() { + if (typeof msal !== 'undefined') { + myMSALObj = new msal.PublicClientApplication(msalConfig); + + // Check if there's a cached user + const accounts = myMSALObj.getAllAccounts(); + if (accounts.length > 0) { + thisUser = accounts[0].username; + // User is already logged in, show content + showProtectedContent(); + } else { + // Need to initialize login elements if not logged in + const signinButton = document.getElementById('signin-button'); + if (signinButton) { + signinButton.addEventListener('click', signIn); + } + } + } else { + console.error("MSAL library not loaded"); + } +} + +// Sign in with popup +export function signIn() { + if (!myMSALObj) { + initializeMSAL(); + } + + myMSALObj.loginPopup(loginRequest) + .then(loginResponse => { + console.log("User logged in:", loginResponse.account.username); + thisUser = loginResponse.account.username; + sessionStorage.setItem('msalUsername', loginResponse.account.username); + showProtectedContent(); + }).catch(error => { + console.error("Error during login:", error); + }); +} + +// Show the protected content (the chat interface) +function showProtectedContent() { + const loginContainer = document.getElementById('login-container'); + const rootContainer = document.getElementById('root'); + + if (loginContainer) { + loginContainer.style.display = 'none'; + } + + if (rootContainer) { + rootContainer.style.display = 'block'; + } + + // Dispatch an event that authentication is complete + window.dispatchEvent(new CustomEvent('authenticationComplete', { + detail: { + username: thisUser + } + })); +} + +// Get current user +export function getCurrentUser() { + if (!thisUser) { + thisUser = sessionStorage.getItem('msalUsername'); + } + return thisUser; +} + +// Check if user is authenticated +export function isAuthenticated() { + if (!myMSALObj) { + initializeMSAL(); + } + + return myMSALObj?.getAllAccounts().length > 0 || !!getCurrentUser(); +} + +// Sign out +export function signOut() { + if (!myMSALObj) { + initializeMSAL(); + } + + const logoutRequest = { + account: myMSALObj.getAccountByUsername(thisUser) + }; + + myMSALObj.logout(logoutRequest).then(() => { + thisUser = null; + sessionStorage.removeItem('msalUsername'); + // Redirect to login page + window.location.reload(); + }); +} + +// Register initialization on window load +window.addEventListener('DOMContentLoaded', initializeMSAL); diff --git a/chat-interface/src/components/ChatInterface.jsx b/chat-interface/src/components/ChatInterface.jsx new file mode 100644 index 0000000..583f685 --- /dev/null +++ b/chat-interface/src/components/ChatInterface.jsx @@ -0,0 +1,140 @@ +// Use environment variables for backend URL +// Define backend URL dynamically based on environment +const BACKEND_URL = import.meta.env.VITE_BACKEND_URL || 'https://ai-sandbox.oliver.solutions/hp_back_v2'; +console.log('ChatInterface using backend URL:', BACKEND_URL); + +const initializeChat = async ({ sessionId, briefFiles, supportingFiles, setError, setIsProcessing, setIsInitialized, setMessages }) => { + if (briefFiles.length === 0) { + setError('Please upload at least one brief file.'); + return; + } + + setIsProcessing(true); + const formData = new FormData(); + + // Add session ID to the form data + formData.append('sessionId', sessionId); + + briefFiles.forEach(file => { + formData.append('brief', file); + }); + + supportingFiles.forEach(file => { + formData.append('supporting', file); + }); + + try { + console.log('Sending initialize request...'); + const response = await fetch(`${BACKEND_URL}/initialize`, { + method: 'POST', + body: formData, // No Content-Type header for FormData + credentials: 'include', // Include credentials (cookies, auth headers) + }); + + console.log('Response status:', response.status); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to initialize chat'); + } + + const data = await response.json(); + console.log('Response data:', data); + + setIsInitialized(true); + setMessages([{ + role: 'assistant', + content: 'Chat initialized! How can I help you?' + }]); + setError(null); // Clear any previous errors + } catch (err) { + console.error('Error:', err); + setError(err.message || 'Failed to initialize chat. Please try again.'); + } finally { + setIsProcessing(false); + } +}; + +const handleSubmit = async ({ inputMessage, sessionId, setMessages, setInputMessage, setIsProcessing, setError, logToConsole }) => { + if (!inputMessage.trim()) return; + + const currentMessage = inputMessage; + setMessages(prev => [...prev, { role: 'user', content: currentMessage }]); + setInputMessage(''); + setIsProcessing(true); + + try { + const response = await fetch(`${BACKEND_URL}/chat`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + message: currentMessage, + sessionId: sessionId + }), + credentials: 'include', // Include credentials for cross-origin + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to get response'); + } + + const dataString = await response.text(); // Get the response as text + const data = JSON.parse(dataString); // Parse the JSON string + + // Extract response data, checking for different response formats + let responseText = ''; + let sources = []; + let reasoning = []; + let images = []; + + if (data.data) { + // New response format + responseText = data.data.response || ''; + sources = data.data.sources || []; + reasoning = data.data.reasoning || []; + images = data.data.images || []; + } else { + // Direct response format + responseText = data.response || ''; + sources = data.sources || []; + reasoning = data.reasoning || []; + images = data.images || []; + } + + // Log the images + if (images && images.length > 0) { + logToConsole('info', 'Images received in response', { imageCount: images.length, images }); + } + + setMessages(prev => { + const newMessages = [...prev]; + newMessages.push({ + role: 'assistant', + content: responseText, + sources: sources, + reasoning: reasoning, + images: images + }); + return newMessages; + }); + setError(null); // Clear any previous error + + } catch (e) { // Include the original JSON string in the error data + console.error('Error processing chat response:', e); + logToConsole('error', 'Chat error', { + error: e.message, + stack: e.stack, + response: e.response + }); + + setError('Failed to process response. Please check the logs for details.'); // Display a more user friendly error in UI + } finally { + setIsProcessing(false); + } +}; + +// Export the functions +export { initializeChat, handleSubmit }; \ No newline at end of file diff --git a/chat-interface/src/components/ConversationManager.jsx b/chat-interface/src/components/ConversationManager.jsx new file mode 100644 index 0000000..023ca51 --- /dev/null +++ b/chat-interface/src/components/ConversationManager.jsx @@ -0,0 +1,206 @@ +// Get the backend URL from environment or use correct path as fallback +// Define backend URL dynamically based on environment +const BACKEND_URL = import.meta.env.VITE_BACKEND_URL || 'https://ai-sandbox.oliver.solutions/hp_back_v2'; +console.log('ConversationManager using backend URL:', BACKEND_URL); + +// Function to load user conversations +export const loadUserConversations = async (username, setConversations, setActiveConversation, loadConversation, setError) => { + try { + const response = await fetch(`${BACKEND_URL}/conversations`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'X-MS-USERNAME': username || '' // Add authenticated username to request + }, + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to load conversations'); + } + + const data = await response.json(); + + if (data.status === 'success') { + setConversations(data.conversations || []); + + // If we have conversations and none are active, select the most recent one + if (data.conversations && data.conversations.length > 0 && setActiveConversation) { + // Sort by last_updated, newest first + const sortedConversations = [...data.conversations].sort((a, b) => + new Date(b.last_updated) - new Date(a.last_updated) + ); + + // Set the most recent conversation as active + setActiveConversation(sortedConversations[0]); + + // Load this conversation if load function provided + if (loadConversation) { + await loadConversation(sortedConversations[0]); + } + } + + return data.conversations || []; + } else { + throw new Error(data.error || 'Failed to load conversations'); + } + } catch (error) { + console.error('Error loading conversations:', error); + if (setError) { + setError('Failed to load conversations. Please try again later.'); + } + return []; + } +}; + +// Function to load a specific conversation's messages +export const loadConversationMessages = async (conversation, username, setMessages, setSessionId, setError) => { + try { + if (!conversation || !conversation.id) { + throw new Error('Invalid conversation'); + } + + const response = await fetch(`${BACKEND_URL}/conversations/${conversation.id}/messages`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'X-MS-USERNAME': username || '' // Add authenticated username to request + }, + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to load conversation messages'); + } + + const data = await response.json(); + + if (data.status === 'success') { + // Format the messages for the UI + const formattedMessages = data.messages.map(msg => ({ + role: msg.role, + content: msg.content, + sources: msg.sources || [], + reasoning: msg.reasoning || [], + images: msg.images || [] + })); + + setMessages(formattedMessages); + + // Set the session ID if provided + if (setSessionId && conversation.session_id) { + setSessionId(conversation.session_id); + localStorage.setItem('chatSessionId', conversation.session_id); + } + + return formattedMessages; + } else { + throw new Error(data.error || 'Failed to load conversation messages'); + } + } catch (error) { + console.error('Error loading conversation messages:', error); + if (setError) { + setError('Failed to load conversation messages. Please try again later.'); + } + return []; + } +}; + +// Function to create a new conversation +export const createNewConversation = async (username, setConversations, setActiveConversation, setSessionId, setMessages, setError) => { + try { + const response = await fetch(`${BACKEND_URL}/conversations/new`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-MS-USERNAME': username || '' // Add authenticated username to request + }, + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to create new conversation'); + } + + const data = await response.json(); + + if (data.status === 'success') { + // Create a new conversation object + const newConversation = { + id: data.conversation_id, + title: "New conversation", + created_at: new Date().toISOString(), + last_updated: new Date().toISOString(), + session_id: data.session_id + }; + + // Update the conversations list + setConversations(prev => [newConversation, ...prev]); + + // Set as active conversation + setActiveConversation(newConversation); + + // Set the session ID + if (setSessionId) { + setSessionId(data.session_id); + localStorage.setItem('chatSessionId', data.session_id); + } + + // Reset messages with welcome message + if (setMessages) { + setMessages([{ + role: 'assistant', + content: 'How can I help you today?' + }]); + } + + return newConversation; + } else { + throw new Error(data.error || 'Failed to create new conversation'); + } + } catch (error) { + console.error('Error creating new conversation:', error); + if (setError) { + setError('Failed to create new conversation. Please try again later.'); + } + return null; + } +}; + +// Function to delete a conversation +export const deleteConversation = async (conversationId, username, setConversations, setError) => { + try { + if (!conversationId) { + throw new Error('Invalid conversation ID'); + } + + const response = await fetch(`${BACKEND_URL}/conversations/${conversationId}`, { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + 'X-MS-USERNAME': username || '' // Add authenticated username to request + }, + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to delete conversation'); + } + + const data = await response.json(); + + if (data.status === 'success') { + // Remove the conversation from the list + setConversations(prev => prev.filter(convo => convo.id !== conversationId)); + return true; + } else { + throw new Error(data.error || 'Failed to delete conversation'); + } + } catch (error) { + console.error('Error deleting conversation:', error); + if (setError) { + setError('Failed to delete conversation. Please try again later.'); + } + return false; + } +}; \ No newline at end of file diff --git a/chat-interface/src/components/ThemeToggle.jsx b/chat-interface/src/components/ThemeToggle.jsx new file mode 100644 index 0000000..e2fe897 --- /dev/null +++ b/chat-interface/src/components/ThemeToggle.jsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { useTheme } from '../main'; +import { Moon, Sun } from 'lucide-react'; + +export default function ThemeToggle() { + const { darkMode, setDarkMode } = useTheme(); + + return ( + + ); +} diff --git a/chat-interface/src/components/ui/alert.jsx b/chat-interface/src/components/ui/alert.jsx new file mode 100644 index 0000000..8437323 --- /dev/null +++ b/chat-interface/src/components/ui/alert.jsx @@ -0,0 +1,37 @@ +import * as React from "react" +import { cn } from "../../lib/utils" + +const Alert = React.forwardRef(({ className, variant = "default", children, ...props }, ref) => { + const variantClasses = { + default: "bg-gray-100 text-gray-900", + destructive: "bg-red-100 text-red-900", + } + + return ( +

+ {children} +

+ ) +}) +Alert.displayName = "Alert" + +const AlertDescription = React.forwardRef(({ className, ...props }, ref) => ( +

+)) +AlertDescription.displayName = "AlertDescription" + +export { Alert, AlertDescription } + diff --git a/chat-interface/src/index.css b/chat-interface/src/index.css new file mode 100644 index 0000000..5e4ccd3 --- /dev/null +++ b/chat-interface/src/index.css @@ -0,0 +1,131 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 84% 4.9%; + --muted: 210 40% 96.1%; + --muted-foreground: 215.4 16.3% 46.9%; + --border: 214.3 31.8% 91.4%; + --radius: 0.5rem; + } + + .dark { + --background: 222.2 84% 4.9%; + --foreground: 210 40% 98%; + --muted: 217.2 32.6% 17.5%; + --muted-foreground: 215 20.2% 65.1%; + --border: 217.2 32.6% 17.5%; + } + + body { + @apply bg-black text-foreground; + border: none !important; + border-top: none !important; + border-bottom: none !important; + } +} + +/* Netflix theme colors */ +.netflix-red { + color: #E50914; +} + +.netflix-bg { + background-color: #141414; +} + +/* Prevent any border from appearing */ +.no-borders, .no-borders::before, .no-borders::after { + border: none !important; + border-top: none !important; + border-bottom: none !important; + border-left: none !important; + border-right: none !important; + outline: none !important; +} + +/* Markdown content styling */ +.markdown-content { + @apply text-base leading-relaxed; +} + +.markdown-content h1 { + @apply text-2xl font-bold mt-4 mb-2; +} + +.markdown-content h2 { + @apply text-xl font-bold mt-3 mb-2; +} + +.markdown-content h3 { + @apply text-lg font-bold mt-3 mb-1; +} + +.markdown-content p { + @apply my-2; +} + +.markdown-content ul { + @apply list-disc pl-5 my-2; +} + +.markdown-content ol { + @apply list-decimal pl-5 my-2; +} + +.markdown-content li { + @apply my-1; +} + +.markdown-content a { + @apply text-blue-600 hover:underline; +} + +.markdown-content code { + @apply font-mono bg-gray-200 px-1 py-0.5 rounded text-sm; +} + +.markdown-content pre { + @apply bg-gray-800 text-white p-3 rounded my-3 overflow-auto; +} + +.markdown-content pre code { + @apply bg-transparent text-white text-sm; +} + +.markdown-content blockquote { + @apply border-l-4 border-gray-300 pl-4 my-3 italic text-gray-700; +} + +.markdown-content table { + @apply border-collapse border border-gray-300 my-3 w-full; +} + +.markdown-content th { + @apply border border-gray-300 bg-gray-100 p-2 font-semibold; +} + +.markdown-content td { + @apply border border-gray-300 p-2; +} + +/* Different background for assistant markdown vs user text */ +.bg-blue-500 .markdown-content { + @apply text-white; +} + +.bg-blue-500 .markdown-content code { + @apply bg-blue-600 text-white; +} + +.bg-blue-500 .markdown-content a { + @apply text-white underline; +} + +.bg-blue-500 .markdown-content blockquote { + @apply border-white/50 text-white/90; +} + diff --git a/chat-interface/src/lib/utils.js b/chat-interface/src/lib/utils.js new file mode 100644 index 0000000..d510289 --- /dev/null +++ b/chat-interface/src/lib/utils.js @@ -0,0 +1,40 @@ +// Original utils.js file content +import { clsx } from "clsx" +import { twMerge } from "tailwind-merge" + +export function cn(...inputs) { + return twMerge(clsx(inputs)) +} + +/** + * Enhanced fetch function with a 300-second timeout + * @param {string} url - URL to fetch + * @param {Object} options - Fetch options + * @returns {Promise} - Fetch promise with timeout + */ +export async function fetchWithTimeout(url, options = {}) { + const timeout = 300000; // 300 seconds in milliseconds + + const controller = new AbortController(); + const { signal } = controller; + + const timeoutId = setTimeout(() => { + controller.abort(); + }, timeout); + + try { + const response = await fetch(url, { + ...options, + signal, + }); + + clearTimeout(timeoutId); + return response; + } catch (error) { + clearTimeout(timeoutId); + if (error.name === 'AbortError') { + throw new Error(`Request timed out after ${timeout / 1000} seconds`); + } + throw error; + } +} \ No newline at end of file diff --git a/chat-interface/src/main.jsx b/chat-interface/src/main.jsx new file mode 100644 index 0000000..3e85a5a --- /dev/null +++ b/chat-interface/src/main.jsx @@ -0,0 +1,75 @@ +import React, { useEffect, useState, createContext, useContext } from 'react' +import ReactDOM from 'react-dom/client' +import App from './App' +import './index.css' +import { initializeMSAL, isAuthenticated } from './auth' + +// Initialize MSAL +initializeMSAL(); + +// Create Theme Context +export const ThemeContext = createContext(); + +export const ThemeProvider = ({ children }) => { + const [darkMode, setDarkMode] = useState(() => { + // Check local storage for user preference + const savedMode = localStorage.getItem('darkMode'); + return savedMode === 'true'; + }); + + // Apply dark mode class to html element + useEffect(() => { + if (darkMode) { + document.documentElement.classList.add('dark'); + } else { + document.documentElement.classList.remove('dark'); + } + // Save preference to localStorage + localStorage.setItem('darkMode', darkMode); + }, [darkMode]); + + return ( + + {children} + + ); +}; + +// Custom hook for using theme context +export const useTheme = () => useContext(ThemeContext); + +const AuthenticatedApp = () => { + const [authenticated, setAuthenticated] = useState(isAuthenticated()); + + useEffect(() => { + // Listen for authentication complete event + const handleAuth = () => { + setAuthenticated(true); + }; + + window.addEventListener('authenticationComplete', handleAuth); + + // Check if already authenticated + if (isAuthenticated()) { + handleAuth(); + } + + return () => { + window.removeEventListener('authenticationComplete', handleAuth); + }; + }, []); + + // The App component will only be rendered when user is authenticated + return authenticated ? ( + + + + ) : null; +}; + +ReactDOM.createRoot(document.getElementById('root')).render( + + + , +) + diff --git a/chat-interface/tailwind.config.js b/chat-interface/tailwind.config.js new file mode 100644 index 0000000..2955b1a --- /dev/null +++ b/chat-interface/tailwind.config.js @@ -0,0 +1,28 @@ +/** @type {import('tailwindcss').Config} */ +export default { + content: [ + "./index.html", + "./src/**/*.{js,jsx,ts,tsx}", + ], + darkMode: 'class', // Enable class-based dark mode + theme: { + extend: { + colors: { + border: "hsl(var(--border))", + background: "hsl(var(--background))", + foreground: "hsl(var(--foreground))", + muted: { + DEFAULT: "hsl(var(--muted))", + foreground: "hsl(var(--muted-foreground))" + }, + }, + borderRadius: { + lg: "var(--radius)", + md: "calc(var(--radius) - 2px)", + sm: "calc(var(--radius) - 4px)", + }, + }, + }, + plugins: [], +} + diff --git a/chat-interface/update-backend.sh b/chat-interface/update-backend.sh new file mode 100755 index 0000000..71e7e06 --- /dev/null +++ b/chat-interface/update-backend.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This script updates the backend URL and rebuilds the application + +# Define the backend URL +BACKEND_URL="https://ai-sandbox.oliver.solutions/netflix_back_v2" + +# Create or update the environment files +echo "# Backend API URL +VITE_BACKEND_URL=${BACKEND_URL} + +# Base URL for the app (changes in production) +VITE_APP_BASE_URL=/" > .env + +echo "# Production backend API URL +VITE_BACKEND_URL=${BACKEND_URL} + +# Base URL for the app in production +VITE_APP_BASE_URL=/netflix_v2/" > .env.production + +# Update components files +for file in src/components/ChatInterface.jsx src/components/ConversationManager.jsx src/App.jsx; do + # Replace hardcoded backend URL references + sed -i '' "s|const BACKEND_URL = ['\"]\/netflix_back_v2['\"]|const BACKEND_URL = import.meta.env.VITE_BACKEND_URL || '${BACKEND_URL}'|g" $file + sed -i '' "s|console.log('.*using backend URL (hardcoded):|console.log('Using backend URL:|g" $file +done + +# Update vite.config.js to use the replacement plugin +sed -i '' 's|plugins: \[react()\],|plugins: [react(), replaceBackendUrl],|g' vite.config.js +sed -i '' 's|code: src.replace(/\['"'\]\/netflix_back\['"'\]/g, '"\/netflix_back_v2"'),|code: src.replace(/\['"'\]\/netflix_back\['"'\]/g, `"${BACKEND_URL}"`),|g' vite.config.js + +# Force a complete rebuild by removing all assets +rm -rf dist/assets/* + +# Rebuild the application +npm run build + +echo "Update complete! The backend URL has been set to ${BACKEND_URL}" \ No newline at end of file diff --git a/chat-interface/vite.config.js b/chat-interface/vite.config.js new file mode 100644 index 0000000..51bb0ed --- /dev/null +++ b/chat-interface/vite.config.js @@ -0,0 +1,124 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' +import path from 'path' + +export default defineConfig(({ mode }) => { + const baseUrl = mode === 'production' ? '/hp_chatbot/' : '/'; + + // Create a replace plugin to ensure all '/hp_back' gets replaced with the full URL + const replaceBackendUrl = { + name: 'replace-backend-url', + transform(src, id) { + if (id.endsWith('.js') || id.endsWith('.jsx')) { + return { + code: src.replace(/['"]\/hp_back['"]/g, '"https://ai-sandbox.oliver.solutions/hp_back_v2"'), + map: null + }; + } + } + }; + + return { + plugins: [react(), replaceBackendUrl], + base: baseUrl, + server: { + port: 5173, + proxy: { + // New MongoDB conversation endpoints + '/conversations': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/conversations/new': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + + // Original endpoints + '/chat': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/status': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/initialize': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/reset': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/download-brief': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/init-chunked-upload': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/upload-chunk': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/finalize-upload': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/upload-small-file': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/initialize-from-uploads': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/images': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/list-images': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + }, + '/capture-screenshot': { + target: 'https://ai-sandbox.oliver.solutions/hp_back_v2', + changeOrigin: true, + secure: true, + timeout: 300000 // 5 minutes timeout + } + } + } + }; +}) + diff --git a/chat-interface/web.config b/chat-interface/web.config new file mode 100644 index 0000000..75ec618 --- /dev/null +++ b/chat-interface/web.config @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..df7a987 --- /dev/null +++ b/config.py @@ -0,0 +1,103 @@ +# hp_chatbot/config.py +import os +from pathlib import Path +from dotenv import load_dotenv +import sys + +# Load environment variables from .env file +env_path = Path(__file__).resolve().parent / '.env' +if env_path.exists(): + # Force reload to ensure environment variables are set + load_dotenv(dotenv_path=env_path, override=True) + print(f"Loaded environment variables from {env_path}") +else: + print(f"WARNING: .env file not found at {env_path}", file=sys.stderr) + +# --- Directory Paths --- +BASE_DIR = Path(__file__).resolve().parent + +UPLOAD_DIR = BASE_DIR / 'uploads' +CHUNK_FOLDER = UPLOAD_DIR / 'chunks' +UPLOAD_METADATA_FOLDER = UPLOAD_DIR / 'metadata' +IMAGES_DIRECTORY = UPLOAD_DIR / 'images' + +SUPPORTING_FILES_DIR = BASE_DIR / 'supporting_files' +HP_DOCS_FOLDER = SUPPORTING_FILES_DIR / 'files_for_rag_store' + +INDEX_STORAGE_DIR = BASE_DIR / 'index_storage' +INDEX_PERSIST_PATH = INDEX_STORAGE_DIR / "hp_docs_index" + +LOG_FILE_PATH = BASE_DIR / 'app.log' + +# Create necessary directories +os.makedirs(CHUNK_FOLDER, exist_ok=True) +os.makedirs(UPLOAD_METADATA_FOLDER, exist_ok=True) +os.makedirs(INDEX_STORAGE_DIR, exist_ok=True) +os.makedirs(IMAGES_DIRECTORY, exist_ok=True) + +# --- Application Settings --- +ALLOWED_EXTENSIONS = {'pdf', 'doc', 'docx', 'txt', 'xls', 'xlsx', 'ppt', 'pptx', 'eml'} +APPLICATION_ROOT = os.environ.get('APPLICATION_ROOT', '') # For running behind proxy +MAX_CONTENT_LENGTH = 500 * 1024 * 1024 # 500MB limit (adjust as needed) + +# --- API Keys --- +# Load from environment variables or use defaults (replace placeholders or set env vars) +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "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") +ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "") +LLAMA_CLOUD_API_KEY = os.environ.get("LLAMA_CLOUD_API_KEY", "") + +# Ensure required keys are set +if not OPENAI_API_KEY: + print("ERROR: OPENAI_API_KEY not set in environment or .env file. This is required.", file=sys.stderr) + print("Please add OPENAI_API_KEY=your_key to your .env file.", file=sys.stderr) + print(f"Current environment keys: {list(filter(lambda k: 'key' in k.lower(), os.environ.keys()))}", file=sys.stderr) + +# Always set environment variables, even if empty - the error messages will be handled by the code +os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY +os.environ["ANTHROPIC_API_KEY"] = ANTHROPIC_API_KEY +os.environ["LLAMA_CLOUD_API_KEY"] = LLAMA_CLOUD_API_KEY + +# Print API key status for debugging +print(f"OpenAI API key {'is set' if OPENAI_API_KEY else 'is NOT set'}", file=sys.stderr) + +# --- AI Model Configuration --- +LLM_MODEL = "chatgpt-4o-latest" # Or "gpt-4o" etc. +EMBEDDING_MODEL = "text-embedding-3-small" +LLM_TEMPERATURE = 0.3 +LLM_TIMEOUT = 300.0 # 5 minutes +AGENT_TIMEOUT = 600.0 # 10 minutes for the agent run +TOOL_EXECUTION_TIMEOUT = 300.0 # 5 minutes for individual tool calls + +# --- LlamaParse Configuration --- +LLAMA_PARSE_VENDOR_MODEL = "openai-gpt4o" # Verify model name +LLAMA_PARSE_MAX_TIMEOUT = 3600 # 1 hour + +# --- Indexing Configuration --- +# NODE_PARSER_CHUNK_SIZE = 2048 # Example if using SentenceSplitter +# NODE_PARSER_CHUNK_OVERLAP = 20 +# Use Semantic Splitter by default (see ai_core.py) +SIMILARITY_TOP_K = 10 +SIMILARITY_CUTOFF = 0.0 # Adjust if needed + +# --- CORS Configuration --- +CORS_ALLOWED_ORIGINS = ["http://localhost:5173", "https://ai-sandbox.oliver.solutions"] # HP chatbot CORS origins +CORS_SUPPORTS_CREDENTIALS = True + +# --- Server Configuration --- +SERVER_HOST = "0.0.0.0" if os.environ.get("PRODUCTION", "false").lower() == "true" else "localhost" +SERVER_PORT = int(os.environ.get("PORT", "8746")) # Port for HP chatbot (unique from Netflix) +USE_RELOADER = os.environ.get("PRODUCTION", "false").lower() != "true" +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") # Changed default to INFO + +# Hypercorn specific timeouts (in seconds) +KEEP_ALIVE_TIMEOUT = 300 +READ_TIMEOUT = 300 +WRITE_TIMEOUT = 300 + +# --- MongoDB Configuration --- +# Assumes mongodb_utils handles connection details (e.g., via environment variables) + +# --- Neo4j Configuration --- +NEO4J_URL = os.environ.get("NEO4J_URL", "bolt://localhost:7688") # Separate Neo4j instance for HP +NEO4J_USERNAME = os.environ.get("NEO4J_USERNAME", "neo4j") +NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "hp-graphrag-2024") # HP-specific password \ No newline at end of file diff --git a/docs/graphRAG chatbot technical documentation.pdf b/docs/graphRAG chatbot technical documentation.pdf new file mode 100644 index 0000000..9f361bf Binary files /dev/null and b/docs/graphRAG chatbot technical documentation.pdf differ diff --git a/docs/graphRAG_chatbot_documentation.md b/docs/graphRAG_chatbot_documentation.md new file mode 100644 index 0000000..616967f --- /dev/null +++ b/docs/graphRAG_chatbot_documentation.md @@ -0,0 +1,666 @@ +# HP GraphRAG Chatbot - Technical Documentation + +## Table of Contents + +1. [System Overview](#system-overview) +2. [Architecture](#architecture) +3. [Technology Stack](#technology-stack) +4. [Data Flow](#data-flow) +5. [Database Design](#database-design) +6. [API Reference](#api-reference) +7. [User Flow](#user-flow) +8. [Security](#security) +9. [Deployment](#deployment) +10. [Development](#development) +11. [Troubleshooting](#troubleshooting) + +--- + +## System Overview + +The HP GraphRAG Chatbot is a sophisticated conversational AI system that combines vector search with knowledge graph capabilities to answer questions about HP marketing materials and brand guidelines. It processes multimodal documents (text + images) and uses a hybrid AI agent approach for intelligent information retrieval and response generation. + +### Key Features + +- **Multimodal Document Processing**: Extracts text and images from PDFs, PowerPoint, and other marketing documents +- **GraphRAG Architecture**: Combines vector similarity search with knowledge graph community detection +- **Custom ReAct Agent**: Implements reasoning and action patterns for intelligent query processing +- **Session Management**: Maintains conversation context across multiple interactions +- **Image Display**: Shows relevant document screenshots alongside responses +- **Authentication**: Microsoft Authentication Library (MSAL) integration +- **Conversation History**: Persistent storage and retrieval of chat sessions + +--- + +## Architecture + +```mermaid +graph TB + subgraph "Frontend (React)" + FE[Chat Interface] + AUTH[MSAL Auth] + CONV[Conversation Manager] + UI[UI Components] + end + + subgraph "Backend (Python/Flask)" + API[Flask Routes] + AGENT[ReAct Agent] + GRAPH[GraphRAG Engine] + SESSION[Session Manager] + PARSE[Document Parser] + end + + subgraph "AI/ML Layer" + LLM[OpenAI GPT-4] + EMBED[Text Embeddings] + LLAMAPARSE[LlamaParse] + end + + subgraph "Data Storage" + NEO4J[(Neo4j
Knowledge Graph)] + MONGO[(MongoDB
Conversations)] + VECTOR[(Vector Index
LlamaIndex)] + FILES[File Storage
Images/Documents] + end + + FE --> API + AUTH --> API + CONV --> API + + API --> AGENT + API --> SESSION + API --> PARSE + + AGENT --> GRAPH + GRAPH --> NEO4J + GRAPH --> VECTOR + AGENT --> LLM + + PARSE --> LLAMAPARSE + LLAMAPARSE --> FILES + PARSE --> EMBED + EMBED --> VECTOR + + SESSION --> MONGO + + style FE fill:#e1f5fe + style API fill:#f3e5f5 + style AGENT fill:#e8f5e8 + style NEO4J fill:#fff3e0 + style MONGO fill:#f1f8e9 +``` + +### Component Breakdown + +#### Frontend (React) +- **Chat Interface**: Main conversational UI with message bubbles, image viewing, and input handling +- **Authentication**: MSAL-based Microsoft authentication +- **Conversation Manager**: Handles multiple conversation sessions and history +- **Theme Toggle**: Dark/light mode support + +#### Backend (Python/Flask) +- **Flask Routes**: RESTful API endpoints for chat, authentication, file serving +- **ReAct Agent**: Custom implementation with reasoning, action, and observation cycles +- **GraphRAG Engine**: Hybrid retrieval combining vector search with graph-based community detection +- **Session Manager**: Maps frontend sessions to database conversations +- **Document Parser**: LlamaParse integration for multimodal document processing + +#### Data Layer +- **Neo4j**: Stores knowledge graph with entities, relationships, and communities +- **MongoDB**: Persists user conversations, messages, and session state +- **Vector Index**: LlamaIndex-based semantic search capabilities +- **File Storage**: Local filesystem for processed images and documents + +--- + +## Technology Stack + +### Backend +- **Framework**: Flask + Hypercorn (ASGI) +- **AI/ML**: + - OpenAI GPT-4 (LLM) + - text-embedding-3-small (embeddings) + - LlamaParse (document processing) + - LlamaIndex (vector indexing) +- **Databases**: + - Neo4j (knowledge graph) + - MongoDB (conversations) +- **Languages**: Python 3.9+ + +### Frontend +- **Framework**: React 18 + Vite +- **Styling**: TailwindCSS + Shadcn/ui +- **Authentication**: Microsoft Authentication Library (MSAL) +- **Languages**: JavaScript/JSX + +### Infrastructure +- **Web Server**: Hypercorn (ASGI server) +- **Containerization**: Docker support +- **Deployment**: Azure/Cloud-based + +--- + +## Data Flow + +```mermaid +sequenceDiagram + participant User + participant Frontend + participant API + participant Agent + participant GraphRAG + participant Neo4j + participant Vector + participant OpenAI + participant MongoDB + + User->>Frontend: Send message + Frontend->>API: POST /chat + API->>Agent: Process query + + Agent->>GraphRAG: Retrieve context + GraphRAG->>Vector: Vector similarity search + GraphRAG->>Neo4j: Community detection + GraphRAG->>OpenAI: Generate synthesis + GraphRAG->>Agent: Combined context + + Agent->>OpenAI: Generate response + OpenAI->>Agent: Response + reasoning + + Agent->>API: Structured response + API->>MongoDB: Store conversation + API->>Frontend: Response with sources/images + Frontend->>User: Display response +``` + +### Document Processing Flow + +```mermaid +flowchart TD + UPLOAD[Document Upload] --> PARSE[LlamaParse Processing] + PARSE --> EXTRACT[Extract Text + Images] + EXTRACT --> SPLIT[Semantic Splitting] + SPLIT --> EMBED[Generate Embeddings] + EMBED --> VECTOR[Store in Vector Index] + SPLIT --> GRAPH[Extract Entities/Relations] + GRAPH --> NEO4J[Store in Neo4j] + EXTRACT --> IMAGES[Save Page Images] + IMAGES --> STORAGE[File Storage] + NEO4J --> COMMUNITY[Community Detection] + COMMUNITY --> CACHE[Cache Communities] +``` + +--- + +## Database Design + +### Neo4j Knowledge Graph Schema + +```mermaid +erDiagram + Entity { + string name + string label + string description + dict properties + } + + Relation { + string label + string source_id + string target_id + string description + dict properties + } + + Community { + int community_id + text summary + list entity_ids + } + + Entity ||--o{ Relation : participates_in + Community ||--o{ Entity : contains +``` + +### MongoDB Collections Schema + +```mermaid +erDiagram + Users { + ObjectId _id + string username + string email + datetime created_at + datetime last_login + } + + Conversations { + ObjectId _id + string session_id + ObjectId user_id + string title + datetime created_at + datetime last_updated + boolean is_deleted + } + + Messages { + ObjectId _id + ObjectId conversation_id + string role + text content + array sources + array reasoning + array images + datetime timestamp + } + + Users ||--o{ Conversations : owns + Conversations ||--o{ Messages : contains +``` + +--- + +## API Reference + +### Authentication +All API endpoints require authentication via `X-MS-USERNAME` header (except in development mode). + +### Core Endpoints + +#### POST /chat +Processes chat messages and returns AI responses. + +**Request:** +```json +{ + "message": "string", + "sessionId": "string" +} +``` + +**Response:** +```json +{ + "status": "success", + "data": { + "response": "string", + "sources": [ + { + "content": "string", + "tool_name": "string", + "retrieval_method": "vector_only|graphrag_hybrid" + } + ], + "reasoning": [ + { + "type": "ActionReasoningStep|ObservationReasoningStep", + "action": "string", + "observation": "string" + } + ], + "images": [ + { + "filename": "string", + "document": "string", + "page": "number", + "url_encoded_filename": "string" + } + ] + } +} +``` + +#### GET /status +Returns system initialization status. + +**Response:** +```json +{ + "global_status": "initialized", + "initialized": true, + "timestamp": "2024-01-01T00:00:00.000Z" +} +``` + +#### GET /conversations +Retrieves user's conversation history. + +**Response:** +```json +{ + "status": "success", + "conversations": [ + { + "id": "string", + "title": "string", + "created_at": "datetime", + "last_updated": "datetime", + "session_id": "string" + } + ] +} +``` + +#### GET /conversations/{id}/messages +Retrieves messages for a specific conversation. + +**Response:** +```json +{ + "status": "success", + "conversation_title": "string", + "messages": [ + { + "id": "string", + "role": "user|assistant", + "content": "string", + "timestamp": "datetime", + "sources": [], + "reasoning": [], + "images": [] + } + ] +} +``` + +#### GET /images/{filename} +Serves processed document images. + +#### POST /reset +Resets the global agent's conversation memory. + +#### DELETE /conversations/{id} +Deletes a conversation (soft delete by default). + +--- + +## User Flow + +```mermaid +flowchart TD + START([User Access]) --> AUTH{Authenticated?} + AUTH -->|No| LOGIN[MSAL Login] + LOGIN --> AUTH + AUTH -->|Yes| LOAD[Load Conversations] + + LOAD --> NEWCHAT[Create New Chat] + NEWCHAT --> INTERFACE[Chat Interface] + + INTERFACE --> INPUT[User Input] + INPUT --> PROCESS[Process with GraphRAG] + PROCESS --> RETRIEVE[Hybrid Retrieval] + RETRIEVE --> GENERATE[Generate Response] + GENERATE --> DISPLAY[Display with Images] + DISPLAY --> INPUT + + DISPLAY --> SAVE[Save to History] + SAVE --> UPDATE[Update Conversation] + + INTERFACE --> HISTORY[View History] + HISTORY --> SELECT[Select Conversation] + SELECT --> LOAD_MSG[Load Messages] + LOAD_MSG --> INTERFACE + + INTERFACE --> EXPORT[Export Brief] + INTERFACE --> DELETE[Delete Conversation] +``` + +### Detailed User Journey + +1. **Authentication**: User logs in via Microsoft MSAL +2. **Conversation Creation**: System creates new conversation or loads existing +3. **Query Processing**: + - User sends message + - GraphRAG performs hybrid retrieval + - Vector similarity search finds relevant chunks + - Knowledge graph identifies related communities + - LLM synthesizes response with reasoning +4. **Response Display**: + - Text response with markdown support + - Source attribution tooltips + - Relevant document images + - Reasoning chain (if available) +5. **History Management**: Conversations persisted and retrievable + +--- + +## Security + +### Authentication +- Microsoft Authentication Library (MSAL) integration +- Azure AD tenant-based access control +- Session-based user identification + +### Data Protection +- No sensitive data logged in plain text +- Conversation data encrypted at rest (MongoDB) +- API key management via environment variables +- CORS configuration for cross-origin requests + +### Access Control +- User-scoped conversation access +- Session-based authorization +- Development vs production mode differentiation + +--- + +## Deployment + +### Environment Configuration + +#### Backend (.env) +```bash +# API Keys +OPENAI_API_KEY=your_openai_key +LLAMA_CLOUD_API_KEY=your_llama_cloud_key +ANTHROPIC_API_KEY=your_anthropic_key + +# Database Configuration +NEO4J_URL=bolt://localhost:7688 +NEO4J_USERNAME=neo4j +NEO4J_PASSWORD=hp-graphrag-2024 + +# Server Configuration +PORT=8746 +PRODUCTION=true +LOG_LEVEL=INFO +``` + +#### Frontend (.env) +```bash +VITE_BACKEND_URL=https://ai-sandbox.oliver.solutions/hp_chatbot_back +VITE_APP_BASE_URL=/hp_chatbot/ +``` + +### Deployment Steps + +1. **Database Setup**: + - Neo4j instance on port 7688 + - MongoDB with authentication + - Initialize collections via `init_mongodb.py` + +2. **Backend Deployment**: + ```bash + pip install -r requirements.txt + python main.py + ``` + +3. **Frontend Build**: + ```bash + cd chat-interface + npm install + npm run build + # Deploy dist/ contents to /hp_chatbot/ path + ``` + +4. **Web Server Configuration**: + - Configure reverse proxy (nginx/Apache) + - Set up SSL certificates + - Configure CORS origins + +--- + +## Development + +### Backend Development + +```bash +# Setup virtual environment +python -m venv env +source env/bin/activate # or env\Scripts\activate on Windows + +# Install dependencies +pip install -r requirements.txt + +# Start development server +python main.py +``` + +### Frontend Development + +```bash +cd chat-interface +npm install +npm run dev +``` + +### Key Development Commands + +| Command | Purpose | +|---------|---------| +| `python main.py` | Start backend server | +| `npm run dev` | Start frontend dev server | +| `npm run build` | Build frontend for production | +| `npm run lint` | Lint frontend code | + +### Code Structure + +``` +hp_graphRAG_bot/ +├── Backend (Python) +│ ├── main.py # Application entry point +│ ├── ai_core.py # Core AI engine & ReAct agent +│ ├── graph_rag_integration.py # GraphRAG system +│ ├── routes.py # Flask API routes +│ ├── session_manager.py # Session management +│ ├── mongodb_utils.py # MongoDB operations +│ ├── config.py # Configuration +│ └── shared_state.py # Global state management +├── Frontend (React) +│ └── chat-interface/ +│ ├── src/ +│ │ ├── App.jsx # Main application component +│ │ ├── components/ # React components +│ │ ├── auth.js # MSAL authentication +│ │ └── lib/ # Utilities +│ └── dist/ # Production build +└── Data Storage + ├── uploads/images/ # Processed document images + ├── index_storage/ # Vector index data + └── supporting_files/ # Source documents +``` + +--- + +## Troubleshooting + +### Common Issues + +#### Backend Issues + +**Problem**: `Global workflow agent not initialized` +**Solution**: Check OpenAI API key and Neo4j connectivity +```bash +# Verify environment variables +echo $OPENAI_API_KEY +# Check Neo4j connection +curl http://localhost:7474 +``` + +**Problem**: `LlamaParse timeout during document processing` +**Solution**: Increase timeout settings in config.py +```python +LLAMA_PARSE_MAX_TIMEOUT = 7200 # 2 hours +``` + +**Problem**: `MongoDB connection failed` +**Solution**: Verify MongoDB service and credentials +```bash +# Check MongoDB status +brew services list | grep mongodb +# Test connection +mongosh mongodb://hp:hp@localhost:27017/hp_chatbot +``` + +#### Frontend Issues + +**Problem**: `CORS policy blocking requests` +**Solution**: Update CORS_ALLOWED_ORIGINS in backend config.py + +**Problem**: `Authentication failures` +**Solution**: Verify MSAL configuration and Azure AD settings + +**Problem**: `Images not loading` +**Solution**: Check image file paths and backend /images/ endpoint + +### Debug Endpoints + +**Development Mode Only:** +- `GET /debug-status` - System state inspection +- `POST /reinitialize` - Force agent reinitialization +- `POST /capture-screenshot` - Manual image extraction testing + +### Logging + +All components use structured logging: +```python +log_structured('info', 'Event description', {'key': 'value'}) +``` + +Log files locations: +- Backend: `app.log` +- MongoDB operations: `mongodb.log` + +--- + +## Performance Considerations + +### Scaling +- **Vector Index**: Consider PostgreSQL pgvector for large deployments +- **Neo4j**: Implement read replicas for query scaling +- **MongoDB**: Use connection pooling and sharding +- **Caching**: Redis for session and community caches + +### Optimization +- **GraphRAG Communities**: Pre-computed and cached +- **Image Processing**: Async processing with queue system +- **Memory Management**: Agent memory reset policies +- **Response Time**: Parallel vector and graph retrieval + +--- + +## Future Enhancements + +### Planned Features +1. **Multi-tenant Architecture**: Support multiple organizations +2. **Advanced Analytics**: Usage metrics and conversation insights +3. **Enhanced Multimodal**: Video and audio processing +4. **Real-time Collaboration**: Multi-user conversations +5. **API Extensions**: Webhook integrations and external tool calling +6. **Advanced Security**: Role-based access control and audit logging + +### Technical Debt +- Implement comprehensive test suite +- Add API rate limiting +- Improve error handling consistency +- Optimize database queries +- Add health check endpoints + +--- + +*Documentation Version: 1.0* +*Last Updated: 2024-01-01* +*System Version: HP GraphRAG Chatbot v1.0* \ No newline at end of file diff --git a/document_generator.py b/document_generator.py new file mode 100644 index 0000000..87a922b --- /dev/null +++ b/document_generator.py @@ -0,0 +1,339 @@ +# hp_chatbot/document_generator.py + +import io +import re +import markdown2 +from bs4 import BeautifulSoup +from docx import Document +from docx.shared import Inches, Pt, RGBColor +from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_BREAK +from docx.oxml.shared import OxmlElement, qn +from docx.oxml import parse_xml + +from utils import log_structured + +# --- Helper for Horizontal Line --- +def add_horizontal_line(paragraph): + """Adds a horizontal line after the specified paragraph.""" + p = paragraph._p # the lxml element beneath the paragraph + pPr = p.get_or_add_pPr() # Get or add paragraph properties element + pBdr = OxmlElement('w:pBdr') # Create paragraph border element + # Add a bottom border + bottom_bdr = OxmlElement('w:bottom') + bottom_bdr.set(qn('w:val'), 'single') # Border style + bottom_bdr.set(qn('w:sz'), '6') # Border size (in 1/8 points) + bottom_bdr.set(qn('w:space'), '1') # Space between text and border + bottom_bdr.set(qn('w:color'), 'auto') # Border color + pBdr.append(bottom_bdr) + pPr.append(pBdr) + +# --- Inline Markdown to DOCX Run Formatting --- +def process_inline_formatting(paragraph, text): + """ + Processes simple inline markdown (bold, italic, code) within text + and adds formatted runs to the paragraph. + Handles nested formatting cautiously. + """ + # Regex to find **bold**, *italic*, _italic_, `code` segments + # It captures the marker and the content separately. + pattern = r'(\*\*|`|\*|_)(.*?)(\1)' + last_end = 0 + + for match in re.finditer(pattern, text): + start, end = match.span() + marker = match.group(1) + content = match.group(2) + + # Add preceding text if any + if start > last_end: + paragraph.add_run(text[last_end:start]) + + # Add formatted run + run = paragraph.add_run(content) + if marker == '**': + run.bold = True + elif marker == '*' or marker == '_': + run.italic = True + elif marker == '`': + run.font.name = 'Courier New' + # run.font.size = Pt(10) # Optional: Set size for code + + last_end = end + + # Add any remaining text after the last match + if last_end < len(text): + paragraph.add_run(text[last_end:]) + + +# --- HTML to DOCX Conversion --- +def convert_html_to_docx(doc: Document, html_content: str): + """ + Converts basic HTML content (from markdown conversion) to Word elements. + Handles common tags like paragraphs, headings, lists, bold, italic, code. + """ + # Pre-process HTML slightly for cleaner parsing + html_content = re.sub(r'\s*\n\s*', '\n', html_content).strip() # Normalize whitespace + html_content = f"{html_content}" # Wrap in body for better parsing + soup = BeautifulSoup(html_content, 'html.parser') + + # Recursive function to handle elements + def process_element(element, current_paragraph=None, current_style=None, in_list=False): + # Skip NavigableString if it's just whitespace or newline outside pre + if isinstance(element, str): + text = str(element).strip('\n') # Keep internal spaces, strip leading/trailing newlines + if text: # Only add if there's actual content + if current_paragraph: + run = current_paragraph.add_run(text) + if current_style: + if 'bold' in current_style: run.bold = True + if 'italic' in current_style: run.italic = True + if 'code' in current_style: run.font.name = 'Courier New' + else: + # Text outside paragraph usually means an error or whitespace + # log_structured('debug', f"Orphan text node found: '{text[:50]}...'") + pass # Or create a default paragraph: doc.add_paragraph(text) + return + + # --- Block Level Elements --- + if element.name in ['p', 'div']: + # Avoid creating paragraphs for empty containers unless they contain
+ text_content = element.get_text(strip=True) + has_br = element.find('br') + if text_content or has_br: + para = doc.add_paragraph() + # Apply list indentation if necessary (though lists handle their own paras) + # if in_list: para.paragraph_format.left_indent = Inches(0.5) + new_style = current_style.copy() if current_style else set() + for child in element.children: + process_element(child, para, new_style, in_list) + # else: skip empty p/div + + elif element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: + try: + level = int(element.name[1]) + heading = doc.add_heading(level=level) + # Process children for inline formatting within heading + new_style = current_style.copy() if current_style else set() + for child in element.children: + process_element(child, heading, new_style, in_list) + # If no children processed (just text), add it directly + if not heading.runs: + heading.add_run(element.get_text(strip=True)) + except ValueError: pass # Should not happen with h1-h6 + + elif element.name == 'ul': + for li in element.find_all('li', recursive=False): + # Each li gets its own paragraph with bullet style + para = doc.add_paragraph(style='List Bullet') + new_style = current_style.copy() if current_style else set() + for child in li.children: + process_element(child, para, new_style, in_list=True) + # If li was empty or only contained whitespace + if not para.text.strip(): + para.text = "" # Ensure empty bullet point exists + + elif element.name == 'ol': + # Numbering is handled by the 'List Number' style + for li in element.find_all('li', recursive=False): + para = doc.add_paragraph(style='List Number') + new_style = current_style.copy() if current_style else set() + for child in li.children: + process_element(child, para, new_style, in_list=True) + if not para.text.strip(): + para.text = "" # Ensure empty numbered item exists + + # Note: 'li' is handled within 'ul'/'ol' processing. + + elif element.name == 'pre': + # Often contains a 'code' element, handle that + code_tag = element.find('code') + content = code_tag.get_text() if code_tag else element.get_text() + if content.strip(): + para = doc.add_paragraph(style='CodeStyle') # Requires 'CodeStyle' to be defined + # Preserve whitespace more carefully for

+                  run = para.add_run(content.strip('\n')) # Strip outer newlines only
+                  run.font.name = 'Courier New'
+                  # run.font.size = Pt(10)
+
+        elif element.name == 'blockquote':
+              para = doc.add_paragraph(style='Quote') # Requires 'Quote' style
+              new_style = current_style.copy() if current_style else set()
+              for child in element.children:
+                  process_element(child, para, new_style, in_list)
+
+        elif element.name == 'hr':
+             para = doc.add_paragraph()
+             add_horizontal_line(para)
+
+        elif element.name == 'br':
+            if current_paragraph:
+                current_paragraph.add_run().add_break() # Add line break within paragraph
+
+
+        # --- Inline Elements ---
+        elif element.name in ['strong', 'b']:
+            new_style = current_style.copy() if current_style else set()
+            new_style.add('bold')
+            for child in element.children:
+                process_element(child, current_paragraph, new_style, in_list)
+
+        elif element.name in ['em', 'i']:
+            new_style = current_style.copy() if current_style else set()
+            new_style.add('italic')
+            for child in element.children:
+                process_element(child, current_paragraph, new_style, in_list)
+
+        elif element.name == 'code':
+            # Handle inline code - assumes it's within a paragraph already
+            if current_paragraph:
+                text = element.get_text()
+                if text:
+                    run = current_paragraph.add_run(text)
+                    run.font.name = 'Courier New'
+                    # Add specific inline code style if desired
+            else:
+                 # Code tag not within a paragraph? Create one.
+                 para = doc.add_paragraph(style='CodeStyle')
+                 run = para.add_run(element.get_text())
+                 run.font.name = 'Courier New'
+
+
+        elif element.name == 'a':
+            # Add hyperlink if possible, otherwise just text
+            text = element.get_text(strip=True)
+            href = element.get('href')
+            if current_paragraph and text:
+                # python-docx doesn't have direct hyperlink support easily added here.
+                # Simplest: add text with underline and blue color.
+                run = current_paragraph.add_run(text)
+                run.underline = True
+                run.font.color.rgb = RGBColor(0x05, 0x63, 0xC1) # Standard link blue
+                # For actual hyperlinks, more complex XML manipulation is needed.
+
+
+        # --- Body/Other Tags: Process children ---
+        elif element.name in ['body', 'span', 'div']: # Treat span/div mostly as containers
+            new_style = current_style.copy() if current_style else set()
+            for child in element.children:
+                 process_element(child, current_paragraph, new_style, in_list)
+
+        # --- Ignored Tags ---
+        elif element.name in ['script', 'style', 'head', 'meta', 'title']:
+             pass # Ignore these tags and their content
+
+        else:
+            # Unknown tag: try to process its children if it's a container,
+            # or add its text content if it's inline-like.
+            log_structured('warning', f"Unhandled HTML tag encountered: <{element.name}>", {'content_preview': element.get_text(strip=True)[:50]})
+            # Default behavior: process children recursively
+            new_style = current_style.copy() if current_style else set()
+            for child in element.children:
+                 process_element(child, current_paragraph, new_style, in_list)
+
+
+    # Start processing from the top-level elements within the parsed body
+    body = soup.find('body')
+    if body:
+        for element in body.children:
+            process_element(element)
+
+
+# --- Main Markdown to DOCX Function ---
+def create_brief_docx(brief_content_markdown: str) -> io.BytesIO:
+    """
+    Creates a Word document (.docx) in memory from markdown content.
+
+    Args:
+        brief_content_markdown: The markdown string content.
+
+    Returns:
+        An io.BytesIO buffer containing the Word document.
+    """
+    doc = Document()
+
+    # --- Define Styles (Optional but recommended) ---
+    styles = doc.styles
+    # Normal style
+    style = styles['Normal']
+    font = style.font
+    font.name = 'Calibri' # Or HP specific font if available
+    font.size = Pt(11)
+
+    # Code style (example)
+    try:
+        code_style = styles.add_style('CodeStyle', 1) # 1 for paragraph style
+        code_style.font.name = 'Courier New'
+        code_style.font.size = Pt(10)
+        # Prevent spell check for code blocks
+        code_style.element.rPr.rFonts.set(qn('w:ascii'), 'Courier New')
+        code_style.element.rPr.rFonts.set(qn('w:hAnsi'), 'Courier New')
+        # code_style.element.xpath('./w:rPr/w:lang')[0].set(qn('w:noProof'), '1') # Requires lxml maybe
+        p_fmt = code_style.paragraph_format
+        p_fmt.space_before = Pt(6)
+        p_fmt.space_after = Pt(6)
+    except ValueError:
+         log_structured('warning', "'CodeStyle' already exists. Using existing.")
+         code_style = styles['CodeStyle'] # Use existing if it fails to add
+
+    # Quote style (example)
+    try:
+         quote_style = styles.add_style('QuoteStyle', 1)
+         quote_style.font.italic = True
+         quote_style.paragraph_format.left_indent = Inches(0.5)
+         quote_style.paragraph_format.space_before = Pt(6)
+         quote_style.paragraph_format.space_after = Pt(6)
+    except ValueError:
+         log_structured('warning', "'QuoteStyle' already exists. Using existing.")
+         quote_style = styles['QuoteStyle']
+
+
+    # --- Document Header ---
+    title = doc.add_heading('Marketing Brief', 0)
+    title.alignment = WD_ALIGN_PARAGRAPH.CENTER
+
+    date_para = doc.add_paragraph()
+    date_para.alignment = WD_ALIGN_PARAGRAPH.RIGHT
+    date_run = date_para.add_run(datetime.now().strftime("%B %d, %Y"))
+    date_run.italic = True
+    # Add some space after date
+    date_para.paragraph_format.space_after = Pt(12)
+
+    # Add a horizontal line separator
+    hr_para = doc.add_paragraph()
+    add_horizontal_line(hr_para)
+    hr_para.paragraph_format.space_after = Pt(18) # Space after the line
+
+
+    # --- Convert Markdown to HTML ---
+    # Using markdown2 with recommended extras for broad compatibility
+    extras = [
+        "tables", "fenced-code-blocks", "header-ids", "footnotes",
+        "task_list", "code-friendly", "cuddled-lists", "markdown-in-html",
+        "strike", "spoiler", "target-blank-links", "smarty-pants" # Added smarty-pants
+    ]
+    html_content = markdown2.markdown(brief_content_markdown, extras=extras)
+
+    log_structured('debug', 'Converted markdown to HTML for DOCX generation', {
+        'md_preview': brief_content_markdown[:200],
+        'html_preview': html_content[:300]
+    })
+
+    # --- Convert HTML to Word Document Elements ---
+    try:
+        convert_html_to_docx(doc, html_content)
+    except Exception as conversion_err:
+         log_structured('error', "Error during HTML to DOCX conversion", {
+             'error': str(conversion_err),
+             'traceback': traceback.format_exc()
+         })
+         # Add error message to the document itself
+         doc.add_paragraph("Error: Could not fully convert content from HTML to DOCX.", style='Emphasis')
+         doc.add_paragraph(str(conversion_err))
+
+
+    # --- Save to Buffer ---
+    doc_buffer = io.BytesIO()
+    doc.save(doc_buffer)
+    doc_buffer.seek(0)
+
+    return doc_buffer
\ No newline at end of file
diff --git a/graphRAG.py b/graphRAG.py
new file mode 100644
index 0000000..c28b533
--- /dev/null
+++ b/graphRAG.py
@@ -0,0 +1,581 @@
+import os
+import json
+import re
+import asyncio
+import networkx as nx
+from collections import defaultdict
+from typing import Any, List, Callable, Optional, Union, Dict
+from dotenv import load_dotenv
+
+# Import LlamaIndex components
+from llama_index.core import Document, Settings
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core import PropertyGraphIndex
+from llama_index.core.async_utils import run_jobs
+from llama_index.core.indices.property_graph.utils import default_parse_triplets_fn
+from llama_index.core.graph_stores.types import EntityNode, KG_NODES_KEY, KG_RELATIONS_KEY, Relation
+from llama_index.core.llms.llm import LLM
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.prompts.default_prompts import DEFAULT_KG_TRIPLET_EXTRACT_PROMPT
+from llama_index.core.schema import TransformComponent, BaseNode
+from llama_index.core.query_engine import CustomQueryEngine
+from llama_index.llms.openai import OpenAI
+from llama_index.core.llms import ChatMessage
+from llama_index.graph_stores.neo4j import Neo4jPropertyGraphStore
+from llama_index.core import SimpleDirectoryReader
+
+# Community detection (using NetworkX instead of graspologic as a fallback)
+try:
+    from community import best_partition  # python-louvain package
+except ImportError:
+    print("Community detection package not found, using NetworkX built-in community detection")
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Use API key from environment as fallback
+if not os.environ.get("OPENAI_API_KEY"):
+    os.environ["OPENAI_API_KEY"] = "sk-proj-wXcoIn81Vwg4Iaw0vhmYT3BlbkFJmt1eOxeEAF1juUfhzMtk"
+
+# Define the GraphRAGExtractor class
+class GraphRAGExtractor(TransformComponent):
+    """Extract triples from a graph.
+
+    Uses an LLM and a simple prompt + output parsing to
+    extract paths (i.e. triples) and entity, relation descriptions
+    from text.
+
+    Args:
+        llm (LLM):
+            The language model to use.
+        extract_prompt (Union[str, PromptTemplate]):
+            The prompt to use for extracting triples.
+        parse_fn (callable):
+            A function to parse the output of the language
+            model.
+        num_workers (int):
+            The number of workers to use for parallel
+            processing.
+        max_paths_per_chunk (int):
+            The maximum number of paths to extract per chunk.
+    """
+
+    llm: LLM
+    extract_prompt: PromptTemplate
+    parse_fn: Callable
+    num_workers: int
+    max_paths_per_chunk: int
+
+    def __init__(
+        self,
+        llm: Optional[LLM] = None,
+        extract_prompt: Optional[Union[str, PromptTemplate]] = None,
+        parse_fn: Callable = default_parse_triplets_fn,
+        max_paths_per_chunk: int = 10,
+        num_workers: int = 4,
+    ) -> None:
+        """Init params."""
+        from llama_index.core import Settings
+
+        if isinstance(extract_prompt, str):
+            extract_prompt = PromptTemplate(extract_prompt)
+
+        super().__init__(
+            llm=llm or Settings.llm,
+            extract_prompt=extract_prompt or DEFAULT_KG_TRIPLET_EXTRACT_PROMPT,
+            parse_fn=parse_fn,
+            num_workers=num_workers,
+            max_paths_per_chunk=max_paths_per_chunk,
+        )
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "GraphExtractor"
+
+    def __call__(
+        self, nodes: List[BaseNode], show_progress: bool = False, **kwargs: Any
+    ) -> List[BaseNode]:
+        """Extract triples from nodes."""
+        return asyncio.run(
+            self.acall(nodes, show_progress=show_progress, **kwargs)
+        )
+
+    async def _aextract(self, node: BaseNode) -> BaseNode:
+        """Extract triples from a node."""
+        assert hasattr(node, "text")
+
+        text = node.get_content(metadata_mode="llm")
+        try:
+            llm_response = await self.llm.apredict(
+                self.extract_prompt,
+                text=text,
+                max_knowledge_triplets=self.max_paths_per_chunk,
+            )
+            entities, entities_relationship = self.parse_fn(llm_response)
+        except ValueError:
+            entities = []
+            entities_relationship = []
+
+        existing_nodes = node.metadata.pop(KG_NODES_KEY, [])
+        existing_relations = node.metadata.pop(KG_RELATIONS_KEY, [])
+        
+        entity_metadata = node.metadata.copy()
+        for entity, entity_type, description in entities:
+            entity_metadata["entity_description"] = description
+            entity_node = EntityNode(
+                name=entity, label=entity_type,
+                properties=entity_metadata
+            )
+            existing_nodes.append(entity_node)
+
+        relation_metadata = node.metadata.copy()
+        for triple in entities_relationship:
+            subj, obj, rel, description = triple
+            relation_metadata["relationship_description"] = description
+            rel_node = Relation(
+                label=rel,
+                source_id=subj,
+                target_id=obj,
+                properties=relation_metadata,
+            )
+            existing_relations.append(rel_node)
+
+        node.metadata[KG_NODES_KEY] = existing_nodes
+        node.metadata[KG_RELATIONS_KEY] = existing_relations
+        return node
+
+    async def acall(
+        self, nodes: List[BaseNode], show_progress: bool = False, **kwargs: Any
+    ) -> List[BaseNode]:
+        """Extract triples from nodes async."""
+        jobs = []
+        for node in nodes:
+            jobs.append(self._aextract(node))
+
+        return await run_jobs(
+            jobs,
+            workers=self.num_workers,
+            show_progress=show_progress,
+            desc="Extracting paths from text",
+        )
+
+# Define the GraphRAGStore class
+class GraphRAGStore(Neo4jPropertyGraphStore):
+    community_summary = {}
+    entity_info = None
+    max_cluster_size = 5
+
+    def generate_community_summary(self, text):
+        """Generate summary for a given text using an LLM."""
+        messages = [
+            ChatMessage(
+                role="system",
+                content=(
+                    "You are provided with a set of "
+                    "relationships from a knowledge graph, each represented as "
+                    "entity1->entity2->relation-"
+                    ">relationship_description. Your task is to create a summary of "
+                    "these "
+                    "relationships. The summary should include "
+                    "the names of the entities involved and a concise synthesis "
+                    "of the relationship descriptions. The "
+                    "goal is to capture the most critical and relevant details that "
+                    "highlight the nature and significance of "
+                    "each relationship. Ensure that the summary is coherent and "
+                    "integrates the information in a way that "
+                    "emphasizes the key aspects of the relationships."
+                ),
+            ),
+            ChatMessage(role="user", content=text),
+        ]
+        response = OpenAI().chat(messages)
+        clean_response = re.sub(r"^assistant:\s*", "", str(response)).strip()
+        return clean_response
+
+    def build_communities(self):
+        """Builds communities from the graph and summarizes them."""
+        nx_graph = self._create_nx_graph()
+        
+        # Use either Leiden algorithm (from graspologic) or an alternative
+        try:
+            from graspologic.partition import hierarchical_leiden
+            community_hierarchical_clusters = hierarchical_leiden(
+                nx_graph, max_cluster_size=self.max_cluster_size
+            )
+            self.entity_info, community_info = self._collect_community_info(
+                nx_graph, community_hierarchical_clusters
+            )
+        except ImportError:
+            # Fallback to community detection using NetworkX or python-louvain
+            try:
+                from community import best_partition
+                partition = best_partition(nx_graph)
+                # Reformat partition data to expected structure
+                clusters = []
+                for node, cluster_id in partition.items():
+                    class Cluster:
+                        def __init__(self, node, cluster):
+                            self.node = node
+                            self.cluster = cluster
+                    clusters.append(Cluster(node, cluster_id))
+                self.entity_info, community_info = self._collect_community_info(
+                    nx_graph, clusters
+                )
+            except ImportError:
+                # Use NetworkX's built-in community detection
+                from networkx.algorithms import community
+                communities = community.greedy_modularity_communities(nx_graph)
+                clusters = []
+                for i, comm in enumerate(communities):
+                    for node in comm:
+                        class Cluster:
+                            def __init__(self, node, cluster):
+                                self.node = node
+                                self.cluster = cluster
+                        clusters.append(Cluster(node, i))
+                self.entity_info, community_info = self._collect_community_info(
+                    nx_graph, clusters
+                )
+                
+        self._summarize_communities(community_info)
+
+    def _create_nx_graph(self):
+        """Converts internal graph representation to NetworkX graph."""
+        nx_graph = nx.Graph()
+        triplets = self.get_triplets()
+        for entity1, relation, entity2 in triplets:
+            nx_graph.add_node(entity1.name)
+            nx_graph.add_node(entity2.name)
+            nx_graph.add_edge(
+                relation.source_id,
+                relation.target_id,
+                relationship=relation.label,
+                description=relation.properties.get("relationship_description", "No description provided"),
+            )
+        return nx_graph
+
+    def _collect_community_info(self, nx_graph, clusters):
+        """
+        Collect information for each node based on their community,
+        allowing entities to belong to multiple clusters.
+        """
+        entity_info = defaultdict(set)
+        community_info = defaultdict(list)
+
+        for item in clusters:
+            node = item.node
+            cluster_id = item.cluster
+
+            # Update entity_info
+            entity_info[node].add(cluster_id)
+
+            for neighbor in nx_graph.neighbors(node):
+                edge_data = nx_graph.get_edge_data(node, neighbor)
+                if edge_data:
+                    detail = f"{node} -> {neighbor} -> {edge_data['relationship']} -> {edge_data['description']}"
+                    community_info[cluster_id].append(detail)
+
+        # Convert sets to lists for easier serialization if needed
+        entity_info = {k: list(v) for k, v in entity_info.items()}
+
+        return dict(entity_info), dict(community_info)
+
+    def _summarize_communities(self, community_info):
+        """Generate and store summaries for each community."""
+        for community_id, details in community_info.items():
+            details_text = "\n".join(details) + "." # Ensure it ends with a period
+            self.community_summary[community_id] = self.generate_community_summary(details_text)
+
+    def get_community_summaries(self):
+        """Returns the community summaries, building them if not already done."""
+        if not self.community_summary:
+            self.build_communities()
+        return self.community_summary
+
+# Define the GraphRAGQueryEngine class
+class GraphRAGQueryEngine(CustomQueryEngine):
+    graph_store: Union[GraphRAGStore, Any]  # Accept any type of graph store
+    index: PropertyGraphIndex
+    llm: LLM
+    similarity_top_k: int = 20
+
+    def custom_query(self, query_str: str) -> str:
+        """Process query using either community-based approach or direct retrieval."""
+        # Check if we're using GraphRAGStore with communities or SimplePropertyGraphStore
+        if hasattr(self.graph_store, 'get_community_summaries'):
+            # GraphRAG approach with communities
+            entities = self.get_entities(query_str, self.similarity_top_k)
+            
+            community_ids = self.retrieve_entity_communities(
+                self.graph_store.entity_info, entities
+            )
+            community_summaries = self.graph_store.get_community_summaries()
+            community_answers = [
+                self.generate_answer_from_summary(community_summary, query_str)
+                for id, community_summary in community_summaries.items()
+                if id in community_ids
+            ]
+
+            final_answer = self.aggregate_answers(community_answers)
+            return final_answer
+        else:
+            # Simple approach for SimplePropertyGraphStore
+            # Just get relevant nodes and generate answer
+            nodes = self.index.as_retriever(
+                similarity_top_k=self.similarity_top_k
+            ).retrieve(query_str)
+            
+            if not nodes:
+                return "I couldn't find any relevant information to answer your question."
+            
+            # Combine text from all retrieved nodes
+            context = "\n\n".join([node.get_content() for node in nodes])
+            
+            # Generate answer using the LLM
+            prompt = f"Based on the following information, please answer this question: {query_str}\n\nInformation:\n{context}"
+            messages = [
+                ChatMessage(role="system", content=prompt),
+                ChatMessage(role="user", content="Please provide a comprehensive answer based on the information provided.")
+            ]
+            response = self.llm.chat(messages)
+            return str(response).strip()
+
+    def get_entities(self, query_str, similarity_top_k):
+        nodes_retrieved = self.index.as_retriever(
+            similarity_top_k=similarity_top_k
+        ).retrieve(query_str)
+
+        entities = set()
+        pattern = r"^(\w+(?:\s+\w+)*)\s*->\s*([a-zA-Z\s]+?)\s*->\s*(\w+(?:\s+\w+)*)$"
+
+        for node in nodes_retrieved:
+            matches = re.findall(
+                pattern, node.text, re.MULTILINE | re.IGNORECASE
+            )
+            for match in matches:
+                subject = match[0]
+                obj = match[2]
+                entities.add(subject)
+                entities.add(obj)
+
+        return list(entities)
+
+    def retrieve_entity_communities(self, entity_info, entities):
+        """
+        Retrieve cluster information for given entities,
+        allowing for multiple clusters per entity.
+
+        Args:
+            entity_info (dict): Dictionary mapping entities to their cluster IDs (list).
+            entities (list): List of entity names to retrieve information for.
+
+        Returns:
+            List of community or cluster IDs to which an entity belongs.
+        """
+        community_ids = []
+
+        for entity in entities:
+            if entity in entity_info:
+                community_ids.extend(entity_info[entity])
+
+        return list(set(community_ids))
+
+    def generate_answer_from_summary(self, community_summary, query):
+        """Generate an answer from a community summary based on a given query using LLM."""
+        prompt = (
+            f"Given the community summary: {community_summary}, "
+            f"how would you answer the following query? Query: {query}"
+        )
+        messages = [
+            ChatMessage(role="system", content=prompt),
+            ChatMessage(
+                role="user",
+                content="I need an answer based on the above information.",
+            ),
+        ]
+        response = self.llm.chat(messages)
+        cleaned_response = re.sub(r"^assistant:\s*", "", str(response)).strip()
+        return cleaned_response
+
+    def aggregate_answers(self, community_answers):
+        """Aggregate individual community answers into a final, coherent response."""
+        prompt = "Combine the following intermediate answers into a final, concise response."
+        messages = [
+            ChatMessage(role="system", content=prompt),
+            ChatMessage(
+                role="user",
+                content=f"Intermediate answers: {community_answers}",
+            ),
+        ]
+        final_response = self.llm.chat(messages)
+        cleaned_final_response = re.sub(
+            r"^assistant:\s*", "", str(final_response)
+        ).strip()
+        return cleaned_final_response
+
+def custom_parse_fn(response_str: str) -> Any:
+    """Custom parser for LLM responses that extract entities and relationships"""
+    json_pattern = r"\{.*\}"
+    match = re.search(json_pattern, response_str, re.DOTALL)
+    entities = []
+    relationships = []
+    
+    if not match:
+        return entities, relationships
+    
+    json_str = match.group(0)
+    try:
+        data = json.loads(json_str)
+        entities = [
+            (
+                entity["entity_name"],
+                entity["entity_type"],
+                entity.get("entity_description", f"Description of {entity['entity_name']}"),
+            )
+            for entity in data.get("entities", [])
+        ]
+        relationships = [
+            (
+                relation["source_entity"],
+                relation["target_entity"],
+                relation["relation"],
+                relation.get("relationship_description", f"Relationship between {relation['source_entity']} and {relation['target_entity']}"),
+            )
+            for relation in data.get("relationships", [])
+        ]
+        return entities, relationships
+    except (json.JSONDecodeError, KeyError) as e:
+        print(f"Error parsing response: {e}")
+        print(f"Problematic JSON: {json_str[:200]}...")
+        return entities, relationships
+
+# Define the prompt template for triple extraction
+KG_TRIPLET_EXTRACT_TMPL = """
+-Goal-
+Given a text document, identify all entities and their entity types from the text and all relationships among the identified entities.
+
+Given the text, extract up to {max_knowledge_triplets} entity-relation triplets.
+
+-Steps-
+1. Identify all entities. For each identified entity, extract the following information:
+- entity_name: Name of the entity, capitalized
+- entity_type: Type of the entity
+- entity_description: Comprehensive description of the entity's attributes and activities
+
+2. From the entities identified in step 1, identify all pairs of (source_entity, target_entity) that are *clearly related* to each other.
+For each pair of related entities, extract the following information:
+- source_entity: name of the source entity, as identified in step 1
+- target_entity: name of the target entity, as identified in step 1
+- relation: relationship between source_entity and target_entity
+- relationship_description: explanation as to why you think the source entity and the target entity are related to each other
+
+3. Output Formatting:
+- Return the result in valid JSON format with two keys: 'entities' (list of entity objects) and 'relationships' (list of relationship objects).
+- Exclude any text outside the JSON structure (e.g., no explanations or comments).
+- If no entities or relationships are identified, return empty lists: { "entities": [], "relationships": [] }.
+
+-Real Data-
+######################
+text: {text}
+######################
+output:
+"""
+
+def main():
+    print("Starting GraphRAG document processing...")
+    
+    # Load documents from specified directory
+    documents = SimpleDirectoryReader(
+        input_dir="supporting_files/files_for_rag_store"
+    ).load_data()
+    
+    print(f"Loaded {len(documents)} documents")
+    
+    # Create nodes using a sentence splitter
+    splitter = SentenceSplitter(chunk_size=1024, chunk_overlap=20)
+    nodes = splitter.get_nodes_from_documents(documents)
+    
+    print(f"Created {len(nodes)} nodes from documents")
+    
+    # Initialize the LLM
+    llm = OpenAI(model="gpt-4")
+    
+    # Create the knowledge graph extractor
+    kg_extractor = GraphRAGExtractor(
+        llm=llm,
+        extract_prompt=KG_TRIPLET_EXTRACT_TMPL,
+        max_paths_per_chunk=2,
+        parse_fn=custom_parse_fn,
+    )
+    
+    # Connect to Neo4j running in Docker
+    neo4j_username = "neo4j"
+    neo4j_password = "tavern-easy-museum-arthur-coconut-3483"
+    neo4j_url = "bolt://localhost:7687"
+    
+    print(f"Connecting to Neo4j at {neo4j_url} with username '{neo4j_username}'")
+    
+    # Create GraphRAGStore (our extended Neo4j store)
+    try:
+        graph_store = GraphRAGStore(
+            username=neo4j_username, 
+            password=neo4j_password,
+            url=neo4j_url
+        )
+        print("Successfully connected to Neo4j database")
+    except Exception as e:
+        print(f"Error connecting to Neo4j: {e}")
+        print("Falling back to in-memory graph store. Some features may be limited.")
+        # Fallback to in-memory graph store if Neo4j connection fails
+        from llama_index.core.graph_stores import SimplePropertyGraphStore
+        graph_store = SimplePropertyGraphStore()
+    
+    # Build the index
+    index = PropertyGraphIndex(
+        nodes=nodes,
+        kg_extractors=[kg_extractor],
+        property_graph_store=graph_store,
+        show_progress=True,
+    )
+    
+    print("Building graph communities...")
+    try:
+        # Build communities for graph-based querying
+        # Only for GraphRAGStore, not for SimplePropertyGraphStore
+        if hasattr(graph_store, 'build_communities'):
+            graph_store.build_communities()
+            print("Communities built successfully")
+        else:
+            print("Skipping community building (not using Neo4j)")
+    except Exception as e:
+        print(f"Error building communities: {e}")
+    
+    # Create the query engine
+    query_engine = GraphRAGQueryEngine(
+        graph_store=graph_store,
+        llm=llm,
+        index=index,
+        similarity_top_k=10,
+    )
+    
+    # Simple interactive query loop
+    print("\n--- GraphRAG Query System Ready ---")
+    print("Type 'exit' to quit")
+    
+    while True:
+        query = input("\nEnter your query: ")
+        
+        if query.lower() in ('exit', 'quit'):
+            break
+            
+        try:
+            response = query_engine.custom_query(query)
+            print("\nResponse:")
+            print(response)
+        except Exception as e:
+            print(f"Error processing query: {e}")
+    
+    print("GraphRAG session ended")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/graph_rag_integration.py b/graph_rag_integration.py
new file mode 100644
index 0000000..a0dd2e0
--- /dev/null
+++ b/graph_rag_integration.py
@@ -0,0 +1,882 @@
+"""
+HP GraphRAG Integration
+
+Integrates GraphRAG functionality into the HP RAG pipeline.
+- GraphRAG for knowledge graph construction from semantically split nodes
+- Community detection and summarization for improved context retrieval
+"""
+
+import os
+import json
+import re
+import asyncio
+import networkx as nx
+from collections import defaultdict
+from typing import Any, List, Callable, Optional, Union, Dict
+from pathlib import Path
+
+# Import LlamaIndex components
+from llama_index.core import Document, Settings
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core import PropertyGraphIndex
+from llama_index.core.async_utils import run_jobs
+from llama_index.core.indices.property_graph.utils import default_parse_triplets_fn
+from llama_index.core.graph_stores.types import EntityNode, KG_NODES_KEY, KG_RELATIONS_KEY, Relation
+from llama_index.core.llms.llm import LLM
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.prompts.default_prompts import DEFAULT_KG_TRIPLET_EXTRACT_PROMPT
+from llama_index.core.schema import TransformComponent, BaseNode
+from llama_index.core.query_engine import CustomQueryEngine
+from llama_index.llms.openai import OpenAI
+from llama_index.core.llms import ChatMessage
+from llama_index.graph_stores.neo4j import Neo4jPropertyGraphStore
+from llama_index.core import SimpleDirectoryReader
+from llama_index.core.vector_stores.types import VectorStoreInfo, MetadataInfo
+from llama_index.core.retrievers import VectorIndexRetriever
+
+# Community detection (using NetworkX instead of graspologic as a fallback)
+try:
+    from community import best_partition  # python-louvain package
+except ImportError:
+    print("Community detection package not found, using NetworkX built-in community detection")
+
+# Import from our modules
+from utils import logger, log_structured
+from config import NEO4J_URL, NEO4J_USERNAME, NEO4J_PASSWORD
+import config
+
+# Define the GraphRAGExtractor class
+class GraphRAGExtractor(TransformComponent):
+    """Extract triples from a graph.
+
+    Uses an LLM and a simple prompt + output parsing to
+    extract paths (i.e. triples) and entity, relation descriptions
+    from text.
+
+    Args:
+        llm (LLM):
+            The language model to use.
+        extract_prompt (Union[str, PromptTemplate]):
+            The prompt to use for extracting triples.
+        parse_fn (callable):
+            A function to parse the output of the language
+            model.
+        num_workers (int):
+            The number of workers to use for parallel
+            processing.
+        max_paths_per_chunk (int):
+            The maximum number of paths to extract per chunk.
+    """
+
+    llm: LLM
+    extract_prompt: PromptTemplate
+    parse_fn: Callable
+    num_workers: int
+    max_paths_per_chunk: int
+
+    def __init__(
+        self,
+        llm: Optional[LLM] = None,
+        extract_prompt: Optional[Union[str, PromptTemplate]] = None,
+        parse_fn: Callable = default_parse_triplets_fn,
+        max_paths_per_chunk: int = 10,
+        num_workers: int = 8,
+    ) -> None:
+        """Init params."""
+        from llama_index.core import Settings
+
+        if isinstance(extract_prompt, str):
+            extract_prompt = PromptTemplate(extract_prompt)
+
+        super().__init__(
+            llm=llm or Settings.llm,
+            extract_prompt=extract_prompt or DEFAULT_KG_TRIPLET_EXTRACT_PROMPT,
+            parse_fn=parse_fn,
+            num_workers=num_workers,
+            max_paths_per_chunk=max_paths_per_chunk,
+        )
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "GraphExtractor"
+
+    def __call__(
+        self, nodes: List[BaseNode], show_progress: bool = False, **kwargs: Any
+    ) -> List[BaseNode]:
+        """Extract triples from nodes."""
+        return asyncio.run(
+            self.acall(nodes, show_progress=show_progress, **kwargs)
+        )
+
+    async def _aextract(self, node: BaseNode) -> BaseNode:
+        """Extract triples from a node."""
+        assert hasattr(node, "text")
+
+        text = node.get_content(metadata_mode="llm")
+        try:
+            llm_response = await self.llm.apredict(
+                self.extract_prompt,
+                text=text,
+                max_knowledge_triplets=self.max_paths_per_chunk,
+            )
+            entities, entities_relationship = self.parse_fn(llm_response)
+        except ValueError:
+            entities = []
+            entities_relationship = []
+
+        existing_nodes = node.metadata.pop(KG_NODES_KEY, [])
+        existing_relations = node.metadata.pop(KG_RELATIONS_KEY, [])
+        
+        entity_metadata = node.metadata.copy()
+        for entity, entity_type, description in entities:
+            entity_metadata["entity_description"] = description
+            entity_node = EntityNode(
+                name=entity, label=entity_type,
+                properties=entity_metadata
+            )
+            existing_nodes.append(entity_node)
+
+        relation_metadata = node.metadata.copy()
+        for triple in entities_relationship:
+            subj, obj, rel, description = triple
+            relation_metadata["relationship_description"] = description
+            rel_node = Relation(
+                label=rel,
+                source_id=subj,
+                target_id=obj,
+                properties=relation_metadata,
+            )
+            existing_relations.append(rel_node)
+
+        node.metadata[KG_NODES_KEY] = existing_nodes
+        node.metadata[KG_RELATIONS_KEY] = existing_relations
+        return node
+
+    async def acall(
+        self, nodes: List[BaseNode], show_progress: bool = False, **kwargs: Any
+    ) -> List[BaseNode]:
+        """Extract triples from nodes async."""
+        jobs = []
+        for node in nodes:
+            jobs.append(self._aextract(node))
+
+        return await run_jobs(
+            jobs,
+            workers=self.num_workers,
+            show_progress=show_progress,
+            desc="Extracting paths from text",
+        )
+
+# Define the GraphRAGStore class (integrating with Neo4j)
+import pickle
+import os
+from pathlib import Path
+
+class GraphRAGStore:
+    community_summary = {}
+    entity_info = None
+    max_cluster_size = 5
+    property_graph_store = None
+    communities_built = False  # Track if communities have been built
+    
+    # Path for cached community data
+    CACHE_DIR = Path("index_storage/graphrag_cache")
+    COMMUNITY_CACHE_FILE = CACHE_DIR / "community_summary.pickle"
+    ENTITY_INFO_CACHE_FILE = CACHE_DIR / "entity_info.pickle"
+    
+    def __init__(self, property_graph_store):
+        """Initialize with a property_graph_store (Neo4j or in-memory)."""
+        self.property_graph_store = property_graph_store
+        self.community_summary = {}
+        self.entity_info = None
+        self.communities_built = False
+        
+        # Ensure cache directory exists
+        os.makedirs(self.CACHE_DIR, exist_ok=True)
+        
+    def add_nodes(self, nodes):
+        """Add nodes to the property graph store."""
+        return self.property_graph_store.add_nodes(nodes)
+        
+    def add_relationships(self, relationships):
+        """Add relationships to the property graph store."""
+        return self.property_graph_store.add_relationships(relationships)
+    
+    def get_triplets(self):
+        """Get triplets from the property graph store."""
+        return self.property_graph_store.get_triplets()
+
+    def generate_community_summary(self, text):
+        """Generate summary for a given text using an LLM with handling for large contexts."""
+        
+        # Check if text is too long and chunk if needed
+        if len(text) > 30000:  # Approximate character limit
+            log_structured('info', f'Community text is large ({len(text)} chars). Chunking for summarization.')
+            # Split into smaller chunks (simple approach)
+            chunks = [text[i:i+30000] for i in range(0, len(text), 30000)]
+            summaries = []
+            
+            for i, chunk in enumerate(chunks):
+                try:
+                    # Use GPT-4o-mini model for better cost efficiency
+                    llm = OpenAI(model="gpt-4o-mini")
+                    messages = [
+                        ChatMessage(
+                            role="system",
+                            content="Summarize these knowledge graph relationships concisely."
+                        ),
+                        ChatMessage(role="user", content=chunk),
+                    ]
+                    response = llm.chat(messages)
+                    summaries.append(str(response).strip())
+                    log_structured('info', f'Successfully summarized community chunk {i+1}/{len(chunks)}')
+                except Exception as e:
+                    log_structured('error', f'Error summarizing community chunk {i+1}/{len(chunks)}: {e}')
+                    
+            # Then summarize the summaries
+            if summaries:
+                final_summary_text = "\n\n".join(summaries)
+                try:
+                    llm = OpenAI(model="gpt-4o-mini")
+                    messages = [
+                        ChatMessage(
+                            role="system",
+                            content="Create a coherent summary from these partial summaries."
+                        ),
+                        ChatMessage(role="user", content=final_summary_text),
+                    ]
+                    response = llm.chat(messages)
+                    return str(response).strip()
+                except Exception as e:
+                    log_structured('error', f'Error creating final summary from chunks: {e}')
+                    # Return the concatenated summaries if we can't summarize them
+                    return final_summary_text
+            else:
+                return "Unable to generate community summary due to size limitations."
+        
+        # For normal size text, use the larger model directly
+        try:
+            # Use GPT-4o-mini model for better cost efficiency
+            llm = OpenAI(model="gpt-4o-mini")
+            messages = [
+                ChatMessage(
+                    role="system",
+                    content=(
+                        "You are provided with a set of "
+                        "relationships from a knowledge graph, each represented as "
+                        "entity1->entity2->relation-"
+                        ">relationship_description. Your task is to create a summary of "
+                        "these relationships. The summary should include "
+                        "the names of the entities involved and a concise synthesis "
+                        "of the relationship descriptions. The "
+                        "goal is to capture the most critical and relevant details that "
+                        "highlight the nature and significance of "
+                        "each relationship. Ensure that the summary is coherent and "
+                        "integrates the information in a way that "
+                        "emphasizes the key aspects of the relationships."
+                    ),
+                ),
+                ChatMessage(role="user", content=text),
+            ]
+            response = llm.chat(messages)
+            clean_response = re.sub(r"^assistant:\s*", "", str(response)).strip()
+            return clean_response
+        except Exception as e:
+            log_structured('error', f'Error generating community summary: {e}')
+            return f"Error summarizing community: {str(e)}"
+
+    def build_communities(self):
+        """Builds communities from the graph and summarizes them."""
+        # Skip if communities are already built in this session
+        if self.communities_built:
+            log_structured('info', 'Communities already built in this session, skipping rebuild')
+            return
+            
+        # First check if we can load from cache
+        if self.load_from_cache():
+            log_structured('info', 'Using cached community data instead of rebuilding')
+            self.communities_built = True
+            return
+            
+        log_structured('info', 'Building communities from graph data')
+        nx_graph = self._create_nx_graph()
+        
+        # Use either Leiden algorithm (from graspologic) or an alternative
+        try:
+            from graspologic.partition import hierarchical_leiden
+            community_hierarchical_clusters = hierarchical_leiden(
+                nx_graph, max_cluster_size=self.max_cluster_size
+            )
+            self.entity_info, community_info = self._collect_community_info(
+                nx_graph, community_hierarchical_clusters
+            )
+        except ImportError:
+            # Fallback to community detection using NetworkX or python-louvain
+            try:
+                from community import best_partition
+                partition = best_partition(nx_graph)
+                # Reformat partition data to expected structure
+                clusters = []
+                for node, cluster_id in partition.items():
+                    class Cluster:
+                        def __init__(self, node, cluster):
+                            self.node = node
+                            self.cluster = cluster
+                    clusters.append(Cluster(node, cluster_id))
+                self.entity_info, community_info = self._collect_community_info(
+                    nx_graph, clusters
+                )
+            except ImportError:
+                # Use NetworkX's built-in community detection
+                from networkx.algorithms import community
+                communities = community.greedy_modularity_communities(nx_graph)
+                clusters = []
+                for i, comm in enumerate(communities):
+                    for node in comm:
+                        class Cluster:
+                            def __init__(self, node, cluster):
+                                self.node = node
+                                self.cluster = cluster
+                        clusters.append(Cluster(node, i))
+                self.entity_info, community_info = self._collect_community_info(
+                    nx_graph, clusters
+                )
+                
+        self._summarize_communities(community_info)
+        
+        # Cache the results after building
+        self.save_to_cache()
+        
+        # Mark communities as built for this session
+        self.communities_built = True
+
+    def _create_nx_graph(self):
+        """Converts internal graph representation to NetworkX graph."""
+        nx_graph = nx.Graph()
+        triplets = self.get_triplets()
+        for entity1, relation, entity2 in triplets:
+            nx_graph.add_node(entity1.name)
+            nx_graph.add_node(entity2.name)
+            nx_graph.add_edge(
+                relation.source_id,
+                relation.target_id,
+                relationship=relation.label,
+                description=relation.properties.get("relationship_description", "No description provided"),
+            )
+        return nx_graph
+
+    def _collect_community_info(self, nx_graph, clusters):
+        """
+        Collect information for each node based on their community,
+        allowing entities to belong to multiple clusters.
+        """
+        entity_info = defaultdict(set)
+        community_info = defaultdict(list)
+
+        for item in clusters:
+            node = item.node
+            cluster_id = item.cluster
+
+            # Update entity_info
+            entity_info[node].add(cluster_id)
+
+            for neighbor in nx_graph.neighbors(node):
+                edge_data = nx_graph.get_edge_data(node, neighbor)
+                if edge_data:
+                    detail = f"{node} -> {neighbor} -> {edge_data['relationship']} -> {edge_data['description']}"
+                    community_info[cluster_id].append(detail)
+
+        # Convert sets to lists for easier serialization if needed
+        entity_info = {k: list(v) for k, v in entity_info.items()}
+
+        return dict(entity_info), dict(community_info)
+
+    def _summarize_communities(self, community_info):
+        """Generate and store summaries for each community."""
+        for community_id, details in community_info.items():
+            details_text = "\n".join(details) + "." # Ensure it ends with a period
+            self.community_summary[community_id] = self.generate_community_summary(details_text)
+
+    def save_to_cache(self):
+        """Save community data to disk cache."""
+        try:
+            # Save community summary
+            with open(self.COMMUNITY_CACHE_FILE, 'wb') as f:
+                pickle.dump(self.community_summary, f)
+            
+            # Save entity info
+            with open(self.ENTITY_INFO_CACHE_FILE, 'wb') as f:
+                pickle.dump(self.entity_info, f)
+                
+            log_structured('info', 'Successfully cached GraphRAG community data', 
+                        {'community_count': len(self.community_summary),
+                         'entity_count': len(self.entity_info) if self.entity_info else 0})
+            return True
+        except Exception as e:
+            log_structured('error', f'Error saving GraphRAG cache: {e}')
+            return False
+    
+    def load_from_cache(self):
+        """Load community data from disk cache if available."""
+        if not self.COMMUNITY_CACHE_FILE.exists() or not self.ENTITY_INFO_CACHE_FILE.exists():
+            log_structured('info', 'GraphRAG cache files not found, will build communities from scratch')
+            return False
+        
+        try:
+            # Load community summary
+            with open(self.COMMUNITY_CACHE_FILE, 'rb') as f:
+                self.community_summary = pickle.load(f)
+            
+            # Load entity info
+            with open(self.ENTITY_INFO_CACHE_FILE, 'rb') as f:
+                self.entity_info = pickle.load(f)
+                
+            log_structured('info', 'Successfully loaded GraphRAG community data from cache', 
+                        {'community_count': len(self.community_summary),
+                         'entity_count': len(self.entity_info) if self.entity_info else 0})
+            
+            # Mark communities as built when successfully loaded from cache
+            self.communities_built = True
+            return True
+        except Exception as e:
+            log_structured('error', f'Error loading GraphRAG cache: {e}')
+            # Reset to empty in case of partial load
+            self.community_summary = {}
+            self.entity_info = None
+            self.communities_built = False
+            return False
+            
+    def get_community_summaries(self):
+        """Returns the community summaries, building them if not already done."""
+        if not self.community_summary:
+            # Try to load from cache first
+            if not self.load_from_cache():
+                # If cache load fails, build from scratch
+                self.build_communities()
+                # Cache the results for next time
+                self.save_to_cache()
+        return self.community_summary
+
+# Define the GraphRAGQueryEngine class
+from typing import Dict, Any
+
+class GraphRAGQueryEngine:
+    """Query engine that combines vector retrieval with graph-based community retrieval."""
+    
+    def __init__(
+        self, 
+        vector_retriever: VectorIndexRetriever,
+        graph_store: GraphRAGStore,
+        llm: Optional[LLM] = None,
+        similarity_top_k: int = 20
+    ):
+        """Initialize with both a vector retriever and graph store."""
+        # Initialize all required fields
+        self.vector_retriever = vector_retriever
+        self.graph_store = graph_store
+        self.llm = llm or Settings.llm
+        self.similarity_top_k = similarity_top_k
+        
+        # Check if communities are built, but don't try to build them here
+        # since that might cause errors with large graphs
+        if not hasattr(self.graph_store, 'entity_info') or self.graph_store.entity_info is None:
+            log_structured('warning', 'GraphRAGQueryEngine initialized without community data. Vector retrieval will still work, but community retrieval may be limited.')
+
+    def custom_query(self, query_str: str) -> Dict:
+        """Process query using both vector retrieval and community-based approach."""
+        log_structured('info', 'GraphRAG query engine: Starting dual retrieval', {'query': query_str})
+        
+        # Step 1: Get vector search results
+        vector_nodes = self.vector_retriever.retrieve(query_str)
+        vector_context = "\n\n".join([node.node.get_content() for node in vector_nodes])
+        log_structured('info', 'GraphRAG query engine: Vector retrieval complete', 
+                     {'node_count': len(vector_nodes)})
+        
+        # Step 2: Get GraphRAG community results (if communities exist)
+        graphrag_context = ""
+        community_ids = []
+        
+        if hasattr(self.graph_store, 'entity_info') and self.graph_store.entity_info is not None:
+            try:
+                entities = self.get_entities(query_str, vector_nodes)
+                community_ids = self.retrieve_entity_communities(self.graph_store.entity_info, entities)
+                
+                try:
+                    community_summaries = self.graph_store.get_community_summaries()
+                    
+                    if community_ids:
+                        filtered_summaries = {id: summary for id, summary in community_summaries.items() 
+                                          if id in community_ids}
+                        graphrag_context = "\n\n".join(filtered_summaries.values())
+                        log_structured('info', 'GraphRAG query engine: Community retrieval complete', 
+                                    {'community_count': len(filtered_summaries)})
+                    else:
+                        log_structured('info', 'GraphRAG query engine: No relevant communities found')
+                except Exception as e:
+                    log_structured('error', f'Error getting community summaries: {e}')
+                    # Continue without graph context
+            except Exception as e:
+                log_structured('error', f'Error during community retrieval: {e}')
+                # Continue with just vector context
+        else:
+            log_structured('warning', 'GraphRAG query engine: No community data available. Using only vector retrieval.')
+        
+        # Step 3: Combine contexts and generate answer
+        combined_result = {
+            "vector_context": vector_context,
+            "graphrag_context": graphrag_context,
+            "vector_nodes": vector_nodes,
+            "community_ids": community_ids
+        }
+        
+        return combined_result
+
+    def get_entities(self, query_str, vector_nodes):
+        """Extract entities from vector nodes that match the query."""
+        entities = set()
+        
+        # Extract entities from the retrieved nodes
+        for node_with_score in vector_nodes:
+            node = node_with_score.node
+            if hasattr(node, 'metadata') and KG_NODES_KEY in node.metadata:
+                for entity_node in node.metadata[KG_NODES_KEY]:
+                    if hasattr(entity_node, 'name'):
+                        entities.add(entity_node.name)
+        
+        # If no entities were found in metadata, try extracting them from text
+        if not entities:
+            pattern = r"(?:^|\s)([A-Z][a-zA-Z0-9\s]+)(?:\s|$)"
+            for node_with_score in vector_nodes:
+                matches = re.findall(pattern, node_with_score.node.get_content())
+                entities.update(matches)
+        
+        log_structured('debug', 'GraphRAG query engine: Extracted entities', 
+                     {'entities': list(entities), 'count': len(entities)})
+        return list(entities)
+
+    def retrieve_entity_communities(self, entity_info, entities):
+        """
+        Retrieve cluster information for given entities,
+        allowing for multiple clusters per entity.
+
+        Args:
+            entity_info (dict): Dictionary mapping entities to their cluster IDs (list).
+            entities (list): List of entity names to retrieve information for.
+
+        Returns:
+            List of community or cluster IDs to which an entity belongs.
+        """
+        community_ids = []
+
+        for entity in entities:
+            if entity in entity_info:
+                community_ids.extend(entity_info[entity])
+            else:
+                # Try case-insensitive matching as fallback
+                for stored_entity, clusters in entity_info.items():
+                    if stored_entity.lower() == entity.lower():
+                        community_ids.extend(clusters)
+                        break
+
+        return list(set(community_ids))
+
+def custom_parse_fn(response_str: str) -> Any:
+    """Custom parser for LLM responses that extract entities and relationships"""
+    json_pattern = r"\{.*\}"
+    match = re.search(json_pattern, response_str, re.DOTALL)
+    entities = []
+    relationships = []
+    
+    if not match:
+        return entities, relationships
+    
+    json_str = match.group(0)
+    try:
+        data = json.loads(json_str)
+        entities = [
+            (
+                entity["entity_name"],
+                entity["entity_type"],
+                entity.get("entity_description", f"Description of {entity['entity_name']}"),
+            )
+            for entity in data.get("entities", [])
+        ]
+        relationships = [
+            (
+                relation["source_entity"],
+                relation["target_entity"],
+                relation["relation"],
+                relation.get("relationship_description", f"Relationship between {relation['source_entity']} and {relation['target_entity']}"),
+            )
+            for relation in data.get("relationships", [])
+        ]
+        return entities, relationships
+    except (json.JSONDecodeError, KeyError) as e:
+        log_structured('error', f"Error parsing response: {e}", {'json_str': json_str[:200]})
+        return entities, relationships
+
+# Define the prompt template for triple extraction
+KG_TRIPLET_EXTRACT_TMPL = """
+-Goal-
+Given a text document, identify all entities and their entity types from the text and all relationships among the identified entities.
+
+Given the text, extract up to {max_knowledge_triplets} entity-relation triplets.
+
+-Steps-
+1. Identify all entities. For each identified entity, extract the following information:
+- entity_name: Name of the entity, capitalized
+- entity_type: Type of the entity
+- entity_description: Comprehensive description of the entity's attributes and activities
+
+2. From the entities identified in step 1, identify all pairs of (source_entity, target_entity) that are *clearly related* to each other.
+For each pair of related entities, extract the following information:
+- source_entity: name of the source entity, as identified in step 1
+- target_entity: name of the target entity, as identified in step 1
+- relation: relationship between source_entity and target_entity
+- relationship_description: explanation as to why you think the source entity and the target entity are related to each other
+
+3. Output Formatting:
+- Return the result in valid JSON format with two keys: 'entities' (list of entity objects) and 'relationships' (list of relationship objects).
+- Exclude any text outside the JSON structure (e.g., no explanations or comments).
+- If no entities or relationships are identified, return empty lists: { "entities": [], "relationships": [] }.
+
+-Real Data-
+######################
+text: {text}
+######################
+output:
+"""
+
+def create_graph_components(llm, nodes=None, max_paths_per_chunk=10, force_reindex=False):
+    """
+    Create GraphRAG components for the HP RAG pipeline.
+    
+    Args:
+        llm: The LLM to use for graph extraction and querying
+        nodes: List of nodes to create the graph from (only used if indexing is needed)
+        max_paths_per_chunk: Maximum number of paths to extract per chunk
+        force_reindex: If True, always recreate the index even if content exists
+        
+    Returns:
+        tuple: (graph_store, property_graph_index)
+    """
+    log_structured('info', 'Creating GraphRAG components')
+    
+    # Note: The graph_store object created here will automatically:
+    # 1. Try to load community data from cache files when build_communities() is called
+    # 2. Save to cache after building communities if loading failed
+    
+    # Connect to Neo4j - hard error if not available
+    property_graph_store = None
+    try:
+        log_structured('info', f'Connecting to Neo4j at {NEO4J_URL}')
+        property_graph_store = Neo4jPropertyGraphStore(
+            username=NEO4J_USERNAME, 
+            password=NEO4J_PASSWORD,
+            url=NEO4J_URL
+        )
+        log_structured('info', 'Successfully connected to Neo4j database')
+    except Exception as e:
+        log_structured('critical', f'FATAL ERROR: Cannot connect to Neo4j: {e}')
+        raise RuntimeError(f"Neo4j connection failed. This application requires Neo4j to be running. Error: {e}")
+    
+    # Create GraphRAGStore wrapper
+    graph_store = GraphRAGStore(property_graph_store)
+    
+    # Check if Neo4j already has content
+    triplets = graph_store.get_triplets()
+    has_existing_content = len(triplets) > 0
+    
+    log_structured('info', f'Neo4j check: Found {len(triplets)} triplets')
+    
+    if has_existing_content and not force_reindex:
+        log_structured('info', f'Neo4j already contains {len(triplets)} triplets. Skipping indexing.')
+        
+        # Create a minimal PropertyGraphIndex without indexing
+        property_graph_index = PropertyGraphIndex(
+            nodes=[],  # Empty nodes since we're not indexing
+            property_graph_store=property_graph_store,
+        )
+        
+        # Build communities from existing data (if not already built)
+        if not graph_store.communities_built:
+            log_structured('info', 'Building graph communities from existing Neo4j data')
+            try:
+                graph_store.build_communities()
+                log_structured('info', 'Communities built successfully')
+            except Exception as e:
+                log_structured('error', f'Error building communities: {e}')
+        else:
+            log_structured('info', 'Communities already built, skipping rebuild')
+    else:
+        # Need to perform indexing
+        if not nodes:
+            log_structured('error', 'No nodes provided for indexing and Neo4j is empty or force_reindex=True')
+            raise ValueError("Nodes must be provided for indexing when Neo4j is empty or force_reindex=True")
+        
+        # Create the knowledge graph extractor
+        kg_extractor = GraphRAGExtractor(
+            llm=llm,
+            extract_prompt=KG_TRIPLET_EXTRACT_TMPL,
+            max_paths_per_chunk=max_paths_per_chunk,
+            parse_fn=custom_parse_fn,
+        )
+        
+        if has_existing_content and force_reindex:
+            log_structured('info', 'Force reindexing requested. Clearing existing Neo4j data.')
+            try:
+                # Try to clear the graph using Neo4j's native query
+                # Note: This requires the Neo4j APOC plugin to be installed
+                from neo4j import GraphDatabase
+                driver = GraphDatabase.driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD))
+                with driver.session() as session:
+                    session.run("MATCH (n) DETACH DELETE n")
+                driver.close()
+                log_structured('info', 'Successfully cleared Neo4j database')
+            except Exception as e:
+                log_structured('warning', f'Error clearing Neo4j database: {e}. Proceeding with indexing anyway.')
+        
+        # Build the property graph index
+        log_structured('info', 'Building PropertyGraphIndex', {'node_count': len(nodes)})
+        property_graph_index = PropertyGraphIndex(
+            nodes=nodes,
+            kg_extractors=[kg_extractor],
+            property_graph_store=property_graph_store,
+            show_progress=True,
+        )
+        
+        # Build communities
+        log_structured('info', 'Building graph communities')
+        try:
+            graph_store.build_communities()
+            log_structured('info', 'Communities built successfully')
+        except Exception as e:
+            log_structured('error', f'Error building communities: {e}')
+    
+    return graph_store, property_graph_index
+
+def create_graphrag_query_engine(vector_retriever, graph_store, llm, similarity_top_k=20):
+    """
+    Create GraphRAG query engine that combines vector and graph-based retrieval.
+    
+    Args:
+        vector_retriever: VectorIndexRetriever for standard retrieval
+        graph_store: GraphRAGStore for community-based retrieval
+        llm: LLM for generating answer
+        similarity_top_k: Number of top results to retrieve
+        
+    Returns:
+        GraphRAGQueryEngine: Query engine for hybrid retrieval
+    """
+    from utils import log_structured
+    
+    try:
+        # Explicitly validate inputs before passing to constructor
+        if vector_retriever is None:
+            raise ValueError("vector_retriever cannot be None")
+        if graph_store is None:
+            raise ValueError("graph_store cannot be None")
+        if llm is None:
+            raise ValueError("llm cannot be None")
+            
+        # Log for debugging
+        log_structured('debug', 'Creating GraphRAGQueryEngine with parameters', {
+            'vector_retriever_type': type(vector_retriever).__name__,
+            'graph_store_type': type(graph_store).__name__,
+            'llm_type': type(llm).__name__,
+            'similarity_top_k': similarity_top_k
+        })
+        
+        # Create the engine
+        return GraphRAGQueryEngine(
+            vector_retriever=vector_retriever,
+            graph_store=graph_store,
+            llm=llm,
+            similarity_top_k=similarity_top_k,
+        )
+    except Exception as e:
+        log_structured('error', f'Error in create_graphrag_query_engine: {e}')
+        raise  # Re-raise the exception for proper handling
+
+def generate_final_answer(query, retrieval_result, llm):
+    """
+    Generate a final answer using both vector and graph-based context.
+    
+    Args:
+        query: The user's query
+        retrieval_result: Result from GraphRAGQueryEngine with vector and graph contexts
+        llm: LLM for generating the final response
+        
+    Returns:
+        str: The final answer
+    """
+    vector_context = retrieval_result.get("vector_context", "")
+    graphrag_context = retrieval_result.get("graphrag_context", "")
+    
+    # Log the contexts for debugging (truncated for brevity)
+    log_structured('debug', 'Generating final answer with dual context', {
+        'query': query,
+        'vector_context_length': len(vector_context),
+        'graphrag_context_length': len(graphrag_context)
+    })
+    
+    if not vector_context and not graphrag_context:
+        return "I couldn't find any relevant information to answer your question."
+    
+    # If no model was provided or we're forcing to use a specific model
+    if llm is None or not hasattr(llm, 'chat'):
+        # Fallback to gpt-4o-mini for better cost efficiency
+        llm = OpenAI(model="gpt-4o-mini")
+        log_structured('info', 'Using gpt-4o-mini model for final answer generation')
+    
+    prompt = f"""
+    Based on the following information from two different sources, please answer this question: {query}
+
+    SOURCE 1 - VECTOR RETRIEVAL:
+    {vector_context}
+
+    SOURCE 2 - KNOWLEDGE GRAPH COMMUNITIES:
+    {graphrag_context}
+    
+    Answer the question based on all the provided information. If there are differences between the sources, 
+    try to reconcile them or note the discrepancy. Please be concise and direct.
+    """
+    
+    messages = [
+        ChatMessage(role="system", content=prompt),
+        ChatMessage(role="user", content="Please provide a comprehensive answer based on all the information provided.")
+    ]
+    
+    response = llm.chat(messages)
+    
+    # Extract just the message content, not the entire response object
+    if hasattr(response, 'message') and hasattr(response.message, 'content'):
+        content = response.message.content
+    elif hasattr(response, 'content'):
+        content = response.content
+    else:
+        # Fallback: convert to string but clean it
+        content = str(response)
+    
+    # Clean any remaining thinking patterns from the response
+    import re
+    thinking_patterns = [
+        r'(?i)Thought:.*?Action:.*?Action Input:.*',  # Remove the specific pattern
+        r'(?i)^Thought:.*',  # Remove any line starting with "Thought:"
+        r'(?i)Action:.*?Action Input:.*',  # Remove Action/Action Input patterns
+        r'(?i)^(Thought|Action|Observation):.*',  # Remove ReAct patterns
+    ]
+    
+    for pattern in thinking_patterns:
+        content = re.sub(pattern, '', content, flags=re.DOTALL | re.MULTILINE)
+    
+    # Clean up extra whitespace
+    content = re.sub(r'\n{3,}', '\n\n', content)
+    content = content.strip()
+    
+    # Final safety check
+    if not content or re.search(r'(?i)^(Thought|Action|Observation):', content):
+        log_structured('warning', 'GraphRAG final answer still contains thinking patterns, using fallback')
+        content = "I found relevant information in the HP marketing materials that can help answer your question. Please let me know if you need more specific details."
+    
+    return content
\ No newline at end of file
diff --git a/init_mongodb.py b/init_mongodb.py
new file mode 100644
index 0000000..6de29cf
--- /dev/null
+++ b/init_mongodb.py
@@ -0,0 +1,127 @@
+"""
+MongoDB Initialization Script for HP Chatbot
+
+This script initializes the MongoDB database with the necessary collections for the HP chatbot.
+It creates collections for users, conversations, and messages.
+
+Usage:
+    python init_mongodb.py
+"""
+
+import pymongo
+import logging
+from datetime import datetime
+import os
+import sys
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('mongodb_init.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# MongoDB connection information
+MONGO_URI = "mongodb://hp:hp@localhost:27017/?authSource=hp_chatbot"  # HP user with hp_chatbot auth source
+DB_NAME = "hp_chatbot"
+
+# Collection names
+USERS_COLLECTION = "users"
+CONVERSATIONS_COLLECTION = "conversations"
+MESSAGES_COLLECTION = "messages"
+
+def init_mongodb():
+    """Initialize MongoDB database and collections."""
+    try:
+        # Connect to MongoDB
+        logger.info("Connecting to MongoDB...")
+        client = pymongo.MongoClient(MONGO_URI)
+        
+        # Test connection
+        client.admin.command('ping')
+        logger.info("Successfully connected to MongoDB")
+        
+        # Create or access database
+        db = client[DB_NAME]
+        logger.info(f"Using database: {DB_NAME}")
+        
+        # Create collections if they don't exist
+        if USERS_COLLECTION not in db.list_collection_names():
+            db.create_collection(USERS_COLLECTION)
+            logger.info(f"Created collection: {USERS_COLLECTION}")
+            
+            # Create indexes for users collection
+            db[USERS_COLLECTION].create_index("username", unique=True)
+            # Create a unique sparse index for email - only enforces uniqueness when email exists
+            db[USERS_COLLECTION].create_index("email", unique=True, sparse=True)
+            logger.info("Created indexes for users collection")
+        
+        if CONVERSATIONS_COLLECTION not in db.list_collection_names():
+            db.create_collection(CONVERSATIONS_COLLECTION)
+            logger.info(f"Created collection: {CONVERSATIONS_COLLECTION}")
+            
+            # Create indexes for conversations collection
+            db[CONVERSATIONS_COLLECTION].create_index("user_id")
+            db[CONVERSATIONS_COLLECTION].create_index("session_id", unique=True)
+            db[CONVERSATIONS_COLLECTION].create_index("created_at")
+            db[CONVERSATIONS_COLLECTION].create_index("last_updated")
+            logger.info("Created indexes for conversations collection")
+        
+        if MESSAGES_COLLECTION not in db.list_collection_names():
+            db.create_collection(MESSAGES_COLLECTION)
+            logger.info(f"Created collection: {MESSAGES_COLLECTION}")
+            
+            # Create indexes for messages collection
+            db[MESSAGES_COLLECTION].create_index("conversation_id")
+            db[MESSAGES_COLLECTION].create_index("timestamp")
+            logger.info("Created indexes for messages collection")
+        
+        logger.info("MongoDB initialization completed successfully")
+        return True
+        
+    except pymongo.errors.ConnectionFailure as e:
+        logger.error(f"Could not connect to MongoDB: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"An error occurred during MongoDB initialization: {e}")
+        return False
+
+def display_collection_info(client):
+    """Display information about the collections in the database."""
+    db = client[DB_NAME]
+    
+    logger.info("=== Database Structure ===")
+    for collection_name in db.list_collection_names():
+        count = db[collection_name].count_documents({})
+        logger.info(f"Collection: {collection_name}, Documents: {count}")
+        
+        # Display indexes
+        indexes = db[collection_name].index_information()
+        logger.info(f"  Indexes: {list(indexes.keys())}")
+
+if __name__ == "__main__":
+    if init_mongodb():
+        # Display collection information
+        client = pymongo.MongoClient(MONGO_URI)
+        display_collection_info(client)
+        
+        # Add sample user if none exist (optional)
+        db = client[DB_NAME]
+        if db[USERS_COLLECTION].count_documents({}) == 0:
+            sample_user = {
+                "username": "sample_user",
+                "email": "sample@example.com",
+                "created_at": datetime.utcnow(),
+                "last_login": datetime.utcnow()
+            }
+            db[USERS_COLLECTION].insert_one(sample_user)
+            logger.info("Added sample user for testing")
+        
+        logger.info("Initialization complete. The database is ready for use.")
+    else:
+        logger.error("Failed to initialize MongoDB. See logs for details.")
+        sys.exit(1)
\ No newline at end of file
diff --git a/json_utils.py b/json_utils.py
new file mode 100644
index 0000000..2bc19bc
--- /dev/null
+++ b/json_utils.py
@@ -0,0 +1,133 @@
+# hp_chatbot/json_utils.py
+import json
+import llama_index
+from llama_index.core.tools import ToolOutput
+from llama_index.core.agent.react.types import (
+    ActionReasoningStep,
+    ObservationReasoningStep,
+    ResponseReasoningStep,
+    BaseReasoningStep,
+)
+from llama_index.core.llms import ChatMessage, LLM, ChatResponse as LlamaResponse
+from llama_index.core.base.response.schema import Response
+from flask.json.provider import JSONProvider
+from bson import ObjectId # Import ObjectId if used in responses/data
+from datetime import datetime
+
+class CustomJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON Encoder to handle LlamaIndex objects, BSON ObjectId, and other types.
+    """
+    def default(self, obj):
+        try:
+            # Specific LlamaIndex Types
+            if isinstance(obj, ToolOutput):
+                return {
+                    'content': str(obj.content) if obj.content is not None else "",
+                    'tool_name': getattr(obj, 'tool_name', None),
+                    'raw_output': str(getattr(obj, 'raw_output', None)), # Safely convert raw_output
+                    'type': 'tool_output',
+                    'metadata': getattr(obj, 'metadata', {})
+                }
+            elif isinstance(obj, (llama_index.core.llms.ChatMessage, ChatMessage)):
+                 return {
+                     'role': str(obj.role),
+                     'content': str(obj.content),
+                     'additional_kwargs': obj.additional_kwargs if hasattr(obj, 'additional_kwargs') else {}
+                 }
+            elif isinstance(obj, (LlamaResponse, Response)):
+                 return {
+                    'content': str(getattr(obj, 'response', getattr(obj, 'message', None))),
+                    'metadata': getattr(obj, 'metadata', {}),
+                    'type': 'llm_response'
+                 }
+            elif isinstance(obj, ActionReasoningStep):
+                return {
+                    'type': 'action_step',
+                    'action': obj.action,
+                    'action_input': obj.action_input, # Should be serializable dict
+                    'thought': getattr(obj, 'thought', None)
+                }
+            elif isinstance(obj, ObservationReasoningStep):
+                return {
+                    'type': 'observation_step',
+                    'observation': str(obj.observation), # Ensure observation is string
+                    'thought': getattr(obj, 'thought', None)
+                }
+            elif isinstance(obj, ResponseReasoningStep):
+                return {
+                    'type': 'response_step',
+                    'response': str(obj.response), # Ensure response is string
+                    'is_streaming': getattr(obj, 'is_streaming', False),
+                    'thought': getattr(obj, 'thought', None)
+                }
+            elif isinstance(obj, BaseReasoningStep): # Catch-all for other steps
+                 return {
+                    'type': 'base_reasoning_step',
+                    'thought': getattr(obj, 'thought', None),
+                    'is_done': getattr(obj, 'is_done', False),
+                 }
+            # Handle LlamaIndex Document/Node related objects if needed
+            elif isinstance(obj, llama_index.core.schema.Document):
+                 return {
+                     'doc_id': obj.id_,
+                     'text_preview': obj.text[:100] + "..." if obj.text else "",
+                     'metadata': obj.metadata, # Metadata should be serializable
+                     'type': 'llama_document'
+                 }
+            elif isinstance(obj, llama_index.core.schema.NodeWithScore):
+                 return {
+                     'node': self.default(obj.node), # Recursively serialize the node
+                     'score': obj.score,
+                     'type': 'node_with_score'
+                 }
+            elif isinstance(obj, llama_index.core.schema.TextNode):
+                 return {
+                     'node_id': obj.id_,
+                     'text_preview': obj.text[:100] + "..." if obj.text else "",
+                     'metadata': obj.metadata,
+                     'type': 'text_node'
+                 }
+
+            # Common Python Types
+            elif isinstance(obj, datetime):
+                return obj.isoformat()
+            elif isinstance(obj, ObjectId):
+                return str(obj)
+            elif isinstance(obj, bytes):
+                 return "" # Or encode to base64 if needed
+
+            # General Fallback for objects with __dict__
+            elif hasattr(obj, '__dict__'):
+                # Filter out private/callable attributes, be cautious with recursion
+                try:
+                    d = {k: v for k, v in obj.__dict__.items()
+                         if not k.startswith('_') and not callable(v)}
+                    # Basic check to prevent deep recursion errors
+                    if len(d) > 50: # Arbitrary limit
+                         return f""
+                    return d
+                except Exception:
+                     return f""
+
+            # Final fallback using standard JSON encoding
+            return super().default(obj)
+
+        except Exception as e:
+            # Log the error? Be careful about logging sensitive data
+            # print(f"DEBUG: JSON encoding error for type {type(obj).__name__}: {e}")
+            return f""
+
+
+class CustomJSONProvider(JSONProvider):
+    """
+    Flask JSON Provider using the CustomJSONEncoder.
+    """
+    def dumps(self, obj, **kwargs):
+        kwargs.setdefault('cls', CustomJSONEncoder)
+        kwargs.setdefault('ensure_ascii', False) # Often useful for non-English text
+        kwargs.setdefault('indent', None) # No indent for production APIs
+        return json.dumps(obj, **kwargs)
+
+    def loads(self, s, **kwargs):
+        return json.loads(s, **kwargs)
\ No newline at end of file
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..e87c3d9
--- /dev/null
+++ b/main.py
@@ -0,0 +1,168 @@
+# hp_chatbot/main.py
+
+import asyncio
+import os
+import sys
+from flask import Flask
+from flask_cors import CORS
+
+# Ensure the project directory is in the Python path
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)
+
+# Import necessary components from our modules
+from config import (
+    APPLICATION_ROOT, MAX_CONTENT_LENGTH,
+    CORS_ALLOWED_ORIGINS, CORS_SUPPORTS_CREDENTIALS,
+    SERVER_HOST, SERVER_PORT, USE_RELOADER, LOG_LEVEL,
+    KEEP_ALIVE_TIMEOUT, READ_TIMEOUT, WRITE_TIMEOUT
+)
+from utils import logger, log_structured
+from json_utils import CustomJSONProvider
+from ai_core import initialize_global_index # Import the initialization function
+from shared_state import global_workflow_agent, is_agent_available # Import shared state
+from routes import register_routes
+from init_mongodb import init_mongodb # Your MongoDB initialization script
+
+# --- Flask App Initialization ---
+app = Flask(__name__)
+
+# Apply custom JSON provider for handling special types (LlamaIndex objects, etc.)
+app.json_provider_class = CustomJSONProvider
+app.json = CustomJSONProvider(app)
+
+# Configuration
+app.config['MAX_CONTENT_LENGTH'] = MAX_CONTENT_LENGTH
+if APPLICATION_ROOT:
+    app.config['APPLICATION_ROOT'] = APPLICATION_ROOT
+    # If using APPLICATION_ROOT, you might need to adjust route prefixes
+    # or use a Blueprint with url_prefix=APPLICATION_ROOT
+    log_structured('info', f"Flask Application Root set to: {APPLICATION_ROOT}")
+
+# CORS Configuration
+CORS(app,
+     resources={r"/*": {"origins": CORS_ALLOWED_ORIGINS}},
+     supports_credentials=CORS_SUPPORTS_CREDENTIALS,
+     # Expose custom headers if needed by the frontend
+     # expose_headers=["Content-Disposition"] # Example for downloads
+)
+log_structured('info', f"CORS configured for origins: {CORS_ALLOWED_ORIGINS}")
+
+
+# --- Register Routes ---
+# Pass the app object to the function in routes.py
+register_routes(app)
+log_structured('info', "Flask routes registered.")
+
+
+# --- Startup Function ---
+async def startup_event() -> bool:
+    """Tasks to run when the application starts.
+    
+    Returns:
+        bool: True if all startup tasks completed successfully, False otherwise
+    """
+    log_structured('info', "Application startup sequence initiated.")
+    all_success = True
+
+    # 1. Initialize MongoDB Connection & Schema (using your script)
+    log_structured('info', "Initializing MongoDB connection...")
+    mongo_success = False
+    try:
+        if init_mongodb():
+            log_structured('info', "MongoDB initialized successfully.")
+            mongo_success = True
+        else:
+            log_structured('warning', "MongoDB initialization script finished, but reported issues.")
+            all_success = False
+    except Exception as db_err:
+        log_structured('critical', "FATAL: MongoDB initialization failed.", {'error': str(db_err)})
+        all_success = False
+        # We'll continue in a degraded state
+
+    # 2. Initialize Global AI Index and Agent
+    log_structured('info', "Initializing global AI index and agent...")
+    index_success = await initialize_global_index()
+    
+    # Explicitly check the status after initialization
+    if not is_agent_available():
+        log_structured('critical', "After initialize_global_index, global_workflow_agent is still unavailable, even though function may have reported success")
+        all_success = False
+    elif not index_success:
+        log_structured('warning', "AI initialization reported failure, but will continue in degraded state")
+        all_success = False
+    else:
+        log_structured('info', "AI initialization successful, global_workflow_agent is available")
+
+    log_structured('info', f"Application startup sequence complete. Overall success: {all_success}")
+    return all_success
+
+
+# --- Shutdown Function (Optional) ---
+async def shutdown_event():
+    """Tasks to run when the application stops."""
+    log_structured('info', "Application shutdown sequence initiated.")
+    # Add any cleanup tasks here (e.g., closing connections if not handled elsewhere)
+    # Note: Hypercorn might not always guarantee graceful shutdown execution.
+    log_structured('info', "Application shutdown sequence complete.")
+
+
+# --- Main Execution Block ---
+if __name__ == '__main__':
+    from hypercorn.config import Config as HypercornConfig
+    from hypercorn.asyncio import serve as hypercorn_serve
+
+    # Create Hypercorn config object
+    config = HypercornConfig()
+
+    # Basic settings
+    config.bind = [f"{SERVER_HOST}:{SERVER_PORT}"]
+    config.use_reloader = USE_RELOADER
+    config.accesslog = '-' # Log to stdout/stderr
+    config.errorlog = '-' # Log to stdout/stderr
+    config.loglevel = LOG_LEVEL.upper()
+    config.worker_class = 'asyncio'
+
+    # Timeouts (ensure these are floats or ints)
+    config.keep_alive_timeout = float(KEEP_ALIVE_TIMEOUT)
+    config.read_timeout = float(READ_TIMEOUT)
+    config.write_timeout = float(WRITE_TIMEOUT)
+
+    # Request size limits (check Hypercorn docs for exact names, might vary slightly)
+    # These might apply to HTTP/1.1 or HTTP/2 differently.
+    # config.h11_max_incomplete_size = MAX_CONTENT_LENGTH # Example for HTTP/1.1
+    # config.h2_max_concurrent_streams = 100 # Example for HTTP/2
+    # config.max_app_buffer_size = MAX_CONTENT_LENGTH # Another potential setting
+    # It's safer to configure these via a reverse proxy (like Nginx) in production.
+    # Hypercorn's defaults are usually reasonable. Let's comment these out for now.
+
+    # Assign startup and shutdown handlers
+    config.startup_hooks = [startup_event]
+    config.shutdown_hooks = [shutdown_event]
+
+    log_structured('info', f"Starting Hypercorn server on {SERVER_HOST}:{SERVER_PORT}")
+    log_structured('info', f"Reload mode: {'Enabled' if USE_RELOADER else 'Disabled'}")
+
+    # Execute startup task before running the server
+    log_structured('info', "Manually executing startup sequence before server start")
+    startup_success = asyncio.run(startup_event())
+    
+    # Double-check that the agent is initialized
+    if not is_agent_available():
+        log_structured('critical', "After startup, global_workflow_agent is still unavailable. Forcing re-initialization...")
+        # Try once more to initialize
+        index_success = asyncio.run(initialize_global_index())
+        if not index_success or not is_agent_available():
+            log_structured('critical', "Emergency initialization also failed. Server will run but chat functionality will be impaired.")
+        else:
+            log_structured('info', "Emergency initialization succeeded.")
+
+    # Run the server
+    try:
+        asyncio.run(hypercorn_serve(app, config))
+    except KeyboardInterrupt:
+         log_structured('info', "Server stopped manually (KeyboardInterrupt).")
+    except Exception as run_err:
+         log_structured('critical', "Hypercorn server failed to run.", {'error': str(run_err)})
+         sys.exit(1)
\ No newline at end of file
diff --git a/mongodb_utils.py b/mongodb_utils.py
new file mode 100644
index 0000000..75086be
--- /dev/null
+++ b/mongodb_utils.py
@@ -0,0 +1,458 @@
+"""
+MongoDB Utilities for HP Chatbot
+
+This module provides utility functions for interacting with MongoDB in the HP chatbot application.
+It includes functions for connecting to MongoDB, and managing users, conversations, and messages.
+"""
+
+import pymongo
+import logging
+from datetime import datetime
+import uuid
+from typing import Dict, List, Optional, Any, Union
+from bson.objectid import ObjectId
+import json
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('mongodb.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# MongoDB connection information
+MONGO_URI = "mongodb://hp:hp@localhost:27017/?authSource=hp_chatbot"
+DB_NAME = "hp_chatbot"
+
+# Collection names
+USERS_COLLECTION = "users"
+CONVERSATIONS_COLLECTION = "conversations"
+MESSAGES_COLLECTION = "messages"
+
+# Global MongoDB client
+mongo_client = None
+db = None
+
+def get_db():
+    """Get or initialize the MongoDB database connection."""
+    global mongo_client, db
+    
+    if mongo_client is None:
+        try:
+            mongo_client = pymongo.MongoClient(MONGO_URI)
+            mongo_client.admin.command('ping')  # Test connection
+            db = mongo_client[DB_NAME]
+            logger.info("Successfully connected to MongoDB")
+        except Exception as e:
+            logger.error(f"Failed to connect to MongoDB: {e}")
+            raise
+    
+    return db
+
+def close_connection():
+    """Close the MongoDB connection."""
+    global mongo_client
+    
+    if mongo_client:
+        mongo_client.close()
+        mongo_client = None
+        logger.info("MongoDB connection closed")
+
+# User functions
+def get_user_by_username(username: str) -> Optional[Dict]:
+    """Get a user by username."""
+    try:
+        db = get_db()
+        user = db[USERS_COLLECTION].find_one({"username": username})
+        return user
+    except Exception as e:
+        logger.error(f"Error getting user by username: {e}")
+        return None
+
+def create_or_update_user(username: str, email: Optional[str] = None) -> Optional[str]:
+    """Create a new user or update an existing one."""
+    try:
+        db = get_db()
+        
+        # Check if user exists
+        existing_user = db[USERS_COLLECTION].find_one({"username": username})
+        
+        if existing_user:
+            # Update last login
+            db[USERS_COLLECTION].update_one(
+                {"username": username},
+                {"$set": {"last_login": datetime.utcnow()}}
+            )
+            return str(existing_user["_id"])
+        else:
+            # Create new user
+            new_user = {
+                "username": username,
+                "created_at": datetime.utcnow(),
+                "last_login": datetime.utcnow()
+            }
+            
+            # Only include email if it's not None to avoid unique constraint issues
+            if email:
+                new_user["email"] = email
+                
+            result = db[USERS_COLLECTION].insert_one(new_user)
+            return str(result.inserted_id)
+    except Exception as e:
+        logger.error(f"Error creating or updating user: {e}")
+        # If the error is a duplicate key error, try to find the existing user
+        if "duplicate key error" in str(e) and "username" in str(e):
+            try:
+                existing_user = db[USERS_COLLECTION].find_one({"username": username})
+                if existing_user:
+                    return str(existing_user["_id"])
+            except:
+                pass
+        return None
+
+# Conversation functions
+def get_conversation(session_id: str) -> Optional[Dict]:
+    """Get a conversation by session ID."""
+    try:
+        db = get_db()
+        conversation = db[CONVERSATIONS_COLLECTION].find_one({"session_id": session_id})
+        return conversation
+    except Exception as e:
+        logger.error(f"Error getting conversation: {e}")
+        return None
+
+def get_conversation_by_id(conversation_id: str) -> Optional[Dict]:
+    """Get a conversation by its MongoDB ID."""
+    try:
+        db = get_db()
+        # Convert string ID to ObjectId
+        try:
+            obj_id = ObjectId(conversation_id)
+            conversation = db[CONVERSATIONS_COLLECTION].find_one({"_id": obj_id})
+            return conversation
+        except Exception as e:
+            logger.error(f"Error converting conversation ID to ObjectId: {e}")
+            return None
+    except Exception as e:
+        logger.error(f"Error getting conversation by ID: {e}")
+        return None
+
+def get_user_conversations(user_id: str) -> List[Dict]:
+    """Get all conversations for a user that are not marked as deleted."""
+    try:
+        db = get_db()
+        conversations = list(db[CONVERSATIONS_COLLECTION].find(
+            {
+                "user_id": user_id,
+                # Only return conversations that either don't have is_deleted or have it set to False
+                "$or": [
+                    {"is_deleted": {"$exists": False}},
+                    {"is_deleted": False}
+                ]
+            }
+        ).sort("last_updated", pymongo.DESCENDING))
+        return conversations
+    except Exception as e:
+        logger.error(f"Error getting user conversations: {e}")
+        return []
+
+def create_conversation(session_id: str, user_id: str, title: str = "New conversation") -> Optional[str]:
+    """Create a new conversation."""
+    try:
+        db = get_db()
+        
+        # Check if conversation already exists with this session_id
+        existing = db[CONVERSATIONS_COLLECTION].find_one({"session_id": session_id})
+        if existing:
+            return str(existing["_id"])
+        
+        # Create new conversation
+        new_conversation = {
+            "session_id": session_id,
+            "user_id": user_id,
+            "title": title,
+            "created_at": datetime.utcnow(),
+            "last_updated": datetime.utcnow()
+        }
+        result = db[CONVERSATIONS_COLLECTION].insert_one(new_conversation)
+        return str(result.inserted_id)
+    except Exception as e:
+        logger.error(f"Error creating conversation: {e}")
+        return None
+
+def update_conversation_title(conversation_id: str, title: str) -> bool:
+    """Update the title of a conversation."""
+    try:
+        db = get_db()
+        db[CONVERSATIONS_COLLECTION].update_one(
+            {"_id": ObjectId(conversation_id)},
+            {"$set": {"title": title, "last_updated": datetime.utcnow()}}
+        )
+        return True
+    except Exception as e:
+        logger.error(f"Error updating conversation title: {e}")
+        return False
+
+def update_conversation_timestamp(conversation_id: str) -> bool:
+    """Update the last_updated timestamp of a conversation."""
+    try:
+        db = get_db()
+        db[CONVERSATIONS_COLLECTION].update_one(
+            {"_id": ObjectId(conversation_id)},
+            {"$set": {"last_updated": datetime.utcnow()}}
+        )
+        return True
+    except Exception as e:
+        logger.error(f"Error updating conversation timestamp: {e}")
+        return False
+
+# Message functions
+def add_message(conversation_id: str, role: str, content: str, 
+                sources: Optional[List] = None, reasoning: Optional[List] = None, 
+                images: Optional[List] = None) -> Optional[str]:
+    """Add a message to a conversation."""
+    try:
+        db = get_db()
+        
+        # Prepare the message document
+        message = {
+            "conversation_id": conversation_id,
+            "role": role,
+            "content": content,
+            "timestamp": datetime.utcnow()
+        }
+        
+        # Add optional fields with serialization
+        if sources:
+            # Serialize sources
+            serialized_sources = json.loads(json.dumps(sources, default=serialize_custom_objects))
+            message["sources"] = serialized_sources
+            
+        if reasoning:
+            # Serialize reasoning steps
+            serialized_reasoning = json.loads(json.dumps(reasoning, default=serialize_custom_objects))
+            message["reasoning"] = serialized_reasoning
+            
+        if images:
+            # Serialize images
+            serialized_images = json.loads(json.dumps(images, default=serialize_custom_objects))
+            message["images"] = serialized_images
+        
+        # Insert the message
+        result = db[MESSAGES_COLLECTION].insert_one(message)
+        
+        # Update the conversation timestamp
+        update_conversation_timestamp(conversation_id)
+        
+        return str(result.inserted_id)
+    except Exception as e:
+        logger.error(f"Error adding message: {e}")
+        return None
+
+def serialize_custom_objects(obj):
+    """
+    Custom serialization function for MongoDB.
+    Handles special types like ActionReasoningStep and other custom classes.
+    """
+    if hasattr(obj, '__dict__'):
+        # For ActionReasoningStep, ObservationReasoningStep, etc.
+        if obj.__class__.__name__.endswith('ReasoningStep'):
+            result = {
+                'type': obj.__class__.__name__
+            }
+            
+            # Add attributes based on the specific type
+            if hasattr(obj, 'action'):
+                result['action'] = obj.action
+            if hasattr(obj, 'action_input'):
+                result['action_input'] = obj.action_input
+            if hasattr(obj, 'observation'):
+                result['observation'] = obj.observation
+            if hasattr(obj, 'response'):
+                result['response'] = obj.response
+            if hasattr(obj, 'thought'):
+                result['thought'] = obj.thought
+                
+            return result
+        
+        # For other objects with __dict__
+        return {k: v for k, v in obj.__dict__.items() 
+                if not k.startswith('_') and not callable(v)}
+    
+    # For objects with content property
+    if hasattr(obj, 'content'):
+        return str(obj.content)
+        
+    # For objects with string representation
+    try:
+        return str(obj)
+    except:
+        return f""
+
+def get_conversation_messages(conversation_id: str) -> List[Dict]:
+    """Get all messages in a conversation."""
+    try:
+        db = get_db()
+        messages = list(db[MESSAGES_COLLECTION].find(
+            {"conversation_id": conversation_id}
+        ).sort("timestamp", pymongo.ASCENDING))
+        return messages
+    except Exception as e:
+        logger.error(f"Error getting conversation messages: {e}")
+        return []
+
+def generate_conversation_title(conversation_id: str, content: List[Dict]) -> Optional[str]:
+    """
+    Generate a title for a conversation based on its content using AI.
+    
+    Args:
+        conversation_id: The ID of the conversation
+        content: List of messages in the conversation
+    
+    Returns:
+        A generated title, or None if generation failed
+    """
+    try:
+        from llama_index.llms.openai import OpenAI as LlamaOpenAI
+        
+        # Extract text from the conversation (first few messages)
+        conversation_text = "\n".join([
+            f"{msg['role']}: {msg['content']}" 
+            for msg in content[:5]  # Use first 5 messages or fewer
+        ])
+        
+        # Create LLM instance
+        llm = LlamaOpenAI(
+            model="chatgpt-4o-latest",
+            temperature=0.3,
+        )
+        
+        # Generate title
+        prompt = f"""
+        Based on the following conversation, generate a short, descriptive title (max 5 words):
+        
+        {conversation_text}
+        
+        Title:
+        """
+        
+        response = llm.complete(prompt)
+        title = response.text.strip()
+        
+        # Update the conversation with the new title
+        update_conversation_title(conversation_id, title)
+        
+        return title
+    except Exception as e:
+        logger.error(f"Error generating conversation title: {e}")
+        return "New conversation"  # Fallback title
+
+def delete_conversation(conversation_id: str, hard_delete: bool = False) -> bool:
+    """
+    Delete a conversation and its messages.
+    
+    Args:
+        conversation_id: The ID of the conversation to delete
+        hard_delete: If True, physically delete the records; if False, mark as deleted
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        db = get_db()
+        
+        if hard_delete:
+            # Permanently delete all messages in the conversation
+            db[MESSAGES_COLLECTION].delete_many({"conversation_id": conversation_id})
+            
+            # Permanently delete the conversation
+            db[CONVERSATIONS_COLLECTION].delete_one({"_id": ObjectId(conversation_id)})
+        else:
+            # Mark the conversation as deleted
+            db[CONVERSATIONS_COLLECTION].update_one(
+                {"_id": ObjectId(conversation_id)},
+                {"$set": {"is_deleted": True}}
+            )
+        
+        return True
+    except Exception as e:
+        logger.error(f"Error deleting conversation: {e}")
+        return False
+
+# Session state management
+def get_session_state(session_id: str) -> Optional[Dict]:
+    """
+    Get the session state from MongoDB.
+    
+    Args:
+        session_id: The session ID
+        
+    Returns:
+        The session state or None if not found
+    """
+    try:
+        conversation = get_conversation(session_id)
+        if conversation:
+            # Return a minimal session state
+            return {
+                "initialized": True,
+                "conversation_id": str(conversation["_id"]),
+                "user_id": conversation["user_id"]
+            }
+        return None
+    except Exception as e:
+        logger.error(f"Error getting session state: {e}")
+        return None
+
+def create_session_state(session_id: str, user_id: str, conversation_id: Optional[str] = None) -> Optional[Dict]:
+    """
+    Create a new session state in MongoDB.
+    
+    Args:
+        session_id: The session ID
+        user_id: The user ID
+        conversation_id: Optional conversation ID. If not provided, a new conversation will be created.
+        
+    Returns:
+        The created session state or None if creation failed
+    """
+    try:
+        if not conversation_id:
+            conversation_id = create_conversation(session_id, user_id)
+            
+        if conversation_id:
+            return {
+                "initialized": True,
+                "conversation_id": conversation_id,
+                "user_id": user_id
+            }
+        return None
+    except Exception as e:
+        logger.error(f"Error creating session state: {e}")
+        return None
+
+def update_session_state(session_id: str, state: Dict) -> bool:
+    """
+    Update the session state in MongoDB.
+    
+    Args:
+        session_id: The session ID
+        state: The new state to save
+        
+    Returns:
+        True if the update was successful, False otherwise
+    """
+    try:
+        conversation = get_conversation(session_id)
+        if conversation:
+            # If we need to store additional session state beyond the conversation
+            # we could add a separate collection for that
+            return True
+        return False
+    except Exception as e:
+        logger.error(f"Error updating session state: {e}")
+        return False
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..baafad6
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,106 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.16
+aiosignal==1.3.2
+annotated-types==0.7.0
+anyio==4.9.0
+asgiref==3.8.1
+asyncio==3.4.3
+attrs==25.3.0
+banks==2.1.1
+beautifulsoup4==4.13.3
+blinker==1.9.0
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
+colorama==0.4.6
+dataclasses-json==0.6.7
+Deprecated==1.2.18
+dirtyjson==1.0.8
+distro==1.9.0
+dnspython==2.7.0
+filetype==1.2.0
+Flask==3.1.0
+flask-cors==5.0.1
+frozenlist==1.5.0
+fsspec==2025.3.2
+future==1.0.0
+greenlet==3.1.1
+griffe==1.7.2
+h11==0.14.0
+h2==4.2.0
+hpack==4.1.0
+httpcore==1.0.7
+httpx==0.28.1
+Hypercorn==0.17.3
+hyperframe==6.1.0
+idna==3.10
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jiter==0.9.0
+joblib==1.4.2
+llama-cloud==0.1.17
+llama-cloud-services==0.6.9
+llama-index==0.12.33
+llama-index-agent-openai==0.4.6
+llama-index-cli==0.4.1
+llama-index-core==0.12.33.post1
+llama-index-embeddings-openai==0.3.1
+llama-index-graph-stores-neo4j==0.4.6
+llama-index-indices-managed-llama-cloud==0.6.11
+llama-index-llms-openai==0.3.30
+llama-index-multi-modal-llms-openai==0.4.3
+llama-index-program-openai==0.3.1
+llama-index-question-gen-openai==0.3.0
+llama-index-readers-file==0.4.7
+llama-index-readers-llama-parse==0.4.0
+llama-parse==0.6.4.post1
+lxml==5.3.2
+markdown2==2.5.3
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+multidict==6.3.2
+mypy-extensions==1.0.0
+neo4j==5.28.1
+nest-asyncio==1.6.0
+networkx==3.4.2
+nltk==3.9.1
+numpy==2.2.4
+openai==1.71.0
+packaging==24.2
+pandas==2.2.3
+pillow==11.1.0
+platformdirs==4.3.7
+priority==2.0.0
+propcache==0.3.1
+pydantic==2.11.2
+pydantic_core==2.33.1
+pymongo==4.7.0
+pypdf==5.4.0
+python-dateutil==2.9.0.post0
+python-docx==1.1.2
+python-dotenv==1.1.0
+python-louvain==0.16
+pytz==2025.2
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+setuptools==79.0.0
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+SQLAlchemy==2.0.40
+striprtf==0.0.26
+tenacity==9.1.2
+tiktoken==0.9.0
+tqdm==4.67.1
+typing-inspect==0.9.0
+typing-inspection==0.4.0
+typing_extensions==4.13.1
+tzdata==2025.2
+urllib3==2.3.0
+uuid==1.30
+Werkzeug==3.1.3
+wheel==0.45.1
+wrapt==1.17.2
+wsproto==1.2.0
+yarl==1.19.0
diff --git a/routes.py b/routes.py
new file mode 100644
index 0000000..18ad928
--- /dev/null
+++ b/routes.py
@@ -0,0 +1,1032 @@
+# hp_chatbot/routes.py
+
+import os
+import asyncio
+import traceback
+import uuid
+import shutil
+import urllib.parse
+from datetime import datetime
+from pathlib import Path
+
+from flask import (
+    Flask, request, jsonify, make_response, send_from_directory, send_file,
+    Blueprint # Consider using Blueprint for larger apps
+)
+from werkzeug.utils import secure_filename # Keep if file uploads are reintroduced
+
+# Import from our modules
+from utils import logger, log_structured, allowed_file # Keep allowed_file if uploads return
+from json_utils import CustomJSONEncoder # Used indirectly by make_response/jsonify with custom provider
+from session_manager import get_or_create_session_state, clear_chat_state_cache
+from ai_core import initialize_global_index # Import initialization function
+from shared_state import global_workflow_agent, global_index, is_agent_available # Import shared state variables
+from document_generator import create_brief_docx
+from config import IMAGES_DIRECTORY, HP_DOCS_FOLDER, BASE_DIR, SUPPORTING_FILES_DIR # Import necessary configs
+from llama_index.core.tools import ToolOutput  # Import ToolOutput for type checking
+from llama_index.core.llms import ChatMessage  # Import ChatMessage for memory management
+
+# Import MongoDB utilities
+from mongodb_utils import (
+    add_message, get_conversation_messages, get_user_conversations,
+    get_conversation_by_id, create_conversation, create_or_update_user,
+    delete_conversation as db_delete_conversation, # Alias to avoid conflict
+    generate_conversation_title, update_session_state
+)
+
+# Create a Blueprint or define routes on the app passed from main.py
+# Using direct app registration for simplicity here.
+# If using Blueprint:
+# api_bp = Blueprint('api', __name__)
+# @api_bp.route('/chat', methods=['POST', 'OPTIONS'])
+# async def chat(): ...
+# In main.py: app.register_blueprint(api_bp)
+
+def register_routes(app: Flask):
+
+    @app.route('/chat', methods=['POST', 'OPTIONS'])
+    async def chat():
+        """Handle chat messages, interact with the AI agent, include image handling."""
+        # Declare global variables we'll need to access
+        global global_workflow_agent
+        
+        if request.method == 'OPTIONS':
+            # Pre-flight request. Reply successfully:
+            resp = make_response(jsonify({'status': 'ok'}))
+            # Add necessary CORS headers if not handled globally by Flask-CORS middleware
+            # resp.headers['Access-Control-Allow-Origin'] = request.headers.get('Origin')
+            # resp.headers['Access-Control-Allow-Credentials'] = 'true'
+            # resp.headers['Access-Control-Allow-Methods'] = 'POST, OPTIONS'
+            # resp.headers['Access-Control-Allow-Headers'] = 'Content-Type, X-MS-USERNAME, Authorization' # Add any custom headers
+            return resp
+
+        session_id = None # Initialize for error logging
+        try:
+            data = request.get_json()
+            if not data or 'message' not in data:
+                log_structured('warning', 'Chat request missing message payload')
+                return jsonify({'error': 'No message provided'}), 400
+
+            session_id = data.get('sessionId')
+            if not session_id:
+                log_structured('warning', 'Chat request missing sessionId')
+                return jsonify({'error': 'Session ID is required'}), 400
+
+            # Use X-MS-USERNAME or a similar header for authenticated username
+            username = request.headers.get('X-MS-USERNAME') # Adjust header name if different
+            # Check if we're in development mode
+            if not username and os.environ.get("PRODUCTION", "false").lower() == "false":
+                # For local development, create a default username
+                username = f"dev_user@local"
+                log_structured('info', f'Development mode: Using default username: {username}')
+            elif not username:
+                 # Allow anonymous access based on session or return error?
+                 # For now, allow anonymous tied to session_id
+                 log_structured('info', f'Chat request from anonymous user (session: {session_id})')
+                 # username = f"anonymous_{session_id[:8]}" # Or handle as error if auth is mandatory
+                 # return jsonify({'error': 'Authentication required (X-MS-USERNAME header missing)'}), 401
+
+            log_structured('info', f'Chat request received', {'session_id': session_id, 'username': username or 'anonymous'})
+
+            # Get or create session state (maps session_id to user_id, conversation_id)
+            session_state = get_or_create_session_state(session_id, username)
+
+            # Ensure session is initialized in both development and production modes
+            if not session_state.get('initialized'):
+                log_structured('info', 'Session not yet initialized. Initializing now.', {'session_id': session_id})
+                # Initialize session properly
+                session_state['initialized'] = True
+                # Update the session state in the database to mark as initialized
+                update_session_state(session_id, {'initialized': True})
+
+            conversation_id = session_state.get('conversation_id')
+            user_id = session_state.get('user_id') # Get user_id for context if needed
+
+            if not conversation_id or not user_id:
+                 log_structured('error', 'Session state missing conversation_id or user_id', {'session_id': session_id, 'state': session_state})
+                 return jsonify({'error': 'Session setup incomplete. Please refresh or start a new chat.'}), 500
+
+            user_message = data['message']
+
+            # Store user message
+            add_message(conversation_id, 'user', user_message)
+
+            log_structured('debug', 'Starting agent workflow run', {
+                'session_id': session_id,
+                'conversation_id': conversation_id,
+                'user_id': user_id,
+                'message_preview': user_message[:100]
+            })
+
+            # Check if global agent is available using the shared state helper
+            if not is_agent_available():
+                log_structured('error', 'Global workflow agent is not initialized')
+                
+                # Try to initialize it on-demand if not available
+                try:
+                    log_structured('info', 'Attempting to initialize global index and agent on-demand')
+                    index_success = await initialize_global_index()
+                    
+                    # Let initialization complete, then check the global variable
+                    if index_success:
+                        log_structured('info', 'Index initialization reported success')
+                        
+                        # Force reload the global module variable
+                        import importlib
+                        import ai_core
+                        importlib.reload(ai_core)
+                        from ai_core import global_workflow_agent
+                        
+                        # Check if global_workflow_agent is now available
+                        if global_workflow_agent:
+                            # Test the agent with a simple function call
+                            try:
+                                # Just check if the run method exists and is callable
+                                if hasattr(global_workflow_agent, 'run') and callable(global_workflow_agent.run):
+                                    log_structured('info', 'Successfully initialized global agent on-demand')
+                                else:
+                                    log_structured('error', 'Agent initialized but run method is not callable')
+                            except Exception as agent_test_err:
+                                log_structured('error', f'Agent functional test failed: {str(agent_test_err)}')
+                        else:
+                            log_structured('error', 'Agent still not available after successful index initialization')
+                    else:
+                        log_structured('error', 'Index initialization reported failure')
+                except Exception as init_err:
+                    log_structured('error', f'Error initializing agent on-demand: {str(init_err)}')
+                
+                # Final check
+                import ai_core
+                # We're already using the global variable from the function declaration
+                # Just reassign it from the imported module
+                global_workflow_agent = ai_core.global_workflow_agent
+                if not global_workflow_agent:
+                    log_structured('critical', 'Failed to get or initialize global workflow agent')
+                    
+                    # Provide a more detailed error message for debugging
+                    error_details = {
+                        'error': 'Chat agent is not available. Please try again later.',
+                        'details': 'The AI agent failed to initialize properly. This might be due to API key issues or other backend problems.',
+                        'code': 'AGENT_UNAVAILABLE'
+                    }
+                    return jsonify(error_details), 503
+            
+            # Use the agent to process the query (with one final check to be sure)
+            if not is_agent_available():
+                log_structured('critical', 'Global agent is unavailable after initialization checks')
+                error_details = {
+                    'error': 'Chat agent became unavailable. Please try again.',
+                    'code': 'AGENT_DISAPPEARED'
+                }
+                return jsonify(error_details), 500
+            
+            # Import fresh reference to agent
+            from shared_state import global_workflow_agent
+            if global_workflow_agent is None:
+                log_structured('critical', 'Global agent is None after direct import from shared_state')
+                error_details = {
+                    'error': 'Chat agent became unavailable. Please try again.',
+                    'code': 'AGENT_DISAPPEARED'
+                }
+                return jsonify(error_details), 500
+                
+            log_structured('info', 'Running agent to process query')
+            
+            # Get a fresh reference to the agent from shared state
+            from shared_state import global_workflow_agent as current_agent
+            
+            # Double-check that the agent is available
+            if current_agent is None or not hasattr(current_agent, 'run'):
+                log_structured('critical', 'Agent disappeared at runtime: null check before .run() call')
+                return jsonify({'error': 'Chat agent became unavailable just before processing. Please try again.', 'code': 'AGENT_GONE'}), 500
+            
+            # Load conversation history into the agent's memory
+            if conversation_id:
+                try:
+                    conv_messages = get_conversation_messages(conversation_id)
+                    # Skip if we already have messages in memory (avoid duplication)
+                    if current_agent.memory and len(current_agent.memory.get()) <= 1:
+                        log_structured('info', f'Loading conversation history: {len(conv_messages)} messages', 
+                                     {'conversation_id': conversation_id})
+                        
+                        # Clear memory first to avoid duplicate history
+                        current_agent.memory.reset()
+                        
+                        # Add messages to memory in the correct order
+                        for msg in conv_messages:
+                            # Skip the current message which will be added by the agent
+                            if msg.get('content') == user_message and msg.get('role') == 'user':
+                                continue
+                            role = msg.get('role')
+                            content = msg.get('content', '')
+                            if role and content:
+                                current_agent.memory.put(ChatMessage(role=role, content=content))
+                except Exception as hist_err:
+                    log_structured('error', f'Error loading conversation history: {str(hist_err)}', 
+                                 {'conversation_id': conversation_id})
+                    # Continue without history rather than failing the request
+            
+            # Call the run method on the current agent
+            raw_response = await current_agent.run(user_message)
+
+            log_structured('debug', 'Raw response received from workflow agent', {
+                'session_id': session_id, 'conversation_id': conversation_id,
+                'response_type': type(raw_response).__name__,
+                'response_keys': list(raw_response.keys()) if isinstance(raw_response, dict) else None,
+            })
+
+            # --- Process Response and Extract Images ---
+            final_response_text = "Sorry, I encountered an issue generating a response."
+            final_sources = []
+            final_reasoning = []
+            images = []
+            seen_images = set() # Track unique images added
+
+            if isinstance(raw_response, dict):
+                final_response_text = str(raw_response.get('response', ''))
+                response_sources = raw_response.get('sources', []) # Should be list of ToolOutput
+                final_reasoning = raw_response.get('reasoning', []) # Should be list of ReasoningSteps
+
+                log_structured('debug', f'Processing {len(response_sources)} sources for metadata and images', {'session_id': session_id})
+
+                # Extract source details and potential images
+                if response_sources and isinstance(response_sources, list):
+                    for source_idx, tool_output_source in enumerate(response_sources):
+                        if tool_output_source is None:
+                            continue
+                            
+                        source_info = {'content': "Source content unavailable", 'metadata': {}} # Default
+                        if isinstance(tool_output_source, ToolOutput):
+                            source_info['content'] = str(tool_output_source.content) if tool_output_source.content else "No content"
+                            source_info['tool_name'] = getattr(tool_output_source, 'tool_name', 'unknown_tool')
+                            source_info['raw_output_preview'] = str(tool_output_source.raw_output)[:100] if hasattr(tool_output_source, 'raw_output') and tool_output_source.raw_output else "N/A"
+                            
+                            # Log GraphRAG retrieval information if present
+                            if getattr(tool_output_source, 'tool_name', '') == 'GraphRAG':
+                                raw_output = getattr(tool_output_source, 'raw_output', {})
+                                if isinstance(raw_output, dict):
+                                    # Extract and log vector and GraphRAG contexts
+                                    vector_context = raw_output.get('vector_context', '')
+                                    graphrag_context = raw_output.get('graphrag_context', '')
+                                    
+                                    # Add retrieval source information to the log
+                                    log_structured('info', 'GraphRAG Retrieval Results', {
+                                        'session_id': session_id,
+                                        'vector_retrieval_size': len(vector_context),
+                                        'graphrag_retrieval_size': len(graphrag_context),
+                                        'vector_retrieval_preview': vector_context[:200] + '...' if len(vector_context) > 200 else vector_context,
+                                        'graphrag_retrieval_preview': graphrag_context[:200] + '...' if len(graphrag_context) > 200 else graphrag_context,
+                                        'community_ids': raw_output.get('community_ids', [])
+                                    })
+                                    
+                                    # Add context source information to the source_info
+                                    source_info['retrieval_method'] = 'graphrag_hybrid'
+                                    source_info['vector_context_length'] = len(vector_context)
+                                    source_info['graphrag_context_length'] = len(graphrag_context)
+                                    source_info['used_communities'] = raw_output.get('community_ids', [])
+                            else:
+                                # Standard vector retrieval
+                                source_info['retrieval_method'] = 'vector_only'
+                            
+                            # Try accessing source_nodes from raw_output if it exists
+                            raw_output = getattr(tool_output_source, 'raw_output', None)
+                            if raw_output and hasattr(raw_output, 'source_nodes'):
+                                nodes_metadata = []
+                                # Safely iterate over source_nodes
+                                source_nodes = getattr(raw_output, 'source_nodes', [])
+                                if source_nodes:
+                                    for node_idx, node_with_score in enumerate(source_nodes):
+                                        node = getattr(node_with_score, 'node', None)
+                                        if node and hasattr(node, 'metadata'):
+                                            node_meta = node.metadata or {}
+                                            nodes_metadata.append(node_meta) # Collect metadata for source info
+                                            
+                                            # Check for images within this node's metadata
+                                            node_image_paths = node_meta.get('image_paths', [])
+                                            if isinstance(node_image_paths, list):
+                                                for img_filename in node_image_paths:
+                                                    if img_filename and isinstance(img_filename, str) and img_filename not in seen_images:
+                                                        # Ensure IMAGES_DIRECTORY is a Path object
+                                                        image_dir = Path(IMAGES_DIRECTORY)
+                                                        image_full_path = image_dir / img_filename
+                                                        
+                                                        if image_full_path.is_file():
+                                                            # Extract doc name/page from filename (adjust parsing as needed)
+                                                            doc_name = node_meta.get('filename', 'Unknown Document')
+                                                            page_num = node_meta.get('source_page', 'Unknown Page')
+
+                                                            # URL encode the filename for the frontend
+                                                            url_encoded_filename = urllib.parse.quote(img_filename)
+                                                            
+                                                            image_info = {
+                                                                'filename': img_filename,
+                                                                'url_encoded_filename': url_encoded_filename,
+                                                                'document': doc_name,
+                                                                'page': page_num,
+                                                                'source': doc_name # Redundant? Maybe use tool name?
+                                                            }
+                                                            images.append(image_info)
+                                                            seen_images.add(img_filename)
+                                                            
+                                                            # Get node ID safely
+                                                            node_id = getattr(node, 'id_', 'unknown_node')
+                                                            log_structured('info', f'Added image from node metadata: {img_filename}', {'session_id': session_id, 'node_id': node_id})
+                                                        else:
+                                                            log_structured('warning', f'Image file from metadata not found: {img_filename}', {'session_id': session_id, 'path_checked': str(image_full_path)})
+                                            else:
+                                                if node_image_paths: # Log if it exists but isn't a list
+                                                    node_id = getattr(node, 'id_', 'unknown_node')
+                                                    log_structured('warning', f"'image_paths' in node metadata is not a list.", {'session_id': session_id, 'node_id': node_id, 'metadata_value': str(node_image_paths)})
+
+                            # Add collected node metadata (or tool metadata) to the source info
+                            # Initialize nodes_metadata if it doesn't exist
+                            if not locals().get('nodes_metadata'):
+                                nodes_metadata = []
+                            source_info['metadata'] = {'nodes': nodes_metadata} if nodes_metadata else getattr(tool_output_source, 'metadata', {})
+
+                        else:
+                            log_structured('warning', f'Source element is not a ToolOutput object', {'type': type(tool_output_source).__name__})
+                            source_info['content'] = str(tool_output_source) # Fallback
+
+                        # Add the processed source info regardless of type
+                        final_sources.append(source_info)
+
+            else:
+                 # Handle cases where agent might return just a string (less ideal)
+                 final_response_text = str(raw_response)
+                 log_structured('warning', 'Agent response was not a dict, sources/reasoning unavailable.', {'response_type': type(raw_response).__name__})
+
+            log_structured('info', f'Final image count for response: {len(images)}', {'session_id': session_id, 'images': images})
+
+            # Structure the response to be sent to the frontend
+            processed_response = {
+                'response': final_response_text,
+                'sources': final_sources, # Use the processed sources
+                'reasoning': final_reasoning, # Use the extracted reasoning steps
+                'images': images # Use the validated list of image info dicts
+            }
+
+            # Store assistant response in MongoDB
+            # Note: Ensure final_sources and final_reasoning are serializable by CustomJSONEncoder
+            add_message(
+                conversation_id,
+                'assistant',
+                final_response_text,
+                sources=final_sources,
+                reasoning=final_reasoning, # Pass reasoning steps
+                images=images # Pass image info list
+            )
+
+            # Update conversation title (if needed)
+            conv_messages = get_conversation_messages(conversation_id)
+            # Generate title after first user/assistant pair usually
+            if conv_messages and len(conv_messages) == 2:
+                try:
+                    new_title = generate_conversation_title(conversation_id, conv_messages)
+                    log_structured('info', f'Generated initial conversation title: "{new_title}"', {'conversation_id': conversation_id})
+                except Exception as title_err:
+                    log_structured('error', f'Error generating conversation title: {title_err}', {'conversation_id': conversation_id})
+                    # Non-fatal error
+
+            # Use make_response to ensure custom JSON encoder is used
+            response = make_response(jsonify({
+                'status': 'success',
+                'data': processed_response
+            }))
+            response.headers['Content-Type'] = 'application/json'
+            return response
+
+        except asyncio.TimeoutError:
+             log_structured('error', 'Chat request timed out', {'session_id': session_id})
+             return jsonify({'status': 'error', 'error': 'Request timed out', 'detail': 'The request took too long to process.'}), 504 # Gateway Timeout
+        except Exception as e:
+            log_structured('error', 'Unhandled error in /chat endpoint', {
+                'session_id': session_id, 'error': str(e), 'traceback': traceback.format_exc()
+            })
+            response = make_response(jsonify({
+                'status': 'error', 'error': 'An internal server error occurred.', 'detail': str(e)
+            }), 500)
+            response.headers['Content-Type'] = 'application/json'
+            return response
+
+    @app.route('/images/', methods=['GET'])
+    def serve_image(filename):
+        """Serve an image file from the designated image directory."""
+        # Log the requested image
+        log_structured('debug', f'Image request received for: {filename}')
+        
+        # For filenames with spaces and special characters, don't use secure_filename
+        # Just check for basic path traversal attempts
+        if '..' in filename or filename.startswith('/'):
+            log_structured('warning', 'Possible path traversal attempt', {'requested': filename})
+            return jsonify({'error': 'Invalid filename'}), 400
+
+        try:
+            # Try the main images directory first (using raw filename)
+            image_path = os.path.join(IMAGES_DIRECTORY, filename)
+            if os.path.exists(image_path) and os.path.isfile(image_path):
+                log_structured('debug', f'Serving image from IMAGES_DIRECTORY: {filename}')
+                return send_from_directory(IMAGES_DIRECTORY, filename, as_attachment=False)
+            
+            # If not found, check for public images in the chat interface public directory
+            public_images_dir = BASE_DIR / "chat-interface" / "public" / "images"
+            public_image_path = os.path.join(public_images_dir, filename)
+            if os.path.exists(public_image_path) and os.path.isfile(public_image_path):
+                log_structured('debug', f'Found image in public directory: {filename}')
+                return send_from_directory(public_images_dir, filename, as_attachment=False)
+            
+            # If still not found, look for the image in supporting files
+            supporting_images_dir = SUPPORTING_FILES_DIR
+            supporting_image_path = os.path.join(supporting_images_dir, filename)
+            if os.path.exists(supporting_image_path) and os.path.isfile(supporting_image_path):
+                log_structured('debug', f'Found image in supporting files: {filename}')
+                return send_from_directory(supporting_images_dir, filename, as_attachment=False)
+            
+            # Try with URL-decoded filename
+            import urllib.parse
+            decoded_filename = urllib.parse.unquote(filename)
+            if decoded_filename != filename:
+                decoded_image_path = os.path.join(IMAGES_DIRECTORY, decoded_filename)
+                if os.path.exists(decoded_image_path) and os.path.isfile(decoded_image_path):
+                    log_structured('debug', f'Serving image with decoded filename: {decoded_filename}')
+                    return send_from_directory(IMAGES_DIRECTORY, decoded_filename, as_attachment=False)
+            
+            log_structured('error', f'Image not found in any directory: {filename}')
+            return jsonify({'error': 'Image not found'}), 404
+        except Exception as e:
+            log_structured('error', f'Error serving image {filename}', {'error': str(e)})
+            return jsonify({'error': 'Server error serving image'}), 500
+
+    @app.route('/list-images', methods=['GET'])
+    def list_images():
+        """List all available images in the images directory."""
+        try:
+            if not os.path.exists(IMAGES_DIRECTORY):
+                log_structured('error', 'Images directory does not exist', {'path': str(IMAGES_DIRECTORY)})
+                return jsonify({'status': 'error', 'message': 'Images directory not found'}), 404
+
+            image_files = []
+            valid_extensions = ('.png', '.jpg', '.jpeg', '.gif', '.webp') # Add more if needed
+            
+            # Use os.listdir which is more reliable with special characters
+            for filename in os.listdir(IMAGES_DIRECTORY):
+                file_path = os.path.join(IMAGES_DIRECTORY, filename)
+                if os.path.isfile(file_path) and os.path.splitext(filename)[1].lower() in valid_extensions:
+                    try:
+                        stat_info = os.stat(file_path)
+                        image_files.append({
+                            'filename': filename,
+                            'url': f'/images/{urllib.parse.quote(filename)}', # URL encode for safety
+                            'size': stat_info.st_size,
+                            'created': datetime.fromtimestamp(stat_info.st_ctime).isoformat()
+                        })
+                    except Exception as stat_err:
+                        log_structured('error', f'Could not get stats for image file: {filename}', {'error': str(stat_err)})
+
+            # Sort by creation time, newest first
+            image_files.sort(key=lambda x: x['created'], reverse=True)
+
+            return jsonify({
+                'status': 'success',
+                'count': len(image_files),
+                'images': image_files
+            })
+        except Exception as e:
+            log_structured('error', 'Error listing images', {'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'status': 'error', 'message': 'Failed to list images'}), 500
+
+
+    # Example route for manual testing (consider removing in production)
+    @app.route('/capture-screenshot', methods=['POST'])
+    async def capture_screenshot():
+        """Manually trigger LlamaParse image capture for a specific file (for testing)."""
+        # This uses parts of ai_core.process_documents_in_directory - refactor might be needed
+        # For simplicity, we'll re-implement the image parsing part here
+        # WARNING: This endpoint allows triggering processing on arbitrary paths if not secured.
+        # Consider adding authentication/authorization or removing it.
+        try:
+            data = request.get_json()
+            file_path_str = data.get('file_path') if data else None
+
+            if not file_path_str:
+                return jsonify({'status': 'error', 'message': 'Missing file_path'}), 400
+
+            # Basic check: Ensure the file is within the expected docs folder for safety
+            target_path = Path(file_path_str).resolve()
+            allowed_dir = HP_DOCS_FOLDER.resolve()
+            if not str(target_path).startswith(str(allowed_dir)):
+                 log_structured('warning', 'Attempt to access file outside allowed directory', {'requested': file_path_str, 'allowed': str(allowed_dir)})
+                 return jsonify({'status': 'error', 'message': 'File path is not allowed'}), 403
+
+            if not target_path.is_file():
+                return jsonify({'status': 'error', 'message': f'File not found: {file_path_str}'}), 404
+
+            # --- Re-initialize LlamaParse for image capture ---
+            parser_images = None
+            image_results = []
+            temp_dir = None
+            try:
+                from config import LLAMA_PARSE_VENDOR_MODEL, LLAMA_PARSE_MAX_TIMEOUT
+                import httpx
+                from llama_parse import LlamaParse # Local import for clarity
+
+                custom_client_images = httpx.AsyncClient(timeout=LLAMA_PARSE_MAX_TIMEOUT)
+                parser_images = LlamaParse(
+                    result_type="markdown", add_page_breaks=False,
+                    system_prompt="Generate page images.",
+                    use_vendor_multimodal_model=True,
+                    vendor_multimodal_model_name=LLAMA_PARSE_VENDOR_MODEL,
+                    premium_mode=False, max_timeout=LLAMA_PARSE_MAX_TIMEOUT,
+                    custom_client=custom_client_images, verbose=True
+                )
+
+                log_structured('info', f'Attempting to capture images from file: {file_path_str} (Manual Trigger)')
+                temp_dir = IMAGES_DIRECTORY / f"temp_manual_{uuid.uuid4().hex[:8]}"
+                os.makedirs(temp_dir, exist_ok=True)
+
+                # --- Execute image parsing (sync methods in thread) ---
+                loop = asyncio.get_running_loop()
+                from concurrent.futures import ThreadPoolExecutor
+                with ThreadPoolExecutor() as pool:
+                     md_json_objs = await loop.run_in_executor(pool, parser_images.get_json_result, str(target_path))
+                     image_dicts = await loop.run_in_executor(pool, parser_images.get_images, md_json_objs, str(temp_dir))
+
+                # Ensure image_dicts is not None and is a list
+                image_dicts = image_dicts or []
+                
+                log_structured('info', f'Manual capture: LlamaParse reported {len(image_dicts)} images for {target_path.name}')
+
+                # --- Process and save images ---
+                if isinstance(image_dicts, list):  # Safety check
+                    for idx, img_info in enumerate(image_dicts):
+                        if not isinstance(img_info, dict):
+                            log_structured('warning', f'Image info is not a dict: {img_info}')
+                            continue
+                        
+                        page_index_0_based = img_info.get('page', idx)
+                        page_num_1_based = page_index_0_based + 1
+                        source_img_path_str = img_info.get('path')
+
+                        if source_img_path_str and os.path.exists(source_img_path_str):
+                            source_img_path = Path(source_img_path_str)
+                            try:
+                                image_filename = f"{target_path.stem}_manual_page{page_num_1_based}_{uuid.uuid4().hex[:6]}.png"
+                                dest_path = IMAGES_DIRECTORY / image_filename
+                                shutil.copy2(source_img_path, dest_path)
+                                image_results.append({
+                                    'success': True, 'page_number': page_num_1_based,
+                                    'filename': image_filename, 'url': f'/images/{image_filename}'
+                                })
+                                log_structured('info', f'Saved manual test image for page {page_num_1_based}', {'path': str(dest_path)})
+                            except Exception as img_err:
+                                log_structured('error', f'Failed to save manual test image page {page_num_1_based}', {'error': str(img_err)})
+                                image_results.append({'success': False, 'page_number': page_num_1_based, 'error': str(img_err)})
+                        else:
+                            image_results.append({'success': False, 'page_number': page_num_1_based, 'error': 'No image path found or path invalid'})
+
+            except Exception as parse_err:
+                 log_structured('error', f'Error during manual image capture for {file_path_str}', {'error': str(parse_err), 'traceback': traceback.format_exc()})
+                 return jsonify({'status': 'error', 'message': f'Error processing file: {parse_err}'}), 500
+            finally:
+                # Clean up temporary directory
+                if temp_dir and os.path.exists(temp_dir):
+                    try: shutil.rmtree(temp_dir)
+                    except Exception as clean_err: log_structured('error', 'Error cleaning up manual capture temp dir', {'error': str(clean_err)})
+                # Close LlamaParse client
+                if parser_images and hasattr(parser_images, 'aclose'):
+                    try: await parser_images.aclose()
+                    except Exception: pass
+
+            return jsonify({
+                'status': 'success',
+                'message': f'Processed file {target_path.name}, attempted to capture {len(image_dicts or [])} images.',
+                'image_results': image_results
+            })
+
+        except Exception as e:
+            log_structured('error', 'Error in /capture-screenshot endpoint', {'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'status': 'error', 'message': f'Internal server error: {str(e)}'}), 500
+
+
+    @app.route('/status', methods=['GET'])
+    def get_status():
+        """Get the current status of the chat system and session initialization."""
+        # Use shared state module instead of globals
+        session_id = request.args.get('sessionId')
+
+        # SUPER SIMPLIFIED - ALWAYS RETURN INITIALIZED
+        log_structured('info', 'Status check: ALWAYS returning initialized=true')
+        
+        status_data = {
+            'global_status': 'initialized',
+            'initialized': True,
+            'is_initialized': True,
+            'timestamp': datetime.now().isoformat(),
+            'override': 'FORCE_INITIALIZED'
+        }
+
+        if session_id:
+            # Optionally check if this specific session is known (cached or in DB)
+            try:
+                 # Use the session manager, but don't force creation if it doesn't exist
+                 state = get_or_create_session_state(session_id, username=None) # Check existing without forcing creation
+                 
+                 # Check if session exists and has the required fields
+                 session_exists = state and state.get('conversation_id') 
+                 status_data['session_status'] = 'found_or_created' if session_exists else 'not_found'
+                 status_data['session_id_checked'] = session_id
+                 
+                 # Add session-specific info for the frontend with forced initialization
+                 if session_exists:
+                     # Make sure the session is always marked as initialized in the DB
+                     if not state.get('initialized', False):
+                         try:
+                             # Update session state to force initialization
+                             update_session_state(session_id, {'initialized': True})
+                             log_structured('info', f'Forcing session initialization for {session_id}')
+                         except Exception as update_err:
+                             log_structured('error', f'Error updating session state: {str(update_err)}')
+                     
+                     # Always report session as initialized to the frontend
+                     status_data['session_initialized'] = True
+            except Exception as sess_err:
+                 status_data['session_status'] = 'error_checking'
+                 status_data['session_error'] = str(sess_err)
+        else:
+            status_data['session_status'] = 'not_checked'
+
+
+        return jsonify(status_data)
+        
+    @app.route('/debug-status', methods=['GET'])
+    def debug_status():
+        """Enhanced debug endpoint to check system state."""
+        # This is a development-only endpoint; disable in production
+        if os.environ.get("PRODUCTION", "false").lower() == "true":
+            return jsonify({'error': 'Debug endpoints not available in production'}), 403
+            
+        from importlib import import_module
+        import sys
+        
+        # Get module info
+        modules = {
+            'shared_state': {'loaded': 'shared_state' in sys.modules},
+            'ai_core': {'loaded': 'ai_core' in sys.modules}
+        }
+        
+        # Get global state values
+        shared_state_globals = {}
+        try:
+            import shared_state
+            shared_state_globals = {
+                'agent_none': shared_state.global_workflow_agent is None,
+                'index_none': shared_state.global_index is None,
+                'graph_store_none': shared_state.global_graph_store is None,
+                'agent_available': shared_state.is_agent_available(),
+                'agent_type': type(shared_state.global_workflow_agent).__name__ if shared_state.global_workflow_agent else None,
+                'agent_has_run': hasattr(shared_state.global_workflow_agent, 'run') if shared_state.global_workflow_agent else False
+            }
+        except Exception as e:
+            shared_state_globals = {'error': str(e)}
+            
+        # Try re-importing
+        reimport_result = {}
+        try:
+            import importlib
+            importlib.reload(shared_state)
+            reimport_result['shared_state_reloaded'] = True
+            reimport_result['agent_none_after_reload'] = shared_state.global_workflow_agent is None
+            reimport_result['agent_available_after_reload'] = shared_state.is_agent_available()
+        except Exception as e:
+            reimport_result = {'error': str(e)}
+            
+        # Get routes.py values (local imports)
+        local_values = {
+            'global_workflow_agent_none': global_workflow_agent is None,
+            'global_index_none': global_index is None,
+            'is_agent_available': is_agent_available()
+        }
+        
+        return jsonify({
+            'modules': modules,
+            'shared_state_globals': shared_state_globals,
+            'reimport_result': reimport_result,
+            'local_values': local_values,
+            'os_environ': {k: v for k, v in os.environ.items() if not k.startswith('OPENAI') and not k.startswith('SECRET')}
+        })
+        
+    @app.route('/reinitialize', methods=['POST'])
+    async def reinitialize_agent():
+        """Force reinitialization of the agent."""
+        # This is a development-only endpoint; disable in production
+        if os.environ.get("PRODUCTION", "false").lower() == "true":
+            return jsonify({'error': 'Debug endpoints not available in production'}), 403
+        
+        from ai_core import initialize_global_index
+        
+        try:
+            # Log current state before reinitializing
+            log_structured('info', 'Reinitializing global agent', {
+                'agent_available_before': is_agent_available(),
+                'agent_none_before': global_workflow_agent is None,
+                'index_none_before': global_index is None
+            })
+            
+            # Attempt to reinitialize
+            success = await initialize_global_index()
+            
+            # Import fresh state after initialization
+            import importlib
+            import shared_state
+            importlib.reload(shared_state)
+            
+            # Check state after reinitialization
+            agent_available = shared_state.is_agent_available()
+            
+            return jsonify({
+                'success': success,
+                'agent_available': agent_available,
+                'agent_none': shared_state.global_workflow_agent is None,
+                'index_none': shared_state.global_index is None,
+                'message': 'Agent reinitialization attempted'
+            })
+        
+        except Exception as e:
+            log_structured('error', 'Error during agent reinitialization', {
+                'error': str(e),
+                'traceback': traceback.format_exc()
+            })
+            return jsonify({
+                'success': False, 
+                'error': str(e),
+                'message': 'Error during reinitialization'
+            }), 500
+
+
+    @app.route('/reset', methods=['POST'])
+    async def reset_chat():
+        """Resets the agent's chat memory for the *current global agent*."""
+        # NOTE: This resets the memory of the SINGLE global agent instance.
+        # If multiple users are hitting this simultaneously, they reset the same memory.
+        # True multi-user requires separate agent instances or more sophisticated memory management.
+        # This endpoint might be more suitable for single-user testing or demos.
+        # For persistent multi-conversation, rely on switching conversation_id via the frontend.
+        session_id = None
+        try:
+            data = request.get_json()
+            session_id = data.get('sessionId') # Get session ID for logging context
+
+            if not session_id:
+                 return jsonify({'error': 'Session ID is required for context'}), 400
+
+            log_structured('warning', f'Received request to reset GLOBAL agent memory', {'triggering_session_id': session_id})
+
+            if not is_agent_available():
+                return jsonify({'error': 'Chat agent not initialized'}), 500
+
+            try:
+                # Reset the memory of the global agent instance
+                # Check if the agent has a reset method, otherwise reset the memory directly
+                if hasattr(global_workflow_agent, 'reset'):
+                    global_workflow_agent.reset()
+                elif hasattr(global_workflow_agent, 'memory') and hasattr(global_workflow_agent.memory, 'reset'):
+                    global_workflow_agent.memory.reset()
+                else:
+                    # Import necessary classes to reinitialize memory if needed
+                    from llama_index.core.memory import ChatMemoryBuffer
+                    global_workflow_agent.memory = ChatMemoryBuffer.from_defaults(llm=global_workflow_agent.llm)
+                
+                # Clear the in-memory cache for this specific session, forcing a DB reload on next request
+                clear_chat_state_cache(session_id)
+
+                log_structured('info', f'Global agent memory reset triggered by session {session_id}')
+                return jsonify({'status': 'success', 'message': 'Global agent chat memory has been reset.'})
+            except Exception as e:
+                log_structured('error', f'Error resetting global agent memory', {'session_id': session_id, 'error': str(e)})
+                return jsonify({'error': f'Failed to reset agent memory: {str(e)}'}), 500
+
+        except Exception as e:
+            log_structured('error', 'Error in /reset endpoint', {'session_id': session_id, 'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'error': str(e)}), 500
+
+    # --- Conversation Management Endpoints ---
+
+    @app.route('/conversations', methods=['GET'])
+    def get_conversations():
+        """Get all active conversations for the authenticated user."""
+        username = request.headers.get('X-MS-USERNAME')
+        # Check if we're in development mode
+        if not username and os.environ.get("PRODUCTION", "false").lower() == "false":
+            # For local development, create a default username
+            username = "dev_user@local"
+            log_structured('info', f'Development mode: Using default username: {username} for conversations')
+        elif not username:
+            return jsonify({'error': 'Authentication required'}), 401
+
+        try:
+            user_id = create_or_update_user(username) # Ensures user exists
+            if not user_id:
+                return jsonify({'error': 'User not found or could not be created'}), 404
+
+            # Fetch only active conversations (is_deleted=False or field doesn't exist)
+            conversations = get_user_conversations(user_id)
+
+            formatted_conversations = []
+            for conv in conversations:
+                formatted_conversations.append({
+                    'id': str(conv['_id']),
+                    'title': conv.get('title', 'Untitled Conversation'),
+                    'created_at': conv.get('created_at', datetime.min).isoformat(),
+                    'last_updated': conv.get('last_updated', datetime.min).isoformat(),
+                    'session_id': conv.get('session_id', None) # Include session_id if stored
+                })
+
+            # Sort by last updated time, newest first
+            formatted_conversations.sort(key=lambda x: x['last_updated'], reverse=True)
+
+            return jsonify({'status': 'success', 'conversations': formatted_conversations})
+
+        except Exception as e:
+            log_structured('error', 'Error getting user conversations', {'username': username, 'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'error': 'Failed to retrieve conversations'}), 500
+
+
+    @app.route('/conversations//messages', methods=['GET'])
+    def get_conversation_msgs(conversation_id_str):
+        """Get all messages for a specific conversation."""
+        username = request.headers.get('X-MS-USERNAME')
+        # Check if we're in development mode
+        if not username and os.environ.get("PRODUCTION", "false").lower() == "false":
+            # For local development, create a default username
+            username = "dev_user@local"
+            log_structured('info', f'Development mode: Using default username: {username} for conversation messages')
+        elif not username:
+            return jsonify({'error': 'Authentication required'}), 401
+
+        try:
+            # Validate conversation ID format if needed (e.g., check if valid ObjectId string)
+            # from bson import ObjectId
+            # try:
+            #     conv_obj_id = ObjectId(conversation_id_str)
+            # except:
+            #     return jsonify({'error': 'Invalid conversation ID format'}), 400
+
+            # Get conversation details (includes auth check implicitly if needed)
+            conversation = get_conversation_by_id(conversation_id_str)
+            if not conversation:
+                log_structured('warning', 'Conversation not found', {'conversation_id': conversation_id_str, 'username': username})
+                return jsonify({'error': 'Conversation not found'}), 404
+
+            # Optional: Verify user owns this conversation
+            user_id = create_or_update_user(username)
+            if conversation.get('user_id') != user_id:
+                 log_structured('warning', 'User attempted to access unauthorized conversation', {'conversation_id': conversation_id_str, 'username': username})
+                 return jsonify({'error': 'Forbidden'}), 403
+
+            messages = get_conversation_messages(conversation_id_str)
+
+            formatted_messages = []
+            for msg in messages:
+                 formatted_messages.append({
+                     'id': str(msg['_id']),
+                     'role': msg.get('role'),
+                     'content': msg.get('content'),
+                     'timestamp': msg.get('timestamp', datetime.min).isoformat(),
+                     'sources': msg.get('sources', []), # Ensure these are serializable
+                     'reasoning': msg.get('reasoning', []),# Ensure these are serializable
+                     'images': msg.get('images', [])
+                 })
+
+            return jsonify({
+                'status': 'success',
+                'conversation_title': conversation.get('title', 'Untitled'),
+                'messages': formatted_messages
+            })
+
+        except Exception as e:
+            log_structured('error', 'Error getting conversation messages', {'conversation_id': conversation_id_str, 'username': username, 'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'error': 'Failed to retrieve messages'}), 500
+
+    @app.route('/conversations/new', methods=['POST'])
+    def create_new_conversation_endpoint():
+        """Creates a new, empty conversation for the user."""
+        username = request.headers.get('X-MS-USERNAME')
+        # Check if we're in development mode
+        if not username and os.environ.get("PRODUCTION", "false").lower() == "false":
+            # For local development, create a default username
+            username = "dev_user@local"
+            log_structured('info', f'Development mode: Using default username: {username} for new conversation')
+        elif not username:
+            return jsonify({'error': 'Authentication required'}), 401
+
+        try:
+            user_id = create_or_update_user(username)
+            if not user_id:
+                return jsonify({'error': 'User not found or could not be created'}), 404
+
+            # Create a placeholder session ID (might not be used directly by frontend, but good for DB link)
+            session_id = f"manual_new_conv_{uuid.uuid4().hex[:8]}"
+            # Create conversation with a default title
+            new_conv_title = f"New Chat ({datetime.now().strftime('%Y-%m-%d %H:%M')})"
+            conversation_id = create_conversation(session_id, user_id, title=new_conv_title)
+
+            if not conversation_id:
+                return jsonify({'error': 'Failed to create conversation'}), 500
+
+            log_structured('info', 'New conversation created manually', {'username': username, 'conversation_id': conversation_id})
+
+            # Return the ID and title of the newly created conversation
+            return jsonify({
+                'status': 'success',
+                'conversation_id': str(conversation_id),
+                'title': new_conv_title
+            })
+
+        except Exception as e:
+            log_structured('error', 'Error creating new conversation', {'username': username, 'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'error': 'Failed to create conversation'}), 500
+
+
+    @app.route('/conversations/', methods=['DELETE'])
+    def delete_conversation_route(conversation_id_str):
+        """Deletes (soft by default) a specific conversation."""
+        username = request.headers.get('X-MS-USERNAME')
+        # Check if we're in development mode
+        if not username and os.environ.get("PRODUCTION", "false").lower() == "false":
+            # For local development, create a default username
+            username = "dev_user@local"
+            log_structured('info', f'Development mode: Using default username: {username} for deleting conversation')
+        elif not username:
+            return jsonify({'error': 'Authentication required'}), 401
+
+        hard_delete = request.args.get('hard_delete', 'false').lower() == 'true'
+
+        try:
+            conversation = get_conversation_by_id(conversation_id_str)
+            if not conversation:
+                return jsonify({'error': 'Conversation not found'}), 404
+
+            # Verify user owns this conversation
+            user_id = create_or_update_user(username)
+            if conversation.get('user_id') != user_id:
+                 return jsonify({'error': 'Forbidden'}), 403
+
+            success = db_delete_conversation(conversation_id_str, hard_delete=hard_delete)
+
+            if success:
+                log_structured('info', f'Conversation {"hard" if hard_delete else "soft"} deleted', {
+                    'conversation_id': conversation_id_str, 'username': username
+                })
+                # Clear any cached state related to this conversation's session_id if known
+                session_id_to_clear = conversation.get('session_id')
+                if session_id_to_clear:
+                     clear_chat_state_cache(session_id_to_clear)
+                return jsonify({'status': 'success'})
+            else:
+                 log_structured('error', 'Failed to delete conversation in DB', {'conversation_id': conversation_id_str})
+                 return jsonify({'error': 'Failed to delete conversation'}), 500
+
+        except Exception as e:
+            log_structured('error', 'Error deleting conversation', {'conversation_id': conversation_id_str, 'username': username, 'error': str(e), 'traceback': traceback.format_exc()})
+            return jsonify({'error': 'Failed to delete conversation'}), 500
+
+
+    # --- Brief Download Endpoint ---
+    @app.route('/download-brief', methods=['POST', 'OPTIONS'])
+    async def download_brief():
+        """Generates and downloads a marketing brief as a Word document."""
+        if request.method == 'OPTIONS':
+             return jsonify({'status': 'ok'}) # Handle pre-flight
+
+        session_id = None
+        try:
+            data = request.get_json()
+            if not data or 'brief_content' not in data:
+                return jsonify({'error': 'No brief content provided'}), 400
+
+            session_id = data.get('sessionId', 'unknown_session') # Get session for context
+            brief_content_markdown = data['brief_content']
+
+            log_structured('info', 'Generating brief document', {'session_id': session_id})
+
+            # Create the Word document in memory using the generator function
+            doc_buffer = create_brief_docx(brief_content_markdown)
+
+            # Send the file
+            download_name = f'marketing_brief_{session_id}_{datetime.now().strftime("%Y%m%d")}.docx'
+            return send_file(
+                doc_buffer,
+                mimetype='application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                as_attachment=True,
+                download_name=download_name
+            )
+
+        except Exception as e:
+            log_structured('error', 'Error generating or sending brief document', {
+                'session_id': session_id, 'error': str(e), 'traceback': traceback.format_exc()
+            })
+            return jsonify({
+                'status': 'error',
+                'error': 'Failed to generate brief document',
+                'detail': str(e)
+            }), 500
\ No newline at end of file
diff --git a/session_manager.py b/session_manager.py
new file mode 100644
index 0000000..8c35872
--- /dev/null
+++ b/session_manager.py
@@ -0,0 +1,162 @@
+# hp_chatbot/session_manager.py
+
+from typing import Dict, Any, Optional
+import uuid
+import time
+from datetime import datetime
+
+# Import MongoDB utilities from the separate file
+from mongodb_utils import (
+    get_db, create_or_update_user, get_user_by_username,
+    create_conversation, get_conversation, get_conversation_by_id, get_user_conversations,
+    get_conversation_messages, add_message, update_conversation_title,
+    generate_conversation_title, delete_conversation,
+    get_session_state as db_get_session_state, # Rename to avoid conflict
+    create_session_state as db_create_session_state,
+    update_session_state as db_update_session_state
+)
+
+# Import necessary components from ai_core
+# Use forward reference for ReActAgent2 if needed, or import normally if load order allows
+from ai_core import global_workflow_agent, ReActAgent2
+
+# Import logging
+from utils import log_structured
+
+# --- In-memory Session Cache ---
+# Stores session-specific data like associated conversation_id and user_id
+# Key: session_id (string), Value: dict {'conversation_id': ObjectId, 'user_id': ObjectId}
+# Note: The actual agent *instance* is now global (global_workflow_agent),
+# but its *memory* provides conversation context. Resetting the agent's memory
+# effectively resets the conversation for that agent instance.
+# We still need to map a *frontend session ID* to a *persistent conversation ID* in the DB.
+chat_state: Dict[str, Dict[str, Any]] = {}
+
+def get_or_create_session_state(session_id: str, username: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Gets or creates session state, mapping session_id to user and conversation in MongoDB.
+    Returns a dictionary containing 'conversation_id' and 'user_id'.
+    The 'workflow_agent' is now global and not stored per session here.
+
+    Args:
+        session_id: The unique identifier from the frontend/client.
+        username: Optional username for linking to a user.
+
+    Returns:
+        A dictionary like {'conversation_id': ObjectId, 'user_id': ObjectId, 'initialized': bool}
+    """
+    global global_workflow_agent # Access the global agent
+
+    # 1. Check in-memory cache first
+    if session_id in chat_state:
+        cached_state = chat_state[session_id]
+        # Ensure it has the necessary keys before returning
+        if 'conversation_id' in cached_state and 'user_id' in cached_state:
+             log_structured('debug', f'Session cache hit for {session_id}', {'cached_state': cached_state})
+             cached_state['initialized'] = global_workflow_agent is not None
+             return cached_state
+        else:
+             log_structured('warning', f'Cached state for {session_id} is incomplete. Re-fetching.', {'cached_state': cached_state})
+             # Remove incomplete entry and proceed to DB check
+             del chat_state[session_id]
+
+
+    # 2. Check persistent storage (MongoDB) for this session_id
+    mongo_session_data = None
+    try:
+        mongo_session_data = db_get_session_state(session_id)
+    except Exception as db_err:
+        log_structured('error', f'Error accessing MongoDB for session {session_id}: {str(db_err)}')
+        # Continue with fallback approach
+
+    user_id = None
+    conversation_id = None
+
+    if mongo_session_data:
+        log_structured('info', f'Loaded existing session state from MongoDB for {session_id}', {
+            'db_data': mongo_session_data # Be careful logging sensitive data
+        })
+        try:
+            user_id = mongo_session_data.get('user_id')
+            conversation_id = mongo_session_data.get('conversation_id')
+        except Exception as parse_err:
+            log_structured('error', f'Error parsing MongoDB session data: {str(parse_err)}')
+            mongo_session_data = None
+
+        # Validate retrieved IDs
+        if not user_id or not conversation_id:
+             log_structured('error', f'Incomplete session data found in DB for {session_id}. Recreating.', {'db_data': mongo_session_data})
+             # Force creation of a new conversation/session state below
+             mongo_session_data = None # Treat as if not found
+
+    # 3. If not found in cache or DB, or if data was invalid, create new state
+    if not mongo_session_data:
+        log_structured('info', f'No valid session state found for {session_id}. Creating new.', {'username': username})
+
+        # Determine User ID
+        effective_username = username if username else f"anonymous_{session_id[:8]}"
+        
+        try:
+            user_id = create_or_update_user(effective_username)
+        except Exception as user_err:
+            log_structured('error', f'Failed to create user in MongoDB: {str(user_err)}')
+            # Create a temporary ID for in-memory operation
+            user_id = f"temp_user_{uuid.uuid4().hex}"
+            log_structured('info', f'Using temporary user ID: {user_id}')
+            
+        if not user_id:
+            log_structured('error', f"Failed to create or update user: {effective_username}")
+            # Create a fallback user ID for in-memory operation
+            user_id = f"fallback_user_{uuid.uuid4().hex}"
+            log_structured('info', f'Using fallback user ID: {user_id}')
+
+        # Create a new Conversation linked to this user
+        # Use a default title, it will be updated after the first interaction
+        new_conv_title = f"New Chat ({datetime.now().strftime('%Y-%m-%d %H:%M')})"
+        
+        try:
+            conversation_id = create_conversation(session_id, user_id, title=new_conv_title)
+        except Exception as conv_err:
+            log_structured('error', f'Failed to create conversation in MongoDB: {str(conv_err)}')
+            # Create a temporary conversation ID for in-memory operation
+            conversation_id = f"temp_conv_{uuid.uuid4().hex}"
+            log_structured('info', f'Using temporary conversation ID: {conversation_id}')
+            
+        if not conversation_id:
+            log_structured('error', f"Failed to create conversation for session {session_id}, user {user_id}")
+            # Create a fallback conversation ID for in-memory operation
+            conversation_id = f"fallback_conv_{uuid.uuid4().hex}"
+            log_structured('info', f'Using fallback conversation ID: {conversation_id}')
+
+        # Store the new session state linkage in MongoDB
+        try:
+            db_create_session_state(session_id, user_id, conversation_id)
+        except Exception as session_err:
+            log_structured('warning', f"Failed to persist new session state link in DB: {str(session_err)}")
+            # Continue with in-memory operation
+
+        log_structured('info', f'Created new conversation and session state link', {
+            'session_id': session_id, 'user_id': user_id, 'conversation_id': conversation_id
+        })
+
+    # 4. Store in the in-memory cache and return
+    # We store the DB IDs, not the agent object itself
+    current_state = {
+        'initialized': global_workflow_agent is not None,
+        'conversation_id': conversation_id,
+        'user_id': user_id
+    }
+    chat_state[session_id] = current_state
+
+    return current_state
+
+def clear_chat_state_cache(session_id: Optional[str] = None):
+    """ Clears the in-memory chat state cache. """
+    global chat_state
+    if session_id:
+        if session_id in chat_state:
+            del chat_state[session_id]
+            log_structured('info', f'Cleared in-memory cache for session {session_id}')
+    else:
+        chat_state = {}
+        log_structured('info', 'Cleared all in-memory session cache')
\ No newline at end of file
diff --git a/shared_state.py b/shared_state.py
new file mode 100644
index 0000000..1139126
--- /dev/null
+++ b/shared_state.py
@@ -0,0 +1,102 @@
+# hp_chatbot/shared_state.py
+"""
+Shared state module to store global variables that need to be
+accessible across different modules and ensures proper synchronization.
+"""
+
+# Store the AI agent here so it's properly shared across modules
+global_workflow_agent = None
+
+# Store the global index here
+global_index = None
+
+# Store GraphRAG components
+global_graph_store = None
+global_property_graph_index = None
+global_graphrag_query_engine = None
+
+# Helper to set the global agent
+def set_global_agent(agent):
+    """Set the global agent instance."""
+    global global_workflow_agent
+    from utils import log_structured
+    
+    if agent is None:
+        log_structured('error', 'Attempted to set global_workflow_agent to None')
+        return False
+        
+    try:
+        # Check that the agent has a run method
+        if not hasattr(agent, 'run'):
+            log_structured('error', 'Agent being set does not have a run method')
+            return False
+            
+        # Set the global agent
+        global_workflow_agent = agent
+        
+        # Verify it was set correctly
+        has_run = hasattr(global_workflow_agent, 'run')
+        success = global_workflow_agent is not None and has_run
+        
+        log_structured('info', f'Global agent set successfully: {success}', {
+            'has_run_method': has_run,
+            'agent_type': type(agent).__name__
+        })
+        
+        return success
+    except Exception as e:
+        log_structured('error', f'Error setting global agent: {str(e)}')
+        return False
+
+# Helper to set the global index
+def set_global_index(index):
+    """Set the global index instance."""
+    global global_index
+    global_index = index
+    return global_index is not None
+
+# Helper to set the GraphRAG components
+def set_graphrag_components(graph_store, property_graph_index, graphrag_query_engine):
+    """Set the global GraphRAG components."""
+    global global_graph_store, global_property_graph_index, global_graphrag_query_engine
+    
+    from utils import log_structured
+    
+    global_graph_store = graph_store
+    global_property_graph_index = property_graph_index
+    global_graphrag_query_engine = graphrag_query_engine
+    
+    components_set = (global_graph_store is not None and 
+                     global_property_graph_index is not None and 
+                     global_graphrag_query_engine is not None)
+    
+    log_structured('info', f'GraphRAG components set successfully: {components_set}')
+    return components_set
+
+# Helper to get agent status
+def is_agent_available():
+    """
+    Check if the global agent is available.
+    Uses direct reference to ensure we check the current module state.
+    """
+    from utils import log_structured
+    
+    # Access the module-level global_workflow_agent directly
+    # We are using the global_workflow_agent from this module, not importing it
+    # This avoids circular import issues and ensures we're checking the actual current value
+    
+    # IMPORTANT: Declare as global to ensure we're checking the correct module-level variable
+    global global_workflow_agent
+    
+    is_available = global_workflow_agent is not None and hasattr(global_workflow_agent, 'run')
+    
+    # Add detailed logging
+    if not is_available:
+        if global_workflow_agent is None:
+            log_structured('warning', 'Agent availability check failed: global_workflow_agent is None')
+        elif not hasattr(global_workflow_agent, 'run'):
+            log_structured('warning', 'Agent availability check failed: global_workflow_agent has no run method')
+    else:
+        log_structured('debug', 'Agent availability check passed: agent exists and has run method')
+    
+    return is_available
\ No newline at end of file
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000..f18e830
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,178 @@
+# hp_chatbot/utils.py
+import logging
+import json
+import os
+import shutil
+from datetime import datetime, timezone
+from typing import Optional, Dict, Any
+from config import LOG_FILE_PATH, CHUNK_FOLDER, UPLOAD_METADATA_FOLDER, ALLOWED_EXTENSIONS
+from llama_index.core.tools import ToolOutput # Import ToolOutput for serialization check
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, # Set level from config later if needed
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler(LOG_FILE_PATH)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Import after logger is defined to avoid circular dependency if CustomJSONEncoder uses logger
+from json_utils import CustomJSONEncoder
+
+# --- Logging Helper ---
+def log_structured(level: str, event_message: str, data: Optional[Dict[str, Any]] = None):
+    """
+    Structured logging helper
+    Args:
+        level: The logging level ('info', 'error', etc.)
+        event_message: The main message describing the event
+        data: Optional dictionary of additional data to log
+    """
+    # Basic serializer to handle common non-serializable types safely
+    def safe_serialize(obj):
+        if isinstance(obj, ToolOutput):
+             # Let CustomJSONEncoder handle ToolOutput specifics if passed directly
+             return obj # Pass it through for the encoder
+        if isinstance(obj, (datetime)):
+            return obj.isoformat()
+        if isinstance(obj, bytes):
+            return ""
+        if hasattr(obj, '__dict__') and not isinstance(obj, (ToolOutput)): # Avoid double processing ToolOutput
+            # Very basic dict representation, avoid complex objects
+            try:
+                # Limit recursion depth or complexity if needed
+                return {k: safe_serialize(v) for k, v in obj.__dict__.items() if not k.startswith('_') and not callable(v)}
+            except Exception:
+                 return f"