import os import subprocess import sys import tempfile import asyncio import signal import threading # Set up temp directories FIRST, before any imports that might use temp files def setup_early_temp_directories(): """Set up temp directories before Quart imports.""" backend_dir = os.path.dirname(os.path.abspath(__file__)) temp_dir = os.path.join(backend_dir, 'temp') try: os.makedirs(temp_dir, exist_ok=True) os.chmod(temp_dir, 0o755) # Test write permissions test_file = os.path.join(temp_dir, 'test_write_early') with open(test_file, 'w') as f: f.write('test') os.remove(test_file) # Set environment variables BEFORE any imports that use tempfile os.environ['TMPDIR'] = temp_dir os.environ['TEMP'] = temp_dir os.environ['TMP'] = temp_dir tempfile.tempdir = temp_dir print(f"โœ“ Early temp directory setup: {temp_dir}") return temp_dir except Exception as e: print(f"Warning: Early temp directory setup failed: {e}") return None # Set up temp directories before importing app setup_early_temp_directories() from app import create_app from app.models.user import User # Create the ASGI app (which wraps Quart + SocketIO) asgi_app = create_app() # Extract the Quart app from the ASGI wrapper quart_app = asgi_app.quart_app # Initialize database on startup async def initialize_database(): pass # Default user creation removed (security: use populate_db.py script instead) # Use the ASGI app for the server app = asgi_app async def startup(): """Initialize the application on startup.""" await initialize_database() # Signal handlers are now managed within the asyncio event loop for proper async shutdown async def run_server(): """Run the server with enhanced shutdown pattern and diagnostics.""" import hypercorn.asyncio from hypercorn import Config import faulthandler import threading # Enable fault handler for debugging faulthandler.register(signal.SIGUSR1 if hasattr(signal, 'SIGUSR1') else signal.SIGBREAK) print("Starting Quart + SocketIO app with hypercorn ASGI server...") print("๐Ÿ“ก WebSocket functionality enabled") print("๐Ÿค– AI Runner active for autonomous conversations") print("โšก All operations async and non-blocking") print("๐Ÿ›‘ Use Ctrl-C for graceful shutdown") print("๐Ÿ” Debug: Send SIGUSR1 for stack dump if it hangs") print("Started Semblance back end service") # Create hypercorn config with debug settings config = Config() config.bind = ["0.0.0.0:5137"] config.debug = False config.use_reloader = False config.loglevel = "info" # Enable more logging config.lifespan = "on" # Ensure lifespan is enabled config.startup_timeout = 60 config.shutdown_timeout = 5 # Shorter for faster shutdown config.graceful_timeout = 3 # Shorter for faster shutdown config.keep_alive_timeout = 2 # Cut lingering connections faster # Add startup tasks to Quart app @quart_app.before_serving async def startup_task(): await startup() # Add shutdown tasks to Quart app with timeout protection @quart_app.after_serving async def shutdown_task(): print("๐Ÿ›‘ Quart app shutting down...") # 1. Shutdown SocketIO first to stop WebSocket tasks try: print("๐Ÿ”Œ Shutting down SocketIO...") from app.extensions import socketio_server # First disconnect all clients print("๐Ÿ”Œ Disconnecting all SocketIO clients...") try: # Get all sessions and disconnect them for sid in list(socketio_server.manager.get_participants('/', None) or []): try: await socketio_server.disconnect(sid, namespace='/') except Exception as disconnect_err: print(f"Error disconnecting {sid}: {disconnect_err}") print("โœ… All SocketIO clients disconnected") except Exception as disconnect_err: print(f"Error during client disconnection: {disconnect_err}") # Then shutdown the server try: if hasattr(socketio_server, 'shutdown'): await socketio_server.shutdown() print("โœ… SocketIO server shutdown complete") else: # Try shutting down the underlying engine.io server if hasattr(socketio_server, 'eio') and hasattr(socketio_server.eio, 'shutdown'): await socketio_server.eio.shutdown() print("โœ… EngineIO shutdown complete") else: print("โš ๏ธ No shutdown method available - relying on task cancellation") except Exception as shutdown_err: print(f"SocketIO shutdown error: {shutdown_err}") except Exception as e: print(f"โš ๏ธ Overall SocketIO shutdown error: {e}") # 2. Shutdown AI Runner try: await asyncio.wait_for( asyncio.to_thread(shutdown_ai_runner_safe), timeout=3.0 # Shorter timeout ) print("โœ… AI Runner shutdown complete") except asyncio.TimeoutError: print("โฑ๏ธ AI Runner shutdown timed out; continuing") except Exception as e: print(f"โš ๏ธ AI Runner shutdown error: {e}") # 3. Close database connections try: print("๐Ÿ—„๏ธ Closing database connections...") await close_database_connections() print("โœ… Database connections closed") except Exception as e: print(f"โš ๏ธ Database close error: {e}") # 4. Cancel any remaining engineio/socketio tasks await cancel_socketio_tasks() def shutdown_ai_runner_safe(): """Safe AI Runner shutdown that doesn't block the main event loop.""" try: from app.services.ai_runner_service import shutdown_ai_runner shutdown_ai_runner() except Exception as e: print(f"Error in AI Runner shutdown: {e}") async def close_database_connections(): """Close all database connections to stop PyMongo background threads.""" try: # Close the global Motor client singleton from app.db import close_db_connections await asyncio.to_thread(close_db_connections) except Exception as e: print(f"Database connection close error: {e}") async def cancel_socketio_tasks(): """Cancel any remaining SocketIO/EngineIO tasks.""" try: print("๐Ÿงน Cancelling remaining SocketIO tasks...") cancelled_count = 0 for task in list(asyncio.all_tasks()): if task.done() or task is asyncio.current_task(): continue # Check if this is a SocketIO/EngineIO related task try: coro = task.get_coro() module = getattr(coro, '__module__', '') qualname = getattr(coro, '__qualname__', '') full_name = f'{module}.{qualname}' socketio_patterns = [ 'engineio.async_socket', 'engineio.async_server', 'socketio.async_server', 'AsyncSocket._websocket_handler', 'AsyncSocket._send_ping', 'AsyncServer._service_task' ] if any(pattern in full_name for pattern in socketio_patterns): task.cancel() cancelled_count += 1 except Exception: pass # Ignore task inspection errors if cancelled_count > 0: print(f"๐Ÿงน Cancelled {cancelled_count} SocketIO tasks") # Give cancellations time to complete await asyncio.sleep(0.5) except Exception as e: print(f"Task cancellation error: {e}") # Create shutdown event for Hypercorn shutdown_event = asyncio.Event() _second_signal = False # For double Ctrl-C force exit def _raise_shutdown(): """Signal handler with force-exit on second Ctrl-C.""" nonlocal _second_signal if _second_signal: print("๐Ÿ”ฅ Second Ctrl-C - forcing exit!") import os os._exit(1) _second_signal = True print("\\n๐Ÿ›‘ Shutdown signal received...") print("๐Ÿ“Š Active threads:", [t.name for t in threading.enumerate()]) shutdown_event.set() # Watchdog to ensure serve() returns async def shutdown_watchdog(): await asyncio.sleep(4) # Wait 4 seconds (should be enough with enhanced cleanup) if not shutdown_event.is_set(): return # Already completed print("โฑ๏ธ Shutdown taking too long - dumping diagnostics...") # Dump asyncio tasks print("\\n=== Active asyncio tasks ===") for task in asyncio.all_tasks(): if not task.done(): print(f"- {task.get_name()}: {task}") # Dump threads print("\\n=== Active threads ===") for thread in threading.enumerate(): print(f"- {thread.name}: daemon={thread.daemon}, alive={thread.is_alive()}") print("๐Ÿ”ฅ Use second Ctrl-C to force exit if needed") asyncio.create_task(shutdown_watchdog(), name="shutdown-watchdog") # Register signal handlers loop = asyncio.get_running_loop() try: loop.add_signal_handler(signal.SIGINT, _raise_shutdown) loop.add_signal_handler(signal.SIGTERM, _raise_shutdown) except NotImplementedError: # Windows fallback signal.signal(signal.SIGINT, lambda *_: loop.call_soon_threadsafe(_raise_shutdown)) signal.signal(signal.SIGTERM, lambda *_: loop.call_soon_threadsafe(_raise_shutdown)) # Let Hypercorn handle shutdown gracefully with the trigger print("๐Ÿ” Debug: Starting Hypercorn serve with shutdown_trigger...") await hypercorn.asyncio.serve( asgi_app, config, shutdown_trigger=shutdown_event.wait, ) print("๐Ÿ›‘ Hypercorn server stopped") if __name__ == '__main__': try: # Check if hypercorn is available try: import hypercorn.asyncio from hypercorn import Config except ImportError as e: print("โŒ Error: hypercorn is required for WebSocket functionality!") print("Please install hypercorn: pip install hypercorn") print(f"ImportError: {e}") sys.exit(1) # Run the server with proper shutdown handling asyncio.run(run_server()) except KeyboardInterrupt: print("\\n๐Ÿ›‘ Keyboard interrupt - shutting down...") except Exception as e: print(f"โŒ Unexpected error: {e}") try: from app.services.ai_runner_service import shutdown_ai_runner shutdown_ai_runner() except: pass sys.exit(1) print("๐Ÿ‘‹ Server stopped")