Oliver/semblance
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add event loop tracking to LLM client getters

Browse source 
The previous lazy initialization fix wasn't sufficient - the genai.Client
internally caches async structures bound to the event loop at creation time.
With ASGI servers like Hypercorn, subsequent requests may come on different
event loop contexts, causing "Future attached to a different loop" errors.

Now tracks which event loop the client was created on and recreates it if
the loop has changed.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
michael 2025-12-19 16:46:35 -06:00
parent 36e1752fea
commit 94f98b837b
1 changed files with 20 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
backend/app/services/llm_service.py
View file

@ -20,25 +20,39 @@ import io
# Set up the Gemini API key (client created lazily to avoid event loop issues)
GEMINI_API_KEY = os.environ.get('GEMINI_API_KEY', 'AIzaSyAc50jzC3k9K1PmKT1vGFi0sCdhhnqsvl0')
_gemini_client = None
_gemini_client_loop = None
# Set up OpenAI API key (client created lazily to avoid event loop issues)
OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY', 'REDACTED_OPENAI_KEY')
_openai_client = None
_openai_client_loop = None
def get_gemini_client():
"""Get or create the Gemini client lazily within the running event loop."""
global _gemini_client
if _gemini_client is None:
"""Get or create the Gemini client for the current event loop.
Recreates the client if the event loop has changed to avoid
'Future attached to a different loop' errors in ASGI environments.
"""
global _gemini_client, _gemini_client_loop
current_loop = asyncio.get_running_loop()
if _gemini_client is None or _gemini_client_loop is not current_loop:
_gemini_client = genai.Client(api_key=GEMINI_API_KEY)
_gemini_client_loop = current_loop
return _gemini_client
def get_openai_client():
"""Get or create the OpenAI client lazily within the running event loop."""
global _openai_client
if _openai_client is None:
"""Get or create the OpenAI client for the current event loop.
Recreates the client if the event loop has changed to avoid
'Future attached to a different loop' errors in ASGI environments.
"""
global _openai_client, _openai_client_loop
current_loop = asyncio.get_running_loop()
if _openai_client is None or _openai_client_loop is not current_loop:
_openai_client = AsyncOpenAI(api_key=OPENAI_API_KEY, timeout=600.0)
_openai_client_loop = current_loop
return _openai_client
# The default model we're using