ideas-generator/server/utils/responses.js

const OpenAI = require('openai');
const { VectorStore, Document, ResponseSession, Conversation } = require('../models');

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  organization: process.env.OPENAI_ORG_ID,
});

// Direct HTTP client for Responses API until it's officially supported in the Node.js SDK
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));

class ResponsesAPIService {
  constructor() {
    if (!process.env.OPENAI_API_KEY || process.env.OPENAI_API_KEY.includes('your-actual')) {
      console.warn('⚠️  OpenAI API key not configured properly for Responses API');
    }
  }

  /**
   * Make direct HTTP request to Responses API
   */
  async makeResponsesAPIRequest(endpoint, method = 'GET', body = null) {
    const url = `https://api.openai.com${endpoint}`;
    
    const headers = {
      'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
      'Content-Type': 'application/json',
    };

    if (process.env.OPENAI_ORG_ID) {
      headers['OpenAI-Organization'] = process.env.OPENAI_ORG_ID;
    }

    const config = {
      method,
      headers,
    };

    if (body && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
      config.body = JSON.stringify(body);
    }

    console.log(`Making ${method} request to ${url}`);
    console.log(`Request body:`, body ? JSON.stringify(body, null, 2) : 'none');

    const response = await fetch(url, config);
    
    if (!response.ok) {
      const errorText = await response.text();
      console.error(`Responses API Error (${response.status}):`, errorText);
      throw new Error(`Responses API request failed: ${response.status} ${errorText}`);
    }

    return await response.json();
  }

  /**
   * Build tools configuration based on agent settings
   */
  buildToolsConfig(agent) {
    const tools = [];

    // Web Search Tool
    if (agent.webSearchEnabled) {
      tools.push({
        type: 'web_search'
      });
    }

    // File Search Tool  
    if (agent.fileSearchEnabled && agent.vectorStoreIds?.length > 0) {
      tools.push({
        type: 'file_search',
        vector_store_ids: agent.vectorStoreIds,
        max_num_results: agent.maxNumResults || 20
      });
    }

    // Code Interpreter Tool
    if (agent.codeInterpreterEnabled) {
      tools.push({
        type: 'code_interpreter'
      });
    }

    return tools;
  }

  /**
   * Create or get an OpenAI conversation for maintaining context
   */
  async createOrGetConversation(conversationId) {
    try {
      if (conversationId) {
        // Try to retrieve existing conversation from our metadata
        const conversation = await Conversation.findByPk(conversationId);
        if (conversation && conversation.metadata?.openaiConversationId) {
          console.log(`Using existing OpenAI conversation: ${conversation.metadata.openaiConversationId}`);
          return conversation.metadata.openaiConversationId;
        }
      }
      
      // Create new OpenAI conversation
      const openaiConversation = await this.makeResponsesAPIRequest('/v1/conversations', 'POST', {});
      console.log(`Created new OpenAI conversation: ${openaiConversation.id}`);
      
      // Store OpenAI conversation ID in our local conversation metadata
      if (conversationId) {
        const conversation = await Conversation.findByPk(conversationId);
        if (conversation) {
          await conversation.update({
            metadata: {
              ...conversation.metadata,
              openaiConversationId: openaiConversation.id
            }
          });
        }
      }
      
      return openaiConversation.id;
    } catch (error) {
      console.error('Error managing OpenAI conversation:', error);
      // If conversation creation fails, continue without it (fallback)
      return null;
    }
  }

  /**
   * Process files for Responses API (upload PDFs, store images as buffers)
   */
  async processFiles(files) {
    const processedFiles = [];
    
    for (const file of files) {
      try {
        console.log(`Processing file: ${file.originalname} (${file.mimetype})`);
        
        if (file.mimetype.startsWith('image/')) {
          // Images: store buffer for base64 encoding (no upload needed)
          processedFiles.push({
            buffer: file.buffer,
            filename: file.originalname,
            mimetype: file.mimetype,
            size: file.size,
            type: 'image'
          });
          console.log(`✅ Image processed: ${file.originalname}`);
        } else {
          // Documents (PDF, text): upload to OpenAI
          const { Readable } = require('stream');
          const fileStream = Readable.from(file.buffer);
          fileStream.path = file.originalname;
          
          const uploadedFile = await openai.files.create({
            file: fileStream,
            purpose: 'assistants'
          });
          
          processedFiles.push({
            id: uploadedFile.id,
            filename: file.originalname,
            mimetype: file.mimetype,
            size: file.size,
            type: 'document'
          });
          
          console.log(`✅ Document uploaded successfully: ${uploadedFile.id}`);
        }
      } catch (error) {
        console.error(`❌ Error processing file ${file.originalname}:`, error);
        throw new Error(`Failed to process file ${file.originalname}: ${error.message}`);
      }
    }
    
    return processedFiles;
  }

  /**
   * Create a response using the OpenAI Responses API
   */
  async createResponse(input, agent, conversationId, userId, options = {}) {
    try {
      const {
        background = agent.backgroundProcessing || false,
        store = true, // Always store for conversation management
        attachments = [],
        openaiConversationId: providedConversationId
      } = options;

      console.log(`Creating Responses API request for agent: ${agent.name} (${agent.key})`);
      
      // Create or get OpenAI conversation for context persistence
      const openaiConversationId = providedConversationId || await this.createOrGetConversation(conversationId);
      
      const tools = this.buildToolsConfig(agent);
      
      // Handle input format based on whether files are present
      let formattedInput;
      if (attachments && attachments.length > 0) {
        // Format input as array with file attachments
        const inputContent = [];
        
        // Add file attachments first
        attachments.forEach(file => {
          if (file.type === 'image') {
            // For images, use base64 encoding
            const base64 = file.buffer.toString('base64');
            inputContent.push({
              type: 'input_image',
              image_url: `data:${file.mimetype};base64,${base64}`
            });
          } else if (file.type === 'document') {
            // For documents (PDF, text), use input_file with file_id
            inputContent.push({
              type: 'input_file', 
              file_id: file.id
            });
          }
        });
        
        // Add text input
        if (input && input.trim()) {
          inputContent.push({
            type: 'input_text',
            text: input
          });
        }
        
        formattedInput = [{
          role: 'user',
          content: inputContent
        }];
      } else {
        // Text-only input (original format)
        formattedInput = input;
      }
      
      const requestParams = {
        model: agent.model,
        input: formattedInput,
        instructions: agent.systemPrompt,
        store,
      };

      // Add conversation ID for context persistence
      if (openaiConversationId) {
        requestParams.conversation = openaiConversationId;
      }

      // Add tools if any are enabled
      if (tools.length > 0) {
        requestParams.tools = tools;
      }

      // Add background processing if enabled
      if (background) {
        requestParams.background = true;
      }

      // Handle model-specific parameters for reasoning models
      if (agent.model === 'gpt-5' || agent.model.startsWith('o1') || agent.model.startsWith('o3')) {
        console.log(`Reasoning model detected: ${agent.model}, using reasoning.effort = ${agent.reasoningEffort}`);
        requestParams.reasoning = { 
          effort: agent.reasoningEffort || 'medium' 
        };
      }
      
      // Note: Responses API handles temperature and token limits automatically

      console.log(`Responses API request params:`, {
        model: requestParams.model,
        tools: tools.length,
        background,
        store,
        reasoning_effort: requestParams.reasoning?.effort
      });

      // Use direct HTTP request for Responses API
      const response = await this.makeResponsesAPIRequest('/v1/responses', 'POST', requestParams);

      // Store response session for tracking
      await ResponseSession.create({
        openaiResponseId: response.id,
        conversationId,
        assistantId: agent.id,
        userId,
        status: response.status || 'completed',
        backgroundProcessing: background,
        toolsUsed: tools,
        metadata: {
          model: response.model || agent.model,
          inputTokens: response.usage?.input_tokens,
          outputTokens: response.usage?.output_tokens,
          totalTokens: response.usage?.total_tokens,
          toolCalls: response.output?.filter(item => item.type !== 'message').length || 0
        }
      });

      return response;
    } catch (error) {
      console.error('Responses API Error:', {
        model: agent.model,
        error: error.message,
        status: error.status,
        code: error.code,
        type: error.type
      });
      throw error;
    }
  }

  /**
   * Retrieve a response by ID
   */
  async retrieveResponse(responseId) {
    try {
      const response = await this.makeResponsesAPIRequest(`/v1/responses/${responseId}`);
      
      // Update our local record
      const responseSession = await ResponseSession.findOne({
        where: { openaiResponseId: responseId }
      });
      
      if (responseSession) {
        await responseSession.update({
          status: response.status,
          metadata: {
            ...responseSession.metadata,
            lastChecked: new Date(),
            finalUsage: response.usage
          }
        });
      }
      
      return response;
    } catch (error) {
      console.error('Error retrieving response:', error);
      throw error;
    }
  }

  /**
   * Cancel a background response
   */
  async cancelResponse(responseId) {
    try {
      const response = await this.makeResponsesAPIRequest(`/v1/responses/${responseId}/cancel`, 'POST');
      
      // Update our local record
      const responseSession = await ResponseSession.findOne({
        where: { openaiResponseId: responseId }
      });
      
      if (responseSession) {
        await responseSession.update({
          status: 'cancelled',
          metadata: {
            ...responseSession.metadata,
            cancelledAt: new Date()
          }
        });
      }
      
      return response;
    } catch (error) {
      console.error('Error cancelling response:', error);
      throw error;
    }
  }

  /**
   * List responses with optional filtering
   */
  async listResponses(options = {}) {
    try {
      const { limit = 20, after } = options;
      
      const listParams = {
        limit,
      };
      
      if (after) {
        listParams.after = after;
      }
      
      const queryString = new URLSearchParams(listParams).toString();
      return await this.makeResponsesAPIRequest(`/v1/responses?${queryString}`);
    } catch (error) {
      console.error('Error listing responses:', error);
      throw error;
    }
  }

  /**
   * Create streaming response (if supported)
   */
  async createStreamingResponse(input, agent, conversationId, userId, onChunk, options = {}) {
    try {
      console.log(`Creating streaming Responses API request for agent: ${agent.name}`);
      
      // Create or get OpenAI conversation for context persistence
      const openaiConversationId = await this.createOrGetConversation(conversationId);
      
      const tools = this.buildToolsConfig(agent);
      
      const requestParams = {
        model: agent.model,
        input: input,
        instructions: agent.systemPrompt,
        store: true
      };

      // Add conversation ID for context persistence
      if (openaiConversationId) {
        requestParams.conversation = openaiConversationId;
      }

      // Add tools if any are enabled
      if (tools.length > 0) {
        requestParams.tools = tools;
      }

      // Handle model-specific parameters for reasoning models
      if (agent.model === 'gpt-5' || agent.model.startsWith('o1') || agent.model.startsWith('o3')) {
        requestParams.reasoning = { 
          effort: agent.reasoningEffort || 'medium' 
        };
      }

      // For now, fallback to non-streaming since streaming with direct HTTP is complex
      console.log('Streaming not yet implemented with direct HTTP, falling back to regular response');
      const response = await this.createResponse(input, agent, conversationId, userId, { ...options, openaiConversationId });
      
      // Extract text from Responses API format and simulate streaming
      const messageOutput = response.output?.find(item => item.type === 'message');
      const textContent = messageOutput?.content?.find(content => content.type === 'output_text');
      const content = textContent?.text || response.output_text || '';
      
      onChunk({ content, done: true, responseId: response.id, fullResponse: content });
      
      return { responseId: response.id, fullResponse: content };
    } catch (error) {
      console.error('Streaming Responses API Error:', error);
      
      // Fall back to non-streaming if streaming not supported
      if (error.message?.includes('streaming') || error.message?.includes('stream')) {
        console.log('Streaming not supported, falling back to regular response');
        const response = await this.createResponse(input, agent, conversationId, userId, options);
        
        // Simulate streaming for consistent interface
        const content = response.choices?.[0]?.message?.content || '';
        onChunk({ content, done: true, responseId: response.id, fullResponse: content });
        
        return { responseId: response.id, fullResponse: content };
      }
      
      throw error;
    }
  }

  /**
   * List OpenAI vector stores
   */
  async listVectorStores(options = {}) {
    try {
      const { limit = 20, order = 'desc', after, before } = options;
      
      const queryParams = new URLSearchParams({ limit, order });
      if (after) queryParams.append('after', after);
      if (before) queryParams.append('before', before);
      
      const vectorStores = await this.makeResponsesAPIRequest(`/v1/vector_stores?${queryParams.toString()}`);
      console.log(`Retrieved ${vectorStores.data?.length || 0} vector stores from OpenAI`);
      
      return vectorStores;
    } catch (error) {
      console.error('Error listing vector stores:', error);
      throw error;
    }
  }

  /**
   * Get a specific vector store by ID
   */
  async getVectorStore(vectorStoreId) {
    try {
      const vectorStore = await this.makeResponsesAPIRequest(`/v1/vector_stores/${vectorStoreId}`);
      return vectorStore;
    } catch (error) {
      console.error(`Error getting vector store ${vectorStoreId}:`, error);
      throw error;
    }
  }

  /**
   * Test connection to Responses API
   */
  async testConnection() {
    try {
      const response = await this.createResponse(
        'Hello, this is a test message.',
        {
          id: 'test',
          name: 'Test Agent',
          key: 'test',
          model: 'gpt-4o',
          systemPrompt: 'You are a helpful test assistant.',
          temperature: 0.7,
          maxTokens: 50,
          webSearchEnabled: false,
          fileSearchEnabled: false,
          codeInterpreterEnabled: false,
          vectorStoreIds: [],
          backgroundProcessing: false
        },
        'test-conversation',
        'test-user',
        { store: false }
      );
      
      console.log('✅ Responses API connection test successful');
      return { success: true, response };
    } catch (error) {
      console.error('❌ Responses API connection test failed:', error.message);
      return { success: false, error: error.message };
    }
  }
}

module.exports = new ResponsesAPIService();