ideas-generator/server/routes/vectorStores.js

const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const { v4: uuidv4 } = require('uuid');
const { VectorStore, Document, Assistant } = require('../models');
const OpenAI = require('openai');
const responsesService = require('../utils/responses');

const router = express.Router();

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  organization: process.env.OPENAI_ORG_ID,
});

// Configure multer for file uploads
const storage = multer.diskStorage({
  destination: function (req, file, cb) {
    const uploadDir = path.join(__dirname, '../uploads');
    if (!fs.existsSync(uploadDir)) {
      fs.mkdirSync(uploadDir, { recursive: true });
    }
    cb(null, uploadDir);
  },
  filename: function (req, file, cb) {
    const uniqueName = uuidv4() + path.extname(file.originalname);
    cb(null, uniqueName);
  }
});

const upload = multer({
  storage: storage,
  limits: {
    fileSize: 100 * 1024 * 1024, // 100MB limit
  },
  fileFilter: (req, file, cb) => {
    // Allow common document types
    const allowedTypes = [
      'application/pdf',
      'text/plain',
      'text/markdown',
      'application/msword',
      'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
      'text/csv',
      'application/json'
    ];
    
    if (allowedTypes.includes(file.mimetype)) {
      cb(null, true);
    } else {
      cb(new Error('Unsupported file type. Please upload PDF, TXT, MD, DOC, DOCX, CSV, or JSON files.'));
    }
  }
});

// Create a new vector store
router.post('/', async (req, res, next) => {
  try {
    const { name, description, expiresAfter } = req.body;

    if (!name) {
      return res.status(400).json({
        error: 'Validation Error',
        message: 'Vector store name is required'
      });
    }

    console.log(`Creating vector store: ${name}`);

    // Create vector store in OpenAI
    const openaiVectorStore = await openai.beta.vectorStores.create({
      name,
      expires_after: expiresAfter ? {
        anchor: "last_active_at",
        days: expiresAfter.days || 7
      } : undefined
    });

    // Store in our database
    const vectorStore = await VectorStore.create({
      openaiVectorStoreId: openaiVectorStore.id,
      name,
      description,
      fileCount: openaiVectorStore.file_counts?.total || 0,
      bytesUsed: openaiVectorStore.usage_bytes || 0,
      status: openaiVectorStore.status,
      expiresAfter,
      lastActiveAt: new Date(openaiVectorStore.last_active_at * 1000),
      metadata: {
        createdBy: 'admin', // You might want to track who created it
        openaiMetadata: openaiVectorStore.metadata
      }
    });

    console.log(`Vector store created: ${vectorStore.id} (OpenAI: ${openaiVectorStore.id})`);

    res.json({
      vectorStore: {
        id: vectorStore.id,
        name: vectorStore.name,
        description: vectorStore.description,
        fileCount: vectorStore.fileCount,
        bytesUsed: vectorStore.bytesUsed,
        status: vectorStore.status,
        openaiId: vectorStore.openaiVectorStoreId,
        createdAt: vectorStore.createdAt
      }
    });

  } catch (error) {
    console.error('Error creating vector store:', error);
    next(error);
  }
});

// List vector stores
router.get('/', async (req, res, next) => {
  try {
    const vectorStores = await VectorStore.findAll({
      order: [['createdAt', 'DESC']],
      include: [{
        model: Document,
        as: 'documents',
        attributes: ['id', 'name', 'fileSize', 'status']
      }]
    });

    res.json({
      vectorStores: vectorStores.map(vs => ({
        id: vs.id,
        name: vs.name,
        description: vs.description,
        fileCount: vs.fileCount,
        bytesUsed: vs.bytesUsed,
        status: vs.status,
        openaiId: vs.openaiVectorStoreId,
        documents: vs.documents,
        createdAt: vs.createdAt,
        lastActiveAt: vs.lastActiveAt
      }))
    });

  } catch (error) {
    console.error('Error listing vector stores:', error);
    next(error);
  }
});

// Upload file to vector store
router.post('/:vectorStoreId/files', upload.single('file'), async (req, res, next) => {
  try {
    const { vectorStoreId } = req.params;
    const { assistantId, purpose = 'assistants' } = req.body;

    if (!req.file) {
      return res.status(400).json({
        error: 'Validation Error',
        message: 'No file uploaded'
      });
    }

    const vectorStore = await VectorStore.findByPk(vectorStoreId);
    if (!vectorStore) {
      return res.status(404).json({
        error: 'Not Found',
        message: 'Vector store not found'
      });
    }

    console.log(`Uploading file ${req.file.originalname} to vector store ${vectorStore.name}`);

    // Upload file to OpenAI
    const openaiFile = await openai.files.create({
      file: fs.createReadStream(req.file.path),
      purpose: 'assistants'
    });

    console.log(`File uploaded to OpenAI: ${openaiFile.id}`);

    // Add file to vector store
    await openai.beta.vectorStores.files.create(
      vectorStore.openaiVectorStoreId,
      {
        file_id: openaiFile.id
      }
    );

    // Store document record
    const document = await Document.create({
      openaiFileId: openaiFile.id,
      name: req.file.filename,
      originalName: req.file.originalname,
      filePath: req.file.path,
      fileSize: req.file.size,
      mimeType: req.file.mimetype,
      purpose,
      status: 'processed',
      vectorStoreId: vectorStore.id,
      assistantId: assistantId || null,
      uploadedByUserId: null, // You might want to track who uploaded it
      metadata: {
        openaiBytes: openaiFile.bytes,
        openaiCreatedAt: openaiFile.created_at
      }
    });

    // Update vector store file count
    await vectorStore.increment('fileCount');
    await vectorStore.increment('bytesUsed', { by: req.file.size });
    await vectorStore.update({ lastActiveAt: new Date() });

    console.log(`Document stored: ${document.id}`);

    res.json({
      document: {
        id: document.id,
        name: document.originalName,
        fileSize: document.fileSize,
        mimeType: document.mimeType,
        status: document.status,
        openaiFileId: document.openaiFileId,
        createdAt: document.createdAt
      },
      message: 'File uploaded successfully'
    });

  } catch (error) {
    console.error('Error uploading file:', error);
    
    // Clean up uploaded file if there was an error
    if (req.file && fs.existsSync(req.file.path)) {
      fs.unlinkSync(req.file.path);
    }
    
    next(error);
  }
});

// Delete vector store
router.delete('/:vectorStoreId', async (req, res, next) => {
  try {
    const { vectorStoreId } = req.params;

    const vectorStore = await VectorStore.findByPk(vectorStoreId);
    if (!vectorStore) {
      return res.status(404).json({
        error: 'Not Found',
        message: 'Vector store not found'
      });
    }

    console.log(`Deleting vector store: ${vectorStore.name}`);

    // Delete from OpenAI
    try {
      await openai.beta.vectorStores.del(vectorStore.openaiVectorStoreId);
    } catch (openaiError) {
      console.warn(`Failed to delete OpenAI vector store: ${openaiError.message}`);
    }

    // Delete associated documents and clean up files
    const documents = await Document.findAll({
      where: { vectorStoreId: vectorStore.id }
    });

    for (const doc of documents) {
      // Clean up local file
      if (doc.filePath && fs.existsSync(doc.filePath)) {
        fs.unlinkSync(doc.filePath);
      }
      await doc.destroy();
    }

    await vectorStore.destroy();

    console.log(`Vector store deleted: ${vectorStoreId}`);

    res.json({
      message: 'Vector store deleted successfully'
    });

  } catch (error) {
    console.error('Error deleting vector store:', error);
    next(error);
  }
});

// Get vector store documents
router.get('/:vectorStoreId/documents', async (req, res, next) => {
  try {
    const { vectorStoreId } = req.params;

    const documents = await Document.findAll({
      where: { vectorStoreId },
      order: [['createdAt', 'DESC']]
    });

    res.json({
      documents: documents.map(doc => ({
        id: doc.id,
        name: doc.originalName,
        fileSize: doc.fileSize,
        mimeType: doc.mimeType,
        status: doc.status,
        openaiFileId: doc.openaiFileId,
        createdAt: doc.createdAt
      }))
    });

  } catch (error) {
    console.error('Error getting vector store documents:', error);
    next(error);
  }
});

// Get OpenAI vector stores (for agent configuration)
router.get('/openai', async (req, res, next) => {
  try {
    console.log('Fetching vector stores from OpenAI API...');
    const { limit = 20, order = 'desc' } = req.query;
    
    const vectorStoresResponse = await responsesService.listVectorStores({ 
      limit: parseInt(limit), 
      order 
    });
    
    // Format for frontend consumption
    const formattedStores = vectorStoresResponse.data?.map(store => ({
      id: store.id,
      name: store.name || `Vector Store ${store.id.substring(0, 8)}`,
      file_counts: store.file_counts || { total: 0 },
      status: store.status,
      created_at: store.created_at,
      last_active_at: store.last_active_at
    })) || [];

    console.log(`Retrieved ${formattedStores.length} vector stores from OpenAI`);
    
    res.json({
      vectorStores: formattedStores,
      has_more: vectorStoresResponse.has_more || false,
      total: formattedStores.length
    });

  } catch (error) {
    console.error('Error fetching OpenAI vector stores:', error);
    
    // Return empty result on error rather than failing completely
    res.json({
      vectorStores: [],
      has_more: false,
      total: 0,
      error: 'Failed to fetch vector stores from OpenAI'
    });
  }
});

module.exports = router;