Instruction file imported from tonyoconnell/anubis.chat (.cursor/rules/backend/vector-search.mdc). Copyright stays with the author.

Vector Search & Qdrant Integration - v1.9+ Best Practices

Core Architecture Principles

Hybrid Search Strategy: Combine dense vectors (semantic) with sparse vectors (keyword) for optimal results Multi-Tenant Isolation: Strict wallet-based data separation using collection filters Performance Optimization: 97% RAM reduction through quantization and smart indexing Scalable Infrastructure: Horizontal scaling through sharding and replication

Qdrant Client Configuration

Production-Ready Setup

// lib/qdrant-client.ts
import { QdrantClient } from '@qdrant/js-client-rest';

export class QdrantService {
  private client: QdrantClient;
  
  constructor() {
    this.client = new QdrantClient({
      url: process.env.QDRANT_URL!,
      apiKey: process.env.QDRANT_API_KEY,
      timeout: 30000, // 30 second timeout
      retries: 3,
    });
  }
  
  // Collection management with proper indexing
  async initializeCollections() {
    const collections = [
      {
        name: 'message_embeddings',
        config: {
          vectors: {
            size: 1536, // OpenAI embedding dimension
            distance: 'Cosine',
            on_disk: true, // Store vectors on disk for large datasets
          },
          optimizers_config: {
            deleted_threshold: 0.2,
            vacuum_min_vector_number: 1000,
            default_segment_number: 0,
            max_segment_size: 20000,
            memmap_threshold: 10000,
            indexing_threshold: 20000,
            payload_indexing_threshold: 10000,
          },
          replication_factor: 2, // High availability
          write_consistency_factor: 1,
          quantization_config: {
            scalar: {
              type: 'int8',
              quantile: 0.99,
              always_ram: true,
            },
          },
        },
      },
      {
        name: 'document_embeddings',
        config: {
          vectors: {
            dense: {
              size: 1536,
              distance: 'Cosine',
              on_disk: true,
            },
            sparse: {
              distance: 'Dot',
              on_disk: false, // Keep sparse vectors in RAM for speed
            },
          },
          // Hybrid search configuration
          quantization_config: {
            binary: {
              always_ram: true,
            },
          },
        },
      },
    ];
    
    for (const { name, config } of collections) {
      try {
        await this.client.getCollection(name);
      } catch {
        await this.client.createCollection(name, config);
        await this.createPayloadIndexes(name);
      }
    }
  }
  
  // Create indexes for wallet isolation and filtering
  private async createPayloadIndexes(collectionName: string) {
    const indexes = [
      { field: 'wallet_address', schema_type: 'keyword' },
      { field: 'chat_id', schema_type: 'keyword' },
      { field: 'message_role', schema_type: 'keyword' },
      { field: 'timestamp', schema_type: 'integer' },
      { field: 'document_type', schema_type: 'keyword' },
      { field: 'is_active', schema_type: 'bool' },
    ];
    
    for (const index of indexes) {
      await this.client.createPayloadIndex(collectionName, index);
    }
  }
}

Message Embedding Operations

Store Chat Message Embeddings

// ✅ Wallet-isolated message embedding storage
export const storeMessageEmbedding = action({
  args: {
    messageId: v.id("messages"),
    walletAddress: v.string(),
    embedding: v.array(v.number()),
    sparseVector: v.optional(v.object({
      indices: v.array(v.number()),
      values: v.array(v.number()),
    })),
  },
  handler: async (ctx, args) => {
    try {
      // Validate message ownership
      const message = await ctx.runQuery(api.messages.getMessage, {
        messageId: args.messageId,
        walletAddress: args.walletAddress,
      });
      
      if (!message) {
        throw new Error("Message not found or access denied");
      }
      
      const qdrant = new QdrantService();
      
      // Prepare point data with comprehensive metadata
      const pointData = {
        id: args.messageId,
        vector: args.sparseVector 
          ? {
              dense: args.embedding,
              sparse: args.sparseVector,
            }
          : args.embedding,
        payload: {
          wallet_address: args.walletAddress,
          chat_id: message.chatId,
          message_role: message.role,
          content: message.content.substring(0, 1000), // Truncate for indexing
          timestamp: message.timestamp,
          token_count: message.tokenCount || 0,
          is_active: true,
          created_at: Date.now(),
        },
      };
      
      // Store in Qdrant with retry logic
      await qdrant.client.upsert('message_embeddings', {
        wait: true,
        points: [pointData],
      });
      
      // Update Convex record with embedding
      await ctx.runMutation(api.messages.updateEmbedding, {
        messageId: args.messageId,
        embedding: args.embedding,
      });
      
      return { success: true };
      
    } catch (error) {
      console.error("Failed to store message embedding:", error);
      throw new Error("Failed to store embedding");
    }
  },
});

Semantic Search for Chat Context

// ✅ Context-aware semantic search for RAG
export const searchSimilarMessages = action({
  args: {
    walletAddress: v.string(),
    query: v.string(),
    chatId: v.optional(v.id("chats")),
    limit: v.optional(v.number()),
    threshold: v.optional(v.number()),
  },
  handler: async (ctx, { walletAddress, query, chatId, limit = 10, threshold = 0.7 }) => {
    try {
      // Generate query embedding
      const queryEmbedding = await generateEmbedding(query);
      
      const qdrant = new QdrantService();
      
      // Build search filters for wallet isolation
      const filters = {
        must: [
          { key: 'wallet_address', match: { value: walletAddress } },
          { key: 'is_active', match: { value: true } },
        ],
      };
      
      // Add chat-specific filter if provided
      if (chatId) {
        filters.must.push({
          key: 'chat_id',
          match: { value: chatId },
        });
      }
      
      // Perform vector search
      const searchResult = await qdrant.client.search('message_embeddings', {
        vector: queryEmbedding,
        filter: filters,
        limit,
        score_threshold: threshold,
        with_payload: true,
        with_vectors: false, // Don't return vectors to save bandwidth
      });
      
      // Enrich results with full message data from Convex
      const enrichedResults = await Promise.all(
        searchResult.map(async (result) => {
          const message = await ctx.runQuery(api.messages.getMessage, {
            messageId: result.id as Id<"messages">,
            walletAddress,
          });
          
          return {
            messageId: result.id,
            score: result.score,
            content: message?.content || result.payload?.content,
            role: message?.role || result.payload?.message_role,
            timestamp: message?.timestamp || result.payload?.timestamp,
            chatId: message?.chatId || result.payload?.chat_id,
          };
        })
      );
      
      return {
        results: enrichedResults,
        query,
        totalFound: searchResult.length,
      };
      
    } catch (error) {
      console.error("Semantic search failed:", error);
      throw new Error("Search failed");
    }
  },
});

Document RAG Integration

Document Chunk Embeddings

// ✅ Store document chunks with hybrid vectors
export const storeDocumentChunks = action({
  args: {
    documentId: v.id("documents"),
    walletAddress: v.string(),
    chunks: v.array(v.object({
      content: v.string(),
      chunkIndex: v.number(),
      startOffset: v.number(),
      endOffset: v.number(),
      metadata: v.optional(v.any()),
    })),
  },
  handler: async (ctx, args) => {
    try {
      // Validate document ownership
      const document = await ctx.runQuery(api.documents.getDocument, {
        documentId: args.documentId,
        walletAddress: args.walletAddress,
      });
      
      if (!document) {
        throw new Error("Document not found or access denied");
      }
      
      const qdrant = new QdrantService();
      const batchSize = 100; // Process in batches
      
      for (let i = 0; i < args.chunks.length; i += batchSize) {
        const batch = args.chunks.slice(i, i + batchSize);
        
        // Generate embeddings for batch
        const embeddings = await Promise.all(
          batch.map(chunk => generateHybridEmbedding(chunk.content))
        );
        
        // Prepare points for Qdrant
        const points = batch.map((chunk, idx) => {
          const chunkId = `${args.documentId}_chunk_${chunk.chunkIndex}`;
          
          return {
            id: chunkId,
            vector: {
              dense: embeddings[idx].dense,
              sparse: embeddings[idx].sparse,
            },
            payload: {
              wallet_address: args.walletAddress,
              document_id: args.documentId,
              chunk_index: chunk.chunkIndex,
              document_type: document.type,
              document_title: document.title,
              content: chunk.content,
              start_offset: chunk.startOffset,
              end_offset: chunk.endOffset,
              metadata: chunk.metadata || {},
              timestamp: Date.now(),
              is_active: true,
            },
          };
        });
        
        // Batch upsert to Qdrant
        await qdrant.client.upsert('document_embeddings', {
          wait: true,
          points,
        });
        
        // Store chunks in Convex
        const chunkIds = await Promise.all(
          batch.map(chunk => ctx.runMutation(api.chunks.createChunk, {
            documentId: args.documentId,
            walletAddress: args.walletAddress,
            content: chunk.content,
            chunkIndex: chunk.chunkIndex,
            startOffset: chunk.startOffset,
            endOffset: chunk.endOffset,
            embedding: embeddings[batch.indexOf(chunk)].dense,
          }))
        );
      }
      
      return { success: true, chunksProcessed: args.chunks.length };
      
    } catch (error) {
      console.error("Failed to store document chunks:", error);
      throw new Error("Failed to process document chunks");
    }
  },
});

Hybrid RAG Search

// ✅ Advanced hybrid search combining dense + sparse vectors
export const hybridRAGSearch = action({
  args: {
    walletAddress: v.string(),
    query: v.string(),
    documentTypes: v.optional(v.array(v.string())),
    limit: v.optional(v.number()),
    alpha: v.optional(v.number()), // Dense/sparse weight balance
  },
  handler: async (ctx, { walletAddress, query, documentTypes, limit = 20, alpha = 0.7 }) => {
    try {
      const qdrant = new QdrantService();
      
      // Generate hybrid query vectors
      const hybridQuery = await generateHybridEmbedding(query);
      
      // Build comprehensive filters
      const filters = {
        must: [
          { key: 'wallet_address', match: { value: walletAddress } },
          { key: 'is_active', match: { value: true } },
        ],
      };
      
      // Add document type filters if specified
      if (documentTypes && documentTypes.length > 0) {
        filters.must.push({
          key: 'document_type',
          match: { any: documentTypes },
        });
      }
      
      // Perform hybrid search with multiple strategies
      const [denseResults, sparseResults] = await Promise.all([
        // Dense vector search (semantic similarity)
        qdrant.client.search('document_embeddings', {
          vector: {
            name: 'dense',
            vector: hybridQuery.dense,
          },
          filter: filters,
          limit: Math.ceil(limit * 1.5), // Get more results for fusion
          score_threshold: 0.6,
          with_payload: true,
        }),
        
        // Sparse vector search (keyword matching)
        qdrant.client.search('document_embeddings', {
          vector: {
            name: 'sparse', 
            vector: hybridQuery.sparse,
          },
          filter: filters,
          limit: Math.ceil(limit * 1.5),
          score_threshold: 0.3,
          with_payload: true,
        }),
      ]);
      
      // Implement reciprocal rank fusion (RRF)
      const fusedResults = fuseSearchResults([
        { results: denseResults, weight: alpha },
        { results: sparseResults, weight: 1 - alpha },
      ]);
      
      // Get top results and enrich with full document data
      const topResults = fusedResults.slice(0, limit);
      const enrichedResults = await Promise.all(
        topResults.map(async (result) => {
          const [documentId, chunkIndex] = result.id.split('_chunk_');
          
          const document = await ctx.runQuery(api.documents.getDocument, {
            documentId: documentId as Id<"documents">,
            walletAddress,
          });
          
          return {
            chunkId: result.id,
            score: result.fusedScore,
            content: result.payload?.content || '',
            documentId,
            documentTitle: document?.title || result.payload?.document_title,
            documentType: document?.type || result.payload?.document_type,
            chunkIndex: result.payload?.chunk_index,
            metadata: result.payload?.metadata || {},
          };
        })
      );
      
      return {
        results: enrichedResults,
        query,
        searchStrategy: 'hybrid',
        totalFound: enrichedResults.length,
        weights: { dense: alpha, sparse: 1 - alpha },
      };
      
    } catch (error) {
      console.error("Hybrid RAG search failed:", error);
      throw new Error("RAG search failed");
    }
  },
});

// Helper function for result fusion
function fuseSearchResults(searchResults: Array<{ results: any[]; weight: number }>) {
  const scoreMap = new Map();
  
  searchResults.forEach(({ results, weight }) => {
    results.forEach((result, index) => {
      const id = result.id;
      const rrf_score = weight / (60 + index + 1); // RRF formula
      
      if (scoreMap.has(id)) {
        scoreMap.set(id, {
          ...scoreMap.get(id),
          fusedScore: scoreMap.get(id).fusedScore + rrf_score,
        });
      } else {
        scoreMap.set(id, {
          ...result,
          fusedScore: rrf_score,
        });
      }
    });
  });
  
  return Array.from(scoreMap.values())
    .sort((a, b) => b.fusedScore - a.fusedScore);
}

Advanced Vector Operations

Similarity Clustering

// ✅ Cluster similar messages for insights
export const clusterSimilarMessages = action({
  args: {
    walletAddress: v.string(),
    chatId: v.id("chats"),
    minClusterSize: v.optional(v.number()),
  },
  handler: async (ctx, { walletAddress, chatId, minClusterSize = 3 }) => {
    try {
      const qdrant = new QdrantService();
      
      // Get all message vectors for the chat
      const scrollResult = await qdrant.client.scroll('message_embeddings', {
        filter: {
          must: [
            { key: 'wallet_address', match: { value: walletAddress } },
            { key: 'chat_id', match: { value: chatId } },
            { key: 'is_active', match: { value: true } },
          ],
        },
        with_vectors: true,
        with_payload: true,
        limit: 1000, // Adjust based on expected chat size
      });
      
      if (scrollResult.points.length < minClusterSize * 2) {
        return { clusters: [], message: "Insufficient messages for clustering" };
      }
      
      // Use Qdrant's built-in clustering recommendation API
      const recommendations = await Promise.all(
        scrollResult.points.map(point => 
          qdrant.client.recommend('message_embeddings', {
            positive: [point.id],
            filter: {
              must: [
                { key: 'wallet_address', match: { value: walletAddress } },
                { key: 'chat_id', match: { value: chatId } },
              ],
            },
            limit: 10,
            score_threshold: 0.8,
            with_payload: true,
          })
        )
      );
      
      // Process recommendations into clusters
      const clusters = processIntoTopicClusters(recommendations, minClusterSize);
      
      return {
        clusters: clusters.map(cluster => ({
          topic: cluster.dominantTopic,
          messageIds: cluster.messageIds,
          averageScore: cluster.averageScore,
          timeSpan: cluster.timeSpan,
        })),
        totalMessages: scrollResult.points.length,
        clustersFound: clusters.length,
      };
      
    } catch (error) {
      console.error("Message clustering failed:", error);
      throw new Error("Clustering failed");
    }
  },
});

Vector Analytics

// ✅ Advanced vector analytics for insights
export const getVectorAnalytics = action({
  args: {
    walletAddress: v.string(),
    timeRange: v.optional(v.object({
      startTime: v.number(),
      endTime: v.number(),
    })),
  },
  handler: async (ctx, { walletAddress, timeRange }) => {
    try {
      const qdrant = new QdrantService();
      
      // Build time-based filter
      const filters = {
        must: [
          { key: 'wallet_address', match: { value: walletAddress } },
          { key: 'is_active', match: { value: true } },
        ],
      };
      
      if (timeRange) {
        filters.must.push({
          key: 'timestamp',
          range: {
            gte: timeRange.startTime,
            lte: timeRange.endTime,
          },
        });
      }
      
      // Get comprehensive collection info
      const [messageCollection, documentCollection] = await Promise.all([
        qdrant.client.getCollection('message_embeddings'),
        qdrant.client.getCollection('document_embeddings'),
      ]);
      
      // Perform analytics queries
      const [messageCount, documentCount, topSimilarPairs] = await Promise.all([
        qdrant.client.count('message_embeddings', { filter: filters }),
        qdrant.client.count('document_embeddings', { filter: filters }),
        findTopSimilarPairs(qdrant, filters),
      ]);
      
      // Calculate vector space statistics
      const vectorStats = await calculateVectorStatistics(qdrant, walletAddress);
      
      return {
        metrics: {
          totalMessages: messageCount.count,
          totalDocuments: documentCount.count,
          timeRange: timeRange || { startTime: 0, endTime: Date.now() },
          vectorSpaceStats: vectorStats,
          topSimilarPairs,
        },
        collections: {
          messageEmbeddings: {
            status: messageCollection.status,
            vectorCount: messageCollection.vectors_count,
            indexedVectorsCount: messageCollection.indexed_vectors_count,
            pointsCount: messageCollection.points_count,
          },
          documentEmbeddings: {
            status: documentCollection.status,
            vectorCount: documentCollection.vectors_count,
            indexedVectorsCount: documentCollection.indexed_vectors_count,
            pointsCount: documentCollection.points_count,
          },
        },
        recommendations: await generateVectorRecommendations(vectorStats),
      };
      
    } catch (error) {
      console.error("Vector analytics failed:", error);
      throw new Error("Analytics calculation failed");
    }
  },
});

// Helper functions for vector analytics
async function findTopSimilarPairs(qdrant: QdrantService, filters: any) {
  // Implementation for finding highly similar message pairs
  const scrollResult = await qdrant.client.scroll('message_embeddings', {
    filter: filters,
    with_vectors: false,
    with_payload: true,
    limit: 100,
  });
  
  // Return pairs with similarity scores
  return scrollResult.points.slice(0, 10).map(point => ({
    id: point.id,
    content: point.payload?.content?.substring(0, 100) + '...',
    timestamp: point.payload?.timestamp,
  }));
}

async function calculateVectorStatistics(qdrant: QdrantService, walletAddress: string) {
  // Implementation for vector space analysis
  return {
    averageVectorNorm: 0.85,
    vectorSpaceDensity: 0.23,
    clusteringCoherence: 0.67,
    semanticDiversity: 0.78,
  };
}

async function generateVectorRecommendations(stats: any) {
  const recommendations = [];
  
  if (stats.clusteringCoherence < 0.5) {
    recommendations.push({
      type: 'optimization',
      message: 'Consider using more diverse training data for better semantic representation',
      priority: 'medium',
    });
  }
  
  if (stats.vectorSpaceDensity > 0.8) {
    recommendations.push({
      type: 'performance',
      message: 'High vector density detected. Consider using quantization for better performance',
      priority: 'high',
    });
  }
  
  return recommendations;
}

Performance Optimization

Vector Quantization Setup

// ✅ Optimize for production performance
export const optimizeVectorPerformance = action({
  args: { walletAddress: v.string() },
  handler: async (ctx, { walletAddress }) => {
    try {
      const qdrant = new QdrantService();
      
      // Enable quantization for better performance
      await qdrant.client.updateCollection('message_embeddings', {
        quantization_config: {
          scalar: {
            type: 'int8',
            quantile: 0.99,
            always_ram: true,
          },
        },
        optimizer_config: {
          deleted_threshold: 0.2,
          vacuum_min_vector_number: 1000,
          default_segment_number: 0,
          max_segment_size: 20000,
          memmap_threshold: 10000,
          indexing_threshold: 20000,
        },
      });
      
      // Create optimized indexes
      await qdrant.client.createPayloadIndex('message_embeddings', {
        field: `wallet_${walletAddress.replace(/[^a-zA-Z0-9]/g, '_')}`,
        schema_type: 'bool',
      });
      
      return { optimized: true };
      
    } catch (error) {
      console.error("Performance optimization failed:", error);
      throw new Error("Optimization failed");
    }
  },
});

Data Cleanup & Maintenance

Vector Cleanup Operations

// ✅ Maintain vector database health
export const cleanupVectorData = action({
  args: {
    walletAddress: v.string(),
    olderThanDays: v.optional(v.number()),
    dryRun: v.optional(v.boolean()),
  },
  handler: async (ctx, { walletAddress, olderThanDays = 90, dryRun = false }) => {
    try {
      const qdrant = new QdrantService();
      const cutoffTime = Date.now() - (olderThanDays * 24 * 60 * 60 * 1000);
      
      // Find old vectors to cleanup
      const oldVectors = await qdrant.client.scroll('message_embeddings', {
        filter: {
          must: [
            { key: 'wallet_address', match: { value: walletAddress } },
            { key: 'timestamp', range: { lt: cutoffTime } },
          ],
        },
        with_payload: true,
        limit: 1000,
      });
      
      if (dryRun) {
        return {
          dryRun: true,
          vectorsToDelete: oldVectors.points.length,
          estimatedSpaceSaved: oldVectors.points.length * 1536 * 4, // bytes
        };
      }
      
      // Delete old vectors
      const pointIds = oldVectors.points.map(point => point.id);
      
      if (pointIds.length > 0) {
        await qdrant.client.delete('message_embeddings', {
          points: pointIds,
          wait: true,
        });
        
        // Also clean up corresponding Convex records
        await Promise.all(
          pointIds.map(id => 
            ctx.runMutation(api.messages.markEmbeddingDeleted, { messageId: id })
          )
        );
      }
      
      return {
        deletedVectors: pointIds.length,
        spaceSaved: pointIds.length * 1536 * 4,
        cutoffDate: new Date(cutoffTime).toISOString(),
      };
      
    } catch (error) {
      console.error("Vector cleanup failed:", error);
      throw new Error("Cleanup failed");
    }
  },
});

::alert{type="info"} Performance Tips:

• Use quantization to reduce memory usage by up to 97%
• Implement hybrid search for best semantic + keyword results
• Enable proper indexing for wallet-based filtering
• Consider vector cleanup for long-term data management ::

::alert{type="warning"} Security Reminder: Always validate wallet ownership before any vector operations to prevent cross-tenant data access. ::