import { Injectable, Logger } from '@nestjs/common'; import * as http from 'http'; import * as https from 'https'; type MeiliConfig = { host: string; apiKey?: string; indexPrefix: string; }; type HybridSearchOptions = { embedder: string; semanticRatio?: number; }; type OpenAiEmbedderConfig = { embedderName: string; apiKey: string; model: string; documentTemplate: string; }; @Injectable() export class MeilisearchService { private readonly logger = new Logger(MeilisearchService.name); private readonly embedderCache = new Map(); private vectorStoreEnabled = false; isEnabled(): boolean { return Boolean(this.getConfig()); } async searchRecord( tenantId: string, objectApiName: string, query: string, displayField?: string, ): Promise<{ id: string; hit: any } | null> { const config = this.getConfig(); if (!config) return null; const indexName = this.buildIndexName(config, tenantId, objectApiName); const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`; console.log('querying Meilisearch index:', { indexName, query, displayField }); try { const response = await this.requestJson('POST', url, { q: query, limit: 5, }, this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch query failed for index ${indexName}: ${response.status}`, ); return null; } const hits = Array.isArray(response.body?.hits) ? response.body.hits : []; if (hits.length === 0) return null; if (displayField) { const loweredQuery = query.toLowerCase(); const exactMatch = hits.find((hit: any) => { const value = hit?.[displayField]; return value && String(value).toLowerCase() === loweredQuery; }); if (exactMatch?.id) { return { id: exactMatch.id, hit: exactMatch }; } } const match = hits[0]; if (match?.id) { return { id: match.id, hit: match }; } } catch (error) { this.logger.warn(`Meilisearch lookup failed: ${error.message}`); } return null; } async searchRecords( tenantId: string, objectApiName: string, query: string, options?: { limit?: number; offset?: number }, ): Promise<{ hits: any[]; total: number }> { const config = this.getConfig(); if (!config) return { hits: [], total: 0 }; const indexName = this.buildIndexName(config, tenantId, objectApiName); const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`; const limit = Number.isFinite(Number(options?.limit)) ? Number(options?.limit) : 20; const offset = Number.isFinite(Number(options?.offset)) ? Number(options?.offset) : 0; try { const response = await this.requestJson('POST', url, { q: query, limit, offset, }, this.buildHeaders(config)); console.log('Meilisearch response body:', response.body); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch query failed for index ${indexName}: ${response.status}`, ); return { hits: [], total: 0 }; } const hits = Array.isArray(response.body?.hits) ? response.body.hits : []; const total = response.body?.estimatedTotalHits ?? response.body?.nbHits ?? hits.length; return { hits, total }; } catch (error) { this.logger.warn(`Meilisearch query failed: ${error.message}`); return { hits: [], total: 0 }; } } async upsertRecord( tenantId: string, objectApiName: string, record: Record, fieldsToIndex: string[], ): Promise { const config = this.getConfig(); if (!config || !record?.id) return; const indexName = this.buildIndexName(config, tenantId, objectApiName); const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents?primaryKey=id`; const document = this.pickRecordFields(record, fieldsToIndex); try { const response = await this.requestJson('POST', url, [document], this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch upsert failed for index ${indexName}: ${response.status}`, ); } } catch (error) { this.logger.warn(`Meilisearch upsert failed: ${error.message}`); } } async deleteRecord( tenantId: string, objectApiName: string, recordId: string, ): Promise { const config = this.getConfig(); if (!config || !recordId) return; const indexName = this.buildIndexName(config, tenantId, objectApiName); const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents/${encodeURIComponent(recordId)}`; try { const response = await this.requestJson('DELETE', url, undefined, this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch delete failed for index ${indexName}: ${response.status}`, ); } } catch (error) { this.logger.warn(`Meilisearch delete failed: ${error.message}`); } } buildSemanticChunkIndexName(tenantId: string): string { const config = this.getConfig(); const prefix = config?.indexPrefix || 'tenant_'; return `${prefix}${tenantId}_semantic_chunks`.toLowerCase(); } async upsertDocuments(indexName: string, documents: Record[]): Promise { const config = this.getConfig(); if (!config || !Array.isArray(documents) || documents.length === 0) return; const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents?primaryKey=id`; try { const response = await this.requestJson('POST', url, documents, this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { this.logger.warn(`Meilisearch document upsert failed for index ${indexName}: ${response.status}`); return; } // Meilisearch indexes (and embeds) documents asynchronously. Wait for the task // to complete so callers can immediately search and see the new documents. const taskUid = response.body?.taskUid ?? response.body?.uid; if (Number.isFinite(Number(taskUid))) { const succeeded = await this.waitForTask(config, Number(taskUid), 30000); if (!succeeded) { this.logger.warn(`Meilisearch indexing task did not succeed within timeout: taskUid=${taskUid} index=${indexName}`); } } } catch (error) { this.logger.warn(`Meilisearch document upsert failed: ${error.message}`); } } async searchIndex( indexName: string, query: string, limit = 20, hybrid?: HybridSearchOptions, ): Promise<{ hits: any[]; total: number }> { const config = this.getConfig(); if (!config) return { hits: [], total: 0 }; const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`; try { const response = await this.requestJson( 'POST', url, { q: query, limit, showRankingScore: true, ...(hybrid ? { hybrid, showRankingScoreDetails: true } : {}), }, this.buildHeaders(config), ); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch search failed for index ${indexName}: ${response.status}`, ); this.logger.warn( `Meilisearch search payload: ${JSON.stringify({ q: query, limit, hybrid })}`, ); this.logger.warn( `Meilisearch search error body: ${JSON.stringify(response.body)}`, ); // If hybrid is invalid (embedder missing), retry once without hybrid if (hybrid && response.body?.code === 'invalid_embedder') { const fallback = await this.requestJson( 'POST', url, { q: query, limit }, this.buildHeaders(config), ); if (this.isSuccessStatus(fallback.status)) { const hits = Array.isArray(fallback.body?.hits) ? fallback.body.hits : []; const total = fallback.body?.estimatedTotalHits ?? fallback.body?.nbHits ?? hits.length; this.logger.warn( `Meilisearch hybrid failed; fell back to lexical search for index ${indexName}.`, ); return { hits, total }; } } return { hits: [], total: 0 }; } const hits = Array.isArray(response.body?.hits) ? response.body.hits : []; const total = response.body?.estimatedTotalHits ?? response.body?.nbHits ?? hits.length; return { hits, total }; } catch (error) { this.logger.warn(`Meilisearch search failed: ${error.message}`); return { hits: [], total: 0 }; } } private getConfig(): MeiliConfig | null { const host = process.env.MEILI_HOST || process.env.MEILISEARCH_HOST; if (!host) return null; const trimmedHost = host.replace(/\/+$/, ''); const apiKey = process.env.MEILI_API_KEY || process.env.MEILISEARCH_API_KEY; const indexPrefix = process.env.MEILI_INDEX_PREFIX || 'tenant_'; return { host: trimmedHost, apiKey, indexPrefix }; } private buildIndexName(config: MeiliConfig, tenantId: string, objectApiName: string): string { return `${config.indexPrefix}${tenantId}_${objectApiName}`.toLowerCase(); } private buildHeaders(config: MeiliConfig): Record { const headers: Record = { 'Content-Type': 'application/json', Accept: 'application/json', }; if (config.apiKey) { headers['X-Meili-API-Key'] = config.apiKey; headers.Authorization = `Bearer ${config.apiKey}`; } return headers; } private pickRecordFields(record: Record, fields: string[]): Record { const document: Record = { id: record.id }; for (const field of fields) { if (record[field] !== undefined) { document[field] = record[field]; } } return document; } private isSuccessStatus(status: number): boolean { return status >= 200 && status < 300; } private requestJson( method: 'POST' | 'DELETE' | 'PATCH' | 'GET', url: string, payload: any, headers: Record, ): Promise<{ status: number; body: any }> { return new Promise((resolve, reject) => { const parsedUrl = new URL(url); const client = parsedUrl.protocol === 'https:' ? https : http; const request = client.request( { method, hostname: parsedUrl.hostname, port: parsedUrl.port, path: `${parsedUrl.pathname}${parsedUrl.search}`, headers, }, (response) => { let data = ''; response.on('data', (chunk) => { data += chunk; }); response.on('end', () => { if (!data) { resolve({ status: response.statusCode || 0, body: null }); return; } try { const body = JSON.parse(data); resolve({ status: response.statusCode || 0, body }); } catch (error) { reject(error); } }); }, ); request.on('error', reject); if (payload !== undefined && method !== 'GET') { request.write(JSON.stringify(payload)); } request.end(); }); } private async enableVectorStore(): Promise { // Temporarily disabled to avoid the overhead of checking on every save. // Re-enable by removing the early return below. return; if (this.vectorStoreEnabled) return; // eslint-disable-line no-unreachable const meiliConfig = this.getConfig(); if (!meiliConfig) return; const url = `${meiliConfig.host}/experimental-features`; try { const response = await this.requestJson( 'PATCH', url, { vectorStore: true }, this.buildHeaders(meiliConfig), ); if (this.isSuccessStatus(response.status)) { this.vectorStoreEnabled = true; this.logger.log('Meilisearch vector store experimental feature enabled'); } else { this.logger.warn( `Failed to enable Meilisearch vector store: ${response.status} ${JSON.stringify(response.body)}`, ); } } catch (error) { this.logger.warn(`Failed to enable Meilisearch vector store: ${error.message}`); } } async ensureOpenAiEmbedder( indexName: string, config: OpenAiEmbedderConfig, ): Promise { const meiliConfig = this.getConfig(); if (!meiliConfig || !config?.apiKey) return false; await this.enableVectorStore(); const signature = JSON.stringify({ embedderName: config.embedderName, model: config.model, documentTemplate: config.documentTemplate, apiKey: config.apiKey, }); const cacheKey = `${indexName}:${config.embedderName}`; if (this.embedderCache.get(cacheKey) === signature) { return true; } const url = `${meiliConfig.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`; try { const response = await this.requestJson( 'PATCH', url, { [config.embedderName]: { source: 'openAi', model: config.model, apiKey: config.apiKey, documentTemplate: config.documentTemplate, }, }, this.buildHeaders(meiliConfig), ); if (!this.isSuccessStatus(response.status)) { this.logger.warn( `Meilisearch embedder update failed for index ${indexName}: ${response.status}`, ); this.logger.warn( `Meilisearch embedder error body: ${JSON.stringify(response.body)}`, ); return false; } const taskUid = response.body?.taskUid ?? response.body?.uid; if (Number.isFinite(Number(taskUid))) { const succeeded = await this.waitForTask(meiliConfig, Number(taskUid), 8000); if (!succeeded) { this.logger.warn(`Meilisearch embedder task did not succeed: ${taskUid}`); return false; } } const hasEmbedder = await this.hasEmbedder(meiliConfig, indexName, config.embedderName); if (!hasEmbedder) { this.logger.warn(`Meilisearch embedder missing after update: ${config.embedderName}`); return false; } this.embedderCache.set(cacheKey, signature); return true; } catch (error) { this.logger.warn(`Meilisearch embedder update failed: ${error.message}`); return false; } } private async waitForTask( config: MeiliConfig, taskUid: number, timeoutMs = 8000, ): Promise { const url = `${config.host}/tasks/${taskUid}`; const start = Date.now(); while (Date.now() - start < timeoutMs) { const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { return false; } const status = response.body?.status; if (status === 'succeeded') return true; if (status === 'failed' || status === 'canceled') { this.logger.warn(`Meilisearch task ${taskUid} failed: ${JSON.stringify(response.body?.error)}`); return false; } await new Promise((resolve) => setTimeout(resolve, 300)); } return false; } private async hasEmbedder( config: MeiliConfig, indexName: string, embedderName: string, ): Promise { const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`; const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config)); if (!this.isSuccessStatus(response.status)) { return false; } const embedders = response.body || {}; return Boolean(embedders && embedders[embedderName]); } }