484 lines
16 KiB
TypeScript
484 lines
16 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
|
import * as http from 'http';
|
|
import * as https from 'https';
|
|
|
|
type MeiliConfig = {
|
|
host: string;
|
|
apiKey?: string;
|
|
indexPrefix: string;
|
|
};
|
|
|
|
type HybridSearchOptions = {
|
|
embedder: string;
|
|
semanticRatio?: number;
|
|
};
|
|
|
|
type OpenAiEmbedderConfig = {
|
|
embedderName: string;
|
|
apiKey: string;
|
|
model: string;
|
|
documentTemplate: string;
|
|
};
|
|
|
|
@Injectable()
|
|
export class MeilisearchService {
|
|
private readonly logger = new Logger(MeilisearchService.name);
|
|
private readonly embedderCache = new Map<string, string>();
|
|
private vectorStoreEnabled = false;
|
|
|
|
isEnabled(): boolean {
|
|
return Boolean(this.getConfig());
|
|
}
|
|
|
|
async searchRecord(
|
|
tenantId: string,
|
|
objectApiName: string,
|
|
query: string,
|
|
displayField?: string,
|
|
): Promise<{ id: string; hit: any } | null> {
|
|
const config = this.getConfig();
|
|
if (!config) return null;
|
|
|
|
const indexName = this.buildIndexName(config, tenantId, objectApiName);
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`;
|
|
|
|
console.log('querying Meilisearch index:', { indexName, query, displayField });
|
|
|
|
try {
|
|
const response = await this.requestJson('POST', url, {
|
|
q: query,
|
|
limit: 5,
|
|
}, this.buildHeaders(config));
|
|
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch query failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const hits = Array.isArray(response.body?.hits) ? response.body.hits : [];
|
|
if (hits.length === 0) return null;
|
|
|
|
if (displayField) {
|
|
const loweredQuery = query.toLowerCase();
|
|
const exactMatch = hits.find((hit: any) => {
|
|
const value = hit?.[displayField];
|
|
return value && String(value).toLowerCase() === loweredQuery;
|
|
});
|
|
if (exactMatch?.id) {
|
|
return { id: exactMatch.id, hit: exactMatch };
|
|
}
|
|
}
|
|
|
|
const match = hits[0];
|
|
if (match?.id) {
|
|
return { id: match.id, hit: match };
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch lookup failed: ${error.message}`);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async searchRecords(
|
|
tenantId: string,
|
|
objectApiName: string,
|
|
query: string,
|
|
options?: { limit?: number; offset?: number },
|
|
): Promise<{ hits: any[]; total: number }> {
|
|
const config = this.getConfig();
|
|
if (!config) return { hits: [], total: 0 };
|
|
|
|
const indexName = this.buildIndexName(config, tenantId, objectApiName);
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`;
|
|
const limit = Number.isFinite(Number(options?.limit)) ? Number(options?.limit) : 20;
|
|
const offset = Number.isFinite(Number(options?.offset)) ? Number(options?.offset) : 0;
|
|
|
|
try {
|
|
const response = await this.requestJson('POST', url, {
|
|
q: query,
|
|
limit,
|
|
offset,
|
|
}, this.buildHeaders(config));
|
|
|
|
console.log('Meilisearch response body:', response.body);
|
|
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch query failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
return { hits: [], total: 0 };
|
|
}
|
|
|
|
const hits = Array.isArray(response.body?.hits) ? response.body.hits : [];
|
|
const total =
|
|
response.body?.estimatedTotalHits ??
|
|
response.body?.nbHits ??
|
|
hits.length;
|
|
return { hits, total };
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch query failed: ${error.message}`);
|
|
return { hits: [], total: 0 };
|
|
}
|
|
}
|
|
|
|
async upsertRecord(
|
|
tenantId: string,
|
|
objectApiName: string,
|
|
record: Record<string, any>,
|
|
fieldsToIndex: string[],
|
|
): Promise<void> {
|
|
const config = this.getConfig();
|
|
if (!config || !record?.id) return;
|
|
|
|
const indexName = this.buildIndexName(config, tenantId, objectApiName);
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents?primaryKey=id`;
|
|
const document = this.pickRecordFields(record, fieldsToIndex);
|
|
|
|
try {
|
|
const response = await this.requestJson('POST', url, [document], this.buildHeaders(config));
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch upsert failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch upsert failed: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
async deleteRecord(
|
|
tenantId: string,
|
|
objectApiName: string,
|
|
recordId: string,
|
|
): Promise<void> {
|
|
const config = this.getConfig();
|
|
if (!config || !recordId) return;
|
|
|
|
const indexName = this.buildIndexName(config, tenantId, objectApiName);
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents/${encodeURIComponent(recordId)}`;
|
|
|
|
try {
|
|
const response = await this.requestJson('DELETE', url, undefined, this.buildHeaders(config));
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch delete failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch delete failed: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
buildSemanticChunkIndexName(tenantId: string): string {
|
|
const config = this.getConfig();
|
|
const prefix = config?.indexPrefix || 'tenant_';
|
|
return `${prefix}${tenantId}_semantic_chunks`.toLowerCase();
|
|
}
|
|
|
|
async upsertDocuments(indexName: string, documents: Record<string, any>[]): Promise<void> {
|
|
const config = this.getConfig();
|
|
if (!config || !Array.isArray(documents) || documents.length === 0) return;
|
|
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/documents?primaryKey=id`;
|
|
try {
|
|
const response = await this.requestJson('POST', url, documents, this.buildHeaders(config));
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(`Meilisearch document upsert failed for index ${indexName}: ${response.status}`);
|
|
return;
|
|
}
|
|
// Meilisearch indexes (and embeds) documents asynchronously. Wait for the task
|
|
// to complete so callers can immediately search and see the new documents.
|
|
const taskUid = response.body?.taskUid ?? response.body?.uid;
|
|
if (Number.isFinite(Number(taskUid))) {
|
|
const succeeded = await this.waitForTask(config, Number(taskUid), 30000);
|
|
if (!succeeded) {
|
|
this.logger.warn(`Meilisearch indexing task did not succeed within timeout: taskUid=${taskUid} index=${indexName}`);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch document upsert failed: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
async searchIndex(
|
|
indexName: string,
|
|
query: string,
|
|
limit = 20,
|
|
hybrid?: HybridSearchOptions,
|
|
): Promise<{ hits: any[]; total: number }> {
|
|
const config = this.getConfig();
|
|
if (!config) return { hits: [], total: 0 };
|
|
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/search`;
|
|
try {
|
|
const response = await this.requestJson(
|
|
'POST',
|
|
url,
|
|
{
|
|
q: query,
|
|
limit,
|
|
showRankingScore: true,
|
|
...(hybrid ? { hybrid, showRankingScoreDetails: true } : {}),
|
|
},
|
|
this.buildHeaders(config),
|
|
);
|
|
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch search failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
this.logger.warn(
|
|
`Meilisearch search payload: ${JSON.stringify({ q: query, limit, hybrid })}`,
|
|
);
|
|
this.logger.warn(
|
|
`Meilisearch search error body: ${JSON.stringify(response.body)}`,
|
|
);
|
|
// If hybrid is invalid (embedder missing), retry once without hybrid
|
|
if (hybrid && response.body?.code === 'invalid_embedder') {
|
|
const fallback = await this.requestJson(
|
|
'POST',
|
|
url,
|
|
{ q: query, limit },
|
|
this.buildHeaders(config),
|
|
);
|
|
if (this.isSuccessStatus(fallback.status)) {
|
|
const hits = Array.isArray(fallback.body?.hits) ? fallback.body.hits : [];
|
|
const total =
|
|
fallback.body?.estimatedTotalHits ?? fallback.body?.nbHits ?? hits.length;
|
|
this.logger.warn(
|
|
`Meilisearch hybrid failed; fell back to lexical search for index ${indexName}.`,
|
|
);
|
|
return { hits, total };
|
|
}
|
|
}
|
|
return { hits: [], total: 0 };
|
|
}
|
|
|
|
const hits = Array.isArray(response.body?.hits) ? response.body.hits : [];
|
|
const total = response.body?.estimatedTotalHits ?? response.body?.nbHits ?? hits.length;
|
|
return { hits, total };
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch search failed: ${error.message}`);
|
|
return { hits: [], total: 0 };
|
|
}
|
|
}
|
|
|
|
private getConfig(): MeiliConfig | null {
|
|
const host = process.env.MEILI_HOST || process.env.MEILISEARCH_HOST;
|
|
if (!host) return null;
|
|
const trimmedHost = host.replace(/\/+$/, '');
|
|
const apiKey = process.env.MEILI_API_KEY || process.env.MEILISEARCH_API_KEY;
|
|
const indexPrefix = process.env.MEILI_INDEX_PREFIX || 'tenant_';
|
|
return { host: trimmedHost, apiKey, indexPrefix };
|
|
}
|
|
|
|
private buildIndexName(config: MeiliConfig, tenantId: string, objectApiName: string): string {
|
|
return `${config.indexPrefix}${tenantId}_${objectApiName}`.toLowerCase();
|
|
}
|
|
|
|
private buildHeaders(config: MeiliConfig): Record<string, string> {
|
|
const headers: Record<string, string> = {
|
|
'Content-Type': 'application/json',
|
|
Accept: 'application/json',
|
|
};
|
|
if (config.apiKey) {
|
|
headers['X-Meili-API-Key'] = config.apiKey;
|
|
headers.Authorization = `Bearer ${config.apiKey}`;
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
private pickRecordFields(record: Record<string, any>, fields: string[]): Record<string, any> {
|
|
const document: Record<string, any> = { id: record.id };
|
|
for (const field of fields) {
|
|
if (record[field] !== undefined) {
|
|
document[field] = record[field];
|
|
}
|
|
}
|
|
return document;
|
|
}
|
|
|
|
private isSuccessStatus(status: number): boolean {
|
|
return status >= 200 && status < 300;
|
|
}
|
|
|
|
private requestJson(
|
|
method: 'POST' | 'DELETE' | 'PATCH' | 'GET',
|
|
url: string,
|
|
payload: any,
|
|
headers: Record<string, string>,
|
|
): Promise<{ status: number; body: any }> {
|
|
return new Promise((resolve, reject) => {
|
|
const parsedUrl = new URL(url);
|
|
const client = parsedUrl.protocol === 'https:' ? https : http;
|
|
const request = client.request(
|
|
{
|
|
method,
|
|
hostname: parsedUrl.hostname,
|
|
port: parsedUrl.port,
|
|
path: `${parsedUrl.pathname}${parsedUrl.search}`,
|
|
headers,
|
|
},
|
|
(response) => {
|
|
let data = '';
|
|
response.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
response.on('end', () => {
|
|
if (!data) {
|
|
resolve({ status: response.statusCode || 0, body: null });
|
|
return;
|
|
}
|
|
try {
|
|
const body = JSON.parse(data);
|
|
resolve({ status: response.statusCode || 0, body });
|
|
} catch (error) {
|
|
reject(error);
|
|
}
|
|
});
|
|
},
|
|
);
|
|
|
|
request.on('error', reject);
|
|
if (payload !== undefined && method !== 'GET') {
|
|
request.write(JSON.stringify(payload));
|
|
}
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
private async enableVectorStore(): Promise<void> {
|
|
// Temporarily disabled to avoid the overhead of checking on every save.
|
|
// Re-enable by removing the early return below.
|
|
return;
|
|
if (this.vectorStoreEnabled) return; // eslint-disable-line no-unreachable
|
|
const meiliConfig = this.getConfig();
|
|
if (!meiliConfig) return;
|
|
const url = `${meiliConfig.host}/experimental-features`;
|
|
try {
|
|
const response = await this.requestJson(
|
|
'PATCH',
|
|
url,
|
|
{ vectorStore: true },
|
|
this.buildHeaders(meiliConfig),
|
|
);
|
|
if (this.isSuccessStatus(response.status)) {
|
|
this.vectorStoreEnabled = true;
|
|
this.logger.log('Meilisearch vector store experimental feature enabled');
|
|
} else {
|
|
this.logger.warn(
|
|
`Failed to enable Meilisearch vector store: ${response.status} ${JSON.stringify(response.body)}`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`Failed to enable Meilisearch vector store: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
async ensureOpenAiEmbedder(
|
|
indexName: string,
|
|
config: OpenAiEmbedderConfig,
|
|
): Promise<boolean> {
|
|
const meiliConfig = this.getConfig();
|
|
if (!meiliConfig || !config?.apiKey) return false;
|
|
|
|
await this.enableVectorStore();
|
|
|
|
const signature = JSON.stringify({
|
|
embedderName: config.embedderName,
|
|
model: config.model,
|
|
documentTemplate: config.documentTemplate,
|
|
apiKey: config.apiKey,
|
|
});
|
|
const cacheKey = `${indexName}:${config.embedderName}`;
|
|
if (this.embedderCache.get(cacheKey) === signature) {
|
|
return true;
|
|
}
|
|
|
|
const url = `${meiliConfig.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`;
|
|
try {
|
|
const response = await this.requestJson(
|
|
'PATCH',
|
|
url,
|
|
{
|
|
[config.embedderName]: {
|
|
source: 'openAi',
|
|
model: config.model,
|
|
apiKey: config.apiKey,
|
|
documentTemplate: config.documentTemplate,
|
|
},
|
|
},
|
|
this.buildHeaders(meiliConfig),
|
|
);
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
this.logger.warn(
|
|
`Meilisearch embedder update failed for index ${indexName}: ${response.status}`,
|
|
);
|
|
this.logger.warn(
|
|
`Meilisearch embedder error body: ${JSON.stringify(response.body)}`,
|
|
);
|
|
return false;
|
|
}
|
|
const taskUid = response.body?.taskUid ?? response.body?.uid;
|
|
if (Number.isFinite(Number(taskUid))) {
|
|
const succeeded = await this.waitForTask(meiliConfig, Number(taskUid), 8000);
|
|
if (!succeeded) {
|
|
this.logger.warn(`Meilisearch embedder task did not succeed: ${taskUid}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
const hasEmbedder = await this.hasEmbedder(meiliConfig, indexName, config.embedderName);
|
|
if (!hasEmbedder) {
|
|
this.logger.warn(`Meilisearch embedder missing after update: ${config.embedderName}`);
|
|
return false;
|
|
}
|
|
this.embedderCache.set(cacheKey, signature);
|
|
return true;
|
|
} catch (error) {
|
|
this.logger.warn(`Meilisearch embedder update failed: ${error.message}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async waitForTask(
|
|
config: MeiliConfig,
|
|
taskUid: number,
|
|
timeoutMs = 8000,
|
|
): Promise<boolean> {
|
|
const url = `${config.host}/tasks/${taskUid}`;
|
|
const start = Date.now();
|
|
while (Date.now() - start < timeoutMs) {
|
|
const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config));
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
return false;
|
|
}
|
|
const status = response.body?.status;
|
|
if (status === 'succeeded') return true;
|
|
if (status === 'failed' || status === 'canceled') {
|
|
this.logger.warn(`Meilisearch task ${taskUid} failed: ${JSON.stringify(response.body?.error)}`);
|
|
return false;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private async hasEmbedder(
|
|
config: MeiliConfig,
|
|
indexName: string,
|
|
embedderName: string,
|
|
): Promise<boolean> {
|
|
const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`;
|
|
const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config));
|
|
if (!this.isSuccessStatus(response.status)) {
|
|
return false;
|
|
}
|
|
const embedders = response.body || {};
|
|
return Boolean(embedders && embedders[embedderName]);
|
|
}
|
|
}
|