WIP - some progress with semantic linking but still needs a lot of work

backend/src/search/meilisearch.service.ts

@@ -24,6 +24,7 @@ type OpenAiEmbedderConfig = {
 export class MeilisearchService {
   private readonly logger = new Logger(MeilisearchService.name);
   private readonly embedderCache = new Map<string, string>();
+  private vectorStoreEnabled = false;
 
   isEnabled(): boolean {
     return Boolean(this.getConfig());
@@ -186,6 +187,16 @@ export class MeilisearchService {
       const response = await this.requestJson('POST', url, documents, this.buildHeaders(config));
       if (!this.isSuccessStatus(response.status)) {
         this.logger.warn(`Meilisearch document upsert failed for index ${indexName}: ${response.status}`);
+        return;
+      }
+      // Meilisearch indexes (and embeds) documents asynchronously. Wait for the task
+      // to complete so callers can immediately search and see the new documents.
+      const taskUid = response.body?.taskUid ?? response.body?.uid;
+      if (Number.isFinite(Number(taskUid))) {
+        const succeeded = await this.waitForTask(config, Number(taskUid), 30000);
+        if (!succeeded) {
+          this.logger.warn(`Meilisearch indexing task did not succeed within timeout: taskUid=${taskUid} index=${indexName}`);
+        }
       }
     } catch (error) {
       this.logger.warn(`Meilisearch document upsert failed: ${error.message}`);
@@ -215,7 +226,33 @@ export class MeilisearchService {
       );
 
       if (!this.isSuccessStatus(response.status)) {
-        this.logger.warn(`Meilisearch search failed for index ${indexName}: ${response.status}`);
+        this.logger.warn(
+          `Meilisearch search failed for index ${indexName}: ${response.status}`,
+        );
+        this.logger.warn(
+          `Meilisearch search payload: ${JSON.stringify({ q: query, limit, hybrid })}`,
+        );
+        this.logger.warn(
+          `Meilisearch search error body: ${JSON.stringify(response.body)}`,
+        );
+        // If hybrid is invalid (embedder missing), retry once without hybrid
+        if (hybrid && response.body?.code === 'invalid_embedder') {
+          const fallback = await this.requestJson(
+            'POST',
+            url,
+            { q: query, limit },
+            this.buildHeaders(config),
+          );
+          if (this.isSuccessStatus(fallback.status)) {
+            const hits = Array.isArray(fallback.body?.hits) ? fallback.body.hits : [];
+            const total =
+              fallback.body?.estimatedTotalHits ?? fallback.body?.nbHits ?? hits.length;
+            this.logger.warn(
+              `Meilisearch hybrid failed; fell back to lexical search for index ${indexName}.`,
+            );
+            return { hits, total };
+          }
+        }
         return { hits: [], total: 0 };
       }
 
@@ -268,7 +305,7 @@ export class MeilisearchService {
   }
 
   private requestJson(
-    method: 'POST' | 'DELETE' | 'PATCH',
+    method: 'POST' | 'DELETE' | 'PATCH' | 'GET',
     url: string,
     payload: any,
     headers: Record<string, string>,
@@ -305,19 +342,49 @@ export class MeilisearchService {
       );
 
       request.on('error', reject);
-      if (payload !== undefined) {
+      if (payload !== undefined && method !== 'GET') {
         request.write(JSON.stringify(payload));
       }
       request.end();
     });
   }
 
+  private async enableVectorStore(): Promise<void> {
+    // Temporarily disabled to avoid the overhead of checking on every save.
+    // Re-enable by removing the early return below.
+    return;
+    if (this.vectorStoreEnabled) return; // eslint-disable-line no-unreachable
+    const meiliConfig = this.getConfig();
+    if (!meiliConfig) return;
+    const url = `${meiliConfig.host}/experimental-features`;
+    try {
+      const response = await this.requestJson(
+        'PATCH',
+        url,
+        { vectorStore: true },
+        this.buildHeaders(meiliConfig),
+      );
+      if (this.isSuccessStatus(response.status)) {
+        this.vectorStoreEnabled = true;
+        this.logger.log('Meilisearch vector store experimental feature enabled');
+      } else {
+        this.logger.warn(
+          `Failed to enable Meilisearch vector store: ${response.status} ${JSON.stringify(response.body)}`,
+        );
+      }
+    } catch (error) {
+      this.logger.warn(`Failed to enable Meilisearch vector store: ${error.message}`);
+    }
+  }
+
   async ensureOpenAiEmbedder(
     indexName: string,
     config: OpenAiEmbedderConfig,
-  ): Promise<void> {
+  ): Promise<boolean> {
     const meiliConfig = this.getConfig();
-    if (!meiliConfig || !config?.apiKey) return;
+    if (!meiliConfig || !config?.apiKey) return false;
+
+    await this.enableVectorStore();
 
     const signature = JSON.stringify({
       embedderName: config.embedderName,
@@ -327,7 +394,7 @@ export class MeilisearchService {
     });
     const cacheKey = `${indexName}:${config.embedderName}`;
     if (this.embedderCache.get(cacheKey) === signature) {
-      return;
+      return true;
     }
 
     const url = `${meiliConfig.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`;
@@ -349,11 +416,67 @@ export class MeilisearchService {
         this.logger.warn(
           `Meilisearch embedder update failed for index ${indexName}: ${response.status}`,
         );
-        return;
+        this.logger.warn(
+          `Meilisearch embedder error body: ${JSON.stringify(response.body)}`,
+        );
+        return false;
+      }
+      const taskUid = response.body?.taskUid ?? response.body?.uid;
+      if (Number.isFinite(Number(taskUid))) {
+        const succeeded = await this.waitForTask(meiliConfig, Number(taskUid), 8000);
+        if (!succeeded) {
+          this.logger.warn(`Meilisearch embedder task did not succeed: ${taskUid}`);
+          return false;
+        }
+      }
+
+      const hasEmbedder = await this.hasEmbedder(meiliConfig, indexName, config.embedderName);
+      if (!hasEmbedder) {
+        this.logger.warn(`Meilisearch embedder missing after update: ${config.embedderName}`);
+        return false;
       }
       this.embedderCache.set(cacheKey, signature);
+      return true;
     } catch (error) {
       this.logger.warn(`Meilisearch embedder update failed: ${error.message}`);
+      return false;
     }
   }
+
+  private async waitForTask(
+    config: MeiliConfig,
+    taskUid: number,
+    timeoutMs = 8000,
+  ): Promise<boolean> {
+    const url = `${config.host}/tasks/${taskUid}`;
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config));
+      if (!this.isSuccessStatus(response.status)) {
+        return false;
+      }
+      const status = response.body?.status;
+      if (status === 'succeeded') return true;
+      if (status === 'failed' || status === 'canceled') {
+        this.logger.warn(`Meilisearch task ${taskUid} failed: ${JSON.stringify(response.body?.error)}`);
+        return false;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 300));
+    }
+    return false;
+  }
+
+  private async hasEmbedder(
+    config: MeiliConfig,
+    indexName: string,
+    embedderName: string,
+  ): Promise<boolean> {
+    const url = `${config.host}/indexes/${encodeURIComponent(indexName)}/settings/embedders`;
+    const response = await this.requestJson('GET', url, undefined, this.buildHeaders(config));
+    if (!this.isSuccessStatus(response.status)) {
+      return false;
+    }
+    const embedders = response.body || {};
+    return Boolean(embedders && embedders[embedderName]);
+  }
 }