WIP - AI Assistant working

backend/src/main.ts

@@ -109,7 +109,8 @@ async function bootstrap() {
 
           case 'media':
             mediaPacketCount++;
-            if (mediaPacketCount % 50 === 0) {
+            // Only log every 500 packets to reduce noise
+            if (mediaPacketCount % 500 === 0) {
               logger.log(`Received media packet #${mediaPacketCount} for StreamSid: ${streamSid}`);
             }
 

backend/src/voice/voice.gateway.ts

@@ -281,8 +281,13 @@ export class VoiceGateway
    */
   async notifyAiSuggestion(userId: string, data: any) {
     const socket = this.connectedUsers.get(userId);
+    this.logger.log(`notifyAiSuggestion - userId: ${userId}, socket connected: ${!!socket}, total connected users: ${this.connectedUsers.size}`);
     if (socket) {
+      this.logger.log(`Emitting ai:suggestion event with data:`, JSON.stringify(data));
       socket.emit('ai:suggestion', data);
+    } else {
+      this.logger.warn(`No socket connection found for userId: ${userId}`);
+      this.logger.log(`Connected users: ${Array.from(this.connectedUsers.keys()).join(', ')}`);
     }
   }
 

backend/src/voice/voice.service.ts

@@ -483,13 +483,36 @@ export class VoiceService {
         // Add to connections map only after it's open
         this.openaiConnections.set(callSid, ws);
 
+        // Store call state with userId for later use
+        this.callStates.set(callSid, {
+          callSid,
+          tenantId: tenant.id,
+          userId,
+          status: 'in-progress',
+        });
+        this.logger.log(`📝 Stored call state for ${callSid} with userId: ${userId}`);
+
         // Initialize session
         ws.send(JSON.stringify({
           type: 'session.update',
           session: {
             model: config.openai.model || 'gpt-4o-realtime-preview',
             voice: config.openai.voice || 'alloy',
-            instructions: 'You are a helpful AI assistant providing real-time support during phone calls. Provide concise, actionable suggestions to help the user.',
+            instructions: `You are an AI assistant in LISTENING MODE, helping a sales/support agent during their phone call.
+
+IMPORTANT: You are NOT talking to the caller. You are advising the agent who is handling the call.
+
+Your role:
+- Listen to the conversation between the agent and the caller
+- Provide concise, actionable suggestions to help the agent
+- Recommend CRM actions (search contacts, create tasks, update records)
+- Alert the agent to important information or next steps
+- Keep suggestions brief (1-2 sentences max)
+
+Format your suggestions like:
+"💡 Suggestion: [your advice]"
+"⚠️ Alert: [important notice]"
+"📋 Action: [recommended CRM action]"`,
             turn_detection: {
               type: 'server_vad',
             },
@@ -587,25 +610,15 @@ export class VoiceService {
     message: any,
   ) {
     try {
-      // Log all message types for debugging
-      this.logger.debug(`OpenAI message type: ${message.type} for call ${callSid}`);
-
       switch (message.type) {
         case 'conversation.item.created':
-          if (message.item.type === 'message' && message.item.role === 'assistant') {
-            // AI response generated
-            this.logger.log(`AI response for call ${callSid}: ${JSON.stringify(message.item.content)}`);
-          }
+          // Skip logging for now
           break;
 
         case 'response.audio.delta':
-          // OpenAI is sending audio response
-          // This needs to be sent to Twilio Media Stream
-          // Note: We'll need to get the streamSid from the call state
+          // OpenAI is sending audio response (skip logging)
           const state = this.callStates.get(callSid);
           if (state?.streamSid && message.delta) {
-            // The controller will handle sending to Twilio
-            // Store audio delta for controller to pick up
             if (!state.pendingAudio) {
               state.pendingAudio = [];
             }
@@ -614,31 +627,50 @@ export class VoiceService {
           break;
 
         case 'response.audio.done':
-          // Audio response complete
-          this.logger.log(`OpenAI audio response complete for call ${callSid}`);
+          // Skip logging
           break;
 
         case 'response.audio_transcript.delta':
-          // Real-time transcript chunk
-          const deltaState = this.callStates.get(callSid);
-          if (deltaState?.userId && message.delta) {
-            this.logger.log(`📝 Transcript chunk: "${message.delta}"`);
-            // Emit to frontend via gateway
-            if (this.voiceGateway) {
-              await this.voiceGateway.notifyAiTranscript(deltaState.userId, {
-                callSid,
-                transcript: message.delta,
-                isFinal: false,
-              });
-            }
-          }
+          // Skip - not transmitting individual words to frontend
           break;
 
         case 'response.audio_transcript.done':
-          // Final transcript
+          // Final transcript - this contains the AI's actual text suggestions!
           const transcript = message.transcript;
-          this.logger.log(`✅ Final transcript for call ${callSid}: "${transcript}"`);
+          this.logger.log(`💡 AI Suggestion: "${transcript}"`);
+          
+          // Save to database
           await this.updateCallTranscript(callSid, tenantId, transcript);
+          
+          // Also send as suggestion to frontend if it looks like a suggestion
+          if (transcript && transcript.length > 0) {
+            // Determine suggestion type
+            let suggestionType: 'response' | 'action' | 'insight' = 'insight';
+            if (transcript.includes('💡') || transcript.toLowerCase().includes('suggest')) {
+              suggestionType = 'response';
+            } else if (transcript.includes('📋') || transcript.toLowerCase().includes('action')) {
+              suggestionType = 'action';
+            } else if (transcript.includes('⚠️') || transcript.toLowerCase().includes('alert')) {
+              suggestionType = 'insight';
+            }
+            
+            // Emit to frontend
+            const state = this.callStates.get(callSid);
+            this.logger.log(`📊 Call state - userId: ${state?.userId}, gateway: ${!!this.voiceGateway}`);
+            
+            if (state?.userId && this.voiceGateway) {
+              this.logger.log(`📤 Sending to user ${state.userId}`);
+              await this.voiceGateway.notifyAiSuggestion(state.userId, {
+                type: suggestionType,
+                text: transcript,
+                callSid,
+                timestamp: new Date().toISOString(),
+              });
+              this.logger.log(`✅ Suggestion sent to agent`);
+            } else {
+              this.logger.warn(`❌ Cannot send - userId: ${state?.userId}, gateway: ${!!this.voiceGateway}, callStates has ${this.callStates.size} entries`);
+            }
+          }
           break;
 
         case 'response.function_call_arguments.done':
@@ -647,11 +679,17 @@ export class VoiceService {
           break;
 
         case 'session.created':
-          this.logger.log(`OpenAI session created for call ${callSid}`);
-          break;
-
         case 'session.updated':
-          this.logger.log(`OpenAI session updated for call ${callSid}`);
+        case 'response.created':
+        case 'response.output_item.added':
+        case 'response.content_part.added':
+        case 'response.content_part.done':
+        case 'response.output_item.done':
+        case 'response.done':
+        case 'input_audio_buffer.speech_started':
+        case 'input_audio_buffer.speech_stopped':
+        case 'input_audio_buffer.committed':
+          // Skip logging for these (too noisy)
           break;
 
         case 'error':
@@ -659,8 +697,7 @@ export class VoiceService {
           break;
 
         default:
-          // Log other message types for debugging
-          this.logger.debug(`Unhandled OpenAI message type: ${message.type}`);
+          // Only log unhandled types occasionally
           break;
       }
     } catch (error) {