WIP - some progress making phone calls from softphone, we need https to test

backend/src/tenant/tenant.controller.ts

@@ -62,16 +62,27 @@ export class TenantController {
     const centralPrisma = getCentralPrisma();
     const domainRecord = await centralPrisma.domain.findUnique({
       where: { domain },
-      include: { tenant: { select: { id: true } } },
+      include: { tenant: { select: { id: true, integrationsConfig: true } } },
     });
 
     if (!domainRecord?.tenant) {
       throw new Error(`Tenant with domain ${domain} not found`);
     }
 
+    // Merge with existing config to preserve masked values
+    let finalConfig = integrationsConfig;
+    if (domainRecord.tenant.integrationsConfig) {
+      const existingConfig = this.tenantDbService.decryptIntegrationsConfig(
+        domainRecord.tenant.integrationsConfig as any,
+      );
+
+      // Replace masked values with actual values from existing config
+      finalConfig = this.unmaskConfig(integrationsConfig, existingConfig);
+    }
+
     // Encrypt the config
     const encryptedConfig = this.tenantDbService.encryptIntegrationsConfig(
-      integrationsConfig,
+      finalConfig,
     );
 
     // Update in database
@@ -88,6 +99,32 @@ export class TenantController {
     };
   }
 
+  /**
+   * Unmask config by replacing masked values with actual values from existing config
+   */
+  private unmaskConfig(newConfig: any, existingConfig: any): any {
+    const result = { ...newConfig };
+
+    // Unmask Twilio credentials
+    if (result.twilio && existingConfig.twilio) {
+      if (result.twilio.authToken === '••••••••' && existingConfig.twilio.authToken) {
+        result.twilio.authToken = existingConfig.twilio.authToken;
+      }
+      if (result.twilio.apiSecret === '••••••••' && existingConfig.twilio.apiSecret) {
+        result.twilio.apiSecret = existingConfig.twilio.apiSecret;
+      }
+    }
+
+    // Unmask OpenAI credentials
+    if (result.openai && existingConfig.openai) {
+      if (result.openai.apiKey === '••••••••' && existingConfig.openai.apiKey) {
+        result.openai.apiKey = existingConfig.openai.apiKey;
+      }
+    }
+
+    return result;
+  }
+
   /**
    * Mask sensitive fields for API responses
    */
@@ -101,6 +138,7 @@ export class TenantController {
       masked.twilio = {
         ...masked.twilio,
         authToken: masked.twilio.authToken ? '••••••••' : '',
+        apiSecret: masked.twilio.apiSecret ? '••••••••' : '',
       };
     }
 

backend/src/voice/interfaces/integration-config.interface.ts

@@ -2,8 +2,9 @@ export interface TwilioConfig {
   accountSid: string;
   authToken: string;
   phoneNumber: string;
-  apiKeySid?: string;
+  apiKey?: string;  // API Key SID for generating access tokens
-  apiKeySecret?: string;
+  apiSecret?: string;  // API Key Secret
+  twimlAppSid?: string;  // TwiML App SID for Voice SDK
 }
 
 export interface OpenAIConfig {

backend/src/voice/voice.controller.ts

@@ -49,6 +49,25 @@ export class VoiceController {
     };
   }
 
+  /**
+   * Generate Twilio access token for browser client
+   */
+  @Get('token')
+  @UseGuards(JwtAuthGuard)
+  async getAccessToken(
+    @Req() req: any,
+    @TenantId() tenantId: string,
+  ) {
+    const userId = req.user?.userId || req.user?.sub;
+    
+    const token = await this.voiceService.generateAccessToken(tenantId, userId);
+
+    return {
+      success: true,
+      data: { token },
+    };
+  }
+
   /**
    * Get call history
    */
@@ -73,19 +92,23 @@ export class VoiceController {
   }
 
   /**
-   * TwiML for outbound calls
+   * TwiML for outbound calls from browser (Twilio Device)
    */
   @Post('twiml/outbound')
   async outboundTwiml(@Req() req: FastifyRequest, @Res() res: FastifyReply) {
+    const body = req.body as any;
+    const to = body.To;
+
+    this.logger.log(`Outbound call to: ${to}`);
+
+    // TwiML to dial the number and setup media stream for OpenAI
     const twiml = `<?xml version="1.0" encoding="UTF-8"?>
 <Response>
   <Start>
     <Stream url="wss://${req.headers.host}/api/voice/stream" />
   </Start>
   <Say>Connecting your call</Say>
-  <Dial>
+  <Dial>${to}</Dial>
-    <Number>${(req.body as any).To}</Number>
-  </Dial>
 </Response>`;
 
     res.type('text/xml').send(twiml);

backend/src/voice/voice.service.ts

@@ -6,6 +6,9 @@ import * as Twilio from 'twilio';
 import { WebSocket } from 'ws';
 import { v4 as uuidv4 } from 'uuid';
 
+const AccessToken = Twilio.jwt.AccessToken;
+const VoiceGrant = AccessToken.VoiceGrant;
+
 @Injectable()
 export class VoiceService {
   private readonly logger = new Logger(VoiceService.name);
@@ -94,6 +97,35 @@ export class VoiceService {
     return {};
   }
 
+  /**
+   * Generate Twilio access token for browser Voice SDK
+   */
+  async generateAccessToken(tenantDomain: string, userId: string): Promise<string> {
+    const { config, tenantId } = await this.getTwilioClient(tenantDomain);
+
+    if (!config.accountSid || !config.apiKey || !config.apiSecret) {
+      throw new Error('Twilio API credentials not configured. Please add API Key and Secret in Settings > Integrations');
+    }
+
+    // Create an access token
+    const token = new AccessToken(
+      config.accountSid,
+      config.apiKey,
+      config.apiSecret,
+      { identity: userId, ttl: 3600 } // 1 hour expiry
+    );
+
+    // Create a Voice grant
+    const voiceGrant = new VoiceGrant({
+      outgoingApplicationSid: config.twimlAppSid, // TwiML App SID for outbound calls
+      incomingAllow: true, // Allow incoming calls
+    });
+
+    token.addGrant(voiceGrant);
+
+    return token.toJwt();
+  }
+
   /**
    * Initiate outbound call
    */

docs/TWILIO_SETUP.md

@@ -0,0 +1,65 @@
+# Twilio Setup Guide for Softphone
+
+## Prerequisites
+- Twilio account with a phone number
+- Account SID and Auth Token
+
+## Basic Setup (Current - Makes calls but no browser audio)
+
+Currently, the softphone initiates calls through Twilio's REST API, but the audio doesn't flow through the browser. The calls go directly to your mobile device with a simple TwiML message.
+
+## Full Browser Audio Setup (Requires additional configuration)
+
+To enable actual softphone functionality where audio flows through your browser's microphone and speakers, you need:
+
+### Option 1: Twilio Client SDK (Recommended)
+
+1. **Create a TwiML App in Twilio Console**
+   - Go to https://console.twilio.com/us1/develop/voice/manage/twiml-apps
+   - Click "Create new TwiML App"
+   - Name it (e.g., "RouteBox Softphone")
+   - Set Voice URL to: `https://yourdomain.com/api/voice/twiml/outbound`
+   - Set Voice Method to: `POST`
+   - Save and copy the TwiML App SID
+
+2. **Create an API Key**
+   - Go to https://console.twilio.com/us1/account/keys-credentials/api-keys
+   - Click "Create API key"
+   - Give it a friendly name
+   - Copy both the SID and Secret (you won't be able to see the secret again)
+
+3. **Add credentials to Settings > Integrations**
+   - Account SID (from main dashboard)
+   - Auth Token (from main dashboard)
+   - Phone Number (your Twilio number)
+   - API Key SID (from step 2)
+   - API Secret (from step 2)
+   - TwiML App SID (from step 1)
+
+### Option 2: Twilio Media Streams (Alternative - More complex)
+
+Uses WebSocket to stream audio bidirectionally:
+- Requires WebSocket server setup
+- More control over audio processing
+- Can integrate with OpenAI Realtime API more easily
+
+## Current Status
+
+The system works but audio doesn't flow through browser because:
+1. Calls are made via REST API only
+2. No Twilio Client SDK integration yet
+3. TwiML returns simple voice message
+
+To enable browser audio, you need to:
+1. Complete the Twilio setup above
+2. Implement the frontend Twilio Device connection
+3. Modify TwiML to dial the browser client instead of just the phone number
+
+## Quick Test (Current Setup)
+
+1. Save your Account SID, Auth Token, and Phone Number in Settings > Integrations
+2. Click the phone icon in sidebar
+3. Enter a phone number and click "Call"
+4. You should receive a call that says "This is a test call from your softphone"
+
+The call works, but audio doesn't route through your browser - it's just a regular phone call initiated by the API.

frontend/composables/useSoftphone.ts

@@ -1,5 +1,6 @@
-import { ref, computed, onMounted, onUnmounted } from 'vue';
+import { ref, computed, onMounted, onUnmounted, shallowRef } from 'vue';
 import { io, Socket } from 'socket.io-client';
+import { Device, Call as TwilioCall } from '@twilio/voice-sdk';
 import { useAuth } from './useAuth';
 import { toast } from 'vue-sonner';
 
@@ -27,6 +28,8 @@ interface AiSuggestion {
 
 // Module-level shared state for global access
 const socket = ref<Socket | null>(null);
+const twilioDevice = shallowRef<Device | null>(null);
+const twilioCall = shallowRef<TwilioCall | null>(null);
 const isConnected = ref(false);
 const isOpen = ref(false);
 const currentCall = ref<Call | null>(null);
@@ -35,6 +38,8 @@ const transcript = ref<CallTranscript[]>([]);
 const aiSuggestions = ref<AiSuggestion[]>([]);
 const callHistory = ref<Call[]>([]);
 const isInitialized = ref(false);
+const isMuted = ref(false);
+const volume = ref(100);
 
 export function useSoftphone() {
   const auth = useAuth();
@@ -55,6 +60,136 @@ export function useSoftphone() {
   const hasIncomingCall = computed(() => incomingCall.value !== null);
   const callStatus = computed(() => currentCall.value?.status || 'idle');
 
+  /**
+   * Request microphone permission explicitly
+   */
+  const requestMicrophonePermission = async () => {
+    try {
+      // Check if mediaDevices is supported
+      if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        toast.error('Microphone access requires HTTPS. Please access the app via https:// or use localhost for testing.');
+        console.error('navigator.mediaDevices not available. This typically means the page is not served over HTTPS.');
+        return false;
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      // Stop the stream immediately, we just wanted the permission
+      stream.getTracks().forEach(track => track.stop());
+      return true;
+    } catch (error: any) {
+      console.error('Microphone permission denied:', error);
+      if (error.name === 'NotAllowedError') {
+        toast.error('Microphone access denied. Please allow microphone access in your browser settings.');
+      } else if (error.name === 'NotFoundError') {
+        toast.error('No microphone found. Please connect a microphone and try again.');
+      } else {
+        toast.error('Microphone access is required for calls. Please ensure you are using HTTPS or localhost.');
+      }
+      return false;
+    }
+  };
+
+  /**
+   * Initialize Twilio Device
+   */
+  const initializeTwilioDevice = async () => {
+    try {
+      // First, explicitly request microphone permission
+      const hasPermission = await requestMicrophonePermission();
+      if (!hasPermission) {
+        return;
+      }
+
+      const { api } = useApi();
+      const response = await api.get('/voice/token');
+      const token = response.data.token;
+
+      twilioDevice.value = new Device(token, {
+        logLevel: 1,
+        codecPreferences: ['opus', 'pcmu'],
+        enableImprovedSignalingErrorPrecision: true,
+        // Specify audio constraints
+        edge: 'ashburn',
+      });
+
+      // Device events
+      twilioDevice.value.on('registered', () => {
+        console.log('Twilio Device registered');
+        toast.success('Softphone ready');
+      });
+
+      twilioDevice.value.on('error', (error) => {
+        console.error('Twilio Device error:', error);
+        toast.error('Device error: ' + error.message);
+      });
+
+      twilioDevice.value.on('incoming', (call: TwilioCall) => {
+        console.log('Incoming call:', call.parameters);
+        twilioCall.value = call;
+        
+        // Update state
+        incomingCall.value = {
+          callSid: call.parameters.CallSid || '',
+          direction: 'inbound',
+          fromNumber: call.parameters.From || '',
+          toNumber: call.parameters.To || '',
+          status: 'ringing',
+        };
+
+        // Setup call handlers
+        setupCallHandlers(call);
+      });
+
+      // Register the device
+      await twilioDevice.value.register();
+      
+    } catch (error: any) {
+      console.error('Failed to initialize Twilio Device:', error);
+      toast.error('Failed to initialize voice device: ' + error.message);
+    }
+  };
+
+  /**
+   * Setup handlers for a Twilio call
+   */
+  const setupCallHandlers = (call: TwilioCall) => {
+    call.on('accept', () => {
+      console.log('Call accepted');
+      currentCall.value = {
+        callSid: call.parameters.CallSid || '',
+        direction: twilioCall.value === call ? 'inbound' : 'outbound',
+        fromNumber: call.parameters.From || '',
+        toNumber: call.parameters.To || '',
+        status: 'in-progress',
+        startedAt: new Date().toISOString(),
+      };
+      incomingCall.value = null;
+    });
+
+    call.on('disconnect', () => {
+      console.log('Call disconnected');
+      currentCall.value = null;
+      twilioCall.value = null;
+    });
+
+    call.on('cancel', () => {
+      console.log('Call cancelled');
+      incomingCall.value = null;
+      twilioCall.value = null;
+    });
+
+    call.on('reject', () => {
+      console.log('Call rejected');
+      incomingCall.value = null;
+      twilioCall.value = null;
+    });
+
+    call.on('error', (error) => {
+      console.error('Call error:', error);
+      toast.error('Call error: ' + error.message);
+    });
+  };
+
   /**
    * Initialize WebSocket connection
    */
@@ -91,6 +226,9 @@ export function useSoftphone() {
     socket.value.on('connect', () => {
       console.log('Softphone WebSocket connected');
       isConnected.value = true;
+      
+      // Initialize Twilio Device after WebSocket connects
+      initializeTwilioDevice();
     });
 
     socket.value.on('disconnect', () => {
@@ -151,102 +289,110 @@ export function useSoftphone() {
   };
 
   /**
-   * Initiate outbound call
+   * Initiate outbound call using Twilio Device
    */
   const initiateCall = async (toNumber: string) => {
-    if (!socket.value?.connected) {
+    if (!twilioDevice.value) {
-      toast.error('Not connected to voice service');
+      toast.error('Voice device not initialized');
       return;
     }
 
-    return new Promise((resolve, reject) => {
+    try {
-      socket.value!.emit('call:initiate', { toNumber }, (response: any) => {
+      // Make call using Twilio Device
-        if (response.success) {
+      const call = await twilioDevice.value.connect({
-          resolve(response);
+        params: {
-        } else {
+          To: toNumber,
-          reject(new Error(response.error));
         }
       });
-    });
+
+      twilioCall.value = call;
+      setupCallHandlers(call);
+
+      toast.success('Calling ' + toNumber);
+    } catch (error: any) {
+      console.error('Failed to initiate call:', error);
+      toast.error('Failed to initiate call: ' + error.message);
+      throw error;
+    }
   };
 
   /**
    * Accept incoming call
    */
   const acceptCall = async (callSid: string) => {
-    if (!socket.value?.connected) {
+    if (!twilioCall.value) {
-      toast.error('Not connected to voice service');
+      toast.error('No incoming call');
       return;
     }
 
-    return new Promise((resolve, reject) => {
+    try {
-      socket.value!.emit('call:accept', { callSid }, (response: any) => {
+      await twilioCall.value.accept();
-        if (response.success) {
+      toast.success('Call accepted');
-          resolve(response);
+    } catch (error: any) {
-        } else {
+      console.error('Failed to accept call:', error);
-          reject(new Error(response.error));
+      toast.error('Failed to accept call: ' + error.message);
-        }
+    }
-      });
-    });
   };
 
   /**
    * Reject incoming call
    */
   const rejectCall = async (callSid: string) => {
-    if (!socket.value?.connected) {
+    if (!twilioCall.value) {
-      toast.error('Not connected to voice service');
+      toast.error('No incoming call');
       return;
     }
 
-    return new Promise((resolve, reject) => {
+    try {
-      socket.value!.emit('call:reject', { callSid }, (response: any) => {
+      twilioCall.value.reject();
-        if (response.success) {
+      incomingCall.value = null;
-          resolve(response);
+      twilioCall.value = null;
-        } else {
+      toast.info('Call rejected');
-          reject(new Error(response.error));
+    } catch (error: any) {
-        }
+      console.error('Failed to reject call:', error);
-      });
+      toast.error('Failed to reject call: ' + error.message);
-    });
+    }
   };
 
   /**
    * End active call
    */
   const endCall = async (callSid: string) => {
-    if (!socket.value?.connected) {
+    if (!twilioCall.value) {
-      toast.error('Not connected to voice service');
+      toast.error('No active call');
       return;
     }
 
-    return new Promise((resolve, reject) => {
+    try {
-      socket.value!.emit('call:end', { callSid }, (response: any) => {
+      twilioCall.value.disconnect();
-        if (response.success) {
+      currentCall.value = null;
-          resolve(response);
+      twilioCall.value = null;
-        } else {
+      toast.info('Call ended');
-          reject(new Error(response.error));
+    } catch (error: any) {
-        }
+      console.error('Failed to end call:', error);
-      });
+      toast.error('Failed to end call: ' + error.message);
-    });
+    }
+  };
+
+  /**
+   * Toggle mute
+   */
+  const toggleMute = () => {
+    if (!twilioCall.value) return;
+    
+    isMuted.value = !isMuted.value;
+    twilioCall.value.mute(isMuted.value);
   };
 
   /**
    * Send DTMF tone
    */
   const sendDtmf = async (callSid: string, digit: string) => {
-    if (!socket.value?.connected) {
+    if (!twilioCall.value) {
       return;
     }
 
-    return new Promise((resolve, reject) => {
+    twilioCall.value.sendDigits(digit);
-      socket.value!.emit('call:dtmf', { callSid, digit }, (response: any) => {
-        if (response.success) {
-          resolve(response);
-        } else {
-          reject(new Error(response.error));
-        }
-      });
-    });
   };
 
   // Event handlers
@@ -402,20 +548,23 @@ export function useSoftphone() {
     hasIncomingCall,
     currentCall,
     incomingCall,
-    callStatus,
     transcript,
     aiSuggestions,
+    callStatus,
     callHistory,
+    isMuted,
+    volume,
 
-    // Methods
+    // Actions
     open,
     close,
-    connect,
-    disconnect,
     initiateCall,
     acceptCall,
     rejectCall,
     endCall,
     sendDtmf,
+    toggleMute,
+    connect,
+    disconnect,
   };
 }

frontend/package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@internationalized/date": "^3.10.1",
     "@nuxtjs/tailwindcss": "^6.11.4",
+    "@twilio/voice-sdk": "^2.11.2",
     "@vueuse/core": "^10.11.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",

frontend/pages/settings/integrations.vue

@@ -53,6 +53,31 @@
                 placeholder="+1234567890"
               />
             </div>
+            <div class="space-y-2">
+              <Label for="twilio-api-key">API Key SID (for browser calls)</Label>
+              <Input
+                id="twilio-api-key"
+                v-model="twilioConfig.apiKey"
+                placeholder="SKxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+              />
+            </div>
+            <div class="space-y-2">
+              <Label for="twilio-api-secret">API Secret</Label>
+              <Input
+                id="twilio-api-secret"
+                v-model="twilioConfig.apiSecret"
+                type="password"
+                placeholder="Enter your API Key Secret"
+              />
+            </div>
+            <div class="space-y-2">
+              <Label for="twilio-twiml-app">TwiML App SID</Label>
+              <Input
+                id="twilio-twiml-app"
+                v-model="twilioConfig.twimlAppSid"
+                placeholder="APxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+              />
+            </div>
           </CardContent>
         </Card>
 
@@ -123,6 +148,9 @@ const twilioConfig = ref({
   accountSid: '',
   authToken: '',
   phoneNumber: '',
+  apiKey: '',
+  apiSecret: '',
+  twimlAppSid: '',
 });
 
 const openaiConfig = ref({