Fix desktop viewer signaling auth and host live audio publish

Author: ralyodio

apps/desktop/src/renderer/hooks/useWebRTCHostAPI.ts

@@ -45,6 +45,7 @@ const DEFAULT_ICE_SERVERS: RTCIceServer[] = [
 export interface ViewerConnection {
   id: string;
   peerConnection: RTCPeerConnection;
+  hostAudioSender: RTCRtpSender | null;
   dataChannel: RTCDataChannel | null;
   connectionState: ConnectionState;
   controlState: 'view-only' | 'requested' | 'granted';
@@ -136,6 +137,36 @@ export function useWebRTCHostAPI({
   onInputReceivedRef.current = onInputReceived;
   onCursorUpdateRef.current = onCursorUpdate;
 
+  const getPreferredHostAudioTrack = useCallback(
+    (streamOverride?: MediaStream | null): MediaStreamTrack | null => {
+      const stream = streamOverride ?? localStreamRef.current;
+      const streamAudioTrack = stream?.getAudioTracks()[0] ?? null;
+      if (streamAudioTrack) return streamAudioTrack;
+      return hostMicStreamRef.current?.getAudioTracks()[0] ?? null;
+    },
+    []
+  );
+
+  const syncHostAudioSender = useCallback(
+    async (viewer: ViewerConnection, preferredTrack: MediaStreamTrack | null) => {
+      const pc = viewer.peerConnection;
+
+      if (preferredTrack) {
+        if (viewer.hostAudioSender) {
+          if (viewer.hostAudioSender.track?.id !== preferredTrack.id) {
+            await viewer.hostAudioSender.replaceTrack(preferredTrack);
+          }
+        } else {
+          viewer.hostAudioSender = pc.addTrack(preferredTrack, new MediaStream([preferredTrack]));
+        }
+      } else if (viewer.hostAudioSender) {
+        pc.removeTrack(viewer.hostAudioSender);
+        viewer.hostAudioSender = null;
+      }
+    },
+    []
+  );
+
   // Send signal via API
   const sendSignal = useCallback(
     async (signal: SignalMessage) => {
@@ -365,7 +396,7 @@ export function useWebRTCHostAPI({
 
   // Create peer connection for a viewer
   const createPeerConnection = useCallback(
-    (viewerId: string): RTCPeerConnection => {
+    (viewerId: string): { pc: RTCPeerConnection; hostAudioSender: RTCRtpSender | null } => {
       console.log('[WebRTCHost] Creating peer connection for viewer:', viewerId);
 
       const pc = new RTCPeerConnection({
@@ -406,11 +437,13 @@ export function useWebRTCHostAPI({
       }
 
       // Add host mic track so the viewer can hear the host
-      const hostMic = hostMicStreamRef.current;
-      if (hostMic) {
-        hostMic.getAudioTracks().forEach((track) => {
-          pc.addTrack(track, hostMic);
-        });
+      let hostAudioSender: RTCRtpSender | null = null;
+      const preferredHostAudioTrack = getPreferredHostAudioTrack(currentStream);
+      if (preferredHostAudioTrack) {
+        hostAudioSender = pc.addTrack(
+          preferredHostAudioTrack,
+          new MediaStream([preferredHostAudioTrack])
+        );
       }
 
       // Handle incoming tracks from viewer (their mic audio)
@@ -502,9 +535,15 @@ export function useWebRTCHostAPI({
         handleDataChannelMessage(viewerId, event);
       };
 
-      return pc;
+      return { pc, hostAudioSender };
     },
-    [hostId, handleDataChannelMessage, sendSignal, relayAudioToOtherViewers]
+    [
+      getPreferredHostAudioTrack,
+      hostId,
+      handleDataChannelMessage,
+      sendSignal,
+      relayAudioToOtherViewers,
+    ]
   );
 
   // Remove a viewer
@@ -538,11 +577,12 @@ export function useWebRTCHostAPI({
 
       console.log('[WebRTCHost] Viewer joining:', viewerId);
 
-      const pc = createPeerConnection(viewerId);
+      const { pc, hostAudioSender } = createPeerConnection(viewerId);
 
       const viewer: ViewerConnection = {
         id: viewerId,
         peerConnection: pc,
+        hostAudioSender,
         dataChannel: null,
         connectionState: 'connecting',
         controlState: 'view-only',
@@ -813,6 +853,7 @@ export function useWebRTCHostAPI({
     async (stream: MediaStream) => {
       localStreamRef.current = stream;
       const videoTracks = stream.getVideoTracks();
+      const preferredHostAudioTrack = getPreferredHostAudioTrack(stream);
       console.log('[WebRTCHost] Publishing stream:', {
         videoTracks: videoTracks.length,
         audioTracks: stream.getAudioTracks().length,
@@ -848,6 +889,7 @@ export function useWebRTCHostAPI({
 
         try {
           const pc = viewer.peerConnection;
+          await syncHostAudioSender(viewer, preferredHostAudioTrack);
           const existingVideoSenders = pc
             .getSenders()
             .filter((sender) => sender.track?.kind === 'video');
@@ -886,7 +928,7 @@ export function useWebRTCHostAPI({
         }
       }
     },
-    [hostId, sendSignal]
+    [getPreferredHostAudioTrack, hostId, sendSignal, syncHostAudioSender]
   );
 
   // Unpublish the screen share stream (remove video tracks) without closing connections
@@ -898,14 +940,16 @@ export function useWebRTCHostAPI({
         continue;
 
       try {
-        // Remove video senders (keep audio -- mic stays)
+        // Remove video senders (keep audio path active)
         const senders = viewer.peerConnection.getSenders();
         for (const sender of senders) {
           if (sender.track?.kind === 'video') {
             viewer.peerConnection.removeTrack(sender);
           }
         }
 
+        await syncHostAudioSender(viewer, getPreferredHostAudioTrack(null));
+
         // Renegotiate so viewer sees track removal
         const offer = await viewer.peerConnection.createOffer();
         await viewer.peerConnection.setLocalDescription(offer);
@@ -923,7 +967,7 @@ export function useWebRTCHostAPI({
         console.error(`[WebRTCHost] Failed to unpublish stream from ${viewer.id}:`, err);
       }
     }
-  }, [hostId, sendSignal]);
+  }, [getPreferredHostAudioTrack, hostId, sendSignal, syncHostAudioSender]);
 
   // Cleanup on unmount
   useEffect(() => {

apps/desktop/src/renderer/hooks/useWebRTCViewerAPI.ts

@@ -111,6 +111,9 @@ export function useWebRTCViewerAPI({
   const pendingCandidatesRef = useRef<RTCIceCandidateInit[]>([]);
   // Serialize signaling message processing to prevent race conditions
   const signalQueueRef = useRef<Promise<void>>(Promise.resolve());
+  // Use the server-assigned subscriber ID for signaling POSTs (desktop token auth can differ
+  // from the locally generated participantId used to open the SSE stream).
+  const signalSenderIdRef = useRef(participantId);
 
   // Callback refs to avoid circular dependencies
   const handleConnectionFailureRef = useRef<(() => Promise<void>) | undefined>(undefined);
@@ -123,6 +126,8 @@ export function useWebRTCViewerAPI({
   onCursorUpdateRef.current = onCursorUpdate;
   onKickedRef.current = onKicked;
 
+  const getSignalSenderId = useCallback(() => signalSenderIdRef.current, []);
+
   // Calculate network quality from metrics
   const calculateNetworkQuality = useCallback((metrics: QualityMetrics): NetworkQuality => {
     const { packetLoss, roundTripTime } = metrics;
@@ -468,7 +473,7 @@ export function useWebRTCViewerAPI({
               await sendSignal({
                 type: 'answer',
                 sdp: answer.sdp,
-                senderId: participantId,
+                senderId: getSignalSenderId(),
                 targetId: message.senderId,
                 timestamp: Date.now(),
               });
@@ -505,7 +510,7 @@ export function useWebRTCViewerAPI({
         setError('Failed to process signaling message');
       }
     },
-    [participantId, sendSignal]
+    [getSignalSenderId, sendSignal]
   );
 
   // Handle signaling messages from SSE — serialized via promise chain
@@ -536,7 +541,7 @@ export function useWebRTCViewerAPI({
             await sendSignal({
               type: 'offer',
               sdp: offer.sdp,
-              senderId: participantId,
+              senderId: getSignalSenderId(),
               timestamp: Date.now(),
             });
           }
@@ -549,7 +554,7 @@ export function useWebRTCViewerAPI({
       setConnectionState('failed');
       setError('Connection failed after multiple attempts');
     }
-  }, [participantId, sendSignal]);
+  }, [getSignalSenderId, sendSignal]);
 
   handleConnectionFailureRef.current = handleConnectionFailure;
 
@@ -644,7 +649,7 @@ export function useWebRTCViewerAPI({
         void sendSignal({
           type: 'ice-candidate',
           candidate: event.candidate.toJSON(),
-          senderId: participantId,
+          senderId: getSignalSenderId(),
           timestamp: Date.now(),
         });
       }
@@ -693,7 +698,7 @@ export function useWebRTCViewerAPI({
     };
 
     return pc;
-  }, [participantId, onStreamReady, onStreamEnded, sendSignal, setupDataChannel]);
+  }, [getSignalSenderId, onStreamReady, onStreamEnded, sendSignal, setupDataChannel]);
 
   // Disconnect and clean up
   const disconnect = useCallback(() => {
@@ -814,8 +819,20 @@ export function useWebRTCViewerAPI({
       // Use ICE servers from the server (includes TURN) if provided
       try {
         const data = JSON.parse(event.data as string) as {
+          subscriberId?: string;
           iceServers?: RTCIceServer[];
         };
+        if (data.subscriberId) {
+          signalSenderIdRef.current = data.subscriberId;
+          if (data.subscriberId !== participantId) {
+            console.log(
+              '[WebRTCViewer] Using server-assigned signaling senderId:',
+              data.subscriberId
+            );
+          }
+        } else {
+          signalSenderIdRef.current = participantId;
+        }
         if (data.iceServers && data.iceServers.length > 0) {
           iceServersRef.current = data.iceServers;
           console.log('[WebRTCViewer] Received ICE servers from server:', data.iceServers.length);