Feat ssrc rewriting (#2192)

JitsiConference.js

@@ -53,7 +53,8 @@ import BridgeVideoType from './service/RTC/BridgeVideoType';
 import CodecMimeType from './service/RTC/CodecMimeType';
 import { MediaType } from './service/RTC/MediaType';
 import RTCEvents from './service/RTC/RTCEvents';
-import { getSourceNameForJitsiTrack } from './service/RTC/SignalingLayer';
+import { SignalingEvents } from './service/RTC/SignalingEvents';
+import { getMediaTypeFromSourceName, getSourceNameForJitsiTrack } from './service/RTC/SignalingLayer';
 import { VideoType } from './service/RTC/VideoType';
 import {
     ACTION_JINGLE_RESTART,
@@ -156,7 +157,12 @@ export default function JitsiConference(options) {
     this.eventEmitter = new EventEmitter();
     this.options = options;
     this.eventManager = new JitsiConferenceEventManager(this);
-    this.participants = {};
+
+    /**
+     * List of all the participants in the conference.
+     * @type {Map<string, JitsiParticipant>};
+     */
+    this.participants = new Map();
 
     /**
      * The signaling layer instance.
@@ -389,6 +395,17 @@ JitsiConference.prototype._init = function(options = {}) {
     );
 
     this._signalingLayer.setChatRoom(this.room);
+    this._signalingLayer.on(
+        SignalingEvents.SOURCE_UPDATED,
+        (sourceName, endpointId, muted, videoType) => {
+            const participant = this.participants.get(endpointId);
+            const mediaType = getMediaTypeFromSourceName(sourceName);
+
+            if (participant) {
+                participant._setSources(mediaType, muted, sourceName, videoType);
+                this.eventEmitter.emit(JitsiConferenceEvents.PARTICIPANT_SOURCE_UPDATED, participant);
+            }
+        });
 
     // Connection interrupted/restored listeners
     this._onIceConnectionInterrupted
@@ -1198,11 +1215,11 @@ JitsiConference.prototype._fireMuteChangeEvent = function(track) {
     if (this.mutedByFocusActor && track.isAudioTrack()) {
         const actorId = Strophe.getResourceFromJid(this.mutedByFocusActor);
 
-        actorParticipant = this.participants[actorId];
+        actorParticipant = this.participants.get(actorId);
     } else if (this.mutedVideoByFocusActor && track.isVideoTrack()) {
         const actorId = Strophe.getResourceFromJid(this.mutedVideoByFocusActor);
 
-        actorParticipant = this.participants[actorId];
+        actorParticipant = this.participants.get(actorId);
     }
 
     // Send the video type message to the bridge if the track is not removed/added to the pc as part of
@@ -1631,11 +1648,10 @@ JitsiConference.prototype.setLastN = function(lastN) {
 };
 
 /**
- * @return Array<JitsiParticipant> an array of all participants in this
- * conference.
+ * @return Array<JitsiParticipant> an array of all participants in this conference.
  */
 JitsiConference.prototype.getParticipants = function() {
-    return Object.values(this.participants);
+    return Array.from(this.participants.values());
 };
 
 /**
@@ -1661,7 +1677,7 @@ JitsiConference.prototype.getParticipantCount = function(countHidden = false) {
  * @param id the id of the participant.
  */
 JitsiConference.prototype.getParticipantById = function(id) {
-    return this.participants[id];
+    return this.participants.get(id);
 };
 
 /**
@@ -1809,7 +1825,7 @@ JitsiConference.prototype.onMemberJoined = function(
         participant._tracks.push(track);
     }
 
-    this.participants[id] = participant;
+    this.participants.set(id, participant);
     this.eventEmitter.emit(
         JitsiConferenceEvents.USER_JOINED,
         id,
@@ -1922,10 +1938,10 @@ JitsiConference.prototype.onMemberLeft = function(jid, reason) {
         });
     }
 
-    const participant = this.participants[id];
+    const participant = this.participants.get(id);
 
     if (participant) {
-        delete this.participants[id];
+        this.participants.delete(id);
         this.eventEmitter.emit(JitsiConferenceEvents.USER_LEFT, id, participant, reason);
     }
 
@@ -1962,7 +1978,7 @@ JitsiConference.prototype.onMemberKicked = function(
         return;
     }
 
-    const actorParticipant = this.participants[actorId];
+    const actorParticipant = this.participants.get(actorId);
 
     if (isSelfPresence) {
         this.eventEmitter.emit(
@@ -1973,7 +1989,7 @@ JitsiConference.prototype.onMemberKicked = function(
         return;
     }
 
-    const kickedParticipant = this.participants[kickedParticipantId];
+    const kickedParticipant = this.participants.get(kickedParticipantId);
 
     kickedParticipant.setIsReplaced(isReplaceParticipant);
 
@@ -2999,7 +3015,7 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(jingleSession, jingl
 
     let remoteID = Strophe.getResourceFromJid(this.p2pJingleSession.remoteJid);
 
-    const participant = this.participants[remoteID];
+    const participant = this.participants.get(remoteID);
 
     if (participant) {
         remoteID = participant.getStatsID() || remoteID;
@@ -3367,7 +3383,7 @@ JitsiConference.prototype._startP2PSession = function(remoteJid) {
 
     let remoteID = Strophe.getResourceFromJid(this.p2pJingleSession.remoteJid);
 
-    const participant = this.participants[remoteID];
+    const participant = this.participants.get(remoteID);
 
     if (participant) {
         remoteID = participant.getStatsID() || remoteID;

JitsiConferenceEventManager.js

@@ -574,15 +574,15 @@ JitsiConferenceEventManager.prototype.setupRTCListeners = function() {
     });
 
     rtc.addListener(RTCEvents.VIDEO_SSRCS_REMAPPED, msg => {
-        const sess = this.conference.getActiveMediaSession();
-
-        sess.videoSsrcsRemapped(msg);
+        for (const session of this.conference.getMediaSessions()) {
+            session.processSourceMap(msg, MediaType.VIDEO);
+        }
     });
 
     rtc.addListener(RTCEvents.AUDIO_SSRCS_REMAPPED, msg => {
-        const sess = this.conference.getActiveMediaSession();
-
-        sess.audioSsrcsRemapped(msg);
+        for (const session of this.conference.getMediaSessions()) {
+            session.processSourceMap(msg, MediaType.AUDIO);
+        }
     });
 
     rtc.addListener(RTCEvents.ENDPOINT_MESSAGE_RECEIVED,

JitsiConferenceEvents.spec.ts

@@ -31,6 +31,7 @@ describe( "/JitsiConferenceEvents members", () => {
         JVB121_STATUS,
         KICKED,
         PARTICIPANT_KICKED,
+        PARTICIPANT_SOURCE_UPDATED,
         LAST_N_ENDPOINTS_CHANGED,
         FORWARDED_SOURCES_CHANGED,
         LOCK_STATE_CHANGED,
@@ -108,6 +109,7 @@ describe( "/JitsiConferenceEvents members", () => {
         expect( JVB121_STATUS ).toBe( 'conference.jvb121Status' );
         expect( KICKED ).toBe( 'conference.kicked' );
         expect( PARTICIPANT_KICKED ).toBe( 'conference.participant_kicked' );
+        expect( PARTICIPANT_SOURCE_UPDATED ).toBe( 'conference.participant_source_updated' );
         expect( LAST_N_ENDPOINTS_CHANGED ).toBe( 'conference.lastNEndpointsChanged' );
         expect( FORWARDED_SOURCES_CHANGED ).toBe( 'conference.forwardedSourcesChanged' );
         expect( LOCK_STATE_CHANGED ).toBe( 'conference.lock_state_changed' );

JitsiConferenceEvents.ts

@@ -241,6 +241,11 @@ export enum JitsiConferenceEvents {
      */
     PARTICIPANT_PROPERTY_CHANGED = 'conference.participant_property_changed',
 
+    /**
+     * Indicates the state of sources attached to a given remote participant has changed.
+     */
+    PARTICIPANT_SOURCE_UPDATED = 'conference.participant_source_updated',
+
     /**
      * Indicates that the conference has switched between JVB and P2P connections.
      * The first argument of this event is a <tt>boolean</tt> which when set to
@@ -490,6 +495,7 @@ export const E2EE_VERIFICATION_COMPLETED = JitsiConferenceEvents.E2EE_VERIFICATI
 export const JVB121_STATUS = JitsiConferenceEvents.JVB121_STATUS;
 export const KICKED = JitsiConferenceEvents.KICKED;
 export const PARTICIPANT_KICKED = JitsiConferenceEvents.PARTICIPANT_KICKED;
+export const PARTICIPANT_SOURCE_UPDATED = JitsiConferenceEvents.PARTICIPANT_SOURCE_UPDATED;
 export const LAST_N_ENDPOINTS_CHANGED = JitsiConferenceEvents.LAST_N_ENDPOINTS_CHANGED;
 export const FORWARDED_SOURCES_CHANGED = JitsiConferenceEvents.FORWARDED_SOURCES_CHANGED;
 export const LOCK_STATE_CHANGED = JitsiConferenceEvents.LOCK_STATE_CHANGED;

JitsiParticipant.js

@@ -43,6 +43,19 @@ export default class JitsiParticipant {
         this._isReplacing = isReplacing;
         this._isReplaced = isReplaced;
         this._features = new Set();
+
+        /**
+         * Remote sources associated with the participant in the following format.
+         * Map<mediaType, Map<sourceName, sourceInfo>>
+         *
+         * mediaType - 'audio' or 'video'.
+         * sourceName - name of the remote source.
+         * sourceInfo: {
+         *   muted: boolean;
+         *   videoType: string;
+         * }
+         */
+        this._sources = new Map();
     }
 
     /**
@@ -61,6 +74,32 @@ export default class JitsiParticipant {
             true);
     }
 
+    /**
+     * Sets source info.
+     * @param {MediaType} mediaType The media type, 'audio' or 'video'.
+     * @param {boolean} muted The new muted state.
+     * @param {string} sourceName The name of the source.
+     * @param {string} videoType The video type of the source.
+     * @returns {void}
+     */
+    _setSources(mediaType, muted, sourceName, videoType) {
+        let sourceByMediaType = this._sources.get(mediaType);
+        const sourceInfo = {
+            muted,
+            videoType
+        };
+
+        if (sourceByMediaType?.size) {
+            sourceByMediaType.set(sourceName, sourceInfo);
+
+            return;
+        }
+
+        sourceByMediaType = new Map();
+        sourceByMediaType.set(sourceName, sourceInfo);
+        this._sources.set(mediaType, sourceByMediaType);
+    }
+
     /**
      * Returns the bot type for the participant.
      *
@@ -130,6 +169,14 @@ export default class JitsiParticipant {
         return this._role;
     }
 
+    /**
+     * Returns the sources associated with this participant.
+     * @returns Map<string, Map<string, Object>>
+     */
+    getSources() {
+        return this._sources;
+    }
+
     /**
      * @returns {String} The stats ID of this participant.
      */

modules/xmpp/JingleSessionPC.js

@@ -6,6 +6,7 @@ import { JitsiTrackEvents } from '../../JitsiTrackEvents';
 import * as CodecMimeType from '../../service/RTC/CodecMimeType';
 import { MediaDirection } from '../../service/RTC/MediaDirection';
 import { MediaType } from '../../service/RTC/MediaType';
+import { VideoType } from '../../service/RTC/VideoType';
 import {
     ICE_DURATION,
     ICE_STATE_CHANGED
@@ -1793,115 +1794,88 @@ export default class JingleSessionPC extends JingleSession {
     }
 
     /**
-     * Filter remapped SSRCs.
-     * Process owner change for existing SSRCs.
-     * Return new ones for further processing.
+     * Processes the source map message received from the bridge and creates a new remote track for newly signaled
+     * SSRCs or updates the source-name and owner on the remote track for an existing SSRC.
+     *
+     * @param {Object} message - The source map message.
+     * @param {string} mediaType - The media type, 'audio' or 'video'.
+     * @returns {void}
      */
-    getNewSources(msg) {
+    processSourceMap(message, mediaType) {
         const newSources = [];
 
-        for (const s of msg.mappedSources) {
-            if (this.peerconnection.addRemoteSsrc(s.ssrc)) {
-                logger.debug(`New SSRC ${s.ssrc}`);
-                newSources[newSources.length] = s;
+        for (const src of message.mappedSources) {
+            if (this.peerconnection.addRemoteSsrc(src.ssrc)) {
+                newSources.push(src);
             } else {
-                const track = this.peerconnection.getTrackBySSRC(s.ssrc);
+                const { owner, source, ssrc, videoType } = src;
+                const track = this.peerconnection.getTrackBySSRC(ssrc);
 
                 if (track) {
-                    logger.debug(`Existing SSRC ${s.ssrc}: new owner ${s.owner}. name=${s.source}`);
+                    logger.debug(`Existing SSRC ${ssrc}: new owner=${owner}, source-name=${source}`);
 
                     // Update the SSRC owner.
-                    this._signalingLayer.setSSRCOwner(s.ssrc, s.owner);
+                    this._signalingLayer.setSSRCOwner(ssrc, owner);
 
-                    if (s.videoType === 'CAMERA') {
-                        track._setVideoType('camera');
-                    } else if (s.videoType === 'DESKTOP') {
-                        track._setVideoType('desktop');
-                    }
+                    // Update the track with all the relevant info.
+                    track.setSourceName(source);
+                    track.setOwner(owner);
+                    if (mediaType === MediaType.VIDEO) {
+                        const type = videoType === 'CAMERA' ? VideoType.CAMERA : VideoType.DESKTOP;
 
-                    track.setSourceName(s.source);
-                    track.setOwner(s.owner);
+                        track._setVideoType(type);
+                    }
                 } else {
-                    logger.error(`Remapped SSRC ${s.ssrc} not found`);
+                    logger.error(`Remote track attached to a remote SSRC=${ssrc} not found`);
                 }
             }
         }
 
-        return newSources;
-    }
-
-    /**
-     * Process SSRC remappings for video sources.
-     */
-    videoSsrcsRemapped(msg) {
-        const newSources = this.getNewSources(msg);
-
-        if (newSources.length > 0) {
-
+        // Add the new SSRCs to the remote description by generating a source message.
+        if (newSources.length) {
             let node = $build('content', {
                 xmlns: 'urn:xmpp:jingle:1',
-                name: 'video'
+                name: mediaType
             }).c('description', {
                 xmlns: 'urn:xmpp:jingle:apps:rtp:1',
-                media: MediaType.VIDEO
+                media: mediaType
             });
 
-            for (const s of newSources) {
-                const idx = ++this.numRemoteVideoSources;
-                const msid = `remote-video-${idx} remote-video-${idx}`;
+            for (const src of newSources) {
+                const { rtx, ssrc } = src;
+                let msid;
 
-                _addSourceElement(node, s, s.ssrc, msid);
-                if (s.rtx !== '-1') {
-                    _addSourceElement(node, s, s.rtx, msid);
-                    node.c('ssrc-group', {
-                        xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
-                        semantics: 'FID'
-                    })
-                        .c('source', {
-                            xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
-                            ssrc: s.ssrc
-                        })
-                        .up()
-                        .c('source', {
-                            xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
-                            ssrc: s.rtx
-                        })
-                        .up()
-                        .up();
-                }
-            }
+                if (mediaType === MediaType.VIDEO) {
+                    const idx = ++this.numRemoteVideoSources;
 
-            node = node.up();
+                    msid = `remote-video-${idx} remote-video-${idx}`;
 
-            this._addOrRemoveRemoteStream(true /* add */, node.node);
-        }
-    }
-
-    /**
-     * Process SSRC remappings for audio sources.
-     */
-    audioSsrcsRemapped(msg) {
-        const newSources = this.getNewSources(msg);
-
-        if (newSources.length > 0) {
-
-            let node = $build('content', {
-                xmlns: 'urn:xmpp:jingle:1',
-                name: 'audio'
-            }).c('description', {
-                xmlns: 'urn:xmpp:jingle:apps:rtp:1',
-                media: MediaType.AUDIO
-            });
-
-            for (const s of newSources) {
-                const idx = ++this.numRemoteAudioSources;
-                const msid = `remote-audio-${idx} remote-audio-${idx}`;
+                    if (rtx !== '-1') {
+                        _addSourceElement(node, src, rtx, msid);
+                        node.c('ssrc-group', {
+                            xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                            semantics: 'FID'
+                        })
+                            .c('source', {
+                                xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                                ssrc
+                            })
+                            .up()
+                            .c('source', {
+                                xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                                ssrc: rtx
+                            })
+                            .up()
+                            .up();
+                    }
+                } else {
+                    const idx = ++this.numRemoteAudioSources;
 
-                _addSourceElement(node, s, s.ssrc, msid);
+                    msid = `remote-audio-${idx} remote-audio-${idx}`;
+                }
+                _addSourceElement(node, src, ssrc, msid);
             }
-
             node = node.up();
-
             this._addOrRemoveRemoteStream(true /* add */, node.node);
         }
     }

modules/xmpp/SignalingLayerImpl.js

@@ -142,12 +142,14 @@ export default class SignalingLayerImpl extends SignalingLayer {
                 = this._remoteSourceState[endpointId] || (this._remoteSourceState[endpointId] = {});
 
             for (const sourceName of Object.keys(sourceInfoJSON)) {
+                let sourceChanged = false;
                 const mediaType = getMediaTypeFromSourceName(sourceName);
                 const newMutedState = Boolean(sourceInfoJSON[sourceName].muted);
                 const oldSourceState = endpointSourceState[sourceName]
                     || (endpointSourceState[sourceName] = { sourceName });
 
                 if (oldSourceState.muted !== newMutedState) {
+                    sourceChanged = true;
                     oldSourceState.muted = newMutedState;
                     if (emitEventsFromHere && !this._localSourceState[sourceName]) {
                         this.eventEmitter.emit(SignalingEvents.SOURCE_MUTED_CHANGED, sourceName, newMutedState);
@@ -161,6 +163,7 @@ export default class SignalingLayerImpl extends SignalingLayer {
 
                 if (oldSourceState.videoType !== newVideoType) {
                     oldSourceState.videoType = newVideoType;
+                    sourceChanged = true;
 
                     // Since having a mix of eps that do/don't support multi-stream in the same call is supported, emit
                     // SOURCE_VIDEO_TYPE_CHANGED event when the remote source changes videoType.
@@ -168,6 +171,15 @@ export default class SignalingLayerImpl extends SignalingLayer {
                         this.eventEmitter.emit(SignalingEvents.SOURCE_VIDEO_TYPE_CHANGED, sourceName, newVideoType);
                     }
                 }
+
+                if (sourceChanged && FeatureFlags.isSsrcRewritingSupported()) {
+                    this.eventEmitter.emit(
+                        SignalingEvents.SOURCE_UPDATED,
+                        sourceName,
+                        mucNick,
+                        newMutedState,
+                        newVideoType);
+                }
             }
 
             // Cleanup removed source names

modules/xmpp/SignalingLayerImpl.spec.js

@@ -179,7 +179,7 @@ describe('SignalingLayerImpl', () => {
                     true
                 );
             });
-            it('from a user with SourceInfo', () => {
+            it('from a user with SourceInfo and ssrc-rewriting disabled', () => {
                 const emitterSpy = spyOn(signalingLayer.eventEmitter, 'emit');
                 const sourceInfo = {
                     '12345678-a0': {
@@ -203,6 +203,33 @@ describe('SignalingLayerImpl', () => {
                     true
                 );
             });
+
+            it('from a user with sourceInfo and ssrc-rewriting enabled', () => {
+                FeatureFlags.init({ ssrcRewritingEnabled: true });
+                const emitterSpy = spyOn(signalingLayer.eventEmitter, 'emit');
+                const sourceInfo = {
+                    '12345678-a0': {
+                        muted: true
+                    }
+                };
+
+                chatRoom.mockSourceInfoPresence('endpoint1', sourceInfo);
+
+                // <audiomuted/> still included for backwards compat and ChatRoom will emit the presence event
+                chatRoom.emitPresenceListener({
+                    tagName: 'audiomuted',
+                    value: 'true'
+                }, 'endpoint1');
+
+                expect(emitterSpy).toHaveBeenCalledTimes(2);
+                expect(emitterSpy.calls.argsFor(1)).toEqual([
+                    SignalingEvents.SOURCE_UPDATED,
+                    '12345678-a0',
+                    'endpoint1',
+                    true,
+                    undefined
+                ]);
+            });
         });
     });
     describe('getPeerMediaInfo', () => {

service/RTC/SignalingEvents.spec.ts

@@ -7,6 +7,7 @@ describe( "/service/RTC/SignalingEvents members", () => {
         PEER_MUTED_CHANGED,
         PEER_VIDEO_TYPE_CHANGED,
         SOURCE_MUTED_CHANGED,
+        SOURCE_UPDATED,
         SOURCE_VIDEO_TYPE_CHANGED,
         SignalingEvents,
         ...others
@@ -16,6 +17,7 @@ describe( "/service/RTC/SignalingEvents members", () => {
         expect( PEER_MUTED_CHANGED ).toBe( 'signaling.peerMuted' );
         expect( PEER_VIDEO_TYPE_CHANGED ).toBe( 'signaling.peerVideoType' );
         expect( SOURCE_MUTED_CHANGED ).toBe( 'signaling.sourceMuted');
+        expect( SOURCE_UPDATED ).toBe( 'signaling.sourceUpdated' );
         expect( SOURCE_VIDEO_TYPE_CHANGED ).toBe( 'signaling.sourceVideoType');
 
         expect( SignalingEvents ).toBeDefined();

service/RTC/SignalingEvents.ts

@@ -24,6 +24,16 @@ export enum SignalingEvents {
      */
     SOURCE_MUTED_CHANGED = 'signaling.sourceMuted',
 
+    /**
+     * Event triggered when presence for a source is received.
+     *
+     * @param {string} sourceName - The name of the source.
+     * @param {string} endpointId - The endpoint id.
+     * @param {boolean} muted - The new muted state.
+     * @param {string} videoType - The video type of the source.
+     */
+    SOURCE_UPDATED = 'signaling.sourceUpdated',
+
     /**
      * Event triggered when source's video type changes.
      *
@@ -37,4 +47,5 @@ export enum SignalingEvents {
 export const PEER_MUTED_CHANGED = SignalingEvents.PEER_MUTED_CHANGED;
 export const PEER_VIDEO_TYPE_CHANGED = SignalingEvents.PEER_VIDEO_TYPE_CHANGED;
 export const SOURCE_MUTED_CHANGED = SignalingEvents.SOURCE_MUTED_CHANGED;
+export const SOURCE_UPDATED = SignalingEvents.SOURCE_UPDATED;
 export const SOURCE_VIDEO_TYPE_CHANGED = SignalingEvents.SOURCE_VIDEO_TYPE_CHANGED;