feat: use getSynchronizationSources on the receiver for remote audio levels (#1245)

* feat: use getSynchronizationSources on the receiver for remote audio levels
Use getSynchronizationSources if it is supported, fallback to using getStats otherwise.
* feat/ref: Use the local audio levels from LocalStatsCollector
When using getSynchronizationSources, use the audio levels from LocalStatsCollector for NoAudioSignalDetection.js
Remove obsolete code - TalkMutedDetection feature using audio levels is not used anymore

JitsiConference.js

@@ -15,7 +15,6 @@ import * as JitsiTrackEvents from './JitsiTrackEvents';
 import authenticateAndUpgradeRole from './authenticateAndUpgradeRole';
 import P2PDominantSpeakerDetection from './modules/detection/P2PDominantSpeakerDetection';
 import RTC from './modules/RTC/RTC';
-import TalkMutedDetection from './modules/detection/TalkMutedDetection';
 import VADTalkMutedDetection from './modules/detection/VADTalkMutedDetection';
 import VADNoiseDetection from './modules/detection/VADNoiseDetection';
 import VADAudioAnalyser from './modules/detection/VADAudioAnalyser';
@@ -410,12 +409,8 @@ JitsiConference.prototype._init = function(options = {}) {
                 this.eventEmitter.emit(JitsiConferenceEvents.TALK_WHILE_MUTED));
 
             this._audioAnalyser.addVADDetectionService(vadTalkMutedDetection);
-
-
         } else {
-            logger.info('Using audio level based detection for generating talk while muted events');
-            this._talkWhileMutedDetection = new TalkMutedDetection(
-                this, () => this.eventEmitter.emit(JitsiConferenceEvents.TALK_WHILE_MUTED));
+            logger.warn('No VAD Processor was provided. Talk while muted detection service was not initialized!');
         }
     }
 

JitsiTrackEvents.js

@@ -37,3 +37,9 @@ export const TRACK_VIDEOTYPE_CHANGED = 'track.videoTypeChanged';
  * to receive data (i.e. the stream is not stopped).
  */
 export const NO_DATA_FROM_SOURCE = 'track.no_data_from_source';
+
+/**
+ * Indicates that the local audio track is not receiving any audio input from
+ * the microphone that is currently selected.
+ */
+export const NO_AUDIO_INPUT = 'track.no_audio_input';

modules/RTC/JitsiTrack.js

@@ -428,6 +428,16 @@ export default class JitsiTrack extends EventEmitter {
                 JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED,
                 audioLevel,
                 tpc);
+
+        // LocalStatsCollector reports a value of 0.008 for muted mics
+        // and a value of 0 when there is no audio input.
+        } else if (this.audioLevel === 0
+            && audioLevel === 0
+            && this.isLocal()
+            && !this.isWebRTCTrackMuted()) {
+            this.emit(
+                JitsiTrackEvents.NO_AUDIO_INPUT,
+                audioLevel);
         }
     }
 

modules/RTC/TraceablePeerConnection.js

@@ -487,6 +487,33 @@ TraceablePeerConnection.prototype._peerMutedChanged = function(
     }
 };
 
+/**
+ * Obtains audio levels of the remote audio tracks by getting the source
+ * information on the RTCRtpReceivers. The information relevant to the ssrc
+ * is updated each time a RTP packet constaining the ssrc is received.
+ * @returns {Object} containing ssrc and audio level information as a
+ * key-value pair.
+ */
+TraceablePeerConnection.prototype.getAudioLevels = function() {
+    const audioLevels = {};
+    const audioReceivers = this.peerconnection.getReceivers()
+        .filter(receiver => receiver.track && receiver.track.kind === MediaType.AUDIO);
+
+    audioReceivers.forEach(remote => {
+        const ssrc = remote.getSynchronizationSources();
+
+        if (ssrc && ssrc.length) {
+            // As per spec, this audiolevel is a value between 0..1 (linear), where 1.0
+            // represents 0 dBov, 0 represents silence, and 0.5 represents approximately
+            // 6 dBSPL change in the sound pressure level from 0 dBov.
+            // https://www.w3.org/TR/webrtc/#dom-rtcrtpcontributingsource-audiolevel
+            audioLevels[ssrc[0].source] = ssrc[0].audioLevel;
+        }
+    });
+
+    return audioLevels;
+};
+
 /**
  * Obtains local tracks for given {@link MediaType}. If the <tt>mediaType</tt>
  * argument is omitted the list of all local tracks will be returned.

modules/browser/BrowserCapabilities.js

@@ -122,6 +122,14 @@ export default class BrowserCapabilities extends BrowserDetection {
         return this.isChromiumBased() || this.isReactNative() || this.isSafari();
     }
 
+    /**
+     * Checks if the current browser supports audio level stats on the receivers.
+     */
+    supportsReceiverStats() {
+        return typeof window.RTCRtpReceiver !== 'undefined'
+            && Object.keys(RTCRtpReceiver.prototype).indexOf('getSynchronizationSources') > -1;
+    }
+
     /**
      * Checks if the current browser reports round trip time statistics for
      * the ICE candidate pair.

modules/detection/NoAudioSignalDetection.js

@@ -1,7 +1,8 @@
 import EventEmitter from 'events';
 
 import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
-
+import * as JitsiTrackEvents from '../../JitsiTrackEvents';
+import browser from '../browser';
 import * as DetectionEvents from './DetectionEvents';
 
 // We wait a certain time interval for constant silence input from the current device to account for
@@ -30,7 +31,9 @@ export default class NoAudioSignalDetection extends EventEmitter {
         this._timeoutTrigger = null;
         this._hasAudioInput = null;
 
-        conference.statistics.addAudioLevelListener(this._audioLevel.bind(this));
+        if (!browser.supportsReceiverStats()) {
+            conference.statistics.addAudioLevelListener(this._audioLevel.bind(this));
+        }
         conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this));
     }
 
@@ -58,7 +61,6 @@ export default class NoAudioSignalDetection extends EventEmitter {
         // the event.
         if (this._hasAudioInput === null || this._hasAudioInput !== status) {
             this._hasAudioInput = status;
-
             this.emit(DetectionEvents.AUDIO_INPUT_STATE_CHANGE, this._hasAudioInput);
         }
     }
@@ -103,7 +105,6 @@ export default class NoAudioSignalDetection extends EventEmitter {
         // Get currently active local tracks from the TraceablePeerConnection
         const localSSRCs = tpc.localSSRCs.get(this._audioTrack.rtcId);
 
-
         // Only target the current active track in the tpc. For some reason audio levels for previous
         // devices are also picked up from the PeerConnection so we filter them out.
         if (!localSSRCs || !localSSRCs.ssrcs.includes(ssrc)) {
@@ -114,7 +115,6 @@ export default class NoAudioSignalDetection extends EventEmitter {
         // can try to fire again.
         this._handleAudioInputStateChange(audioLevel);
         this._handleNoAudioInputDetection(audioLevel);
-
     }
 
     /**
@@ -129,6 +129,23 @@ export default class NoAudioSignalDetection extends EventEmitter {
             this._audioTrack = track;
             this._eventFired = false;
             this._clearTriggerTimeout();
+
+            // Listen for the audio levels on the newly added audio track
+            if (browser.supportsReceiverStats()) {
+                track.on(
+                    JitsiTrackEvents.NO_AUDIO_INPUT,
+                    audioLevel => {
+                        this._handleNoAudioInputDetection(audioLevel);
+                    }
+                );
+                track.on(
+                    JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED,
+                    audioLevel => {
+                        this._handleNoAudioInputDetection(audioLevel);
+                        this._handleAudioInputStateChange(audioLevel);
+                    }
+                );
+            }
         }
     }
 }

modules/detection/TalkMutedDetection.js

@@ -1,127 +0,0 @@
-import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
-
-/**
- * The value which we use to say, every sound over this threshold
- * is talking on the mic.
- * @type {number}
- */
-const SPEECH_DETECT_THRESHOLD = 0.6;
-
-/**
- * Detect user trying to speek while is locally muted and fires an event.
- */
-export default class TalkMutedDetection {
-    /**
-     * Creates TalkMutedDetection
-     * @param conference the JitsiConference instance that created us.
-     * @param callback the callback to call when detected that the local user is
-     * talking while her microphone is muted.
-     * @constructor
-     */
-    constructor(conference, callback) {
-        /**
-         * The callback to call when detected that the local user is talking
-         * while her microphone is muted.
-         *
-         * @private
-         */
-        this._callback = callback;
-
-        /**
-         * The indicator which determines whether <tt>callback</tt> has been
-         * invoked for the current local audio track of <tt>conference</tt> so
-         * that it is invoked once only.
-         *
-         * @private
-         */
-        this._eventFired = false;
-
-        // XXX I went back and forth on the subject of where to put the access
-        // to statistics. On the one had, (1) statistics is likely intended to
-        // be private to conference and (2) there is a desire to keep the
-        // dependencies of modules to the minimum (i.e. not have
-        // TalkMutedDetection depend on statistics). On the other hand, (1)
-        // statistics is technically not private because
-        // JitsiConferenceEventManager accesses it and (2) TalkMutedDetection
-        // works exactly because it knows that there are no audio levels for
-        // JitsiLocalTrack but there are audio levels for the local participant
-        // through statistics.
-        conference.statistics.addAudioLevelListener(
-            this._audioLevel.bind(this));
-
-        conference.on(
-            JitsiConferenceEvents.TRACK_MUTE_CHANGED,
-            this._trackMuteChanged.bind(this));
-        conference.on(
-            JitsiConferenceEvents.TRACK_ADDED,
-            this._trackAdded.bind(this));
-    }
-
-    /* eslint-disable max-params */
-    /**
-     * Receives audio level events for all send and receive streams.
-     *
-     * @param {TraceablePeerConnection} pc - WebRTC PeerConnection object of the
-     * @param {number} ssrc - The synchronization source identifier (SSRC) of
-     * the endpoint/participant/stream being reported.
-     * @param {number} audioLevel - The audio level of <tt>ssrc</tt>.
-     * @param {boolean} isLocal - <tt>true</tt> if <tt>ssrc</tt> represents a
-     * local/send stream or <tt>false</tt> for a remote/receive stream.
-     */
-    _audioLevel(tpc, ssrc, audioLevel, isLocal) {
-        // We are interested in the local audio stream only and if event is not
-        // sent yet.
-        if (!isLocal || !this.audioTrack || this._eventFired) {
-            return;
-        }
-
-        if (this.audioTrack.isMuted()
-            && audioLevel > SPEECH_DETECT_THRESHOLD) {
-            this._eventFired = true;
-            this._callback();
-        }
-    }
-    /* eslint-enable max-params */
-
-    /**
-     * Determines whether a specific {@link JitsiTrack} represents a local audio
-     * track.
-     *
-     * @param {JitsiTrack} track - The <tt>JitsiTrack</tt> to be checked whether
-     * it represents a local audio track.
-     * @private
-     * @return {boolean} - <tt>true</tt> if the specified <tt>track</tt>
-     * represents a local audio track; otherwise, <tt>false</tt>.
-     */
-    _isLocalAudioTrack(track) {
-        return track.isAudioTrack() && track.isLocal();
-    }
-
-    /**
-     * Notifies this <tt>TalkMutedDetection</tt> that a {@link JitsiTrack} was
-     * added to the associated {@link JitsiConference}. Looks for the local
-     * audio track only.
-     *
-     * @param {JitsiTrack} track - The added <tt>JitsiTrack</tt>.
-     * @private
-     */
-    _trackAdded(track) {
-        if (this._isLocalAudioTrack(track)) {
-            this.audioTrack = track;
-        }
-    }
-
-    /**
-     * Notifies this <tt>TalkMutedDetection</tt> that the mute state of a
-     * {@link JitsiTrack} has changed. Looks for the local audio track only.
-     *
-     * @param {JitsiTrack} track - The <tt>JitsiTrack</tt> whose mute state has
-     * changed.
-     * @private
-     */
-    _trackMuteChanged(track) {
-        if (this._isLocalAudioTrack(track) && track.isMuted()) {
-            this._eventFired = false;
-        }
-    }
-}

modules/statistics/LocalStatsCollector.js

@@ -114,10 +114,12 @@ LocalStatsCollector.prototype.start = function() {
             analyser.getByteTimeDomainData(array);
             const audioLevel = timeDomainDataToAudioLevel(array);
 
-            if (audioLevel !== this.audioLevel) {
-                this.audioLevel = animateLevel(audioLevel, this.audioLevel);
-                this.callback(this.audioLevel);
-            }
+            // Set the audio levels always as NoAudioSignalDetection now
+            // uses audio levels from LocalStatsCollector and waits for
+            // atleast 4 secs for a no audio signal before displaying the
+            // notification on the UI.
+            this.audioLevel = animateLevel(audioLevel, this.audioLevel);
+            this.callback(this.audioLevel);
         },
         this.intervalMilis
     );

modules/statistics/RTPStatsCollector.js

@@ -291,31 +291,49 @@ StatsCollector.prototype.errorCallback = function(error) {
  */
 StatsCollector.prototype.start = function(startAudioLevelStats) {
     if (startAudioLevelStats) {
+        if (browser.supportsReceiverStats()) {
+            logger.info('Using RTCRtpSynchronizationSource for remote audio levels');
+        }
         this.audioLevelsIntervalId = setInterval(
             () => {
-                // Interval updates
-                this.peerconnection.getStats(
-                    report => {
-                        let results = null;
-
-                        if (!report || !report.result
-                            || typeof report.result !== 'function') {
-                            results = report;
-                        } else {
-                            results = report.result();
-                        }
-                        this.currentAudioLevelsReport = results;
-                        if (this._usesPromiseGetStats) {
-                            this.processNewAudioLevelReport();
-                        } else {
-                            this.processAudioLevelReport();
+                if (browser.supportsReceiverStats()) {
+                    const audioLevels = this.peerconnection.getAudioLevels();
+
+                    for (const ssrc in audioLevels) {
+                        if (audioLevels.hasOwnProperty(ssrc)) {
+                            this.eventEmitter.emit(
+                                StatisticsEvents.AUDIO_LEVEL,
+                                this.peerconnection,
+                                Number.parseInt(ssrc, 10),
+                                audioLevels[ssrc],
+                                false /* isLocal */);
                         }
-
-                        this.baselineAudioLevelsReport
-                            = this.currentAudioLevelsReport;
-                    },
-                    error => this.errorCallback(error)
-                );
+                    }
+                } else {
+                    // Interval updates
+                    this.peerconnection.getStats(
+                        report => {
+                            let results = null;
+
+                            if (!report || !report.result
+                                || typeof report.result !== 'function') {
+                                results = report;
+                            } else {
+                                results = report.result();
+                            }
+                            this.currentAudioLevelsReport = results;
+                            if (this._usesPromiseGetStats) {
+                                this.processNewAudioLevelReport();
+                            } else {
+                                this.processAudioLevelReport();
+                            }
+
+                            this.baselineAudioLevelsReport
+                                = this.currentAudioLevelsReport;
+                        },
+                        error => this.errorCallback(error)
+                    );
+                }
             },
             this.audioLevelsIntervalMilis
         );