feat(stats): Add audio output problem detector.

JitsiConference.js

@@ -28,6 +28,7 @@ import Jvb121EventGenerator from './modules/event/Jvb121EventGenerator';
 import RecordingManager from './modules/recording/RecordingManager';
 import RttMonitor from './modules/rttmonitor/rttmonitor';
 import AvgRTPStatsReporter from './modules/statistics/AvgRTPStatsReporter';
+import AudioOutputProblemDetector from './modules/statistics/AudioOutputProblemDetector';
 import SpeakerStatsCollector from './modules/statistics/SpeakerStatsCollector';
 import Statistics from './modules/statistics/statistics';
 import Transcriber from './modules/transcription/transcriber';
@@ -163,6 +164,12 @@ export default function JitsiConference(options) {
     this.avgRtpStatsReporter
         = new AvgRTPStatsReporter(this, options.config.avgRtpStatsN || 15);
 
+    /**
+     * Detects issues with the audio of remote participants.
+     * @type {AudioOutputProblemDetector}
+     */
+    this._audioOutputProblemDetector = new AudioOutputProblemDetector(this);
+
     /**
      * Indicates whether the connection is interrupted or not.
      */
@@ -455,6 +462,11 @@ JitsiConference.prototype.leave = function() {
         this.avgRtpStatsReporter = null;
     }
 
+    if (this._audioOutputProblemDetector) {
+        this._audioOutputProblemDetector.dispose();
+        this._audioOutputProblemDetector = null;
+    }
+
     if (this.rttMonitor) {
         this.rttMonitor.stop();
         this.rttMonitor = null;

modules/connectivity/ConnectionQuality.js

@@ -453,7 +453,8 @@ export default class ConnectionQuality {
             packetLoss: this._localStats.packetLoss,
             connectionQuality: this._localStats.connectionQuality,
             jvbRTT: this._localStats.jvbRTT,
-            serverRegion: this._localStats.serverRegion
+            serverRegion: this._localStats.serverRegion,
+            avgAudioLevels: this._localStats.localAvgAudioLevels
         };
 
         try {
@@ -543,7 +544,8 @@ export default class ConnectionQuality {
             packetLoss: data.packetLoss,
             connectionQuality: data.connectionQuality,
             jvbRTT: data.jvbRTT,
-            serverRegion: data.serverRegion
+            serverRegion: data.serverRegion,
+            avgAudioLevels: data.avgAudioLevels
         };
 
         this.eventEmitter.emit(

modules/statistics/AudioOutputProblemDetector.js

@@ -0,0 +1,133 @@
+import { getLogger } from 'jitsi-meet-logger';
+
+import * as ConferenceEvents from '../../JitsiConferenceEvents';
+import * as ConnectionQualityEvents from '../../service/connectivity/ConnectionQualityEvents';
+import * as MediaType from '../../service/RTC/MediaType';
+import { createAudioOutputProblemEvent } from '../../service/statistics/AnalyticsEvents';
+
+import Statistics from './statistics';
+
+const logger = getLogger(__filename);
+
+/**
+ * Number of remote samples that will be used for comparison with local ones.
+ */
+const NUMBER_OF_REMOTE_SAMPLES = 3;
+
+/**
+ * Collects the average audio levels per participant from the local stats and the stats received by every remote
+ * participant and compares them to detect potential audio problem for a participant.
+ */
+export default class AudioOutputProblemDetector {
+
+    /**
+     * Creates new <tt>AudioOutputProblemDetector</tt> instance.
+     *
+     * @param {JitsiCofnerence} conference - The conference instance to be monitored.
+     */
+    constructor(conference) {
+        this._conference = conference;
+        this._lastReceivedAudioLevel = {};
+        this._reportedParticipants = [];
+        this._onLocalAudioLevelsReport = this._onLocalAudioLevelsReport.bind(this);
+        this._onRemoteAudioLevelReceived = this._onRemoteAudioLevelReceived.bind(this);
+        this._clearUserData = this._clearUserData.bind(this);
+        this._conference.on(ConnectionQualityEvents.REMOTE_STATS_UPDATED, this._onRemoteAudioLevelReceived);
+        this._conference.statistics.addConnectionStatsListener(this._onLocalAudioLevelsReport);
+        this._conference.on(ConferenceEvents.USER_LEFT, this._clearUserData);
+    }
+
+    /**
+     * A listener for audio level data received by a remote participant.
+     *
+     * @param {string} userID - The user id of the participant that sent the data.
+     * @param {number} audioLevel - The average audio level value.
+     * @returns {void}
+     */
+    _onRemoteAudioLevelReceived(userID, { avgAudioLevels }) {
+        if (this._reportedParticipants.indexOf(userID) !== -1) {
+            return;
+        }
+
+        if (!Array.isArray(this._lastReceivedAudioLevel[userID])) {
+            this._lastReceivedAudioLevel[userID] = [ ];
+        } else if (this._lastReceivedAudioLevel[userID].length >= NUMBER_OF_REMOTE_SAMPLES) {
+            this._lastReceivedAudioLevel[userID].shift();
+        }
+
+        this._lastReceivedAudioLevel[userID].push(avgAudioLevels);
+
+    }
+
+    /**
+     * A listener for audio level data retrieved by the local stats.
+     *
+     * @param {TraceablePeerConnection} tpc - The <tt>TraceablePeerConnection</tt> instance used to gather the data.
+     * @param {Object} avgAudioLevels - The average audio levels per participant.
+     * @returns {void}
+     */
+    _onLocalAudioLevelsReport(tpc, { avgAudioLevels }) {
+        if (tpc !== this._conference.getActivePeerConnection()) {
+            return;
+        }
+
+        Object.keys(this._lastReceivedAudioLevel).forEach(userID => {
+            if (this._reportedParticipants.indexOf(userID) !== -1) {
+                // Do not report the participant again.
+                return;
+            }
+
+            const remoteAudioLevels = this._lastReceivedAudioLevel[userID];
+            const participant = this._conference.getParticipantById(userID);
+
+            if (participant) {
+                const tracks = participant.getTracksByMediaType(MediaType.AUDIO);
+
+                if (tracks.length > 0 && participant.isAudioMuted()) {
+                    // We don't need to report an error if everything seems fine with the participant and its tracks but
+                    // the participant is audio muted.
+                    return;
+                }
+            }
+
+            if ((!(userID in avgAudioLevels) || avgAudioLevels[userID] === 0)
+                    && Array.isArray(remoteAudioLevels)
+                    && remoteAudioLevels.length === NUMBER_OF_REMOTE_SAMPLES
+                    && remoteAudioLevels.every(audioLevel => audioLevel > 0)) {
+                const remoteAudioLevelsString = JSON.stringify(remoteAudioLevels);
+
+                Statistics.sendAnalytics(
+                    createAudioOutputProblemEvent(userID, avgAudioLevels[userID], remoteAudioLevelsString));
+                logger.warn(`A potential problem is detected with the audio output for participant ${
+                    userID}, local audio levels: ${avgAudioLevels[userID]}, remote audio levels: ${
+                    remoteAudioLevelsString}`);
+                this._reportedParticipants.push(userID);
+                this._clearUserData();
+            }
+        });
+    }
+
+    /**
+     * Clears the data stored for a participant.
+     *
+     * @param {string} userID - The id of the participant.
+     * @returns {void}
+     */
+    _clearUserData(userID) {
+        delete this._lastReceivedAudioLevel[userID];
+    }
+
+    /**
+     * Disposes the allocated resources.
+     *
+     * @returns {void}
+     */
+    dispose() {
+        this._conference.off(ConnectionQualityEvents.REMOTE_STATS_UPDATED, this._onRemoteAudioLevelReceived);
+        this._conference.off(ConferenceEvents.USER_LEFT, this._clearUserData);
+        this._conference.statistics.removeConnectionStatsListener(this._onLocalAudioLevelsReport);
+        this._lastReceivedAudioLevel = undefined;
+        this._reportedParticipants = undefined;
+        this._conference = undefined;
+    }
+}

modules/statistics/RTPStatsCollector.js

@@ -257,6 +257,7 @@ export default function StatsCollector(
     this.currentAudioLevelsReport = null;
     this.currentStatsReport = null;
     this.previousStatsReport = null;
+    this.audioLevelReportHistory = {};
     this.audioLevelsIntervalId = null;
     this.eventEmitter = eventEmitter;
     this.conferenceStats = new ConferenceStats();
@@ -846,6 +847,29 @@ StatsCollector.prototype._processAndEmitReport = function() {
             calculatePacketLoss(lostPackets.upload, totalPackets.upload)
     };
 
+    const avgAudioLevels = {};
+    let localAvgAudioLevels;
+
+    Object.keys(this.audioLevelReportHistory).forEach(ssrc => {
+        const { data, isLocal } = this.audioLevelReportHistory[ssrc];
+        const avgAudioLevel = data.reduce((sum, currentValue) => sum + currentValue) / data.length;
+
+        if (isLocal) {
+            localAvgAudioLevels = avgAudioLevel;
+        } else {
+            const track = this.peerconnection.getTrackBySSRC(Number(ssrc));
+
+            if (track) {
+                const participantId = track.getParticipantId();
+
+                if (participantId) {
+                    avgAudioLevels[participantId] = avgAudioLevel;
+                }
+            }
+        }
+    });
+    this.audioLevelReportHistory = {};
+
     this.eventEmitter.emit(
         StatisticsEvents.CONNECTION_STATS,
         this.peerconnection,
@@ -855,7 +879,9 @@ StatsCollector.prototype._processAndEmitReport = function() {
             'packetLoss': this.conferenceStats.packetLoss,
             'resolution': resolutions,
             'framerate': framerates,
-            'transport': this.conferenceStats.transport
+            'transport': this.conferenceStats.transport,
+            localAvgAudioLevels,
+            avgAudioLevels
         });
     this.conferenceStats.transport = [];
 };
@@ -942,6 +968,14 @@ StatsCollector.prototype.processAudioLevelReport = function() {
                 audioLevel = audioLevel / 32767;
             }
 
+            if (!(ssrc in this.audioLevelReportHistory)) {
+                this.audioLevelReportHistory[ssrc] = {
+                    isLocal,
+                    data: []
+                };
+            }
+            this.audioLevelReportHistory[ssrc].data.push(audioLevel);
+
             this.eventEmitter.emit(
                 StatisticsEvents.AUDIO_LEVEL,
                 this.peerconnection,

service/statistics/AnalyticsEvents.js

@@ -470,6 +470,26 @@ export const createRttByRegionEvent = function(attributes) {
     };
 };
 
+/**
+ * Creates an event which contains information about the audio output problem (the user id of the affected participant,
+ * the local audio levels and the remote audio levels that triggered the event).
+ *
+ * @param {string} userID - The user id of the affected participant.
+ * @param {*} localAudioLevel - The local audio levels.
+ * @param {*} remoteAudioLevels - The audio levels received from the participant.
+ */
+export function createAudioOutputProblemEvent(userID, localAudioLevel, remoteAudioLevels) {
+    return {
+        type: TYPE_OPERATIONAL,
+        action: 'audio.output.problem',
+        attributes: {
+            userID,
+            localAudioLevel,
+            remoteAudioLevels
+        }
+    };
+}
+
 /**
  * Creates an event which contains an information related to the bridge channel close event.
  *