feat: add facial-expressions in speaker stats (#1724)

* feat: facial expression in speaker stats

* feat: send xmpp message with facial expression to server

* fix: rebase conflicts

* feat: facial expression in speaker stats

* feat: facial expression in speaker stats

* fchore(facial-expressions): remove the send facial expression call from update facial expression function

* feat(facial-expressions): store expresions as a timeline

* refactor(facial-expressions): store expressions by counting them in a map

* feat(facial-expressions): camera time tracker

* add(facial-expression): increase facial expression with duration parameter from payload

* add(facial-expressions): the disgusted expression

* refactor(facial-expressions): remove camera time tracker

* refactor(facial-expressions): change data channel message handel position for facial expressions and renamed some types

* fix(facial-expressions): move facial expression endpoint message handler from statistics.js to ConnectionQuality.js

* fix(facial-expressions): remove unused type

JitsiConference.js

@@ -3645,6 +3645,14 @@ JitsiConference.prototype.getSpeakerStats = function() {
     return this.speakerStatsCollector.getStats();
 };
 
+/**
+ * Sends a facial expression with its duration to the xmpp server.
+ * @param {Object} payload
+ */
+JitsiConference.prototype.sendFacialExpression = function(payload) {
+    this.xmpp.sendFacialExpressionEvent(this.room.roomjid, payload);
+};
+
 /**
  * Sets the constraints for the video that is requested from the bridge.
  *

JitsiConferenceEvents.js

@@ -427,3 +427,8 @@ export const AV_MODERATION_PARTICIPANT_APPROVED = 'conference.av_moderation.part
  * }.
  */
 export const AV_MODERATION_PARTICIPANT_REJECTED = 'conference.av_moderation.participant.rejected';
+
+/**
+ * A new facial expression is added with its duration for a participant
+ */
+export const FACIAL_EXPRESSION_ADDED = 'conference.facial_expression.added';

modules/connectivity/ConnectionQuality.js

@@ -18,6 +18,12 @@ const logger = getLogger(__filename);
  */
 const STATS_MESSAGE_TYPE = 'stats';
 
+/**
+ * The value to use for the "type" field for messages sent
+ * over the data channel that contain facial expression.
+ */
+const FACIAL_EXPRESSION_MESSAGE_TYPE = 'facial_expression';
+
 const kSimulcastFormats = [
     { width: 1920,
         height: 1080,
@@ -223,6 +229,17 @@ export default class ConnectionQuality {
                 this._updateRemoteStats(participant.getId(), payload);
             });
 
+        conference.on(
+            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+            (participant, payload) => {
+                if (payload.type === FACIAL_EXPRESSION_MESSAGE_TYPE) {
+                    this.eventEmitter.emit(
+                        ConferenceEvents.FACIAL_EXPRESSION_ADDED,
+                        participant.getId(),
+                        payload);
+                }
+            });
+
         // Listen to local statistics events originating from the RTC module and update the _localStats field.
         conference.statistics.addConnectionStatsListener(this._updateLocalStats.bind(this));
 

modules/statistics/SpeakerStats.js

@@ -24,6 +24,15 @@ class SpeakerStats {
         this.totalDominantSpeakerTime = 0;
         this._dominantSpeakerStart = 0;
         this._hasLeft = false;
+        this._facialExpressions = {
+            happy: 0,
+            neutral: 0,
+            surprised: 0,
+            angry: 0,
+            fearful: 0,
+            disgusted: 0,
+            sad: 0
+        };
     }
 
     /**
@@ -125,6 +134,35 @@ class SpeakerStats {
         this._hasLeft = true;
         this.setDominantSpeaker(false);
     }
+
+    /**
+     * Gets the facial expressions of the user.
+     *
+     * @returns {Object}
+     */
+    getFacialExpressions() {
+        return this._facialExpressions;
+    }
+
+    /**
+     * Sets the facial expressions of the user.
+     *
+     * @param {Object} facialExpressions - object with facial expressions.
+     * @returns {void}
+     */
+    setFacialExpressions(facialExpressions) {
+        this._facialExpressions = facialExpressions;
+    }
+
+    /**
+     * Adds a new facial expression to speaker stats.
+     *
+     * @param  {string} facialExpression
+     * @param {number} duration
+     */
+    addFacialExpression(facialExpression, duration) {
+        this._facialExpressions[facialExpression] += duration;
+    }
 }
 
 module.exports = SpeakerStats;

modules/statistics/SpeakerStatsCollector.js

@@ -41,6 +41,9 @@ export default class SpeakerStatsCollector {
         conference.addEventListener(
             JitsiConferenceEvents.DISPLAY_NAME_CHANGED,
             this._onDisplayNameChange.bind(this));
+        conference.addEventListener(
+            JitsiConferenceEvents.FACIAL_EXPRESSION_ADDED,
+            this._onFacialExpressionAdd.bind(this));
         if (conference.xmpp) {
             conference.xmpp.addListener(
                 XMPPEvents.SPEAKER_STATS_RECEIVED,
@@ -117,6 +120,22 @@ export default class SpeakerStatsCollector {
         }
     }
 
+    /**
+     * Adds a new facial expression with its duration of a remote user.
+     *
+     * @param {string} userId - The user id of the user that left.
+     * @param {Object} data - The facial expression with its duration.
+     * @returns {void}
+     * @private
+     */
+    _onFacialExpressionAdd(userId, data) {
+        const savedUser = this.stats.users[userId];
+
+        if (savedUser) {
+            savedUser.addFacialExpression(data.facialExpression, data.duration);
+        }
+    }
+
     /**
      * Return a copy of the tracked SpeakerStats models.
      *
@@ -158,6 +177,8 @@ export default class SpeakerStatsCollector {
 
             speakerStatsToUpdate.totalDominantSpeakerTime
                 = newStats[userId].totalDominantSpeakerTime;
+
+            speakerStatsToUpdate.setFacialExpressions(newStats[userId].facialExpressions);
         }
     }
 }

modules/xmpp/xmpp.js

@@ -900,6 +900,29 @@ export default class XMPP extends Listenable {
         this.connection.send(msg);
     }
 
+    /**
+     * Sends facial expression to speaker stats component.
+     * @param {String} roomJid - The room jid where the speaker event occurred.
+     * @param {Object} payload - The expression to be sent to the speaker stats.
+     */
+    sendFacialExpressionEvent(roomJid, payload) {
+        // no speaker stats component advertised
+        if (!this.speakerStatsComponentAddress || !roomJid) {
+            return;
+        }
+
+        const msg = $msg({ to: this.speakerStatsComponentAddress });
+
+        msg.c('facialExpression', {
+            xmlns: 'http://jitsi.org/jitmeet',
+            room: roomJid,
+            expression: payload.facialExpression,
+            duration: payload.duration
+        }).up();
+
+        this.connection.send(msg);
+    }
+
     /**
      * Check if the given argument is a valid JSON ENDPOINT_MESSAGE string by
      * parsing it and checking if it has a field called 'type'.