feat(face-landmarks): add face landmarks timeline (#2157)

* feat(face-landmarks): add timeline

* add duration to xmpp payload

JitsiConference.js

@@ -3710,7 +3710,7 @@ JitsiConference.prototype.getSpeakerStats = function() {
  */
 JitsiConference.prototype.sendFaceLandmarks = function(payload) {
     if (payload.faceExpression) {
-        this.xmpp.sendFaceExpressionEvent(this.room.roomjid, payload);
+        this.xmpp.sendFaceLandmarksEvent(this.room.roomjid, payload);
     }
 };
 

JitsiConferenceEvents.spec.ts

@@ -73,7 +73,6 @@ describe( "/JitsiConferenceEvents members", () => {
         AV_MODERATION_CHANGED,
         AV_MODERATION_PARTICIPANT_APPROVED,
         AV_MODERATION_PARTICIPANT_REJECTED,
-        FACE_LANDMARK_ADDED,
         BREAKOUT_ROOMS_MOVE_TO_ROOM,
         BREAKOUT_ROOMS_UPDATED,
         METADATA_UPDATED,
@@ -151,7 +150,6 @@ describe( "/JitsiConferenceEvents members", () => {
         expect( AV_MODERATION_CHANGED ).toBe( 'conference.av_moderation.changed' );
         expect( AV_MODERATION_PARTICIPANT_APPROVED ).toBe( 'conference.av_moderation.participant.approved' );
         expect( AV_MODERATION_PARTICIPANT_REJECTED ).toBe( 'conference.av_moderation.participant.rejected' );
-        expect( FACE_LANDMARK_ADDED ).toBe( 'conference.face_landmark.added' );
         expect( BREAKOUT_ROOMS_MOVE_TO_ROOM ).toBe( 'conference.breakout-rooms.move-to-room' );
         expect( BREAKOUT_ROOMS_UPDATED ).toBe( 'conference.breakout-rooms.updated' );
         expect( METADATA_UPDATED ).toBe( 'conference.metadata.updated' );
@@ -227,7 +225,6 @@ describe( "/JitsiConferenceEvents members", () => {
         expect( JitsiConferenceEvents.AV_MODERATION_CHANGED ).toBe( 'conference.av_moderation.changed' );
         expect( JitsiConferenceEvents.AV_MODERATION_PARTICIPANT_APPROVED ).toBe( 'conference.av_moderation.participant.approved' );
         expect( JitsiConferenceEvents.AV_MODERATION_PARTICIPANT_REJECTED ).toBe( 'conference.av_moderation.participant.rejected' );
-        expect( JitsiConferenceEvents.FACE_LANDMARK_ADDED ).toBe( 'conference.face_landmark.added' );
         expect( JitsiConferenceEvents.BREAKOUT_ROOMS_MOVE_TO_ROOM ).toBe( 'conference.breakout-rooms.move-to-room' );
         expect( JitsiConferenceEvents.BREAKOUT_ROOMS_UPDATED ).toBe( 'conference.breakout-rooms.updated' );
         expect( JitsiConferenceEvents.METADATA_UPDATED ).toBe( 'conference.metadata.updated' );

JitsiConferenceEvents.ts

@@ -440,11 +440,6 @@ export enum JitsiConferenceEvents {
      */
     AV_MODERATION_PARTICIPANT_REJECTED = 'conference.av_moderation.participant.rejected',
 
-    /**
-     * A new face landmark object is added for a participant
-     */
-    FACE_LANDMARK_ADDED = 'conference.face_landmark.added',
-
     /**
      * Event fired when a participant is requested to join a given (breakout) room.
      */
@@ -531,7 +526,6 @@ export const AV_MODERATION_REJECTED = JitsiConferenceEvents.AV_MODERATION_REJECT
 export const AV_MODERATION_CHANGED = JitsiConferenceEvents.AV_MODERATION_CHANGED;
 export const AV_MODERATION_PARTICIPANT_APPROVED = JitsiConferenceEvents.AV_MODERATION_PARTICIPANT_APPROVED;
 export const AV_MODERATION_PARTICIPANT_REJECTED = JitsiConferenceEvents.AV_MODERATION_PARTICIPANT_REJECTED;
-export const FACE_LANDMARK_ADDED = JitsiConferenceEvents.FACE_LANDMARK_ADDED;
 export const BREAKOUT_ROOMS_MOVE_TO_ROOM = JitsiConferenceEvents.BREAKOUT_ROOMS_MOVE_TO_ROOM;
 export const BREAKOUT_ROOMS_UPDATED = JitsiConferenceEvents.BREAKOUT_ROOMS_UPDATED;
 export const METADATA_UPDATED = JitsiConferenceEvents.METADATA_UPDATED;

modules/connectivity/ConnectionQuality.js

@@ -18,12 +18,6 @@ const logger = getLogger(__filename);
  */
 const STATS_MESSAGE_TYPE = 'stats';
 
-/**
- * The value to use for the "type" field for messages sent
- * over the data channel that contain a face landmark.
- */
-const FACE_LANDMARK_MESSAGE_TYPE = 'face_landmark';
-
 const kSimulcastFormats = [
     { width: 1920,
         height: 1080,
@@ -229,17 +223,6 @@ export default class ConnectionQuality {
                 this._updateRemoteStats(participant.getId(), payload);
             });
 
-        conference.on(
-            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
-            (participant, payload) => {
-                if (payload.type === FACE_LANDMARK_MESSAGE_TYPE) {
-                    this.eventEmitter.emit(
-                        ConferenceEvents.FACE_LANDMARK_ADDED,
-                        participant.getId(),
-                        payload);
-                }
-            });
-
         // Listen to local statistics events originating from the RTC module and update the _localStats field.
         conference.statistics.addConnectionStatsListener(this._updateLocalStats.bind(this));
 

modules/statistics/SpeakerStats.js

@@ -26,15 +26,7 @@ class SpeakerStats {
         this._isDominantSpeaker = false;
         this._isSilent = false;
         this._hasLeft = false;
-        this._faceExpressions = {
-            happy: 0,
-            neutral: 0,
-            surprised: 0,
-            angry: 0,
-            fearful: 0,
-            disgusted: 0,
-            sad: 0
-        };
+        this._faceLandmarks = [];
     }
 
     /**
@@ -154,32 +146,32 @@ class SpeakerStats {
     }
 
     /**
-     * Gets the face expressions of the user.
+     * Gets the face landmarks of the user.
      *
      * @returns {Object}
      */
-    getFaceExpressions() {
-        return this._faceExpressions;
+    getFaceLandmarks() {
+        return this._faceLandmarks;
     }
 
     /**
-     * Sets the face expressions of the user.
+     * Sets the face landmarks of the user.
      *
-     * @param {Object} faceExpressions - object with face expressions.
+     * @param {Object} faceLandmarks - object with face expressions.
      * @returns {void}
      */
-    setFaceExpressions(faceExpressions) {
-        this._faceExpressions = faceExpressions;
+    setFaceLandmarks(faceLandmarks) {
+        this._faceLandmarks = faceLandmarks;
     }
 
     /**
-     * Adds a new face expression to speaker stats.
+     * Adds new face landmarks to speaker stats.
      *
      * @param  {string} faceExpression
      * @param {number} duration
      */
-    addFaceExpression(faceExpression, duration) {
-        this._faceExpressions[faceExpression] += duration;
+    addFaceLandmarks(faceLandmarks) {
+        this._faceLandmarks.push(faceLandmarks);
     }
 }
 

modules/statistics/SpeakerStatsCollector.js

@@ -3,6 +3,14 @@ import { XMPPEvents } from '../../service/xmpp/XMPPEvents';
 
 import SpeakerStats from './SpeakerStats';
 
+
+/**
+ * The value to use for the "type" field for messages sent
+ * over the data channel that contain a face landmark.
+ */
+
+const FACE_LANDMARK_MESSAGE_TYPE = 'face-landmarks';
+
 /**
  * A collection for tracking speaker stats. Attaches listeners
  * to the conference to automatically update on tracked events.
@@ -41,9 +49,14 @@ export default class SpeakerStatsCollector {
         conference.addEventListener(
             JitsiConferenceEvents.DISPLAY_NAME_CHANGED,
             this._onDisplayNameChange.bind(this));
-        conference.addEventListener(
-            JitsiConferenceEvents.FACE_LANDMARK_ADDED,
-            this._onFaceLandmarkAdd.bind(this));
+
+        conference.on(
+            JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+                (participant, { type, faceLandmarks }) => {
+                    if (type === FACE_LANDMARK_MESSAGE_TYPE) {
+                        this._onFaceLandmarkAdd(participant.getId(), faceLandmarks);
+                    }
+                });
         if (conference.xmpp) {
             conference.xmpp.addListener(
                 XMPPEvents.SPEAKER_STATS_RECEIVED,
@@ -132,8 +145,8 @@ export default class SpeakerStatsCollector {
     _onFaceLandmarkAdd(userId, data) {
         const savedUser = this.stats.users[userId];
 
-        if (savedUser && data.faceExpression) {
-            savedUser.addFaceExpression(data.faceExpression, data.duration);
+        if (savedUser && data) {
+            savedUser.addFaceLandmarks(data);
         }
     }
 
@@ -177,7 +190,7 @@ export default class SpeakerStatsCollector {
                 speakerStatsToUpdate.totalDominantSpeakerTime
                     = newStats[userId].totalDominantSpeakerTime;
 
-                speakerStatsToUpdate.setFaceExpressions(newStats[userId].faceExpressions);
+                speakerStatsToUpdate.setFaceLandmarks(newStats[userId].faceLandmarks);
             }
         }
     }

modules/xmpp/xmpp.js

@@ -935,11 +935,11 @@ export default class XMPP extends Listenable {
     }
 
     /**
-     * Sends face expressions to speaker stats component.
+     * Sends face landmarks to speaker stats component.
      * @param {String} roomJid - The room jid where the speaker event occurred.
      * @param {Object} payload - The expression to be sent to the speaker stats.
      */
-    sendFaceExpressionEvent(roomJid, payload) {
+    sendFaceLandmarksEvent(roomJid, payload) {
         // no speaker stats component advertised
         if (!this.speakerStatsComponentAddress || !roomJid) {
             return;
@@ -947,10 +947,11 @@ export default class XMPP extends Listenable {
 
         const msg = $msg({ to: this.speakerStatsComponentAddress });
 
-        msg.c('faceExpression', {
+        msg.c('faceLandmarks', {
             xmlns: 'http://jitsi.org/jitmeet',
             room: roomJid,
-            expression: payload.faceExpression,
+            faceExpression: payload.faceExpression,
+            timestamp: payload.timestamp,
             duration: payload.duration
         }).up();