瀏覽代碼

feat: add facial-expressions in speaker stats (#1724)

* feat: facial expression in speaker stats

* feat: send xmpp message with facial expression to server

* fix: rebase conflicts

* feat: facial expression in speaker stats

* feat: facial expression in speaker stats

* fchore(facial-expressions): remove the send facial expression call from update facial expression function

* feat(facial-expressions): store expresions as a timeline

* refactor(facial-expressions): store expressions by counting them in a map

* feat(facial-expressions): camera time tracker

* add(facial-expression): increase facial expression with duration parameter from payload

* add(facial-expressions): the disgusted expression

* refactor(facial-expressions): remove camera time tracker

* refactor(facial-expressions): change data channel message handel position for facial expressions and renamed some types

* fix(facial-expressions): move facial expression endpoint message handler from statistics.js to ConnectionQuality.js

* fix(facial-expressions): remove unused type
dev1
Gabriel Borlea 3 年之前
父節點
當前提交
6d981ebb6c
沒有連結到貢獻者的電子郵件帳戶。

+ 8
- 0
JitsiConference.js 查看文件

3645
     return this.speakerStatsCollector.getStats();
3645
     return this.speakerStatsCollector.getStats();
3646
 };
3646
 };
3647
 
3647
 
3648
+/**
3649
+ * Sends a facial expression with its duration to the xmpp server.
3650
+ * @param {Object} payload
3651
+ */
3652
+JitsiConference.prototype.sendFacialExpression = function(payload) {
3653
+    this.xmpp.sendFacialExpressionEvent(this.room.roomjid, payload);
3654
+};
3655
+
3648
 /**
3656
 /**
3649
  * Sets the constraints for the video that is requested from the bridge.
3657
  * Sets the constraints for the video that is requested from the bridge.
3650
  *
3658
  *

+ 5
- 0
JitsiConferenceEvents.js 查看文件

427
  * }.
427
  * }.
428
  */
428
  */
429
 export const AV_MODERATION_PARTICIPANT_REJECTED = 'conference.av_moderation.participant.rejected';
429
 export const AV_MODERATION_PARTICIPANT_REJECTED = 'conference.av_moderation.participant.rejected';
430
+
431
+/**
432
+ * A new facial expression is added with its duration for a participant
433
+ */
434
+export const FACIAL_EXPRESSION_ADDED = 'conference.facial_expression.added';

+ 17
- 0
modules/connectivity/ConnectionQuality.js 查看文件

18
  */
18
  */
19
 const STATS_MESSAGE_TYPE = 'stats';
19
 const STATS_MESSAGE_TYPE = 'stats';
20
 
20
 
21
+/**
22
+ * The value to use for the "type" field for messages sent
23
+ * over the data channel that contain facial expression.
24
+ */
25
+const FACIAL_EXPRESSION_MESSAGE_TYPE = 'facial_expression';
26
+
21
 const kSimulcastFormats = [
27
 const kSimulcastFormats = [
22
     { width: 1920,
28
     { width: 1920,
23
         height: 1080,
29
         height: 1080,
223
                 this._updateRemoteStats(participant.getId(), payload);
229
                 this._updateRemoteStats(participant.getId(), payload);
224
             });
230
             });
225
 
231
 
232
+        conference.on(
233
+            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
234
+            (participant, payload) => {
235
+                if (payload.type === FACIAL_EXPRESSION_MESSAGE_TYPE) {
236
+                    this.eventEmitter.emit(
237
+                        ConferenceEvents.FACIAL_EXPRESSION_ADDED,
238
+                        participant.getId(),
239
+                        payload);
240
+                }
241
+            });
242
+
226
         // Listen to local statistics events originating from the RTC module and update the _localStats field.
243
         // Listen to local statistics events originating from the RTC module and update the _localStats field.
227
         conference.statistics.addConnectionStatsListener(this._updateLocalStats.bind(this));
244
         conference.statistics.addConnectionStatsListener(this._updateLocalStats.bind(this));
228
 
245
 

+ 38
- 0
modules/statistics/SpeakerStats.js 查看文件

24
         this.totalDominantSpeakerTime = 0;
24
         this.totalDominantSpeakerTime = 0;
25
         this._dominantSpeakerStart = 0;
25
         this._dominantSpeakerStart = 0;
26
         this._hasLeft = false;
26
         this._hasLeft = false;
27
+        this._facialExpressions = {
28
+            happy: 0,
29
+            neutral: 0,
30
+            surprised: 0,
31
+            angry: 0,
32
+            fearful: 0,
33
+            disgusted: 0,
34
+            sad: 0
35
+        };
27
     }
36
     }
28
 
37
 
29
     /**
38
     /**
125
         this._hasLeft = true;
134
         this._hasLeft = true;
126
         this.setDominantSpeaker(false);
135
         this.setDominantSpeaker(false);
127
     }
136
     }
137
+
138
+    /**
139
+     * Gets the facial expressions of the user.
140
+     *
141
+     * @returns {Object}
142
+     */
143
+    getFacialExpressions() {
144
+        return this._facialExpressions;
145
+    }
146
+
147
+    /**
148
+     * Sets the facial expressions of the user.
149
+     *
150
+     * @param {Object} facialExpressions - object with facial expressions.
151
+     * @returns {void}
152
+     */
153
+    setFacialExpressions(facialExpressions) {
154
+        this._facialExpressions = facialExpressions;
155
+    }
156
+
157
+    /**
158
+     * Adds a new facial expression to speaker stats.
159
+     *
160
+     * @param  {string} facialExpression
161
+     * @param {number} duration
162
+     */
163
+    addFacialExpression(facialExpression, duration) {
164
+        this._facialExpressions[facialExpression] += duration;
165
+    }
128
 }
166
 }
129
 
167
 
130
 module.exports = SpeakerStats;
168
 module.exports = SpeakerStats;

+ 21
- 0
modules/statistics/SpeakerStatsCollector.js 查看文件

41
         conference.addEventListener(
41
         conference.addEventListener(
42
             JitsiConferenceEvents.DISPLAY_NAME_CHANGED,
42
             JitsiConferenceEvents.DISPLAY_NAME_CHANGED,
43
             this._onDisplayNameChange.bind(this));
43
             this._onDisplayNameChange.bind(this));
44
+        conference.addEventListener(
45
+            JitsiConferenceEvents.FACIAL_EXPRESSION_ADDED,
46
+            this._onFacialExpressionAdd.bind(this));
44
         if (conference.xmpp) {
47
         if (conference.xmpp) {
45
             conference.xmpp.addListener(
48
             conference.xmpp.addListener(
46
                 XMPPEvents.SPEAKER_STATS_RECEIVED,
49
                 XMPPEvents.SPEAKER_STATS_RECEIVED,
117
         }
120
         }
118
     }
121
     }
119
 
122
 
123
+    /**
124
+     * Adds a new facial expression with its duration of a remote user.
125
+     *
126
+     * @param {string} userId - The user id of the user that left.
127
+     * @param {Object} data - The facial expression with its duration.
128
+     * @returns {void}
129
+     * @private
130
+     */
131
+    _onFacialExpressionAdd(userId, data) {
132
+        const savedUser = this.stats.users[userId];
133
+
134
+        if (savedUser) {
135
+            savedUser.addFacialExpression(data.facialExpression, data.duration);
136
+        }
137
+    }
138
+
120
     /**
139
     /**
121
      * Return a copy of the tracked SpeakerStats models.
140
      * Return a copy of the tracked SpeakerStats models.
122
      *
141
      *
158
 
177
 
159
             speakerStatsToUpdate.totalDominantSpeakerTime
178
             speakerStatsToUpdate.totalDominantSpeakerTime
160
                 = newStats[userId].totalDominantSpeakerTime;
179
                 = newStats[userId].totalDominantSpeakerTime;
180
+
181
+            speakerStatsToUpdate.setFacialExpressions(newStats[userId].facialExpressions);
161
         }
182
         }
162
     }
183
     }
163
 }
184
 }

+ 23
- 0
modules/xmpp/xmpp.js 查看文件

900
         this.connection.send(msg);
900
         this.connection.send(msg);
901
     }
901
     }
902
 
902
 
903
+    /**
904
+     * Sends facial expression to speaker stats component.
905
+     * @param {String} roomJid - The room jid where the speaker event occurred.
906
+     * @param {Object} payload - The expression to be sent to the speaker stats.
907
+     */
908
+    sendFacialExpressionEvent(roomJid, payload) {
909
+        // no speaker stats component advertised
910
+        if (!this.speakerStatsComponentAddress || !roomJid) {
911
+            return;
912
+        }
913
+
914
+        const msg = $msg({ to: this.speakerStatsComponentAddress });
915
+
916
+        msg.c('facialExpression', {
917
+            xmlns: 'http://jitsi.org/jitmeet',
918
+            room: roomJid,
919
+            expression: payload.facialExpression,
920
+            duration: payload.duration
921
+        }).up();
922
+
923
+        this.connection.send(msg);
924
+    }
925
+
903
     /**
926
     /**
904
      * Check if the given argument is a valid JSON ENDPOINT_MESSAGE string by
927
      * Check if the given argument is a valid JSON ENDPOINT_MESSAGE string by
905
      * parsing it and checking if it has a field called 'type'.
928
      * parsing it and checking if it has a field called 'type'.

Loading…
取消
儲存