fix(av-moderation) Advanced moderation improvements (#9935)

* Update moderation in effect notifications

Only display one notification for each media type. Display notification for keyboard shortcuts as well

* Update muted remotely notification

Display name of moderator in the notification

* Fix indentation on moderation menu

* Update text for video moderation

* Added moderator label in participant pane

* Update microphone icon in participant list

For participants that speak, or are noisy, but aren't dominant speaker, the icon in the participant list will look the same as the dominant speaker icon but will not change their position in the list

* Added sound for asked to unmute notification

* Code review changes

* Code review changes

Use simple var instead of function for audio media state

* Move constants to constants file

* Moved constants from notifications to av-moderation

conference.js

@@ -24,6 +24,8 @@ import {
     redirectToStaticPage,
     reloadWithStoredParams
 } from './react/features/app/actions';
+import { showModeratedNotification } from './react/features/av-moderation/actions';
+import { shouldShowModeratedNotification } from './react/features/av-moderation/functions';
 import {
     AVATAR_URL_COMMAND,
     EMAIL_COMMAND,
@@ -120,7 +122,7 @@ import {
     maybeOpenFeedbackDialog,
     submitFeedback
 } from './react/features/feedback';
-import { showNotification } from './react/features/notifications';
+import { isModerationNotificationDisplayed, showNotification } from './react/features/notifications';
 import { mediaPermissionPromptVisibilityChanged, toggleSlowGUMOverlay } from './react/features/overlay';
 import { suspendDetected } from './react/features/power-monitor';
 import {
@@ -871,13 +873,24 @@ export default {
      * dialogs in case of media permissions error.
      */
     muteAudio(mute, showUI = true) {
+        const state = APP.store.getState();
+
         if (!mute
-                && isUserInteractionRequiredForUnmute(APP.store.getState())) {
+            && isUserInteractionRequiredForUnmute(state)) {
             logger.error('Unmuting audio requires user interaction');
 
             return;
         }
 
+        // check for A/V Moderation when trying to unmute
+        if (!mute && shouldShowModeratedNotification(MEDIA_TYPE.AUDIO, state)) {
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.AUDIO, state)) {
+                APP.store.dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            }
+
+            return;
+        }
+
         // Not ready to modify track's state yet
         if (!this._localTracksInitialized) {
             // This will only modify base/media.audio.muted which is then synced

config.js

@@ -737,6 +737,7 @@ var config = {
 
     // Array<string> of disabled sounds.
     // Possible values:
+    // - 'ASKED_TO_UNMUTE_SOUND'
     // - 'E2EE_OFF_SOUND'
     // - 'E2EE_ON_SOUND'
     // - 'INCOMING_MSG_SOUND'

lang/main.json

@@ -566,9 +566,9 @@
         "moderator": "You're now a moderator",
         "muted": "You have started the conversation muted.",
         "mutedTitle": "You're muted!",
-        "mutedRemotelyTitle": "You've been muted by the moderator",
+        "mutedRemotelyTitle": "You've been muted by {{moderator}}",
         "mutedRemotelyDescription": "You can always unmute when you're ready to speak. Mute back when you're done to keep noise away from the meeting.",
-        "videoMutedRemotelyTitle": "Your camera has been turned off by the moderator",
+        "videoMutedRemotelyTitle": "Your camera has been turned off by {{moderator}}",
         "videoMutedRemotelyDescription": "You can always turn it on again.",
         "passwordRemovedRemotely": "$t(lockRoomPasswordUppercase) removed by another participant",
         "passwordSetRemotely": "$t(lockRoomPasswordUppercase) set by another participant",
@@ -623,7 +623,7 @@
             "stopEveryonesVideo": "Stop everyone's video",
             "stopVideo": "Stop video",
             "unblockEveryoneMicCamera": "Unblock everyone's mic and camera",
-            "videoModeration": "Start video"
+            "videoModeration": "Start their video"
         }
     },
     "passwordSetRemotely": "Set by another participant",

react/features/av-moderation/constants.js

@@ -17,3 +17,15 @@ export const MEDIA_TYPE_TO_PENDING_STORE_KEY: {[key: MediaType]: string} = {
     [MEDIA_TYPE.AUDIO]: 'pendingAudio',
     [MEDIA_TYPE.VIDEO]: 'pendingVideo'
 };
+
+export const ASKED_TO_UNMUTE_SOUND_ID = 'ASKED_TO_UNMUTE_SOUND';
+
+export const AUDIO_MODERATION_NOTIFICATION_ID = 'audio-moderation';
+export const VIDEO_MODERATION_NOTIFICATION_ID = 'video-moderation';
+export const CS_MODERATION_NOTIFICATION_ID = 'screensharing-moderation';
+
+export const MODERATION_NOTIFICATIONS = {
+    [MEDIA_TYPE.AUDIO]: AUDIO_MODERATION_NOTIFICATION_ID,
+    [MEDIA_TYPE.VIDEO]: VIDEO_MODERATION_NOTIFICATION_ID,
+    [MEDIA_TYPE.PRESENTER]: CS_MODERATION_NOTIFICATION_ID
+};

react/features/av-moderation/middleware.js

@@ -1,6 +1,7 @@
 // @flow
 import { batch } from 'react-redux';
 
+import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app';
 import { getConferenceState } from '../base/conference';
 import { JitsiConferenceEvents } from '../base/lib-jitsi-meet';
 import { MEDIA_TYPE } from '../base/media';
@@ -13,6 +14,7 @@ import {
     raiseHand
 } from '../base/participants';
 import { MiddlewareRegistry, StateListenerRegistry } from '../base/redux';
+import { playSound, registerSound, unregisterSound } from '../base/sounds';
 import {
     hideNotification,
     showNotification
@@ -35,21 +37,31 @@ import {
     participantApproved,
     participantPendingAudio
 } from './actions';
+import {
+    ASKED_TO_UNMUTE_SOUND_ID, AUDIO_MODERATION_NOTIFICATION_ID,
+    CS_MODERATION_NOTIFICATION_ID,
+    VIDEO_MODERATION_NOTIFICATION_ID
+} from './constants';
 import {
     isEnabledFromState,
     isParticipantApproved,
     isParticipantPending
 } from './functions';
-
-const VIDEO_MODERATION_NOTIFICATION_ID = 'video-moderation';
-const AUDIO_MODERATION_NOTIFICATION_ID = 'audio-moderation';
-const CS_MODERATION_NOTIFICATION_ID = 'video-moderation';
+import { ASKED_TO_UNMUTE_FILE } from './sounds';
 
 MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
     const { type } = action;
     const { conference } = getConferenceState(getState());
 
     switch (type) {
+    case APP_WILL_MOUNT: {
+        dispatch(registerSound(ASKED_TO_UNMUTE_SOUND_ID, ASKED_TO_UNMUTE_FILE));
+        break;
+    }
+    case APP_WILL_UNMOUNT: {
+        dispatch(unregisterSound(ASKED_TO_UNMUTE_SOUND_ID));
+        break;
+    }
     case LOCAL_PARTICIPANT_MODERATION_NOTIFICATION: {
         let descriptionKey;
         let titleKey;
@@ -160,6 +172,7 @@ StateListenerRegistry.register(
                         customActionNameKey: 'notify.unmute',
                         customActionHandler: () => dispatch(muteLocal(false, MEDIA_TYPE.AUDIO))
                     }));
+                    dispatch(playSound(ASKED_TO_UNMUTE_SOUND_ID));
                 }
             });
 

react/features/av-moderation/sounds.js

@@ -0,0 +1,6 @@
+/**
+ * The name of the bundled audio file which will be played for the raise hand sound.
+ *
+ * @type {string}
+ */
+export const ASKED_TO_UNMUTE_FILE = 'asked-unmute.mp3';

react/features/base/media/actions.js

@@ -4,6 +4,7 @@ import type { Dispatch } from 'redux';
 
 import { showModeratedNotification } from '../../av-moderation/actions';
 import { shouldShowModeratedNotification } from '../../av-moderation/functions';
+import { isModerationNotificationDisplayed } from '../../notifications';
 
 import {
     SET_AUDIO_MUTED,
@@ -113,7 +114,9 @@ export function setVideoMuted(
 
         // check for A/V Moderation when trying to unmute
         if (!muted && shouldShowModeratedNotification(MEDIA_TYPE.VIDEO, state)) {
-            ensureTrack && dispatch(showModeratedNotification(MEDIA_TYPE.VIDEO));
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.VIDEO, state)) {
+                ensureTrack && dispatch(showModeratedNotification(MEDIA_TYPE.VIDEO));
+            }
 
             return;
         }

react/features/base/participants/actions.js

@@ -466,7 +466,7 @@ export function participantUpdated(participant = {}) {
  * @returns {Promise}
  */
 export function participantMutedUs(participant, track) {
-    return dispatch => {
+    return (dispatch, getState) => {
         if (!participant) {
             return;
         }
@@ -474,7 +474,10 @@ export function participantMutedUs(participant, track) {
         const isAudio = track.isAudioTrack();
 
         dispatch(showNotification({
-            titleKey: isAudio ? 'notify.mutedRemotelyTitle' : 'notify.videoMutedRemotelyTitle'
+            titleKey: isAudio ? 'notify.mutedRemotelyTitle' : 'notify.videoMutedRemotelyTitle',
+            titleArguments: {
+                moderator: getParticipantDisplayName(getState, participant.getId())
+            }
         }));
     };
 }

react/features/base/tracks/middleware.js

@@ -3,7 +3,7 @@
 import UIEvents from '../../../../service/UI/UIEvents';
 import { showModeratedNotification } from '../../av-moderation/actions';
 import { shouldShowModeratedNotification } from '../../av-moderation/functions';
-import { hideNotification } from '../../notifications';
+import { hideNotification, isModerationNotificationDisplayed } from '../../notifications';
 import { isPrejoinPageVisible } from '../../prejoin/functions';
 import { getAvailableDevices } from '../devices/actions';
 import {
@@ -142,7 +142,9 @@ MiddlewareRegistry.register(store => next => action => {
             // check for A/V Moderation when trying to start screen sharing
             if ((action.enabled || action.enabled === undefined)
                 && shouldShowModeratedNotification(MEDIA_TYPE.VIDEO, store.getState())) {
-                store.dispatch(showModeratedNotification(MEDIA_TYPE.PRESENTER));
+                if (!isModerationNotificationDisplayed(MEDIA_TYPE.PRESENTER, store.getState())) {
+                    store.dispatch(showModeratedNotification(MEDIA_TYPE.PRESENTER));
+                }
 
                 return;
             }

react/features/notifications/functions.js

@@ -1,5 +1,7 @@
 // @flow
 
+import { MODERATION_NOTIFICATIONS } from '../av-moderation/constants';
+import { MEDIA_TYPE } from '../base/media';
 import { toState } from '../base/redux';
 
 declare var interfaceConfig: Object;
@@ -26,3 +28,18 @@ export function areThereNotifications(stateful: Object | Function) {
 export function joinLeaveNotificationsDisabled() {
     return Boolean(typeof interfaceConfig !== 'undefined' && interfaceConfig?.DISABLE_JOIN_LEAVE_NOTIFICATIONS);
 }
+
+/**
+ * Returns whether or not the moderation notification for the given type is displayed.
+ *
+ * @param {MEDIA_TYPE} mediaType - The media type to check.
+ * @param {Object | Function} stateful - The redux store state.
+ * @returns {boolean}
+ */
+export function isModerationNotificationDisplayed(mediaType: MEDIA_TYPE, stateful: Object | Function) {
+    const state = toState(stateful);
+
+    const { notifications } = state['features/notifications'];
+
+    return Boolean(notifications.find(n => n.uid === MODERATION_NOTIFICATIONS[mediaType]));
+}

react/features/participants-pane/components/FooterContextMenu.js

@@ -55,10 +55,10 @@ const useStyles = makeStyles(() => {
         },
         text: {
             color: '#C2C2C2',
-            padding: '10px 16px 10px 52px'
+            padding: '10px 16px'
         },
         paddedAction: {
-            marginLeft: '36px;'
+            marginLeft: '36px'
         }
     };
 });

react/features/participants-pane/components/web/MeetingParticipantItem.js

@@ -1,15 +1,23 @@
 // @flow
 
-import React from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 
+import { JitsiTrackEvents } from '../../../base/lib-jitsi-meet';
+import { MEDIA_TYPE } from '../../../base/media';
 import {
     getLocalParticipant,
     getParticipantByIdOrUndefined,
-    getParticipantDisplayName
+    getParticipantDisplayName,
+    isParticipantModerator
 } from '../../../base/participants';
 import { connect } from '../../../base/redux';
-import { isParticipantAudioMuted, isParticipantVideoMuted } from '../../../base/tracks';
-import { ACTION_TRIGGER, type MediaState } from '../../constants';
+import {
+    getLocalAudioTrack,
+    getTrackByMediaTypeAndParticipant,
+    isParticipantAudioMuted,
+    isParticipantVideoMuted
+} from '../../../base/tracks';
+import { ACTION_TRIGGER, type MediaState, MEDIA_STATE } from '../../constants';
 import {
     getParticipantAudioMediaState,
     getParticipantVideoMediaState,
@@ -27,6 +35,11 @@ type Props = {
      */
     _audioMediaState: MediaState,
 
+    /**
+     * The audio track related to the participant.
+     */
+    _audioTrack: ?Object,
+
     /**
      * Media state for video.
      */
@@ -136,6 +149,7 @@ type Props = {
  */
 function MeetingParticipantItem({
     _audioMediaState,
+    _audioTrack,
     _videoMediaState,
     _displayName,
     _local,
@@ -155,12 +169,46 @@ function MeetingParticipantItem({
     participantActionEllipsisLabel,
     youText
 }: Props) {
+
+    const [ hasAudioLevels, setHasAudioLevel ] = useState(false);
+    const [ registeredEvent, setRegisteredEvent ] = useState(false);
+
+    const _updateAudioLevel = useCallback(level => {
+        const audioLevel = typeof level === 'number' && !isNaN(level)
+            ? level : 0;
+
+        setHasAudioLevel(audioLevel > 0.009);
+    }, []);
+
+    useEffect(() => {
+        if (_audioTrack && !registeredEvent) {
+            const { jitsiTrack } = _audioTrack;
+
+            if (jitsiTrack) {
+                jitsiTrack.on(JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED, _updateAudioLevel);
+                setRegisteredEvent(true);
+            }
+        }
+
+        return () => {
+            if (_audioTrack && registeredEvent) {
+                const { jitsiTrack } = _audioTrack;
+
+                jitsiTrack && jitsiTrack.off(JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED, _updateAudioLevel);
+            }
+        };
+    }, [ _audioTrack ]);
+
+    const audioMediaState = _audioMediaState === MEDIA_STATE.UNMUTED && hasAudioLevels
+        ? MEDIA_STATE.DOMINANT_SPEAKER : _audioMediaState;
+
     return (
         <ParticipantItem
             actionsTrigger = { ACTION_TRIGGER.HOVER }
-            audioMediaState = { _audioMediaState }
+            audioMediaState = { audioMediaState }
             displayName = { _displayName }
             isHighlighted = { isHighlighted }
+            isModerator = { isParticipantModerator(_participant) }
             local = { _local }
             onLeave = { onLeave }
             openDrawerForParticipant = { openDrawerForParticipant }
@@ -181,7 +229,7 @@ function MeetingParticipantItem({
                     <ParticipantActionEllipsis
                         aria-label = { participantActionEllipsisLabel }
                         onClick = { onContextMenu } />
-                 </>
+                </>
             }
 
             {!overflowDrawer && _localVideoOwner && _participant?.isFakeParticipant && (
@@ -214,8 +262,13 @@ function _mapStateToProps(state, ownProps): Object {
     const _videoMediaState = getParticipantVideoMediaState(participant, _isVideoMuted, state);
     const _quickActionButtonType = getQuickActionButtonType(participant, _isAudioMuted, state);
 
+    const tracks = state['features/base/tracks'];
+    const _audioTrack = participantID === localParticipantId
+        ? getLocalAudioTrack(tracks) : getTrackByMediaTypeAndParticipant(tracks, MEDIA_TYPE.AUDIO, participantID);
+
     return {
         _audioMediaState,
+        _audioTrack,
         _videoMediaState,
         _displayName: getParticipantDisplayName(state, participant?.id),
         _local: Boolean(participant?.local),

react/features/participants-pane/components/web/ParticipantItem.js

@@ -3,6 +3,7 @@
 import React, { type Node, useCallback } from 'react';
 
 import { Avatar } from '../../../base/avatar';
+import { translate } from '../../../base/i18n';
 import {
     ACTION_TRIGGER,
     AudioStateIcons,
@@ -14,10 +15,12 @@ import {
 
 import { RaisedHandIndicator } from './RaisedHandIndicator';
 import {
+    ModeratorLabel,
     ParticipantActionsHover,
     ParticipantActionsPermanent,
     ParticipantContainer,
     ParticipantContent,
+    ParticipantDetailsContainer,
     ParticipantName,
     ParticipantNameContainer,
     ParticipantStates
@@ -58,6 +61,11 @@ type Props = {
      */
     isHighlighted?: boolean,
 
+    /**
+     * Whether or not the participant is a moderator.
+     */
+    isModerator: boolean,
+
     /**
      * True if the participant is local.
      */
@@ -93,6 +101,11 @@ type Props = {
      */
     videoMediaState: MediaState,
 
+    /**
+     * Invoked to obtain translated strings.
+     */
+    t: Function,
+
     /**
      * The translated "you" text.
      */
@@ -105,9 +118,10 @@ type Props = {
  * @param {Props} props - The props of the component.
  * @returns {ReactNode}
  */
-export default function ParticipantItem({
+function ParticipantItem({
     children,
     isHighlighted,
+    isModerator,
     onLeave,
     actionsTrigger = ACTION_TRIGGER.HOVER,
     audioMediaState = MEDIA_STATE.NONE,
@@ -118,6 +132,7 @@ export default function ParticipantItem({
     openDrawerForParticipant,
     overflowDrawer,
     raisedHand,
+    t,
     youText
 }: Props) {
     const ParticipantActions = Actions[actionsTrigger];
@@ -140,12 +155,17 @@ export default function ParticipantItem({
                 participantId = { participantID }
                 size = { 32 } />
             <ParticipantContent>
-                <ParticipantNameContainer>
-                    <ParticipantName>
-                        { displayName }
-                    </ParticipantName>
-                    { local ? <span>&nbsp;({ youText })</span> : null }
-                </ParticipantNameContainer>
+                <ParticipantDetailsContainer>
+                    <ParticipantNameContainer>
+                        <ParticipantName>
+                            { displayName }
+                        </ParticipantName>
+                        { local ? <span>&nbsp;({ youText })</span> : null }
+                    </ParticipantNameContainer>
+                    {isModerator && <ModeratorLabel>
+                        {t('videothumbnail.moderator')}
+                    </ModeratorLabel>}
+                </ParticipantDetailsContainer>
                 { !local && <ParticipantActions children = { children } /> }
                 <ParticipantStates>
                     { raisedHand && <RaisedHandIndicator /> }
@@ -156,3 +176,5 @@ export default function ParticipantItem({
         </ParticipantContainer>
     );
 }
+
+export default translate(ParticipantItem);

react/features/participants-pane/components/web/styled.js

@@ -282,6 +282,7 @@ export const ParticipantContainer = styled.div`
   color: white;
   display: flex;
   font-size: 13px;
+  font-weight: normal;
   height: ${props => props.theme.participantItemHeight}px;
   margin: 0 -${props => props.theme.panePadding}px;
   padding-left: ${props => props.theme.panePadding}px;
@@ -341,10 +342,24 @@ export const ParticipantName = styled.div`
 `;
 
 export const ParticipantNameContainer = styled.div`
+  display: flex;
+  flex: 1;
+  overflow: hidden;
+`;
+
+export const ModeratorLabel = styled.div`
+  font-size: 12px;
+  line-height: 16px;
+  color: #858585;
+`;
+
+export const ParticipantDetailsContainer = styled.div`
   display: flex;
   flex: 1;
   margin-right: 8px;
   overflow: hidden;
+  flex-direction: column;
+  justify-content: flex-start;
 `;
 
 export const RaisedHandIndicatorBackground = styled.div`

react/features/video-menu/actions.any.js

@@ -23,6 +23,7 @@ import {
     getRemoteParticipants,
     muteRemoteParticipant
 } from '../base/participants';
+import { isModerationNotificationDisplayed } from '../notifications';
 
 declare var APP: Object;
 
@@ -47,7 +48,9 @@ export function muteLocal(enable: boolean, mediaType: MEDIA_TYPE) {
 
         // check for A/V Moderation when trying to unmute
         if (!enable && shouldShowModeratedNotification(MEDIA_TYPE.AUDIO, getState())) {
-            dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.AUDIO, getState())) {
+                dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            }
 
             return;
         }