fix(av-moderation) Advanced moderation improvements (#9935)

* Update moderation in effect notifications

Only display one notification for each media type. Display notification for keyboard shortcuts as well

* Update muted remotely notification

Display name of moderator in the notification

* Fix indentation on moderation menu

* Update text for video moderation

* Added moderator label in participant pane

* Update microphone icon in participant list

For participants that speak, or are noisy, but aren't dominant speaker, the icon in the participant list will look the same as the dominant speaker icon but will not change their position in the list

* Added sound for asked to unmute notification

* Code review changes

* Code review changes

Use simple var instead of function for audio media state

* Move constants to constants file

* Moved constants from notifications to av-moderation

conference.js

     redirectToStaticPage,
     reloadWithStoredParams
 } from './react/features/app/actions';
+import { showModeratedNotification } from './react/features/av-moderation/actions';
+import { shouldShowModeratedNotification } from './react/features/av-moderation/functions';
 import {
     AVATAR_URL_COMMAND,
     EMAIL_COMMAND,
     maybeOpenFeedbackDialog,
     submitFeedback
 } from './react/features/feedback';
-import { showNotification } from './react/features/notifications';
+import { isModerationNotificationDisplayed, showNotification } from './react/features/notifications';
 import { mediaPermissionPromptVisibilityChanged, toggleSlowGUMOverlay } from './react/features/overlay';
 import { suspendDetected } from './react/features/power-monitor';
 import {
      * dialogs in case of media permissions error.
      */
     muteAudio(mute, showUI = true) {
+        const state = APP.store.getState();
+
         if (!mute
-                && isUserInteractionRequiredForUnmute(APP.store.getState())) {
+            && isUserInteractionRequiredForUnmute(state)) {
             logger.error('Unmuting audio requires user interaction');
 
             return;
         }
 
+        // check for A/V Moderation when trying to unmute
+        if (!mute && shouldShowModeratedNotification(MEDIA_TYPE.AUDIO, state)) {
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.AUDIO, state)) {
+                APP.store.dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            }
+
+            return;
+        }
+
         // Not ready to modify track's state yet
         if (!this._localTracksInitialized) {
             // This will only modify base/media.audio.muted which is then synced

config.js

 
     // Array<string> of disabled sounds.
     // Possible values:
+    // - 'ASKED_TO_UNMUTE_SOUND'
     // - 'E2EE_OFF_SOUND'
     // - 'E2EE_ON_SOUND'
     // - 'INCOMING_MSG_SOUND'

lang/main.json

         "moderator": "You're now a moderator",
         "muted": "You have started the conversation muted.",
         "mutedTitle": "You're muted!",
-        "mutedRemotelyTitle": "You've been muted by the moderator",
+        "mutedRemotelyTitle": "You've been muted by {{moderator}}",
         "mutedRemotelyDescription": "You can always unmute when you're ready to speak. Mute back when you're done to keep noise away from the meeting.",
-        "videoMutedRemotelyTitle": "Your camera has been turned off by the moderator",
+        "videoMutedRemotelyTitle": "Your camera has been turned off by {{moderator}}",
         "videoMutedRemotelyDescription": "You can always turn it on again.",
         "passwordRemovedRemotely": "$t(lockRoomPasswordUppercase) removed by another participant",
         "passwordSetRemotely": "$t(lockRoomPasswordUppercase) set by another participant",
             "stopEveryonesVideo": "Stop everyone's video",
             "stopVideo": "Stop video",
             "unblockEveryoneMicCamera": "Unblock everyone's mic and camera",
-            "videoModeration": "Start video"
+            "videoModeration": "Start their video"
         }
     },
     "passwordSetRemotely": "Set by another participant",

react/features/av-moderation/constants.js

     [MEDIA_TYPE.AUDIO]: 'pendingAudio',
     [MEDIA_TYPE.VIDEO]: 'pendingVideo'
 };
+
+export const ASKED_TO_UNMUTE_SOUND_ID = 'ASKED_TO_UNMUTE_SOUND';
+
+export const AUDIO_MODERATION_NOTIFICATION_ID = 'audio-moderation';
+export const VIDEO_MODERATION_NOTIFICATION_ID = 'video-moderation';
+export const CS_MODERATION_NOTIFICATION_ID = 'screensharing-moderation';
+
+export const MODERATION_NOTIFICATIONS = {
+    [MEDIA_TYPE.AUDIO]: AUDIO_MODERATION_NOTIFICATION_ID,
+    [MEDIA_TYPE.VIDEO]: VIDEO_MODERATION_NOTIFICATION_ID,
+    [MEDIA_TYPE.PRESENTER]: CS_MODERATION_NOTIFICATION_ID
+};

react/features/av-moderation/middleware.js

 // @flow
 import { batch } from 'react-redux';
 
+import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app';
 import { getConferenceState } from '../base/conference';
 import { JitsiConferenceEvents } from '../base/lib-jitsi-meet';
 import { MEDIA_TYPE } from '../base/media';
     raiseHand
 } from '../base/participants';
 import { MiddlewareRegistry, StateListenerRegistry } from '../base/redux';
+import { playSound, registerSound, unregisterSound } from '../base/sounds';
 import {
     hideNotification,
     showNotification
     participantApproved,
     participantPendingAudio
 } from './actions';
+import {
+    ASKED_TO_UNMUTE_SOUND_ID, AUDIO_MODERATION_NOTIFICATION_ID,
+    CS_MODERATION_NOTIFICATION_ID,
+    VIDEO_MODERATION_NOTIFICATION_ID
+} from './constants';
 import {
     isEnabledFromState,
     isParticipantApproved,
     isParticipantPending
 } from './functions';
-
-const VIDEO_MODERATION_NOTIFICATION_ID = 'video-moderation';
-const AUDIO_MODERATION_NOTIFICATION_ID = 'audio-moderation';
-const CS_MODERATION_NOTIFICATION_ID = 'video-moderation';
+import { ASKED_TO_UNMUTE_FILE } from './sounds';
 
 MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
     const { type } = action;
     const { conference } = getConferenceState(getState());
 
     switch (type) {
+    case APP_WILL_MOUNT: {
+        dispatch(registerSound(ASKED_TO_UNMUTE_SOUND_ID, ASKED_TO_UNMUTE_FILE));
+        break;
+    }
+    case APP_WILL_UNMOUNT: {
+        dispatch(unregisterSound(ASKED_TO_UNMUTE_SOUND_ID));
+        break;
+    }
     case LOCAL_PARTICIPANT_MODERATION_NOTIFICATION: {
         let descriptionKey;
         let titleKey;
                         customActionNameKey: 'notify.unmute',
                         customActionHandler: () => dispatch(muteLocal(false, MEDIA_TYPE.AUDIO))
                     }));
+                    dispatch(playSound(ASKED_TO_UNMUTE_SOUND_ID));
                 }
             });
 

react/features/av-moderation/sounds.js

+/**
+ * The name of the bundled audio file which will be played for the raise hand sound.
+ *
+ * @type {string}
+ */
+export const ASKED_TO_UNMUTE_FILE = 'asked-unmute.mp3';

react/features/base/media/actions.js

 
 import { showModeratedNotification } from '../../av-moderation/actions';
 import { shouldShowModeratedNotification } from '../../av-moderation/functions';
+import { isModerationNotificationDisplayed } from '../../notifications';
 
 import {
     SET_AUDIO_MUTED,
 
         // check for A/V Moderation when trying to unmute
         if (!muted && shouldShowModeratedNotification(MEDIA_TYPE.VIDEO, state)) {
-            ensureTrack && dispatch(showModeratedNotification(MEDIA_TYPE.VIDEO));
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.VIDEO, state)) {
+                ensureTrack && dispatch(showModeratedNotification(MEDIA_TYPE.VIDEO));
+            }
 
             return;
         }

react/features/base/participants/actions.js

  * @returns {Promise}
  */
 export function participantMutedUs(participant, track) {
-    return dispatch => {
+    return (dispatch, getState) => {
         if (!participant) {
             return;
         }
         const isAudio = track.isAudioTrack();
 
         dispatch(showNotification({
-            titleKey: isAudio ? 'notify.mutedRemotelyTitle' : 'notify.videoMutedRemotelyTitle'
+            titleKey: isAudio ? 'notify.mutedRemotelyTitle' : 'notify.videoMutedRemotelyTitle',
+            titleArguments: {
+                moderator: getParticipantDisplayName(getState, participant.getId())
+            }
         }));
     };
 }

react/features/base/tracks/middleware.js

 import UIEvents from '../../../../service/UI/UIEvents';
 import { showModeratedNotification } from '../../av-moderation/actions';
 import { shouldShowModeratedNotification } from '../../av-moderation/functions';
-import { hideNotification } from '../../notifications';
+import { hideNotification, isModerationNotificationDisplayed } from '../../notifications';
 import { isPrejoinPageVisible } from '../../prejoin/functions';
 import { getAvailableDevices } from '../devices/actions';
 import {
             // check for A/V Moderation when trying to start screen sharing
             if ((action.enabled || action.enabled === undefined)
                 && shouldShowModeratedNotification(MEDIA_TYPE.VIDEO, store.getState())) {
-                store.dispatch(showModeratedNotification(MEDIA_TYPE.PRESENTER));
+                if (!isModerationNotificationDisplayed(MEDIA_TYPE.PRESENTER, store.getState())) {
+                    store.dispatch(showModeratedNotification(MEDIA_TYPE.PRESENTER));
+                }
 
                 return;
             }

react/features/notifications/functions.js

 // @flow
 
+import { MODERATION_NOTIFICATIONS } from '../av-moderation/constants';
+import { MEDIA_TYPE } from '../base/media';
 import { toState } from '../base/redux';
 
 declare var interfaceConfig: Object;
 export function joinLeaveNotificationsDisabled() {
     return Boolean(typeof interfaceConfig !== 'undefined' && interfaceConfig?.DISABLE_JOIN_LEAVE_NOTIFICATIONS);
 }
+
+/**
+ * Returns whether or not the moderation notification for the given type is displayed.
+ *
+ * @param {MEDIA_TYPE} mediaType - The media type to check.
+ * @param {Object | Function} stateful - The redux store state.
+ * @returns {boolean}
+ */
+export function isModerationNotificationDisplayed(mediaType: MEDIA_TYPE, stateful: Object | Function) {
+    const state = toState(stateful);
+
+    const { notifications } = state['features/notifications'];
+
+    return Boolean(notifications.find(n => n.uid === MODERATION_NOTIFICATIONS[mediaType]));
+}

react/features/participants-pane/components/FooterContextMenu.js

         },
         text: {
             color: '#C2C2C2',
-            padding: '10px 16px 10px 52px'
+            padding: '10px 16px'
         },
         paddedAction: {
-            marginLeft: '36px;'
+            marginLeft: '36px'
         }
     };
 });

react/features/participants-pane/components/web/MeetingParticipantItem.js

 // @flow
 
-import React from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 
+import { JitsiTrackEvents } from '../../../base/lib-jitsi-meet';
+import { MEDIA_TYPE } from '../../../base/media';
 import {
     getLocalParticipant,
     getParticipantByIdOrUndefined,
-    getParticipantDisplayName
+    getParticipantDisplayName,
+    isParticipantModerator
 } from '../../../base/participants';
 import { connect } from '../../../base/redux';
-import { isParticipantAudioMuted, isParticipantVideoMuted } from '../../../base/tracks';
-import { ACTION_TRIGGER, type MediaState } from '../../constants';
+import {
+    getLocalAudioTrack,
+    getTrackByMediaTypeAndParticipant,
+    isParticipantAudioMuted,
+    isParticipantVideoMuted
+} from '../../../base/tracks';
+import { ACTION_TRIGGER, type MediaState, MEDIA_STATE } from '../../constants';
 import {
     getParticipantAudioMediaState,
     getParticipantVideoMediaState,
      */
     _audioMediaState: MediaState,
 
+    /**
+     * The audio track related to the participant.
+     */
+    _audioTrack: ?Object,
+
     /**
      * Media state for video.
      */
  */
 function MeetingParticipantItem({
     _audioMediaState,
+    _audioTrack,
     _videoMediaState,
     _displayName,
     _local,
     participantActionEllipsisLabel,
     youText
 }: Props) {
+
+    const [ hasAudioLevels, setHasAudioLevel ] = useState(false);
+    const [ registeredEvent, setRegisteredEvent ] = useState(false);
+
+    const _updateAudioLevel = useCallback(level => {
+        const audioLevel = typeof level === 'number' && !isNaN(level)
+            ? level : 0;
+
+        setHasAudioLevel(audioLevel > 0.009);
+    }, []);
+
+    useEffect(() => {
+        if (_audioTrack && !registeredEvent) {
+            const { jitsiTrack } = _audioTrack;
+
+            if (jitsiTrack) {
+                jitsiTrack.on(JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED, _updateAudioLevel);
+                setRegisteredEvent(true);
+            }
+        }
+
+        return () => {
+            if (_audioTrack && registeredEvent) {
+                const { jitsiTrack } = _audioTrack;
+
+                jitsiTrack && jitsiTrack.off(JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED, _updateAudioLevel);
+            }
+        };
+    }, [ _audioTrack ]);
+
+    const audioMediaState = _audioMediaState === MEDIA_STATE.UNMUTED && hasAudioLevels
+        ? MEDIA_STATE.DOMINANT_SPEAKER : _audioMediaState;
+
     return (
         <ParticipantItem
             actionsTrigger = { ACTION_TRIGGER.HOVER }
-            audioMediaState = { _audioMediaState }
+            audioMediaState = { audioMediaState }
             displayName = { _displayName }
             isHighlighted = { isHighlighted }
+            isModerator = { isParticipantModerator(_participant) }
             local = { _local }
             onLeave = { onLeave }
             openDrawerForParticipant = { openDrawerForParticipant }
                     <ParticipantActionEllipsis
                         aria-label = { participantActionEllipsisLabel }
                         onClick = { onContextMenu } />
-                 </>
+                </>
             }
 
             {!overflowDrawer && _localVideoOwner && _participant?.isFakeParticipant && (
     const _videoMediaState = getParticipantVideoMediaState(participant, _isVideoMuted, state);
     const _quickActionButtonType = getQuickActionButtonType(participant, _isAudioMuted, state);
 
+    const tracks = state['features/base/tracks'];
+    const _audioTrack = participantID === localParticipantId
+        ? getLocalAudioTrack(tracks) : getTrackByMediaTypeAndParticipant(tracks, MEDIA_TYPE.AUDIO, participantID);
+
     return {
         _audioMediaState,
+        _audioTrack,
         _videoMediaState,
         _displayName: getParticipantDisplayName(state, participant?.id),
         _local: Boolean(participant?.local),

react/features/participants-pane/components/web/ParticipantItem.js

 import React, { type Node, useCallback } from 'react';
 
 import { Avatar } from '../../../base/avatar';
+import { translate } from '../../../base/i18n';
 import {
     ACTION_TRIGGER,
     AudioStateIcons,
 
 import { RaisedHandIndicator } from './RaisedHandIndicator';
 import {
+    ModeratorLabel,
     ParticipantActionsHover,
     ParticipantActionsPermanent,
     ParticipantContainer,
     ParticipantContent,
+    ParticipantDetailsContainer,
     ParticipantName,
     ParticipantNameContainer,
     ParticipantStates
      */
     isHighlighted?: boolean,
 
+    /**
+     * Whether or not the participant is a moderator.
+     */
+    isModerator: boolean,
+
     /**
      * True if the participant is local.
      */
      */
     videoMediaState: MediaState,
 
+    /**
+     * Invoked to obtain translated strings.
+     */
+    t: Function,
+
     /**
      * The translated "you" text.
      */
  * @param {Props} props - The props of the component.
  * @returns {ReactNode}
  */
-export default function ParticipantItem({
+function ParticipantItem({
     children,
     isHighlighted,
+    isModerator,
     onLeave,
     actionsTrigger = ACTION_TRIGGER.HOVER,
     audioMediaState = MEDIA_STATE.NONE,
     openDrawerForParticipant,
     overflowDrawer,
     raisedHand,
+    t,
     youText
 }: Props) {
     const ParticipantActions = Actions[actionsTrigger];
                 participantId = { participantID }
                 size = { 32 } />
             <ParticipantContent>
-                <ParticipantNameContainer>
-                    <ParticipantName>
-                        { displayName }
-                    </ParticipantName>
-                    { local ? <span>&nbsp;({ youText })</span> : null }
-                </ParticipantNameContainer>
+                <ParticipantDetailsContainer>
+                    <ParticipantNameContainer>
+                        <ParticipantName>
+                            { displayName }
+                        </ParticipantName>
+                        { local ? <span>&nbsp;({ youText })</span> : null }
+                    </ParticipantNameContainer>
+                    {isModerator && <ModeratorLabel>
+                        {t('videothumbnail.moderator')}
+                    </ModeratorLabel>}
+                </ParticipantDetailsContainer>
                 { !local && <ParticipantActions children = { children } /> }
                 <ParticipantStates>
                     { raisedHand && <RaisedHandIndicator /> }
         </ParticipantContainer>
     );
 }
+
+export default translate(ParticipantItem);

react/features/participants-pane/components/web/styled.js

   color: white;
   display: flex;
   font-size: 13px;
+  font-weight: normal;
   height: ${props => props.theme.participantItemHeight}px;
   margin: 0 -${props => props.theme.panePadding}px;
   padding-left: ${props => props.theme.panePadding}px;
 `;
 
 export const ParticipantNameContainer = styled.div`
+  display: flex;
+  flex: 1;
+  overflow: hidden;
+`;
+
+export const ModeratorLabel = styled.div`
+  font-size: 12px;
+  line-height: 16px;
+  color: #858585;
+`;
+
+export const ParticipantDetailsContainer = styled.div`
   display: flex;
   flex: 1;
   margin-right: 8px;
   overflow: hidden;
+  flex-direction: column;
+  justify-content: flex-start;
 `;
 
 export const RaisedHandIndicatorBackground = styled.div`

react/features/video-menu/actions.any.js

     getRemoteParticipants,
     muteRemoteParticipant
 } from '../base/participants';
+import { isModerationNotificationDisplayed } from '../notifications';
 
 declare var APP: Object;
 
 
         // check for A/V Moderation when trying to unmute
         if (!enable && shouldShowModeratedNotification(MEDIA_TYPE.AUDIO, getState())) {
-            dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.AUDIO, getState())) {
+                dispatch(showModeratedNotification(MEDIA_TYPE.AUDIO));
+            }
 
             return;
         }