import { EventEmitter } from 'events'; import { VAD_SCORE_PUBLISHED, VAD_TALK_WHILE_MUTED } from './DetectionEvents'; import { getLogger } from 'jitsi-meet-logger'; import TrackVADEmitter from '../detection/TrackVADEmitter'; import * as JitsiConferenceEvents from '../../JitsiConferenceEvents'; const logger = getLogger(__filename); /** * The value which we use to say, every sound over this threshold * is talking on the mic. * @type {number} */ const VAD_DETECT_THRESHOLD = 0.7; /** * Detect user trying to speek while is locally muted and fires an event. */ export default class VADTalkMutedDetection extends EventEmitter { /** * Creates TalkMutedDetection * @param conference the JitsiConference instance that created us. * @param callback the callback to call when detected that the local user is * talking while her microphone is muted. * @constructor */ constructor(conference, vadProcessor) { super(); logger.info('[ADBG] Created VADTalkMutedDetection.'); /** * The indicator which determines whether callback has been * invoked for the current local audio track of conference so * that it is invoked once only. * * @private */ this._eventFired = false; this._vadProcessor = vadProcessor; this._vadEmitter = null; this._processing = false; this._scoreArray = []; conference.on(JitsiConferenceEvents.TRACK_MUTE_CHANGED, this._trackMuteChanged.bind(this)); conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this)); // TODO do we need to handle the case where tracks are removed, make sure this cleans up properly so // we don't have any leeks i.e. stale JitsiLocalTracks } /* eslint-disable max-params */ /** * Receives audio level events for all send and receive streams. * * @param {TraceablePeerConnection} pc - WebRTC PeerConnection object of the * @param {number} ssrc - The synchronization source identifier (SSRC) of * the endpoint/participant/stream being reported. * @param {number} audioLevel - The audio level of ssrc. * @param {boolean} isLocal - true if ssrc represents a * local/send stream or false for a remote/receive stream. */ _processVADScore(vadScore) { // We are interested in the local audio stream only and if event is not // sent yet. if (this._eventFired) { return; } if (this.audioTrack.isMuted()) { if (vadScore.score > 0.8 && !this._processing) { this._processing = true; this._processTimeout = setTimeout(() => { let scoreSum = 0; for (const score of this._scoreArray) { scoreSum += score; } const avgScore = scoreSum / this._scoreArray.length; if (avgScore > VAD_DETECT_THRESHOLD) { this.emit(VAD_TALK_WHILE_MUTED, ''); this._eventFired = true; console.log('[ADBG] Triggered array size: ', this._scoreArray, '. AVG: ', avgScore); } else { console.log('[ADBG] Not triggered array size: ', this._scoreArray, '. AVG: ', avgScore); } this._scoreArray = []; this._processing = false; }, 1500); } if (this._processing) { this._scoreArray.push(vadScore.score); } } } /* eslint-enable max-params */ /** * Determines whether a specific {@link JitsiTrack} represents a local audio * track. * * @param {JitsiTrack} track - The JitsiTrack to be checked whether * it represents a local audio track. * @private * @return {boolean} - true if the specified track * represents a local audio track; otherwise, false. */ _isLocalAudioTrack(track) { return track.isAudioTrack() && track.isLocal(); } /** * Notifies this TalkMutedDetection that a {@link JitsiTrack} was * added to the associated {@link JitsiConference}. Looks for the local * audio track only. * * @param {JitsiTrack} track - The added JitsiTrack. * @private */ _trackAdded(track) { if (this._isLocalAudioTrack(track)) { logger.info('[ADBG] Audio track added.'); this.audioTrack = track; this._vadProcessor().then(vadProcessor => { TrackVADEmitter.create(track.getDeviceId(), 4096, vadProcessor).then(vadEmitter => { if (this._vadEmitter) { this._vadEmitter.destroy(); } this._vadEmitter = vadEmitter; this._vadEmitter.on(VAD_SCORE_PUBLISHED, this._processVADScore.bind(this)); this._eventFired = false; this._processing = false; clearTimeout(this._processTimeout); }); }); } } /** * Notifies this TalkMutedDetection that the mute state of a * {@link JitsiTrack} has changed. Looks for the local audio track only. * * @param {JitsiTrack} track - The JitsiTrack whose mute state has * changed. * @private */ _trackMuteChanged(track) { if (this._isLocalAudioTrack(track) && track.isMuted()) { logger.info('[ADBG] Audio track muted.'); this._eventFired = false; clearTimeout(this._processTimeout); } } }