Noisy microphone detection (#1013)

* feat: implement vad noise detection

* start detection services on initial track add

* additional refactor / calibrate noise detection

* stop detection services on track remove

* address code review

* address code review p2

JitsiConference.js

@@ -17,6 +17,8 @@ import P2PDominantSpeakerDetection from './modules/detection/P2PDominantSpeakerD
 import RTC from './modules/RTC/RTC';
 import TalkMutedDetection from './modules/detection/TalkMutedDetection';
 import VADTalkMutedDetection from './modules/detection/VADTalkMutedDetection';
+import VADNoiseDetection from './modules/detection/VADNoiseDetection';
+import VADAudioAnalyser from './modules/detection/VADAudioAnalyser';
 import * as DetectionEvents from './modules/detection/DetectionEvents';
 import NoAudioSignalDetection from './modules/detection/NoAudioSignalDetection';
 import browser from './modules/browser';
@@ -375,14 +377,24 @@ JitsiConference.prototype._init = function(options = {}) {
     this.eventManager.setupStatisticsListeners();
 
     if (config.enableTalkWhileMuted) {
+
         // If VAD processor factory method is provided uses VAD based detection, otherwise fallback to audio level
         // based detection.
         if (config.createVADProcessor) {
             logger.info('Using VAD detection for generating talk while muted events');
-            this._talkWhileMutedDetection = new VADTalkMutedDetection(this, config.createVADProcessor);
-            this._talkWhileMutedDetection.on(DetectionEvents.VAD_TALK_WHILE_MUTED, () =>
+
+            if (!this._audioAnalyser) {
+                this._audioAnalyser = new VADAudioAnalyser(this, config.createVADProcessor);
+            }
+
+            const vadTalkMutedDetection = new VADTalkMutedDetection();
+
+            vadTalkMutedDetection.on(DetectionEvents.VAD_TALK_WHILE_MUTED, () =>
                 this.eventEmitter.emit(JitsiConferenceEvents.TALK_WHILE_MUTED));
 
+            this._audioAnalyser.addVADDetectionService(vadTalkMutedDetection);
+
+
         } else {
             logger.info('Using audio level based detection for generating talk while muted events');
             this._talkWhileMutedDetection = new TalkMutedDetection(
@@ -390,6 +402,23 @@ JitsiConference.prototype._init = function(options = {}) {
         }
     }
 
+    if (config.enableNoisyMicDetection) {
+        if (config.createVADProcessor) {
+            if (!this._audioAnalyser) {
+                this._audioAnalyser = new VADAudioAnalyser(this, config.createVADProcessor);
+            }
+
+            const vadNoiseDetection = new VADNoiseDetection();
+
+            vadNoiseDetection.on(DetectionEvents.VAD_NOISY_DEVICE, () =>
+                this.eventEmitter.emit(JitsiConferenceEvents.NOISY_MIC));
+
+            this._audioAnalyser.addVADDetectionService(vadNoiseDetection);
+        } else {
+            logger.warn('No VAD Processor was provided. Noisy microphone detection service was not initialized!');
+        }
+    }
+
     // Generates events based on no audio input detector.
     if (config.enableNoAudioDetection) {
         this._noAudioSignalDetection = new NoAudioSignalDetection(this);

JitsiConferenceEvents.js

@@ -2,6 +2,12 @@
  * The events for the conference.
  */
 
+/**
+ * Event indicates that the current conference audio input switched between audio
+ * input states,i.e. with or without audio input.
+ */
+export const AUDIO_INPUT_STATE_CHANGE = 'conference.audio_input_state_changed';
+
 /**
  * Indicates that authentication status changed.
  */
@@ -147,6 +153,16 @@ export const SERVER_REGION_CHANGED = 'conference.server_region_changed';
  */
 export const MESSAGE_RECEIVED = 'conference.messageReceived';
 
+/**
+ * Event indicates that the current selected input device has no signal
+ */
+export const NO_AUDIO_INPUT = 'conference.no_audio_input';
+
+/**
+ * Event indicates that the current microphone used by the conference is noisy.
+ */
+export const NOISY_MIC = 'conference.noisy_mic';
+
 /**
  * New private text message was received.
  */
@@ -250,17 +266,6 @@ export const SUSPEND_DETECTED = 'conference.suspendDetected';
  */
 export const TALK_WHILE_MUTED = 'conference.talk_while_muted';
 
-/**
- * Event indicates that the current selected input device has no signal
- */
-export const NO_AUDIO_INPUT = 'conference.no_audio_input';
-
-/**
- * Event indicates that the current conference audio input switched between audio
- * input states,i.e. with or without audio input.
- */
-export const AUDIO_INPUT_STATE_CHANGE = 'conference.audio_input_state_changed';
-
 /**
  * A new media track was added to the conference. The event provides the
  * following parameters to its listeners:

doc/API.md

@@ -137,6 +137,7 @@ JitsiMeetJS.setLogLevel(JitsiMeetJS.logLevels.ERROR);
         - TALK_WHILE_MUTED - notifies that a local user is talking while having the microphone muted.
         - NO_AUDIO_INPUT - notifies that the current selected input device has no signal.
         - AUDIO_INPUT_STATE_CHANGE - notifies that the current conference audio input switched between audio input states i.e. with or without audio input.
+        - NOISY_MIC - notifies that the current microphone used by the conference is noisy.
 
     2. connection
         - CONNECTION_FAILED - indicates that the server connection failed.

modules/detection/DetectionEvents.js

@@ -1,9 +1,10 @@
-/** Event triggered by NoAudioSignalDetector when the local audio device associated with a JitsiConference goes silent
- * for a period of time, meaning that the device is either broken or hardware/software muted.
+/**
+ * Event triggered by a audio detector indicating that its active state has changed from active to inactive or vice
+ * versa.
  * @event
- * @type {void}
+ * @type {boolean} - true when service has changed to active false otherwise.
  */
-export const NO_AUDIO_INPUT = 'no_audio_input_detected';
+export const DETECTOR_STATE_CHANGE = 'detector_state_change';
 
 /** Event triggered by {@link NoAudioSignalDetector} when the local audio device associated with a JitsiConference
  * starts receiving audio levels with the value of 0 meaning no audio is being captured on that device, or when
@@ -13,6 +14,20 @@ export const NO_AUDIO_INPUT = 'no_audio_input_detected';
  */
 export const AUDIO_INPUT_STATE_CHANGE = 'audio_input_state_changed';
 
+/** Event triggered by NoAudioSignalDetector when the local audio device associated with a JitsiConference goes silent
+ * for a period of time, meaning that the device is either broken or hardware/software muted.
+ * @event
+ * @type {void}
+ */
+export const NO_AUDIO_INPUT = 'no_audio_input_detected';
+
+/**
+ *  Event generated by {@link VADNoiseDetection} when the tracked device is considered noisy.
+ *  @event
+ *  @type {Object}
+ */
+export const VAD_NOISY_DEVICE = 'detection.vad_noise_device';
+
 /**
  * Event generated by VADReportingService when if finishes creating a VAD report for the monitored devices.
  * The generated objects are of type Array<Object>, one score for each monitored device.
@@ -31,6 +46,7 @@ export const VAD_REPORT_PUBLISHED = 'vad-report-published';
  * @type {Object}
  * @property {Date}   timestamp - Exact time at which processed PCM sample was generated.
  * @property {number} score - VAD score on a scale from 0 to 1 (i.e. 0.7)
+ * @property {Float32Array} pcmData - Raw PCM data with which the VAD score was calculated.
  * @property {string} deviceId - Device id of the associated track.
  */
 export const VAD_SCORE_PUBLISHED = 'detection.vad_score_published';

modules/detection/TrackVADEmitter.js

@@ -2,6 +2,7 @@ import EventEmitter from 'events';
 
 import RTC from '../RTC/RTC';
 
+import { createAudioContext } from './webaudio/WebAudioUtils';
 import { VAD_SCORE_PUBLISHED } from './DetectionEvents';
 
 /**
@@ -48,7 +49,7 @@ export default class TrackVADEmitter extends EventEmitter {
         /**
          * The AudioContext instance with the preferred sample frequency.
          */
-        this._audioContext = new AudioContext({ sampleRate: vadProcessor.getRequiredPCMFrequency() });
+        this._audioContext = createAudioContext({ sampleRate: vadProcessor.getRequiredPCMFrequency() });
 
         /**
          * PCM Sample size expected by the VAD Processor instance. We cache it here as this value is used extensively,
@@ -97,7 +98,7 @@ export default class TrackVADEmitter extends EventEmitter {
     /**
      * Sets up the audio graph in the AudioContext.
      *
-     * @returns {Promise<void>}
+     * @returns {void}
      */
     _initializeAudioContext() {
         this._audioSource = this._audioContext.createMediaStreamSource(this._localTrack.stream);
@@ -132,11 +133,14 @@ export default class TrackVADEmitter extends EventEmitter {
 
         for (; i + this._vadSampleSize < completeInData.length; i += this._vadSampleSize) {
             const pcmSample = completeInData.slice(i, i + this._vadSampleSize);
-            const vadScore = this._vadProcessor.calculateAudioFrameVAD(pcmSample);
+
+            // The VAD processor might change the values inside the array so we make a copy.
+            const vadScore = this._vadProcessor.calculateAudioFrameVAD(pcmSample.slice());
 
             this.emit(VAD_SCORE_PUBLISHED, {
                 timestamp: sampleTimestamp,
                 score: vadScore,
+                pcmData: pcmSample,
                 deviceId: this._localTrack.getDeviceId()
             });
         }

modules/detection/VADAudioAnalyser.js

@@ -0,0 +1,219 @@
+import { EventEmitter } from 'events';
+import { getLogger } from 'jitsi-meet-logger';
+
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
+
+import { VAD_SCORE_PUBLISHED, DETECTOR_STATE_CHANGE } from './DetectionEvents';
+import TrackVADEmitter from './TrackVADEmitter';
+
+const logger = getLogger(__filename);
+
+/**
+ * Sample rate of TrackVADEmitter, it defines how many audio samples are processed at a time.
+ * @type {number}
+ */
+const VAD_EMITTER_SAMPLE_RATE = 4096;
+
+/**
+ * Connects a TrackVADEmitter to the target conference local audio track and manages various services that use
+ * the data to produce audio analytics (VADTalkMutedDetection and VADNoiseDetection).
+ */
+export default class VADAudioAnalyser extends EventEmitter {
+    /**
+     * Creates <tt>VADAudioAnalyser</tt>
+     * @param {JitsiConference} conference - JitsiConference instance that created us.
+     * @param {Object} createVADProcessor - Function that creates a Voice activity detection processor. The processor
+     * needs to implement the following functions:
+     * - <tt>getSampleLength()</tt> - Returns the sample size accepted by getSampleLength.
+     * - <tt>getRequiredPCMFrequency()</tt> - Returns the PCM frequency at which the processor operates.
+     * - <tt>calculateAudioFrameVAD(pcmSample)</tt> - Process a 32 float pcm sample of getSampleLength size.
+     * @constructor
+     */
+    constructor(conference, createVADProcessor) {
+        super();
+
+        /**
+         * Member function that instantiates a VAD processor.
+         */
+        this._createVADProcessor = createVADProcessor;
+
+        /**
+         * Current {@link TrackVADEmitter}. VAD Emitter uses a {@link JitsiLocalTrack} and VAD processor to generate
+         * period voice probability scores.
+         */
+        this._vadEmitter = null;
+
+        /**
+         * Current state of the _vadEmitter
+         */
+        this._isVADEmitterRunning = false;
+
+        /**
+         * Array of currently attached VAD processing services.
+         */
+        this._detectionServices = [];
+
+        /**
+         * Promise used to chain create and destroy operations associated with TRACK_ADDED and TRACK_REMOVED events
+         * coming from the conference.
+         * Because we have an async created component (VAD Processor) we need to make sure that it's initialized before
+         * we destroy it ( when changing the device for instance), or when we use it from an external point of entry
+         * i.e. (TRACK_MUTE_CHANGED event callback).
+         */
+        this._vadInitTracker = Promise.resolve();
+
+        /**
+         * Listens for {@link TrackVADEmitter} events and processes them.
+         */
+        this._processVADScore = this._processVADScore.bind(this);
+
+        conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this));
+        conference.on(JitsiConferenceEvents.TRACK_REMOVED, this._trackRemoved.bind(this));
+        conference.on(JitsiConferenceEvents.TRACK_MUTE_CHANGED, this._trackMuteChanged.bind(this));
+    }
+
+    /**
+     * Attach a VAD detector service to the analyser and handle it's state changes.
+     *
+     * @param {Object} vadTMDetector
+     */
+    addVADDetectionService(vadService) {
+        this._detectionServices.push(vadService);
+        vadService.on(DETECTOR_STATE_CHANGE, () => {
+            // When the state of a detector changes check if there are any active detectors attached so that
+            // the _vadEmitter doesn't run needlessly.
+            const activeDetector = this._detectionServices.filter(detector => detector.isActive() === true);
+
+            // If there are no active detectors running and the vadEmitter is running then stop the emitter as it is
+            // uses a considerable amount of CPU. Otherwise start the service if it's stopped and there is a detector
+            // that needs it.
+            if (!activeDetector.length && this._isVADEmitterRunning) {
+                this._stopVADEmitter();
+            } else if (!this._isVADEmitterRunning) {
+                this._startVADEmitter();
+            }
+        });
+    }
+
+    /**
+     * Start the {@link TrackVADEmitter} and attach the event listener.
+     * @returns {void}
+     */
+    _startVADEmitter() {
+        this._vadEmitter.on(VAD_SCORE_PUBLISHED, this._processVADScore);
+        this._vadEmitter.start();
+        this._isVADEmitterRunning = true;
+    }
+
+    /**
+     * Stop the {@link TrackVADEmitter} and detach the event listener.
+     * @returns {void}
+     */
+    _stopVADEmitter() {
+        this._vadEmitter.removeListener(VAD_SCORE_PUBLISHED, this._processVADScore);
+        this._vadEmitter.stop();
+        this._isVADEmitterRunning = false;
+    }
+
+    /**
+     * Listens for {@link TrackVADEmitter} events and directs them to attached services as needed.
+     *
+     * @param {Object} vadScore -VAD score emitted by {@link TrackVADEmitter}
+     * @param {Date}   vadScore.timestamp - Exact time at which processed PCM sample was generated.
+     * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7)
+     * @param {Float32Array} pcmData - Raw PCM data with which the VAD score was calculated.
+     * @param {string} vadScore.deviceId - Device id of the associated track.
+     * @listens VAD_SCORE_PUBLISHED
+     */
+    _processVADScore(vadScore) {
+        for (const detector of this._detectionServices) {
+            detector.processVADScore(vadScore);
+        }
+    }
+
+    /**
+     * Change the isMuted state of all attached detection services.
+     *
+     * @param {boolean} isMuted
+     */
+    _changeDetectorsMuteState(isMuted) {
+        for (const detector of this._detectionServices) {
+            detector.changeMuteState(isMuted);
+        }
+    }
+
+    /**
+     * Notifies the detector that a track was added to the associated {@link JitsiConference}.
+     * Only take into account local audio tracks.
+     * @param {JitsiTrack} track - The added track.
+     * @returns {void}
+     * @listens TRACK_ADDED
+     */
+    _trackAdded(track) {
+        if (track.isLocalAudioTrack()) {
+            // Keep a track promise so we take into account successive TRACK_ADD events being generated so that we
+            // destroy/create the processing context in the proper order.
+            this._vadInitTracker = this._vadInitTracker.then(() => this._createVADProcessor())
+                .then(vadProcessor =>
+                    TrackVADEmitter.create(track.getDeviceId(), VAD_EMITTER_SAMPLE_RATE, vadProcessor)
+                )
+                .then(vadEmitter => {
+                    logger.debug('Created VAD emitter for track: ', track.getTrackLabel());
+
+                    this._vadEmitter = vadEmitter;
+
+                    // Iterate through the detection services and set their appropriate mute state, depending on
+                    // service this will trigger a DETECTOR_STATE_CHANGE which in turn might start the _vadEmitter.
+                    this._changeDetectorsMuteState(track.isMuted());
+                });
+        }
+    }
+
+    /**
+     * Notifies the detector that the mute state of a {@link JitsiConference} track has changed. Only takes into account
+     * local audio tracks.
+     * @param {JitsiTrack} track - The track whose mute state has changed.
+     * @returns {void}
+     * @listens TRACK_MUTE_CHANGED
+     */
+    _trackMuteChanged(track) {
+        if (track.isLocalAudioTrack()) {
+            // On a mute toggle reset the state.
+            this._vadInitTracker = this._vadInitTracker.then(() => {
+                // Set mute status for the attached detection services.
+                this._changeDetectorsMuteState(track.isMuted());
+            });
+        }
+    }
+
+    /**
+     * Notifies the detector that a track associated with the {@link JitsiConference} was removed. Only takes into
+     * account local audio tracks. Cleans up resources associated with the track and resets the processing context.
+     *
+     * @param {JitsiTrack} track - The removed track.
+     * @returns {void}
+     * @listens TRACK_REMOVED
+     */
+    _trackRemoved(track) {
+        if (track.isLocalAudioTrack()) {
+            // Use the promise to make sure operations are in sequence.
+            this._vadInitTracker = this._vadInitTracker.then(() => {
+                logger.debug('Removing track from VAD detection - ', track.getTrackLabel());
+
+                // Track was removed, clean up and set appropriate states.
+                if (this._vadEmitter) {
+                    this._stopVADEmitter();
+                    this._vadEmitter.destroy();
+                    this._vadEmitter = null;
+                }
+
+                // Reset state of detectors when active track is removed.
+                for (const detector of this._detectionServices) {
+                    detector.reset();
+                }
+            });
+        }
+    }
+
+
+}

modules/detection/VADNoiseDetection.js

@@ -0,0 +1,187 @@
+import { EventEmitter } from 'events';
+
+import { calculateAverage, filterPositiveValues } from '../util/MathUtil';
+
+import { VAD_NOISY_DEVICE, DETECTOR_STATE_CHANGE } from './DetectionEvents';
+
+/**
+ * The average value VAD needs to be under over a period of time to be considered noise.
+ * @type {number}
+ */
+const VAD_NOISE_AVG_THRESHOLD = 0.2;
+
+/**
+ * The average values that audio input need to be over to be considered loud.
+ * @type {number}
+ */
+const NOISY_AUDIO_LEVEL_THRESHOLD = 0.040;
+
+/**
+ * The value that a VAD score needs to be under in order for processing to begin.
+ * @type {number}
+ */
+const VAD_SCORE_TRIGGER = 0.2;
+
+/**
+ * The value that a VAD score needs to be under in order for processing to begin.
+ * @type {number}
+ */
+const AUDIO_LEVEL_SCORE_TRIGGER = 0.020;
+
+/**
+ * Time span over which we calculate an average score used to determine if we trigger the event.
+ * @type {number}
+ */
+const PROCESS_TIME_FRAME_SPAN_MS = 1500;
+
+/**
+ * Detect if provided VAD score and PCM data is considered noise.
+ */
+export default class VADNoiseDetection extends EventEmitter {
+    /**
+     * Creates <tt>VADNoiseDetection</tt>
+     *
+     * @constructor
+     */
+    constructor() {
+        super();
+
+        /**
+         * Flag which denotes the current state of the detection service i.e.if there is already a processing operation
+         * ongoing.
+         */
+        this._processing = false;
+
+        /**
+         * Buffer that keeps the VAD scores for a period of time.
+         */
+        this._scoreArray = [];
+
+        /**
+         * Buffer that keeps audio level samples for a period of time.
+         */
+        this._audioLvlArray = [];
+
+        /**
+         * Current state of the service, if it's not active no processing will occur.
+         */
+        this._active = false;
+
+        this._calculateNoisyScore = this._calculateNoisyScore.bind(this);
+    }
+
+    /**
+     * Compute cumulative VAD score and PCM audio levels once the PROCESS_TIME_FRAME_SPAN_MS timeout has elapsed.
+     * If the score is above the set threshold fire the event.
+     * @returns {void}
+     * @fires VAD_NOISY_DEVICE
+     */
+    _calculateNoisyScore() {
+        const scoreAvg = calculateAverage(this._scoreArray);
+        const audioLevelAvg = calculateAverage(this._audioLvlArray);
+
+        if (scoreAvg < VAD_NOISE_AVG_THRESHOLD && audioLevelAvg > NOISY_AUDIO_LEVEL_THRESHOLD) {
+            this.emit(VAD_NOISY_DEVICE);
+
+            this._setActiveState(false);
+        }
+
+        // We reset the context in case a new process phase needs to be triggered.
+        this.reset();
+    }
+
+    /**
+     * Record the vad score and average volume in the appropriate buffers.
+     *
+     * @param {number} vadScore
+     * @param {number} avgAudioLvl - average audio level of the PCM sample associated with the VAD score.s
+     */
+    _recordValues(vadScore, avgAudioLvl) {
+        this._scoreArray.push(vadScore);
+        this._audioLvlArray.push(avgAudioLvl);
+    }
+
+    /**
+     * Set the active state of the detection service and notify any listeners.
+     *
+     * @param {boolean} active
+     * @fires DETECTOR_STATE_CHANGE
+     */
+    _setActiveState(active) {
+        this._active = active;
+        this.emit(DETECTOR_STATE_CHANGE, this._active);
+    }
+
+    /**
+     * Change the state according to the muted status of the tracked device.
+     *
+     * @param {boolean} isMuted - Is the device muted or not.
+     */
+    changeMuteState(isMuted) {
+        // This service only needs to run when the microphone is not muted.
+        this._setActiveState(!isMuted);
+        this.reset();
+    }
+
+    /**
+     * Check whether or not the service is active or not.
+     *
+     * @returns {boolean}
+     */
+    isActive() {
+        return this._active;
+    }
+
+    /**
+     * Reset the processing context, clear buffers, cancel the timeout trigger.
+     *
+     * @returns {void}
+     */
+    reset() {
+        this._processing = false;
+        this._scoreArray = [];
+        this._audioLvlArray = [];
+        clearTimeout(this._processTimeout);
+    }
+
+    /**
+     * Listens for {@link TrackVADEmitter} events and processes them.
+     *
+     * @param {Object} vadScore -VAD score emitted by {@link TrackVADEmitter}
+     * @param {Date}   vadScore.timestamp - Exact time at which processed PCM sample was generated.
+     * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7)
+     * @param {Float32Array} vadScore.pcmData - Raw PCM Data associated with the VAD score.
+     * @param {string} vadScore.deviceId - Device id of the associated track.
+     * @listens VAD_SCORE_PUBLISHED
+     */
+    processVADScore(vadScore) {
+        if (!this._active) {
+            return;
+        }
+
+        // There is a processing phase on going, add score to buffer array.
+        if (this._processing) {
+            // Filter and calculate sample average so we don't have to process one large array at a time.
+            const posAudioLevels = filterPositiveValues(vadScore.pcmData);
+
+            this._recordValues(vadScore.score, calculateAverage(posAudioLevels));
+
+            return;
+        }
+
+        // If the VAD score for the sample is low and audio level has a high enough level we can start listening for
+        // noise
+        if (vadScore.score < VAD_SCORE_TRIGGER) {
+            const posAudioLevels = filterPositiveValues(vadScore.pcmData);
+            const avgAudioLvl = calculateAverage(posAudioLevels);
+
+            if (avgAudioLvl > AUDIO_LEVEL_SCORE_TRIGGER) {
+                this._processing = true;
+                this._recordValues(vadScore.score, avgAudioLvl);
+
+                // Once the preset timeout executes the final score will be calculated.
+                this._processTimeout = setTimeout(this._calculateNoisyScore, PROCESS_TIME_FRAME_SPAN_MS);
+            }
+        }
+    }
+}

modules/detection/VADTalkMutedDetection.js

@@ -1,12 +1,9 @@
 import { EventEmitter } from 'events';
-import { getLogger } from 'jitsi-meet-logger';
 
-import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
+import { calculateAverage } from '../util/MathUtil';
 
-import { VAD_SCORE_PUBLISHED, VAD_TALK_WHILE_MUTED } from './DetectionEvents';
-import TrackVADEmitter from './TrackVADEmitter';
+import { VAD_TALK_WHILE_MUTED, DETECTOR_STATE_CHANGE } from './DetectionEvents';
 
-const logger = getLogger(__filename);
 
 /**
  * The threshold which the average VAD values for a span of time needs to exceed to trigger an event.
@@ -25,42 +22,24 @@ const VAD_VOICE_LEVEL = 0.9;
  * Sample rate of TrackVADEmitter, it defines how many audio samples are processed at a time.
  * @type {number}
  */
-const VAD_EMITTER_SAMPLE_RATE = 4096;
 
 /**
  * Time span over which we calculate an average score used to determine if we trigger the event.
  * @type {number}
  */
-const PROCESS_TIME_FRAME_SPAN_MS = 1500;
+const PROCESS_TIME_FRAME_SPAN_MS = 700;
 
 /**
- * Detect user trying to speak while is locally muted and fires an event using a TrackVADEmitter.
+ * Detect if provided VAD score which is generated on a muted device is voice and fires an event.
  */
 export default class VADTalkMutedDetection extends EventEmitter {
     /**
      * Creates <tt>VADTalkMutedDetection</tt>
-     * @param {JitsiConference} conference - JitsiConference instance that created us.
-     * @param {Object} createVADProcessor - Function that creates a Voice activity detection processor. The processor
-     * needs to implement the following functions:
-     * - <tt>getSampleLength()</tt> - Returns the sample size accepted by getSampleLength.
-     * - <tt>getRequiredPCMFrequency()</tt> - Returns the PCM frequency at which the processor operates.
-     * - <tt>calculateAudioFrameVAD(pcmSample)</tt> - Process a 32 float pcm sample of getSampleLength size.
      * @constructor
      */
-    constructor(conference, createVADProcessor) {
+    constructor() {
         super();
 
-        /**
-         * Member function that instantiates a VAD processor.
-         */
-        this._createVADProcessor = createVADProcessor;
-
-        /**
-         * Current {@link TrackVADEmitter}. VAD Emitter uses a {@link JitsiLocalTrack} and VAD processor to generate
-         * period voice probability scores.
-         */
-        this._vadEmitter = null;
-
         /**
          * Flag which denotes the current state of the detection service i.e.if there is already a processing operation
          * ongoing.
@@ -73,73 +52,62 @@ export default class VADTalkMutedDetection extends EventEmitter {
         this._scoreArray = [];
 
         /**
-         * Promise used to chain create and destroy operations associated with TRACK_ADDED and TRACK_REMOVED events
-         * coming from the conference.
-         * Because we have an async created component (VAD Processor) we need to make sure that it's initialized before
-         * we destroy it ( when changing the device for instance), or when we use it from an external point of entry
-         * i.e. (TRACK_MUTE_CHANGED event callback).
+         * Current mute state of the audio track being monitored.
          */
-        this._vadInitTracker = Promise.resolve();
+        this._active = false;
 
-        /**
-         * Listens for {@link TrackVADEmitter} events and processes them.
-         */
-        this._processVADScore = this._processVADScore.bind(this);
-
-        /**
-         * {@link JitsiConference} bindings.
-         */
-        conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this));
-        conference.on(JitsiConferenceEvents.TRACK_REMOVED, this._trackRemoved.bind(this));
-        conference.on(JitsiConferenceEvents.TRACK_MUTE_CHANGED, this._trackMuteChanged.bind(this));
+        this._calculateVADScore = this._calculateVADScore.bind(this);
     }
 
     /**
-     * Start the {@link TrackVADEmitter} and attach the event listener.
+     * Compute cumulative VAD score function called once the PROCESS_TIME_FRAME_SPAN_MS timeout has elapsed.
      * @returns {void}
+     * @fires VAD_TALK_WHILE_MUTED
      */
-    _startVADEmitter() {
-        this._vadEmitter.on(VAD_SCORE_PUBLISHED, this._processVADScore);
-        this._vadEmitter.start();
+    _calculateVADScore() {
+        const score = calculateAverage(this._scoreArray);
+
+        if (score > VAD_AVG_THRESHOLD) {
+            this.emit(VAD_TALK_WHILE_MUTED);
+
+            // Event was fired. Stop event emitter and remove listeners so no residue events kick off after this point
+            // and a single VAD_TALK_WHILE_MUTED is generated per mic muted state.
+            this._setActiveState(false);
+        }
+
+        // We reset the context in case a new process phase needs to be triggered.
+        this.reset();
     }
 
     /**
-     * Stop the {@link TrackVADEmitter} and detach the event listener.
-     * @returns {void}
+     * Set the active state of the detection service and notify any listeners.
+     *
+     * @param {boolean} active
+     * @fires DETECTOR_STATE_CHANGE
      */
-    _stopVADEmitter() {
-        this._vadEmitter.removeListener(VAD_SCORE_PUBLISHED, this._processVADScore);
-        this._vadEmitter.stop();
+    _setActiveState(active) {
+        this._active = active;
+        this.emit(DETECTOR_STATE_CHANGE, this._active);
     }
 
     /**
-     * Calculates the average value of a Float32Array.
+     * Change the state according to the muted status of the tracked device.
      *
-     * @param {Float32Array} scoreArray - Array of vad scores.
-     * @returns {number} - Score average.
+     * @param {boolean} isMuted - Is the device muted or not.
      */
-    _calculateAverage(scoreArray) {
-        return scoreArray.length > 0 ? scoreArray.reduce((a, b) => a + b) / scoreArray.length : 0;
+    changeMuteState(isMuted) {
+        // This service only needs to run when the microphone is muted.
+        this._setActiveState(isMuted);
+        this.reset();
     }
 
     /**
-     * Compute cumulative VAD score function called once the PROCESS_TIME_FRAME_SPAN_MS timeout has elapsed.
-     * @returns {void}
-     * @fires VAD_TALK_WHILE_MUTED
+     * Check whether or not the service is active or not.
+     *
+     * @returns {boolean}
      */
-    _calculateVADScore() {
-        const score = this._calculateAverage(this._scoreArray);
-
-        if (score > VAD_AVG_THRESHOLD) {
-            this.emit(VAD_TALK_WHILE_MUTED, {});
-
-            // Event was fired. Stop event emitter and remove listeners so no residue events kick off after this point
-            // and a single VAD_TALK_WHILE_MUTED is generated per mic muted state.
-            this._stopVADEmitter();
-        }
-
-        // We reset the context in case a new process phase needs to be triggered.
-        this._reset();
+    isActive() {
+        return this._active;
     }
 
     /**
@@ -151,106 +119,37 @@ export default class VADTalkMutedDetection extends EventEmitter {
      * @param {string} vadScore.deviceId - Device id of the associated track.
      * @listens VAD_SCORE_PUBLISHED
      */
-    _processVADScore(vadScore) {
-        // Because we remove all listeners on the vadEmitter once the main event is triggered,
-        // there is no need to check for rogue events.
-        if (vadScore.score > VAD_VOICE_LEVEL && !this._processing) {
-            this._processing = true;
-
-            // Start gathering VAD scores for the configured period of time.
-            this._processTimeout = setTimeout(this._calculateVADScore.bind(this), PROCESS_TIME_FRAME_SPAN_MS);
+    processVADScore(vadScore) {
+        if (!this._active) {
+            return;
         }
 
         // There is a processing phase on going, add score to buffer array.
         if (this._processing) {
             this._scoreArray.push(vadScore.score);
-        }
-    }
 
-    /**
-     * Reset the processing context, clear buffer, cancel the timeout trigger.
-     *
-     * @returns {void}
-     */
-    _reset() {
-        this._processing = false;
-        this._scoreArray = [];
-        clearTimeout(this._processTimeout);
-    }
-
-    /**
-     * Notifies the detector that a track was added to the associated {@link JitsiConference}.
-     * Only take into account local audio tracks.
-     * @param {JitsiTrack} track - The added track.
-     * @returns {void}
-     * @listens TRACK_ADDED
-     */
-    _trackAdded(track) {
-        if (track.isLocalAudioTrack()) {
-            // Keep a track promise so we take into account successive TRACK_ADD events being generated so that we
-            // destroy/create the processing context in the proper order.
-            this._vadInitTracker = this._vadInitTracker.then(() => this._createVADProcessor())
-                .then(vadProcessor =>
-                    TrackVADEmitter.create(track.getDeviceId(), VAD_EMITTER_SAMPLE_RATE, vadProcessor)
-                )
-                .then(vadEmitter => {
-                    logger.debug('Created VAD emitter for track: ', track.getTrackLabel());
-
-                    this._vadEmitter = vadEmitter;
-
-                    if (track.isMuted()) {
-                        this._startVADEmitter();
-                    }
-                });
+            return;
         }
-    }
 
-    /**
-     * Notifies the detector that the mute state of a {@link JitsiConference} track has changed. Only takes into account
-     * local audio tracks. In case the track was muted the detector starts the {@link TrackVADEmitter} otherwise it's
-     * stopped.
-     * @param {JitsiTrack} track - The track whose mute state has changed.
-     * @returns {void}
-     * @listens TRACK_MUTE_CHANGED
-     */
-    _trackMuteChanged(track) {
-        if (track.isLocalAudioTrack()) {
-            // On a mute toggle reset the state.
-            this._vadInitTracker = this._vadInitTracker.then(() => {
+        // Because we remove all listeners on the vadEmitter once the main event is triggered,
+        // there is no need to check for rogue events.
+        if (vadScore.score > VAD_VOICE_LEVEL) {
+            this._processing = true;
+            this._scoreArray.push(vadScore.score);
 
-                // Reset the processing context in between muted states so that each individual mute phase can generate
-                // it's own event.
-                this._reset();
-                if (track.isMuted()) {
-                    this._startVADEmitter();
-                } else {
-                    this._stopVADEmitter();
-                }
-            });
+            // Start gathering VAD scores for the configured period of time.
+            this._processTimeout = setTimeout(this._calculateVADScore, PROCESS_TIME_FRAME_SPAN_MS);
         }
     }
 
     /**
-     * Notifies the detector that a track associated with the {@link JitsiConference} was removed. Only takes into
-     * account local audio tracks. Cleans up resources associated with the track and resets the processing context.
+     * Reset the processing context, clear buffer, cancel the timeout trigger.
      *
-     * @param {JitsiTrack} track - The removed track.
      * @returns {void}
-     * @listens TRACK_REMOVED
      */
-    _trackRemoved(track) {
-        if (track.isLocalAudioTrack()) {
-            // Use the promise to make sure operations are in sequence.
-            this._vadInitTracker = this._vadInitTracker.then(() => {
-                logger.debug('Removing track from VAD detection - ', track.getTrackLabel());
-
-                if (this._vadEmitter) {
-                    this._stopVADEmitter();
-                    this._reset();
-                    this._vadEmitter.destroy();
-                    this._vadEmitter = null;
-                }
-            });
-        }
+    reset() {
+        this._processing = false;
+        this._scoreArray = [];
+        clearTimeout(this._processTimeout);
     }
 }

modules/detection/webaudio/WebAudioUtils.js

@@ -0,0 +1,14 @@
+/**
+ * Adapter that creates AudioContext objects depending on the browser.
+ *
+ * @returns {AudioContext} - Return a new AudioContext or undefined if the browser does not support it.
+ */
+export function createAudioContext(options) {
+    const AudioContextImpl = window.AudioContext || window.webkitAudioContext;
+
+    if (!AudioContextImpl) {
+        return undefined;
+    }
+
+    return new AudioContextImpl(options);
+}

modules/util/MathUtil.js

@@ -17,3 +17,24 @@ export function safeCounterIncrement(number) {
 
     return nextValue + 1;
 }
+
+/**
+ * Calculates the average value of am Array of numbers.
+ *
+ * @param {Float32Array} valueArray - Array of numbers.
+ * @returns {number} - Number array average.
+ */
+export function calculateAverage(valueArray) {
+    return valueArray.length > 0 ? valueArray.reduce((a, b) => a + b) / valueArray.length : 0;
+}
+
+
+/**
+ * Returns only the positive values from an array of numbers.
+ *
+ * @param {Float32Array} valueArray - Array of vad scores.
+ * @returns {Array} - Array of positive numbers.
+ */
+export function filterPositiveValues(valueArray) {
+    return valueArray.filter(value => value >= 0);
+}