initial commit

JitsiConference.js

@@ -13,10 +13,11 @@ import JitsiTrackError from './JitsiTrackError';
 import * as JitsiTrackErrors from './JitsiTrackErrors';
 import * as JitsiTrackEvents from './JitsiTrackEvents';
 import authenticateAndUpgradeRole from './authenticateAndUpgradeRole';
-import P2PDominantSpeakerDetection from './modules/P2PDominantSpeakerDetection';
+import P2PDominantSpeakerDetection from './modules/detection/P2PDominantSpeakerDetection';
 import RTC from './modules/RTC/RTC';
-import TalkMutedDetection from './modules/TalkMutedDetection';
-import NoAudioSignalDetection from './modules/NoAudioSignalDetection';
+import VADTalkMutedDetection from './modules/detection/VADTalkMutedDetection';
+import * as DetectionEvents from './modules/detection/DetectionEvents';
+import NoAudioSignalDetection from './modules/detection/NoAudioSignalDetection';
 import browser from './modules/browser';
 import ConnectionQuality from './modules/connectivity/ConnectionQuality';
 import IceFailedNotification
@@ -275,6 +276,9 @@ JitsiConference.resourceCreator = function(jid, isAuthenticatedUser) {
  * @param options.connection {JitsiConnection} overrides this.connection
  */
 JitsiConference.prototype._init = function(options = {}) {
+
+    logger.info('[ADBG] Init concerence v.0.1!');
+
     // Override connection and xmpp properties (Useful if the connection
     // reloaded)
     if (options.connection) {
@@ -373,15 +377,22 @@ JitsiConference.prototype._init = function(options = {}) {
     this.eventManager.setupStatisticsListeners();
 
     if (config.enableTalkWhileMuted) {
-        // eslint-disable-next-line no-new
-        new TalkMutedDetection(
-            this,
-            () =>
+
+        if (config.vadProcessor) {
+            // eslint-disable-next-line no-new
+            this._talkWhileMutedDetection = new VADTalkMutedDetection(this, config.vadProcessor);
+            this._talkWhileMutedDetection.on(DetectionEvents.VAD_TALK_WHILE_MUTED, () =>
                 this.eventEmitter.emit(JitsiConferenceEvents.TALK_WHILE_MUTED));
+
+        } else {
+            logger.info('[ADBG] No vadProcessor set!');
+        }
+
     }
 
     // Generates events based on no audio input detector.
-    if (config.enableNoAudioDetection) {
+    // eslint-disable-next-line
+    if (true) {
         // eslint-disable-next-line no-new
         new NoAudioSignalDetection(this, () =>
             this.eventEmitter.emit(JitsiConferenceEvents.NO_AUDIO_INPUT));

modules/detection/DetectionEvents.js

@@ -0,0 +1,9 @@
+/**
+ * A media track mute status was changed.
+ */
+export const VAD_SCORE_PUBLISHED = 'detection.vad_score_published';
+
+/**
+ * A media track mute status was changed.
+ */
+export const VAD_TALK_WHILE_MUTED = 'detection.vad_talk_while_muted';

modules/NoAudioSignalDetection.js → modules/detection/NoAudioSignalDetection.js

@@ -1,4 +1,4 @@
-import * as JitsiConferenceEvents from '../JitsiConferenceEvents';
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
 
 // We wait a certain time interval for constant silence input from the current device to account for
 // potential abnormalities and for a better use experience i.e. don't generate event the instant
@@ -23,7 +23,6 @@ export default class NoAudioSignalDetection {
 
         conference.statistics.addAudioLevelListener(this._audioLevel.bind(this));
         conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this));
-
     }
 
     /**
@@ -32,7 +31,6 @@ export default class NoAudioSignalDetection {
      * @returns {boolean}
      */
     _hasSilencePeriodElapsed() {
-
         const currentDate = new Date();
         const elapsedSec = (currentDate.getTime() - this._firstSilentSignalDate.getTime()) / 1000;
 
@@ -52,7 +50,7 @@ export default class NoAudioSignalDetection {
         if (!this._firstSilentSignalDate) {
             this._firstSilentSignalDate = new Date();
 
-        // If the configured interval has elapsed trigger the callback
+            // If the configured interval has elapsed trigger the callback
         } else if (this._hasSilencePeriodElapsed()) {
             this._eventFired = true;
             this._callback();
@@ -69,7 +67,6 @@ export default class NoAudioSignalDetection {
      * @param {boolean} isLocal - true for local/send streams or false for remote/receive streams.
      */
     _audioLevel(tpc, ssrc, audioLevel, isLocal) {
-
         // We are interested in the local audio stream if the event was not triggered on this device.
         if (!isLocal || !this._audioTrack || this._eventFired) {
             return;

modules/P2PDominantSpeakerDetection.js → modules/detection/P2PDominantSpeakerDetection.js

@@ -1,5 +1,5 @@
-import * as JitsiConferenceEvents from '../JitsiConferenceEvents';
-import RTCEvents from '../service/RTC/RTCEvents';
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
+import RTCEvents from '../../service/RTC/RTCEvents';
 
 /**
  * The value which we use to say, every sound over this threshold

modules/TalkMutedDetection.js → modules/detection/TalkMutedDetection.js

@@ -1,4 +1,4 @@
-import * as JitsiConferenceEvents from '../JitsiConferenceEvents';
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
 
 /**
  * The value which we use to say, every sound over this threshold

modules/detection/TrackVADEmitter.js

@@ -0,0 +1,155 @@
+import EventEmitter from 'events';
+import RTC from '../RTC/RTC';
+import { VAD_SCORE_PUBLISHED } from './DetectionEvents';
+
+/**
+ * Connects an audio JitsiLocalTrack to a vadProcessor using WebAudio ScriptProcessorNode.
+ * Once an object is created audio from the local track flows through the ScriptProcessorNode as raw PCM.
+ * The PCM is processed by the injected vad module and a voice activity detection score is obtained, the
+ * score is published to consumers via an EventEmitter.
+ * After work is done with this service the destroy method needs to be called for a proper cleanup.
+ */
+export default class TrackVADEmitter extends EventEmitter {
+    /**
+     * Constructor.
+     *
+     * @param {number} procNodeSampleRate - Sample rate of the ScriptProcessorNode. Possible values  256, 512, 1024,
+     *  2048, 4096, 8192, 16384. Passing other values will default to closes neighbor.
+     * @param {Object} vadProcessor - adapter that allows us to calculate VAD score
+     * for PCM samples.
+     * @param {Object} jitsiLocalTrack - JitsiLocalTrack corresponding to micDeviceId.
+     */
+    constructor(procNodeSampleRate, vadProcessor, jitsiLocalTrack) {
+        super();
+        this._procNodeSampleRate = procNodeSampleRate;
+        this._vadProcessor = vadProcessor;
+        this._localTrack = jitsiLocalTrack;
+        this._micDeviceId = jitsiLocalTrack.getDeviceId();
+        this._bufferResidue = new Float32Array([]);
+        this._audioContext = new AudioContext({ sampleRate: 44100 });
+
+        this._vadSampleSize = vadProcessor.getSampleLength();
+        this._onAudioProcess = this._onAudioProcess.bind(this);
+
+        this._initializeAudioContext();
+        this._connectAudioGraph();
+    }
+
+    /**
+     * Factory method that sets up all the necessary components for the creation of the TrackVADEmitter.
+     *
+     * @param {string} micDeviceId - Target microphone device id.
+     * @param {number} procNodeSampleRate - Sample rate of the proc node.
+     * @returns {Promise<TrackVADEmitter>} - Promise resolving in a new instance of TrackVADEmitter.
+     */
+    static create(micDeviceId, procNodeSampleRate, vadProcessor) {
+        return RTC.obtainAudioAndVideoPermissions({
+            devices: [ 'audio' ],
+            micDeviceId
+        }).then(localTrack => {
+            // We only expect one audio track when specifying a device id.
+            if (!localTrack[0]) {
+                throw new Error(`Failed to create jitsi local track for device id: ${micDeviceId}`);
+            }
+
+            return new TrackVADEmitter(procNodeSampleRate, vadProcessor, localTrack[0]);
+
+            // We have no exception handling at this point as there is nothing to clean up, the vadProcessor
+            // life cycle is handled by whoever created this instance.
+        });
+    }
+
+    /**
+     * Sets up the audio graph in the AudioContext.
+     *
+     * @returns {Promise<void>}
+     */
+    _initializeAudioContext() {
+        this._audioSource = this._audioContext.createMediaStreamSource(this._localTrack.stream);
+
+        // TODO AudioProcessingNode is deprecated check and replace with alternative.
+        // We don't need stereo for determining the VAD score so we create a single channel processing node.
+        this._audioProcessingNode = this._audioContext.createScriptProcessor(this._procNodeSampleRate, 1, 1);
+        this._audioProcessingNode.onaudioprocess = this._onAudioProcess;
+    }
+
+    /**
+     * TODO maybe move this logic to the VAD Processor.
+     * ScriptProcessorNode callback, the input parameters contains the PCM audio that is then sent to rnnoise.
+     * Rnnoise only accepts PCM samples of 480 bytes whereas the webaudio processor node can't sample at a multiple
+     * of 480 thus after each _onAudioProcess callback there will remain and PCM buffer residue equal
+     * to _procNodeSampleRate / 480 which will be added to the next sample buffer and so on.
+     *
+     * @param {AudioProcessingEvent} audioEvent - Audio event.
+     * @returns {void}
+     */
+    _onAudioProcess(audioEvent) {
+        // Prepend the residue PCM buffer from the previous process callback.
+        const inData = audioEvent.inputBuffer.getChannelData(0);
+        const completeInData = [ ...this._bufferResidue, ...inData ];
+        const sampleTimestamp = Date.now();
+
+        let i = 0;
+
+        for (; i + this._vadSampleSize < completeInData.length; i += this._vadSampleSize) {
+            const pcmSample = completeInData.slice(i, i + this._vadSampleSize);
+            const vadScore = this._vadProcessor.calculateAudioFrameVAD(pcmSample);
+
+            this.emit(VAD_SCORE_PUBLISHED, {
+                timestamp: sampleTimestamp,
+                score: vadScore,
+                deviceId: this._micDeviceId
+            });
+        }
+
+        this._bufferResidue = completeInData.slice(i, completeInData.length);
+    }
+
+    /**
+     * Connects the nodes in the AudioContext to start the flow of audio data.
+     *
+     * @returns {void}
+     */
+    _connectAudioGraph() {
+        this._audioSource.connect(this._audioProcessingNode);
+        this._audioProcessingNode.connect(this._audioContext.destination);
+    }
+
+    /**
+     * Disconnects the nodes in the AudioContext.
+     *
+     * @returns {void}
+     */
+    _disconnectAudioGraph() {
+        // Even thought we disconnect the processing node it seems that some callbacks remain queued,
+        // resulting in calls with and uninitialized context.
+        // eslint-disable-next-line no-empty-function
+        this._audioProcessingNode.onaudioprocess = () => {};
+        this._audioProcessingNode.disconnect();
+        this._audioSource.disconnect();
+    }
+
+    /**
+     * Cleanup potentially acquired resources.
+     *
+     * @returns {void}
+     */
+    _cleanupResources() {
+        this._disconnectAudioGraph();
+        this._localTrack.stopStream();
+    }
+
+    /**
+     * Destroy TrackVADEmitter instance (release resources and stop callbacks).
+     *
+     * @returns {void}
+     */
+    destroy() {
+        if (this._destroyed) {
+            return;
+        }
+
+        this._cleanupResources();
+        this._destroyed = true;
+    }
+}

modules/detection/VADTalkMutedDetection.js

@@ -0,0 +1,163 @@
+import { EventEmitter } from 'events';
+import { VAD_SCORE_PUBLISHED, VAD_TALK_WHILE_MUTED } from './DetectionEvents';
+import { getLogger } from 'jitsi-meet-logger';
+import TrackVADEmitter from '../detection/TrackVADEmitter';
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
+
+const logger = getLogger(__filename);
+
+/**
+ * The value which we use to say, every sound over this threshold
+ * is talking on the mic.
+ * @type {number}
+ */
+const VAD_DETECT_THRESHOLD = 0.7;
+
+/**
+ * Detect user trying to speek while is locally muted and fires an event.
+ */
+export default class VADTalkMutedDetection extends EventEmitter {
+    /**
+     * Creates TalkMutedDetection
+     * @param conference the JitsiConference instance that created us.
+     * @param callback the callback to call when detected that the local user is
+     * talking while her microphone is muted.
+     * @constructor
+     */
+    constructor(conference, vadProcessor) {
+        super();
+        logger.info('[ADBG] Created VADTalkMutedDetection.');
+
+        /**
+         * The indicator which determines whether <tt>callback</tt> has been
+         * invoked for the current local audio track of <tt>conference</tt> so
+         * that it is invoked once only.
+         *
+         * @private
+         */
+        this._eventFired = false;
+
+        this._vadProcessor = vadProcessor;
+
+        this._vadEmitter = null;
+
+        this._processing = false;
+
+        this._scoreArray = [];
+
+        conference.on(JitsiConferenceEvents.TRACK_MUTE_CHANGED, this._trackMuteChanged.bind(this));
+        conference.on(JitsiConferenceEvents.TRACK_ADDED, this._trackAdded.bind(this));
+
+        // TODO do we need to handle the case where tracks are removed, make sure this cleans up properly so
+        // we don't have any leeks i.e. stale JitsiLocalTracks
+    }
+
+    /* eslint-disable max-params */
+    /**
+     * Receives audio level events for all send and receive streams.
+     *
+     * @param {TraceablePeerConnection} pc - WebRTC PeerConnection object of the
+     * @param {number} ssrc - The synchronization source identifier (SSRC) of
+     * the endpoint/participant/stream being reported.
+     * @param {number} audioLevel - The audio level of <tt>ssrc</tt>.
+     * @param {boolean} isLocal - <tt>true</tt> if <tt>ssrc</tt> represents a
+     * local/send stream or <tt>false</tt> for a remote/receive stream.
+     */
+    _processVADScore(vadScore) {
+        // We are interested in the local audio stream only and if event is not
+        // sent yet.
+        if (this._eventFired) {
+            return;
+        }
+
+        if (this.audioTrack.isMuted()) {
+            if (vadScore.score > 0.8 && !this._processing) {
+                this._processing = true;
+
+                this._processTimeout = setTimeout(() => {
+                    let scoreSum = 0;
+
+                    for (const score of this._scoreArray) {
+                        scoreSum += score;
+                    }
+
+                    const avgScore = scoreSum / this._scoreArray.length;
+
+                    if (avgScore > VAD_DETECT_THRESHOLD) {
+                        this.emit(VAD_TALK_WHILE_MUTED, '');
+                        this._eventFired = true;
+                        console.log('[ADBG] Triggered array size: ', this._scoreArray, '. AVG: ', avgScore);
+                    } else {
+                        console.log('[ADBG] Not triggered array size: ', this._scoreArray, '. AVG: ', avgScore);
+                    }
+
+                    this._scoreArray = [];
+                    this._processing = false;
+                }, 1500);
+            }
+
+            if (this._processing) {
+                this._scoreArray.push(vadScore.score);
+            }
+        }
+    }
+    /* eslint-enable max-params */
+
+    /**
+     * Determines whether a specific {@link JitsiTrack} represents a local audio
+     * track.
+     *
+     * @param {JitsiTrack} track - The <tt>JitsiTrack</tt> to be checked whether
+     * it represents a local audio track.
+     * @private
+     * @return {boolean} - <tt>true</tt> if the specified <tt>track</tt>
+     * represents a local audio track; otherwise, <tt>false</tt>.
+     */
+    _isLocalAudioTrack(track) {
+        return track.isAudioTrack() && track.isLocal();
+    }
+
+    /**
+     * Notifies this <tt>TalkMutedDetection</tt> that a {@link JitsiTrack} was
+     * added to the associated {@link JitsiConference}. Looks for the local
+     * audio track only.
+     *
+     * @param {JitsiTrack} track - The added <tt>JitsiTrack</tt>.
+     * @private
+     */
+    _trackAdded(track) {
+        if (this._isLocalAudioTrack(track)) {
+            logger.info('[ADBG] Audio track added.');
+            this.audioTrack = track;
+            this._vadProcessor().then(vadProcessor => {
+                TrackVADEmitter.create(track.getDeviceId(), 4096, vadProcessor).then(vadEmitter => {
+                    if (this._vadEmitter) {
+                        this._vadEmitter.destroy();
+                    }
+
+                    this._vadEmitter = vadEmitter;
+                    this._vadEmitter.on(VAD_SCORE_PUBLISHED, this._processVADScore.bind(this));
+                    this._eventFired = false;
+                    this._processing = false;
+                    clearTimeout(this._processTimeout);
+                });
+            });
+        }
+    }
+
+    /**
+     * Notifies this <tt>TalkMutedDetection</tt> that the mute state of a
+     * {@link JitsiTrack} has changed. Looks for the local audio track only.
+     *
+     * @param {JitsiTrack} track - The <tt>JitsiTrack</tt> whose mute state has
+     * changed.
+     * @private
+     */
+    _trackMuteChanged(track) {
+        if (this._isLocalAudioTrack(track) && track.isMuted()) {
+            logger.info('[ADBG] Audio track muted.');
+            this._eventFired = false;
+            clearTimeout(this._processTimeout);
+        }
+    }
+}

package.json

@@ -22,6 +22,7 @@
     "current-executing-script": "0.1.3",
     "jitsi-meet-logger": "github:jitsi/jitsi-meet-logger#5ec92357570dc8f0b7ffc1528820721c84c6af8b",
     "js-utils": "github:jitsi/js-utils#446497893023aa8dec403e0e4e35a22cae6bc87d",
+    "lib-jitsi-meet": "^1.0.6",
     "lodash.isequal": "4.5.0",
     "sdp-transform": "2.3.0",
     "strophe.js": "1.2.16",
@@ -56,7 +57,7 @@
   },
   "scripts": {
     "lint": "eslint . && flow",
-    "postinstall": "webpack -p",
+    "postinstall": "webpack",
     "test": "karma start karma.conf.js",
     "test-watch": "karma start karma.conf.js --no-single-run",
     "validate": "npm ls"

webpack.config.js

@@ -61,9 +61,11 @@ const config = {
         concatenateModules: minimize
     },
     output: {
-        filename: `[name]${minimize ? '.min' : ''}.js`,
+        // eslint-disable-next-line
+        filename: `[name]${true ? '.min' : ''}.js`,
         path: process.cwd(),
-        sourceMapFilename: `[name].${minimize ? 'min' : 'js'}.map`
+        // eslint-disable-next-line
+        sourceMapFilename: `[name].${true ? 'min' : 'js'}.map`
     },
     performance: {
         hints: minimize ? 'error' : false,