feat(ScreenshotCaptureEffect) Implement.

conference.js

@@ -122,6 +122,7 @@ import { setSharedVideoStatus } from './react/features/shared-video';
 import { createPresenterEffect } from './react/features/stream-effects/presenter';
 import { endpointMessageReceived } from './react/features/subtitles';
 import { createRnnoiseProcessorPromise } from './react/features/rnnoise';
+import { toggleScreenshotCaptureEffect } from './react/features/screenshot-capture';
 
 const logger = require('jitsi-meet-logger').getLogger(__filename);
 
@@ -1460,6 +1461,8 @@ export default {
             promise = promise.then(() => this.useVideoStream(null));
         }
 
+        APP.store.dispatch(toggleScreenshotCaptureEffect(false));
+
         return promise.then(
             () => {
                 this.videoSwitchInProgress = false;
@@ -1731,6 +1734,7 @@ export default {
             .then(stream => this.useVideoStream(stream))
             .then(() => {
                 this.videoSwitchInProgress = false;
+                APP.store.dispatch(toggleScreenshotCaptureEffect(true));
                 sendAnalytics(createScreenSharingEvent('started'));
                 logger.log('Screen sharing started');
             })

interface_config.js

@@ -188,7 +188,12 @@ var interfaceConfig = {
      *
      * Note: this mode is experimental and subject to breakage.
      */
-    AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only'
+    AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only',
+
+    /**
+     * If we should capture periodic screenshots of the content sharing.
+     */
+    ENABLE_SCREENSHOT_CAPTURE: false
 
     /**
      * How many columns the tile view can expand to. The respected range is

package-lock.json

@@ -13314,6 +13314,14 @@
         "node-modules-regexp": "^1.0.0"
       }
     },
+    "pixelmatch": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.1.0.tgz",
+      "integrity": "sha512-HqtgvuWN12tBzKJf7jYsc38Ha28Q2NYpmBL9WostEGgDHJqbTLkjydZXL1ZHM02ZnB+Dkwlxo87HBY38kMiD6A==",
+      "requires": {
+        "pngjs": "^3.4.0"
+      }
+    },
     "pkg-dir": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
@@ -13375,6 +13383,11 @@
       "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==",
       "dev": true
     },
+    "pngjs": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
+      "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w=="
+    },
     "popper.js": {
       "version": "1.14.4",
       "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.14.4.tgz",

package.json

@@ -61,6 +61,7 @@
     "lodash": "4.17.13",
     "moment": "2.19.4",
     "moment-duration-format": "2.2.2",
+    "pixelmatch": "5.1.0",
     "react": "16.9",
     "react-dom": "16.9",
     "react-emoji-render": "1.0.0",

react/features/base/tracks/functions.js

@@ -1,5 +1,6 @@
 /* global APP */
 
+import { createScreenshotCaptureEffect } from '../../stream-effects/screenshot-capture';
 import { getBlurEffect } from '../../blur';
 import JitsiMeetJS, { JitsiTrackErrors, browser } from '../lib-jitsi-meet';
 import { MEDIA_TYPE } from '../media';
@@ -101,21 +102,30 @@ export function createLocalTracksF(
     const constraints = options.constraints
         ?? state['features/base/config'].constraints;
 
-    // Do not load blur effect if option for ignoring effects is present.
-    // This is needed when we are creating a video track for presenter mode.
-    const loadEffectsPromise = state['features/blur'].blurEnabled
+    const blurPromise = state['features/blur'].blurEnabled
         ? getBlurEffect()
-            .then(blurEffect => [ blurEffect ])
             .catch(error => {
                 logger.error('Failed to obtain the blur effect instance with error: ', error);
 
-                return Promise.resolve([]);
+                return Promise.resolve();
             })
-        : Promise.resolve([]);
+        : Promise.resolve();
+    const screenshotCapturePromise = state['features/screenshot-capture'].capturesEnabled
+        ? createScreenshotCaptureEffect(state)
+            .catch(error => {
+                logger.error('Failed to obtain the screenshot capture effect effect instance with error: ', error);
+
+                return Promise.resolve();
+            })
+        : Promise.resolve();
+    const loadEffectsPromise = Promise.all([ blurPromise, screenshotCapturePromise ]);
 
     return (
-        loadEffectsPromise.then(effects =>
-            JitsiMeetJS.createLocalTracks(
+        loadEffectsPromise.then(effectsArray => {
+            // Filter any undefined values returned by Promise.resolve().
+            const effects = effectsArray.filter(effect => Boolean(effect));
+
+            return JitsiMeetJS.createLocalTracks(
                 {
                     cameraDeviceId,
                     constraints,
@@ -138,7 +148,8 @@ export function createLocalTracksF(
                 logger.error('Failed to create local tracks', options.devices, err);
 
                 return Promise.reject(err);
-            })));
+            });
+        }));
 }
 
 /**

react/features/screenshot-capture/actionTypes.js

@@ -0,0 +1,11 @@
+// @flow
+
+/**
+ * Redux action type dispatched in order to toggle screenshot captures.
+ *
+ * {
+ *      type: SET_SCREENSHOT_CAPTURE
+ * }
+ */
+
+export const SET_SCREENSHOT_CAPTURE = 'SET_SCREENSHOT_CAPTURE';

react/features/screenshot-capture/actions.js

@@ -0,0 +1,52 @@
+// @flow
+
+import { createScreenshotCaptureEffect } from '../stream-effects/screenshot-capture';
+import { getLocalVideoTrack } from '../../features/base/tracks';
+
+import { SET_SCREENSHOT_CAPTURE } from './actionTypes';
+
+/**
+ * Marks the on-off state of screenshot captures.
+ *
+ * @param {boolean} enabled - Whether to turn screen captures on or off.
+ * @returns {{
+    *      type: START_SCREENSHOT_CAPTURE,
+    *      payload: enabled
+    * }}
+*/
+function setScreenshotCapture(enabled) {
+    return {
+        type: SET_SCREENSHOT_CAPTURE,
+        payload: enabled
+    };
+}
+
+/**
+* Action that toggles the screenshot captures.
+*
+* @param {boolean} enabled - Bool that represents the intention to start/stop screenshot captures.
+* @returns {Promise}
+*/
+export function toggleScreenshotCaptureEffect(enabled: boolean) {
+    return function(dispatch: (Object) => Object, getState: () => any) {
+        const state = getState();
+
+        if (state['features/screenshot-capture'].capturesEnabled !== enabled) {
+            const { jitsiTrack } = getLocalVideoTrack(state['features/base/tracks']);
+
+            return createScreenshotCaptureEffect(state)
+                .then(effect =>
+                    jitsiTrack.setEffect(enabled ? effect : undefined)
+                        .then(() => {
+                            dispatch(setScreenshotCapture(enabled));
+                        })
+                        .catch(() => {
+                            dispatch(setScreenshotCapture(!enabled));
+                        })
+                )
+                .catch(() => dispatch(setScreenshotCapture(false)));
+        }
+
+        return Promise.resolve();
+    };
+}

react/features/screenshot-capture/index.js

@@ -0,0 +1,3 @@
+export * from './actions';
+
+import './reducer';

react/features/screenshot-capture/reducer.js

@@ -0,0 +1,23 @@
+// @flow
+
+import { ReducerRegistry } from '../base/redux';
+import { PersistenceRegistry } from '../base/storage';
+
+import { SET_SCREENSHOT_CAPTURE } from './actionTypes';
+
+PersistenceRegistry.register('features/screnshot-capture', true, {
+    capturesEnabled: false
+});
+
+ReducerRegistry.register('features/screenshot-capture', (state = {}, action) => {
+    switch (action.type) {
+    case SET_SCREENSHOT_CAPTURE: {
+        return {
+            ...state,
+            capturesEnabled: action.payload
+        };
+    }
+    }
+
+    return state;
+});

react/features/stream-effects/screenshot-capture/ScreenshotCaptureEffect.js

@@ -0,0 +1,176 @@
+// @flow
+
+import pixelmatch from 'pixelmatch';
+
+import {
+    CLEAR_INTERVAL,
+    INTERVAL_TIMEOUT,
+    PIXEL_LOWER_BOUND,
+    POLL_INTERVAL,
+    SET_INTERVAL
+} from './constants';
+
+import { getCurrentConference } from '../../base/conference';
+import { processScreenshot } from './processScreenshot';
+import { timerWorkerScript } from './worker';
+
+declare var interfaceConfig: Object;
+
+/**
+ * Effect that wraps {@code MediaStream} adding periodic screenshot captures.
+ * Manipulates the original desktop stream and performs custom processing operations, if implemented.
+ */
+export default class ScreenshotCaptureEffect {
+    _state: Object;
+    _currentCanvas: HTMLCanvasElement;
+    _currentCanvasContext: CanvasRenderingContext2D;
+    _videoElement: HTMLVideoElement;
+    _handleWorkerAction: Function;
+    _initScreenshotCapture: Function;
+    _streamWorker: Worker;
+    _streamHeight: any;
+    _streamWidth: any;
+    _storedImageData: Uint8ClampedArray;
+
+    /**
+     * Initializes a new {@code ScreenshotCaptureEffect} instance.
+     *
+     * @param {Object} state - The redux state.
+     */
+    constructor(state: Object) {
+        this._state = state;
+        this._currentCanvas = document.createElement('canvas');
+        this._currentCanvasContext = this._currentCanvas.getContext('2d');
+        this._videoElement = document.createElement('video');
+
+        // Bind handlers such that they access the same instance.
+        this._handleWorkerAction = this._handleWorkerAction.bind(this);
+        this._initScreenshotCapture = this._initScreenshotCapture.bind(this);
+        this._streamWorker = new Worker(timerWorkerScript);
+        this._streamWorker.onmessage = this._handleWorkerAction;
+    }
+
+    /**
+     * Checks if the local track supports this effect.
+     *
+     * @param {JitsiLocalTrack} jitsiLocalTrack - Targeted local track.
+     * @returns {boolean} - Returns true if this effect can run on the specified track, false otherwise.
+     */
+    isEnabled(jitsiLocalTrack: Object) {
+        return (
+            interfaceConfig.ENABLE_SCREENSHOT_CAPTURE
+            && jitsiLocalTrack.isVideoTrack()
+            && jitsiLocalTrack.videoType === 'desktop'
+        );
+    }
+
+    /**
+     * Starts the screenshot capture event on a loop.
+     *
+     * @param {MediaStream} stream - The desktop stream from which screenshots are to be sent.
+     * @returns {MediaStream} - The same stream, with the interval set.
+     */
+    startEffect(stream: MediaStream) {
+        const desktopTrack = stream.getVideoTracks()[0];
+        const { height, width }
+            = desktopTrack.getSettings() ?? desktopTrack.getConstraints();
+
+        this._streamHeight = height;
+        this._streamWidth = width;
+        this._currentCanvas.height = parseInt(height, 10);
+        this._currentCanvas.width = parseInt(width, 10);
+        this._videoElement.height = parseInt(height, 10);
+        this._videoElement.width = parseInt(width, 10);
+        this._videoElement.srcObject = stream;
+        this._videoElement.play();
+
+        // Store first capture for comparisons in {@code this._handleScreenshot}.
+        this._videoElement.addEventListener('loadeddata', this._initScreenshotCapture);
+
+        return stream;
+    }
+
+    /**
+     * Stops the ongoing {@code ScreenshotCaptureEffect} by clearing the {@code Worker} interval.
+     *
+     * @returns {void}
+     */
+    stopEffect() {
+        this._streamWorker.postMessage({ id: CLEAR_INTERVAL });
+        this._videoElement.removeEventListener('loadeddata', this._initScreenshotCapture);
+    }
+
+    /**
+     * Method that is called as soon as the first frame of the video loads from stream.
+     * The method is used to store the {@code ImageData} object from the first frames
+     * in order to use it for future comparisons based on which we can process only certain
+     * screenshots.
+     *
+     * @private
+     * @returns {void}
+     */
+    _initScreenshotCapture() {
+        const storedCanvas = document.createElement('canvas');
+        const storedCanvasContext = storedCanvas.getContext('2d');
+
+        storedCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight);
+        const { data } = storedCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight);
+
+        this._storedImageData = data;
+        this._streamWorker.postMessage({
+            id: SET_INTERVAL,
+            timeMs: POLL_INTERVAL
+        });
+    }
+
+    /**
+     * Handler of the {@code EventHandler} message that calls the appropriate method based on the parameter's id.
+     *
+     * @private
+     * @param {EventHandler} message - Message received from the Worker.
+     * @returns {void}
+     */
+    _handleWorkerAction(message: Object) {
+        return message.data.id === INTERVAL_TIMEOUT && this._handleScreenshot();
+    }
+
+    /**
+     * Method that decides whether an image should be processed based on a preset pixel lower bound.
+     *
+     * @private
+     * @param {integer} nbPixels - The number of pixels of the candidate image.
+     * @returns {boolean} - Whether the image should be processed or not.
+     */
+    _shouldProcessScreenshot(nbPixels: number) {
+        return nbPixels >= PIXEL_LOWER_BOUND;
+    }
+
+    /**
+     * Screenshot handler.
+     *
+     * @private
+     * @returns {void}
+     */
+    _handleScreenshot() {
+        this._currentCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight);
+        const { data } = this._currentCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight);
+        const diffPixels = pixelmatch(data, this._storedImageData, null, this._streamWidth, this._streamHeight);
+
+        if (this._shouldProcessScreenshot(diffPixels)) {
+            const conference = getCurrentConference(this._state);
+            const sessionId = conference.getMeetingUniqueId();
+            const { connection, timeEstablished } = this._state['features/base/connection'];
+            const jid = connection.getJid();
+            const timeLapseSeconds = timeEstablished && Math.floor((Date.now() - timeEstablished) / 1000);
+            const { jwt } = this._state['features/base/jwt'];
+
+            this._storedImageData = data;
+            processScreenshot(this._currentCanvas, {
+                jid,
+                jwt,
+                sessionId,
+                timeLapseSeconds
+            });
+        }
+    }
+}

react/features/stream-effects/screenshot-capture/constants.js

@@ -0,0 +1,42 @@
+// @flow
+
+/**
+ * Number of pixels that signal if two images should be considered different.
+ */
+export const PIXEL_LOWER_BOUND = 100000;
+
+/**
+ * Number of milliseconds that represent how often screenshots should be taken.
+ */
+export const POLL_INTERVAL = 30000;
+
+/**
+ * SET_INTERVAL constant is used to set interval and it is set in
+ * the id property of the request.data property. timeMs property must
+ * also be set. request.data example:
+ *
+ * {
+ *      id: SET_INTERVAL,
+ *      timeMs: 33
+ * }
+ */
+export const SET_INTERVAL = 1;
+
+/**
+ * CLEAR_INTERVAL constant is used to clear the interval and it is set in
+ * the id property of the request.data property.
+ *
+ * {
+ *      id: CLEAR_INTERVAL
+ * }
+ */
+export const CLEAR_INTERVAL = 2;
+
+/**
+ * INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
+ *
+ * {
+ *      id: INTERVAL_TIMEOUT
+ * }
+ */
+export const INTERVAL_TIMEOUT = 3;

react/features/stream-effects/screenshot-capture/index.js

@@ -0,0 +1,19 @@
+// @flow
+
+import ScreenshotCaptureEffect from './ScreenshotCaptureEffect';
+import { toState } from '../../base/redux';
+
+/**
+ * Creates a new instance of ScreenshotCaptureEffect.
+ *
+ * @param {Object | Function} stateful - The redux store, state, or
+ * {@code getState} function.
+ * @returns {Promise<ScreenshotCaptureEffect>}
+ */
+export function createScreenshotCaptureEffect(stateful: Object | Function) {
+    if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
+        return Promise.reject(new Error('ScreenshotCaptureEffect not supported!'));
+    }
+
+    return Promise.resolve(new ScreenshotCaptureEffect(toState(stateful)));
+}

react/features/stream-effects/screenshot-capture/processScreenshot.js

@@ -0,0 +1,12 @@
+// @flow
+
+/**
+ * Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}.
+ *
+ * @param {HTMLCanvasElement} canvas - The canvas containing a screenshot to be processed.
+ * @param {Object} options - Custom options required for processing.
+ * @returns {void}
+ */
+export function processScreenshot(canvas: HTMLCanvasElement, options: Object) { // eslint-disable-line no-unused-vars
+    return;
+}

react/features/stream-effects/screenshot-capture/worker.js

@@ -0,0 +1,30 @@
+// @flow
+
+import {
+    CLEAR_INTERVAL,
+    INTERVAL_TIMEOUT,
+    SET_INTERVAL
+} from './constants';
+
+const code = `
+    var timer;
+
+    onmessage = function(request) {
+        switch (request.data.id) {
+        case ${SET_INTERVAL}: {
+            timer = setInterval(() => {
+                postMessage({ id: ${INTERVAL_TIMEOUT} });
+            }, request.data.timeMs);
+            break;
+        }
+        case ${CLEAR_INTERVAL}: {
+            if (timer) {
+                clearInterval(timer);
+            }
+            break;
+        }
+        }
+    };
+`;
+
+export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));