jvc0/react/features/face-centering/actions.js

import 'image-capture';

import { getCurrentConference } from '../base/conference';
import { getLocalParticipant, getParticipantCount } from '../base/participants';
import { getLocalVideoTrack } from '../base/tracks';
import { getBaseUrl } from '../base/util';
import '../facial-recognition/createImageBitmap';

import {
    START_FACE_RECOGNITION,
    STOP_FACE_RECOGNITION,
    UPDATE_FACE_COORDINATES
} from './actionTypes';
import {
    FACE_BOX_MESSAGE,
    SEND_IMAGE_INTERVAL_MS
} from './constants';
import { sendDataToWorker, sendFaceBoxToParticipants } from './functions';
import logger from './logger';

/**
 * Interval object for sending new image data to worker.
 */
let interval;

/**
 * Object containing  a image capture of the local track.
 */
let imageCapture;

/**
 * Object where the face centering worker is stored.
 */
let worker;

/**
 * Loads the worker.
 *
 * @returns {Function}
 */
export function loadWorker() {
    return async function(dispatch: Function, getState: Function) {
        if (navigator.product === 'ReactNative') {
            logger.warn('Unsupported environment for face centering');

            return;
        }

        const baseUrl = getBaseUrl();
        let workerUrl = `${baseUrl}libs/face-centering-worker.min.js`;

        const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });

        workerUrl = window.URL.createObjectURL(workerBlob);
        worker = new Worker(workerUrl, { name: 'Face Centering Worker' });
        worker.onmessage = function(e: Object) {
            const { type, value } = e.data;

            // receives a message with the face(s) bounding box.
            if (type === FACE_BOX_MESSAGE) {
                const state = getState();
                const conference = getCurrentConference(state);
                const localParticipant = getLocalParticipant(state);

                if (getParticipantCount(state) > 1) {
                    sendFaceBoxToParticipants(conference, value);
                }

                dispatch({
                    type: UPDATE_FACE_COORDINATES,
                    faceBox: value,
                    id: localParticipant.id
                });
            }
        };

        dispatch(startFaceRecognition());
    };
}

/**
 * Starts the recognition and detection of face position.
 *
 * @param {Track | undefined} track - Track for which to start detecting faces.
 *
 * @returns {Function}
 */
export function startFaceRecognition(track) {
    return async function(dispatch: Function, getState: Function) {
        if (!worker) {
            return;
        }
        const state = getState();
        const { recognitionActive } = state['features/face-centering'];

        if (recognitionActive) {
            logger.log('Face centering already active.');

            return;
        }

        const localVideoTrack = track || getLocalVideoTrack(state['features/base/tracks']);

        if (!localVideoTrack) {
            logger.warn('Face centering is disabled due to missing local track.');

            return;
        }

        dispatch({ type: START_FACE_RECOGNITION });
        logger.log('Start face recognition');

        const stream = localVideoTrack.jitsiTrack.getOriginalStream();
        const firstVideoTrack = stream.getVideoTracks()[0];

        imageCapture = new ImageCapture(firstVideoTrack);
        const { disableLocalVideoFlip, faceCoordinatesSharing } = state['features/base/config'];

        interval = setInterval(() => {
            sendDataToWorker(worker, imageCapture, faceCoordinatesSharing?.threshold, !disableLocalVideoFlip);
        }, faceCoordinatesSharing?.captureInterval || SEND_IMAGE_INTERVAL_MS);
    };
}

/**
 * Stops the recognition and detection of face position.
 *
 * @returns {Function}
 */
export function stopFaceRecognition() {
    return function(dispatch: Function) {
        clearInterval(interval);
        interval = null;
        imageCapture = null;

        dispatch({ type: STOP_FACE_RECOGNITION });
        logger.log('Stop face recognition');
    };
}