feat(remotecontrol): Implement basic remote control support

app.js

@@ -22,6 +22,8 @@ import conference from './conference';
 import API from './modules/API/API';
 
 import translation from "./modules/translation/translation";
+// For remote control testing:
+// import remoteControlController from "./modules/remotecontrol/Controller";
 
 const APP = {
     // Used by do_external_connect.js if we receive the attach data after

conference.js

@@ -17,6 +17,9 @@ import UIUtil from './modules/UI/util/UIUtil';
 
 import analytics from './modules/analytics/analytics';
 
+// For remote control testing:
+// import remoteControlReceiver from './modules/remotecontrol/Receiver';
+
 const ConnectionEvents = JitsiMeetJS.events.connection;
 const ConnectionErrors = JitsiMeetJS.errors.connection;
 
@@ -981,6 +984,8 @@ export default {
         let externalInstallation = false;
 
         if (shareScreen) {
+            // For remote control testing:
+            // remoteControlReceiver.start();
             createLocalTracks({
                 devices: ['desktop'],
                 desktopSharingExtensionExternalInstallation: {
@@ -1070,6 +1075,8 @@ export default {
                     dialogTitleKey, dialogTxt, false);
             });
         } else {
+            // For remote control testing:
+            // remoteControlReceiver.stop();
             createLocalTracks({ devices: ['video'] }).then(
                 ([stream]) => this.useVideoStream(stream)
             ).then(() => {
@@ -1600,12 +1607,23 @@ export default {
     },
     /**
     * Adds any room listener.
-    * @param eventName one of the ConferenceEvents
-    * @param callBack the function to be called when the event occurs
+    * @param {string} eventName one of the ConferenceEvents
+    * @param {Function} listener the function to be called when the event
+    * occurs
+    */
+    addConferenceListener(eventName, listener) {
+        room.on(eventName, listener);
+    },
+
+    /**
+    * Removes any room listener.
+    * @param {string} eventName one of the ConferenceEvents
+    * @param {Function} listener the listener to be removed.
     */
-    addConferenceListener(eventName, callBack) {
-        room.on(eventName, callBack);
+    removeConferenceListener(eventName, listener) {
+        room.off(eventName, listener);
     },
+
     /**
      * Inits list of current devices and event listener for device change.
      * @private
@@ -1813,5 +1831,17 @@ export default {
         APP.settings.setAvatarUrl(url);
         APP.UI.setUserAvatarUrl(room.myUserId(), url);
         sendData(commands.AVATAR_URL, url);
+    },
+
+    /**
+     * Sends a message via the data channel.
+     * @param to {string} the id of the endpoint that should receive the
+     * message. If "" the message will be sent to all participants.
+     * @param payload {object} the payload of the message.
+     * @throws NetworkError or InvalidStateError or Error if the operation
+     * fails.
+     */
+    sendEndpointMessage (to, payload) {
+        room.sendEndpointMessage(to, payload);
     }
 };

modules/API/API.js

@@ -201,6 +201,14 @@ export default {
         triggerEvent("video-ready-to-close", {});
     },
 
+    /**
+     * Sends remote control event.
+     * @param {object} event the event.
+     */
+    sendRemoteControlEvent(event) {
+        sendMessage({method: "remote-control-event", params: event});
+    },
+
     /**
      * Removes the listeners.
      */

modules/keycode/keycode.js

@@ -0,0 +1,160 @@
+/**
+ * Enumerates the supported keys.
+ */
+export const KEYS = {
+    BACKSPACE: "backspace" ,
+    DELETE : "delete",
+    RETURN : "enter",
+    TAB : "tab",
+    ESCAPE : "escape",
+    UP : "up",
+    DOWN : "down",
+    RIGHT : "right",
+    LEFT : "left",
+    HOME : "home",
+    END : "end",
+    PAGEUP : "pageup",
+    PAGEDOWN : "pagedown",
+
+    F1 : "f1",
+    F2 : "f2",
+    F3 : "f3",
+    F4 : "f4",
+    F5 : "f5",
+    F6 : "f6",
+    F7 : "f7",
+    F8 : "f8",
+    F9 : "f9",
+    F10 : "f10",
+    F11 : "f11",
+    F12 : "f12",
+    META : "command",
+    CMD_L: "command",
+    CMD_R: "command",
+    ALT : "alt",
+    CONTROL : "control",
+    SHIFT : "shift",
+    CAPS_LOCK: "caps_lock", //not supported by robotjs
+    SPACE : "space",
+    PRINTSCREEN : "printscreen",
+    INSERT : "insert",
+
+    NUMPAD_0 : "numpad_0",
+    NUMPAD_1 : "numpad_1",
+    NUMPAD_2 : "numpad_2",
+    NUMPAD_3 : "numpad_3",
+    NUMPAD_4 : "numpad_4",
+    NUMPAD_5 : "numpad_5",
+    NUMPAD_6 : "numpad_6",
+    NUMPAD_7 : "numpad_7",
+    NUMPAD_8 : "numpad_8",
+    NUMPAD_9 : "numpad_9",
+
+    COMMA: ",",
+
+    PERIOD: ".",
+    SEMICOLON: ";",
+    QUOTE: "'",
+    BRACKET_LEFT: "[",
+    BRACKET_RIGHT: "]",
+    BACKQUOTE: "`",
+    BACKSLASH: "\\",
+    MINUS: "-",
+    EQUAL: "=",
+    SLASH: "/"
+};
+
+/**
+ * Mapping between the key codes and keys deined in KEYS.
+ * The mappings are based on
+ * https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode#Specifications
+ */
+let keyCodeToKey = {
+    8: KEYS.BACKSPACE,
+    9: KEYS.TAB,
+    13: KEYS.RETURN,
+    16: KEYS.SHIFT,
+    17: KEYS.CONTROL,
+    18: KEYS.ALT,
+    20: KEYS.CAPS_LOCK,
+    27: KEYS.ESCAPE,
+    32: KEYS.SPACE,
+    33: KEYS.PAGEUP,
+    34: KEYS.PAGEDOWN,
+    35: KEYS.END,
+    36: KEYS.HOME,
+    37: KEYS.LEFT,
+    38: KEYS.UP,
+    39: KEYS.RIGHT,
+    40: KEYS.DOWN,
+    42: KEYS.PRINTSCREEN,
+    44: KEYS.PRINTSCREEN,
+    45: KEYS.INSERT,
+    46: KEYS.DELETE,
+    59: KEYS.SEMICOLON,
+    61: KEYS.EQUAL,
+    91: KEYS.CMD_L,
+    92: KEYS.CMD_R,
+    93: KEYS.CMD_R,
+    96: KEYS.NUMPAD_0,
+    97: KEYS.NUMPAD_1,
+    98: KEYS.NUMPAD_2,
+    99: KEYS.NUMPAD_3,
+    100: KEYS.NUMPAD_4,
+    101: KEYS.NUMPAD_5,
+    102: KEYS.NUMPAD_6,
+    103: KEYS.NUMPAD_7,
+    104: KEYS.NUMPAD_8,
+    105: KEYS.NUMPAD_9,
+    112: KEYS.F1,
+    113: KEYS.F2,
+    114: KEYS.F3,
+    115: KEYS.F4,
+    116: KEYS.F5,
+    117: KEYS.F6,
+    118: KEYS.F7,
+    119: KEYS.F8,
+    120: KEYS.F9,
+    121: KEYS.F10,
+    122: KEYS.F11,
+    123: KEYS.F12,
+    124: KEYS.PRINTSCREEN,
+    173: KEYS.MINUS,
+    186: KEYS.SEMICOLON,
+    187: KEYS.EQUAL,
+    188: KEYS.COMMA,
+    189: KEYS.MINUS,
+    190: KEYS.PERIOD,
+    191: KEYS.SLASH,
+    192: KEYS.BACKQUOTE,
+    219: KEYS.BRACKET_LEFT,
+    220: KEYS.BACKSLASH,
+    221: KEYS.BRACKET_RIGHT,
+    222: KEYS.QUOTE,
+    224: KEYS.META,
+    229: KEYS.SEMICOLON
+};
+
+/**
+ * Generate codes for digit keys (0-9)
+ */
+for(let i = 0; i < 10; i++) {
+    keyCodeToKey[i + 48] = `${i}`;
+}
+
+/**
+ * Generate codes for letter keys (a-z)
+ */
+for(let i = 0; i < 26; i++) {
+    let keyCode = i + 65;
+    keyCodeToKey[keyCode] = String.fromCharCode(keyCode).toLowerCase();
+}
+
+/**
+ * Returns key associated with the keyCode from the passed event.
+ * @param {KeyboardEvent} event the event
+ * @returns {KEYS} the key on the keyboard.
+ */
+export function keyboardEventToKey(event) {
+    return keyCodeToKey[event.which];
+}

modules/remotecontrol/Controller.js

@@ -0,0 +1,136 @@
+/* global $, APP */
+import * as KeyCodes from "../keycode/keycode";
+
+/**
+ * Extract the keyboard key from the keyboard event.
+ * @param event {KeyboardEvent} the event.
+ * @returns {KEYS} the key that is pressed or undefined.
+ */
+function getKey(event) {
+    return KeyCodes.keyboardEventToKey(event);
+}
+
+/**
+ * Extract the modifiers from the keyboard event.
+ * @param event {KeyboardEvent} the event.
+ * @returns {Array} with possible values: "shift", "control", "alt", "command".
+ */
+function getModifiers(event) {
+    let modifiers = [];
+    if(event.shiftKey) {
+        modifiers.push("shift");
+    }
+
+    if(event.ctrlKey) {
+        modifiers.push("control");
+    }
+
+
+    if(event.altKey) {
+        modifiers.push("alt");
+    }
+
+    if(event.metaKey) {
+        modifiers.push("command");
+    }
+
+    return modifiers;
+}
+
+/**
+ * This class represents the controller party for a remote controller session.
+ * It listens for mouse and keyboard events and sends them to the receiver
+ * party of the remote control session.
+ */
+class Controller {
+    /**
+     * Creates new instance.
+     */
+    constructor() {}
+
+    /**
+     * Starts processing the mouse and keyboard events.
+     * @param {JQuery.selector} area the selector which will be used for
+     * attaching the listeners on.
+     */
+    start(area) {
+        this.area = area;
+        this.area.mousemove(event => {
+            const position = this.area.position();
+            this._sendEvent({
+                type: "mousemove",
+                x: (event.pageX - position.left)/this.area.width(),
+                y: (event.pageY - position.top)/this.area.height()
+            });
+        });
+        this.area.mousedown(this._onMouseClickHandler.bind(this, "mousedown"));
+        this.area.mouseup(this._onMouseClickHandler.bind(this, "mouseup"));
+        this.area.dblclick(
+            this._onMouseClickHandler.bind(this, "mousedblclick"));
+        this.area.contextmenu(() => false);
+        this.area[0].onmousewheel = event => {
+            this._sendEvent({
+                type: "mousescroll",
+                x: event.deltaX,
+                y: event.deltaY
+            });
+        };
+        $(window).keydown(this._onKeyPessHandler.bind(this, "keydown"));
+        $(window).keyup(this._onKeyPessHandler.bind(this, "keyup"));
+    }
+
+    /**
+     * Stops processing the mouse and keyboard events.
+     */
+    stop() {
+        this.area.off( "mousemove" );
+        this.area.off( "mousedown" );
+        this.area.off( "mouseup" );
+        this.area.off( "contextmenu" );
+        this.area.off( "dblclick" );
+        $(window).off( "keydown");
+        $(window).off( "keyup");
+        this.area[0].onmousewheel = undefined;
+    }
+
+    /**
+     * Handler for mouse click events.
+     * @param {String} type the type of event ("mousedown"/"mouseup")
+     * @param {Event} event the mouse event.
+     */
+    _onMouseClickHandler(type, event) {
+        this._sendEvent({
+            type: type,
+            button: event.which
+        });
+    }
+
+    /**
+     * Handler for key press events.
+     * @param {String} type the type of event ("keydown"/"keyup")
+     * @param {Event} event the key event.
+     */
+    _onKeyPessHandler(type, event) {
+        this._sendEvent({
+            type: type,
+            key: getKey(event),
+            modifiers: getModifiers(event),
+        });
+    }
+
+    /**
+     * Sends remote control event to the controlled participant.
+     * @param {Object} event the remote control event.
+     */
+    _sendRemoteControlEvent(event) {
+        try{
+            APP.conference.sendEndpointMessage("",
+                {type: "remote-control-event", event});
+        } catch (e) {
+            // failed to send the event.
+        }
+    }
+}
+
+
+export default new Controller();

modules/remotecontrol/Receiver.js

@@ -0,0 +1,47 @@
+/* global APP, JitsiMeetJS */
+const ConferenceEvents = JitsiMeetJS.events.conference;
+
+/**
+ * This class represents the receiver party for a remote controller session.
+ * It handles "remote-control-event" events and sends them to the
+ * API module. From there the events can be received from wrapper application
+ * and executed.
+ */
+class Receiver {
+    /**
+     * Creates new instance.
+     * @constructor
+     */
+    constructor() {}
+
+    /**
+     * Attaches listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED events.
+     */
+    start() {
+        APP.conference.addConferenceListener(
+            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+            this._onRemoteControlEvent);
+    }
+
+    /**
+     * Removes the listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED
+     * events.
+     */
+    stop() {
+        APP.conference.removeConferenceListener(
+            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+            this._onRemoteControlEvent);
+    }
+
+    /**
+     * Sends "remote-control-event" events to to the API module.
+     * @param {JitsiParticipant} participant the controller participant
+     * @param {Object} event the remote control event.
+     */
+    _onRemoteControlEvent(participant, event) {
+        if(event.type === "remote-control-event")
+            APP.API.sendRemoteControlEvent(event.event);
+    }
+}
+
+export default new Receiver();