[WEB] Show final translated speech to text results as subtitles (#3276)

* Shows final translated speech to text results as subtitles

* Use conference from redux state and removes addTranscriptMessage

react/features/subtitles/actionTypes.js

@@ -1,14 +1,3 @@
-/**
- * The type of (redux) action which indicates that a transcript with
- * a new message_id is received.
- *
- * {
- *      type: ADD_TRANSCRIPT_MESSAGE,
- *      transcriptMessageID: string,
- *      participantName: string
- * }
- */
-export const ADD_TRANSCRIPT_MESSAGE = Symbol('ADD_TRANSCRIPT_MESSAGE');
 
 /**
  * The type of (redux) action which indicates that an endpoint message
@@ -35,7 +24,7 @@ export const REMOVE_TRANSCRIPT_MESSAGE = Symbol('REMOVE_TRANSCRIPT_MESSAGE');
 
 /**
  * The type of (redux) action which indicates that a transcript with an
- * existing message_id to be updated is received.
+ * given message_id to be added or updated is received.
  *
  * {
  *      type: UPDATE_TRANSCRIPT_MESSAGE,

react/features/subtitles/actions.js

@@ -1,32 +1,11 @@
 // @flow
 
 import {
-    ADD_TRANSCRIPT_MESSAGE,
     ENDPOINT_MESSAGE_RECEIVED,
     REMOVE_TRANSCRIPT_MESSAGE,
     UPDATE_TRANSCRIPT_MESSAGE
 } from './actionTypes';
 
-/**
- * Signals that a transcript with a new message_id is received.
- *
- * @param {string} transcriptMessageID - The new message_id.
- * @param {string} participantName - The participant name of the sender.
- * @returns {{
- *      type: ADD_TRANSCRIPT_MESSAGE,
- *      transcriptMessageID: string,
- *      participantName: string
- * }}
- */
-export function addTranscriptMessage(transcriptMessageID: string,
-        participantName: string) {
-    return {
-        type: ADD_TRANSCRIPT_MESSAGE,
-        transcriptMessageID,
-        participantName
-    };
-}
-
 /**
  * Signals that a participant sent an endpoint message on the data channel.
  *
@@ -63,7 +42,7 @@ export function removeTranscriptMessage(transcriptMessageID: string) {
 }
 
 /**
- * Signals that a transcript with an existing message_id to be updated
+ * Signals that a transcript with the given message_id to be added or updated
  * is received.
  *
  * @param {string} transcriptMessageID -The transcript message_id to be updated.

react/features/subtitles/middleware.js

@@ -1,14 +1,35 @@
+// @flow
+
 import { MiddlewareRegistry } from '../base/redux';
 
 import { ENDPOINT_MESSAGE_RECEIVED } from './actionTypes';
 import {
-    addTranscriptMessage,
     removeTranscriptMessage,
     updateTranscriptMessage
 } from './actions';
 
 const logger = require('jitsi-meet-logger').getLogger(__filename);
 
+declare var APP: Object;
+
+/**
+ * The type of json-message which indicates that json carries a
+ * transcription result.
+ */
+const JSON_TYPE_TRANSCRIPTION_RESULT = 'transcription-result';
+
+/**
+ * The type of json-message which indicates that json carries a
+ * translation result.
+ */
+const JSON_TYPE_TRANSLATION_RESULT = 'translation-result';
+
+/**
+ * The local participant property which is used to store the language
+ * preference for translation for a participant.
+ */
+const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';
+
 /**
 * Time after which the rendered subtitles will be removed.
 */
@@ -47,31 +68,46 @@ MiddlewareRegistry.register(store => next => action => {
  */
 function _endpointMessageReceived({ dispatch, getState }, next, action) {
     const json = action.json;
+    const translationLanguage
+        = getState()['features/base/conference'].conference
+            .getLocalParticipantProperty(P_NAME_TRANSLATION_LANGUAGE);
 
     try {
+        const transcriptMessageID = json.message_id;
+        const participantName = json.participant.name;
+        const isInterim = json.is_interim;
+        const stability = json.stability;
+
+        if (json.type === JSON_TYPE_TRANSLATION_RESULT
+            && json.language === translationLanguage) {
+            // Displays final results in the target language if translation is
+            // enabled.
+
+            const newTranscriptMessage = {
+                participantName,
+                final: json.text
+            };
+
+            dispatch(updateTranscriptMessage(transcriptMessageID,
+                newTranscriptMessage));
+
+            setTimeout(() => {
+                dispatch(removeTranscriptMessage(transcriptMessageID));
+            }, REMOVE_AFTER_MS);
+
+        } else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT
+            && !translationLanguage) {
+            // Displays interim and final results without any translation if
+            // translations are disabled.
 
-        // Let's first check if the given object has the correct
-        // type in the json, which identifies it as a json message sent
-        // from Jigasi with speech-to-to-text results
-        if (json.type === 'transcription-result') {
-            // Extract the useful data from the json.
-            const isInterim = json.is_interim;
-            const participantName = json.participant.name;
-            const stability = json.stability;
             const text = json.transcript[0].text;
-            const transcriptMessageID = json.message_id;
-
-            // If this is the first result with the unique message ID,
-            // we add it to the state along with the name of the participant
-            // who said given text
-            if (!getState()['features/subtitles']
-                .transcriptMessages.has(transcriptMessageID)) {
-                dispatch(addTranscriptMessage(transcriptMessageID,
-                    participantName));
-            }
-            const { transcriptMessages } = getState()['features/subtitles'];
+
+            // We update the previous transcript message with the same
+            // message ID or adds a new transcript message if it does not
+            // exist in the map.
             const newTranscriptMessage
-                = { ...transcriptMessages.get(transcriptMessageID) };
+                = { ...getState()['features/subtitles'].transcriptMessages
+                    .get(transcriptMessageID) || { participantName } };
 
             // If this is final result, update the state as a final result
             // and start a count down to remove the subtitle from the state

react/features/subtitles/reducer.js

@@ -1,7 +1,6 @@
 import { ReducerRegistry } from '../base/redux';
 
 import {
-    ADD_TRANSCRIPT_MESSAGE,
     REMOVE_TRANSCRIPT_MESSAGE,
     UPDATE_TRANSCRIPT_MESSAGE
 } from './actionTypes';
@@ -20,9 +19,6 @@ const defaultState = {
 ReducerRegistry.register('features/subtitles', (
         state = defaultState, action) => {
     switch (action.type) {
-    case ADD_TRANSCRIPT_MESSAGE:
-        return _addTranscriptMessage(state, action);
-
     case REMOVE_TRANSCRIPT_MESSAGE:
         return _removeTranscriptMessage(state, action);
 
@@ -33,28 +29,6 @@ ReducerRegistry.register('features/subtitles', (
     return state;
 });
 
-/**
- * Reduces a specific Redux action ADD_TRANSCRIPT_MESSAGE of the feature
- * transcription.
- *
- * @param {Object} state - The Redux state of the feature transcription.
- * @param {Action} action -The Redux action ADD_TRANSCRIPT_MESSAGE to reduce.
- * @returns {Object} The new state of the feature transcription after the
- * reduction of the specified action.
- */
-function _addTranscriptMessage(state,
-        { transcriptMessageID, participantName }) {
-    const newTranscriptMessages = new Map(state.transcriptMessages);
-
-    // Adds a new key,value pair to the Map once a new message arrives.
-    newTranscriptMessages.set(transcriptMessageID, { participantName });
-
-    return {
-        ...state,
-        transcriptMessages: newTranscriptMessages
-    };
-}
-
 /**
  * Reduces a specific Redux action REMOVE_TRANSCRIPT_MESSAGE of the feature
  * transcription.