lib-jitsi-meet/modules/transcription/transcriber.js

var AudioRecorder = require( './audioRecorder');
var SphinxService = require(
    './transcriptionServices/SphinxTranscriptionService');

var BEFORE_STATE = "before";
var RECORDING_STATE = "recording";
var TRANSCRIBING_STATE = "transcribing";
var FINISHED_STATE = "finished";

//the amount of characters each line in the transcription will have
var MAXIMUM_SENTENCE_LENGTH = 80;

/**
 * This is the main object for handing the Transcription. It interacts with
 * the audioRecorder to record every person in a conference and sends the
 * recorder audio to a transcriptionService. The returned speech-to-text result
 * will be merged to create a transcript
 * @param {AudioRecorder} audioRecorder An audioRecorder recording a conference
 */
var transcriber = function() {
    //the object which can record all audio in the conference
    this.audioRecorder = new AudioRecorder();
    //this object can send the recorder audio to a speech-to-text service
    this.transcriptionService =  new SphinxService();
    //holds a counter to keep track if merging can start
    this.counter = null;
    //holds the date when transcription started which makes it possible
    //to calculate the offset between recordings
    this.startTime = null;
    //will hold the transcription once it is completed
    this.transcription = null;
    //this will be a method which will be called once the transcription is done
    //with the transcription as parameter
    this.callback =  null;
    //stores all the retrieved speech-to-text results to merge together
    //this value will store an Array<Word> object
    this.results = [];
    // Stores the current state of the transcription process
    this.state = BEFORE_STATE;
    //Used in the updateTranscription method to add a new line when the
    //sentence becomes to long
    this.lineLength = 0;
};

/**
 * Method to start the transcription process. It will tell the audioRecorder
 * to start storing all audio streams and record the start time for merging
 * purposes
 */
transcriber.prototype.start = function start() {
    if(this.state !== BEFORE_STATE){
        throw new Error("The transcription can only start when it's in the" +
            "\"" + BEFORE_STATE + "\" state. It's currently in the " +
            "\"" + this.state + "\" state");
    }
    this.state = RECORDING_STATE;
    this.audioRecorder.start();
    this.startTime = new Date();
};

/**
 * Method to stop the transcription process. It will tell the audioRecorder to
 * stop, and get all the recorded audio to send it to the transcription service

 * @param callback a callback which will receive the transcription
 */
transcriber.prototype.stop = function stop(callback) {
    if(this.state !== RECORDING_STATE){
        throw new Error("The transcription can only stop when it's in the" +
            "\"" + RECORDING_STATE + "\" state. It's currently in the " +
            "\"" + this.state + "\" state");
    }
    //stop the recording
    console.log("stopping recording and sending audio files");
    this.audioRecorder.stop();
    //and send all recorded audio the the transcription service
    var t = this;

    var callBack = blobCallBack.bind(this);
    this.audioRecorder.getRecordingResults().forEach(function(recordingResult){
        t.transcriptionService.send(recordingResult, callBack);
        t.counter++;
    });
    //set the state to "transcribing" so that maybeMerge() functions correctly
    this.state = TRANSCRIBING_STATE;
    //and store the callback for later
    this.callback = callback;
};

/**
 * This method gets the answer from the transcription service, calculates the
 * offset and adds is to every Word object. It will also start the merging
 * when every send request has been received
 *
 * note: Make sure to bind this as a Transcription object
 *
 * @param {RecordingResult} answer a RecordingResult object with a defined
 * WordArray
 */
var blobCallBack = function(answer){
    console.log("retrieved an answer from the transcription service. The" +
        " answer has an array of length: " + answer.wordArray.length);
    //first add the offset between the start of the transcription and
    //the start of the recording to all start and end times
    if(answer.wordArray.length > 0) {
        var offset = answer.startTime.getUTCMilliseconds() -
            this.startTime.getUTCMilliseconds();
        //transcriber time will always be earlier
        if (offset < 0) {
            offset = 0; //presume 0 if it somehow not earlier
        }

        var array = "[";
        answer.wordArray.forEach(function(wordObject) {
            wordObject.begin += offset;
            wordObject.end += offset;
            array += wordObject.word+",";
        });
        array += "]";
        console.log(array);
        //give a name value to the Array object so that the merging can access
        //the name value without having to use the whole recordingResult object
        //in the algorithm
        answer.wordArray.name = answer.name;
    }
    //then store the array and decrease the counter
    this.results.push(answer.wordArray);
    this.counter--;
    console.log("current counter: " + this.counter);
    //and check if all results have been received.
    this.maybeMerge();
};

/**
 * this method will check if the counter is zero. If it is, it will call
 * the merging method
 */
transcriber.prototype.maybeMerge = function(){
    if(this.state === TRANSCRIBING_STATE && this.counter === 0){
        //make sure to include the events in the result arrays before
        //merging starts
        this.merge();
    }
};

/**
 * This method will merge all speech-to-text arrays together in one
 * readable transcription string
 */
transcriber.prototype.merge = function() {
    console.log("starting merge process!\n The length of the array: " +
        this.results.length);
    this.transcription = "";
    //the merging algorithm will look over all Word objects who are at pos 0 in
    //every array. It will then select the one closest in time to the
    //previously placed word, while removing the selected word from its array
    //note: words can be skipped the skipped word's begin and end time somehow
    //end up between the closest word start and end time
    var arrays = this.results;
    //arrays of Word objects
    var potentialWords = []; //array of the first Word objects
    //check if any arrays are already empty and remove them
    hasPopulatedArrays(arrays);

    //populate all the potential Words for a first time
    arrays.forEach(function (array){
        pushWordToSortedArray(potentialWords, array);
    });

    //keep adding words to transcription until all arrays are exhausted
    var lowestWordArray;
    var wordToAdd;
    var foundSmaller;
    while(hasPopulatedArrays(arrays)){
        //first select the lowest array;
        lowestWordArray = arrays[0];
        arrays.forEach(function(wordArray){
           if(wordArray[0].begin < lowestWordArray[0].begin){
               lowestWordArray = wordArray;
           }
        });
        //put the word in the transcription
        wordToAdd = lowestWordArray.shift();
        this.updateTranscription(wordToAdd,lowestWordArray.name);

        //keep going until a word in another array has a smaller time
        //or the array is empty
        while(!foundSmaller && lowestWordArray.length > 0){
            arrays.forEach(function(wordArray){
                if(wordArray[0].begin < lowestWordArray[0].begin){
                    foundSmaller = true;
                }
            });
            //add next word if no smaller time has been found
            if(!foundSmaller){
                wordToAdd = lowestWordArray.shift();
                this.updateTranscription(wordToAdd, null);
            }
        }

    }

    //set the state to finished and do the necessary left-over tasks
    this.state = FINISHED_STATE;
    if(this.callback){
        this.callback(this.transcription);
    }
};

/**
 * Appends a word object to the transcription. It will make a new line with a
 * name if a name is specified
 * @param {Word} word the Word object holding the word to append
 * @param {String|null} name the name of a new speaker. Null if not applicable
 */
transcriber.prototype.updateTranscription = function(word, name){
    if(name !== undefined && name !== null){
        this.transcription += "\n" + name + ":";
        this.lineLength = name.length + 1; //+1 for the semi-colon
    }
    if(this.lineLength + word.word.length > MAXIMUM_SENTENCE_LENGTH){
        this.transcription += "\n    ";
        this.lineLength = 4; //because of the 4 spaces after the new line
    }
    this.transcription += " " + word.word;
    this.lineLength += word.word.length + 1; //+1 for the space
};

/**
 * Check if the given 2 dimensional array has any non-zero Word-arrays in them.
 * All zero-element arrays inside will be removed
 * If any non-zero-element arrays are found, the method will return true.
 * otherwise it will return false
 * @param {Array<Array>} twoDimensionalArray the array to check
 * @returns {boolean} true if any non-zero arrays inside, otherwise false
 */
var hasPopulatedArrays = function(twoDimensionalArray){
    var i;
    for(i = 0; i < twoDimensionalArray.length; i++){
        if(twoDimensionalArray[i].length === 0){
            twoDimensionalArray.splice(i, 1);
        }
    }
    return twoDimensionalArray.length > 0;
};

/**
 * Push a word to the right location in a sorted array. The array is sorted
 * from lowest to highest start time. Every word is stored in an object which
 * includes the name of the person saying the word.
 *
 * @param {Array<Word>} array the sorted array to push to
 * @param {Word} word the word to push into the array
 */
var pushWordToSortedArray = function(array, word){
    if(array.length === 0) {
        array.push(word);
    }
    else{
        if(array[array.length - 1].begin <= word.begin){
            array.push(word);
            return;
        }
        var i;
        for(i = 0; i < array.length; i++){
            if(word.begin < array[i].begin){
                array.splice(i, 0, word);
                return;
            }
        }
        array.push(word); //fail safe
    }
};

/**
 * Gives the transcriber a JitsiTrack holding an audioStream to transcribe.
 * The JitsiTrack is given to the audioRecorder. If it doesn't hold an
 * audiostream, it will not be added by the audioRecorder
 * @param {JitsiTrack} track the track to give to the audioRecorder
 */
transcriber.prototype.addTrack = function(track){
    this.audioRecorder.addTrack(track);
};

/**
 * Remove the given track from the auioRecorder
 * @param track
 */
transcriber.prototype.removeTrack = function(track){
    this.audioRecorder.removeTrack(track);
};

/**
 * Will return the created transcription if it's avialable or throw an error
 * when it's not done yet
 * @returns {String} the transcription as a String
 */
transcriber.prototype.getTranscription = function(){
    if(this.state !== FINISHED_STATE){
        throw new Error("The transcription can only be retrieved when it's in" +
            " the\"" + FINISHED_STATE + "\" state. It's currently in the " +
            "\"" + this.state + "\" state");
    }
    return this.transcription;
};

/**
 * Returns the current state of the transcription process
 */
transcriber.prototype.getState = function(){
    return this.state;
};

/**
 * Resets the state to the "before" state, such that it's again possible to
 * call the start method
 */
transcriber.prototype.reset = function() {
    this.state = BEFORE_STATE;
    this.counter = null;
    this.transcription = null;
    this.startTime = null;
    this.callback = null;
    this.results = [];
    this.lineLength = 0;
};

module.exports = transcriber;