var AudioRecorder = require( './audioRecorder'); var SphinxService = require( './transcriptionServices/SphinxTranscriptionService'); var BEFORE_STATE = "before"; var RECORDING_STATE = "recording"; var TRANSCRIBING_STATE = "transcribing"; var FINISHED_STATE = "finished"; //the amount of characters each line in the transcription will have var MAXIMUM_SENTENCE_LENGTH = 80; /** * This is the main object for handing the Transcription. It interacts with * the audioRecorder to record every person in a conference and sends the * recorder audio to a transcriptionService. The returned speech-to-text result * will be merged to create a transcript * @param {AudioRecorder} audioRecorder An audioRecorder recording a conference */ var transcriber = function() { //the object which can record all audio in the conference this.audioRecorder = new AudioRecorder(); //this object can send the recorder audio to a speech-to-text service this.transcriptionService = new SphinxService(); //holds a counter to keep track if merging can start this.counter = null; //holds the date when transcription started which makes it possible //to calculate the offset between recordings this.startTime = null; //will hold the transcription once it is completed this.transcription = null; //this will be a method which will be called once the transcription is done //with the transcription as parameter this.callback = null; //stores all the retrieved speech-to-text results to merge together //this value will store an Array object this.results = []; // Stores the current state of the transcription process this.state = BEFORE_STATE; //Used in the updateTranscription method to add a new line when the //sentence becomes to long this.lineLength = 0; }; /** * Method to start the transcription process. It will tell the audioRecorder * to start storing all audio streams and record the start time for merging * purposes */ transcriber.prototype.start = function start() { if(this.state !== BEFORE_STATE){ throw new Error("The transcription can only start when it's in the" + "\"" + BEFORE_STATE + "\" state. It's currently in the " + "\"" + this.state + "\" state"); } this.state = RECORDING_STATE; this.audioRecorder.start(); this.startTime = new Date(); }; /** * Method to stop the transcription process. It will tell the audioRecorder to * stop, and get all the recorded audio to send it to the transcription service * @param callback a callback which will receive the transcription */ transcriber.prototype.stop = function stop(callback) { if(this.state !== RECORDING_STATE){ throw new Error("The transcription can only stop when it's in the" + "\"" + RECORDING_STATE + "\" state. It's currently in the " + "\"" + this.state + "\" state"); } //stop the recording console.log("stopping recording and sending audio files"); this.audioRecorder.stop(); //and send all recorded audio the the transcription service var t = this; var callBack = blobCallBack.bind(this); this.audioRecorder.getRecordingResults().forEach(function(recordingResult){ t.transcriptionService.send(recordingResult, callBack); t.counter++; }); //set the state to "transcribing" so that maybeMerge() functions correctly this.state = TRANSCRIBING_STATE; //and store the callback for later this.callback = callback; }; /** * This method gets the answer from the transcription service, calculates the * offset and adds is to every Word object. It will also start the merging * when every send request has been received * * note: Make sure to bind this as a Transcription object * * @param {RecordingResult} answer a RecordingResult object with a defined * WordArray */ var blobCallBack = function(answer){ console.log("retrieved an answer from the transcription service. The" + " answer has an array of length: " + answer.wordArray.length); //first add the offset between the start of the transcription and //the start of the recording to all start and end times if(answer.wordArray.length > 0) { var offset = answer.startTime.getUTCMilliseconds() - this.startTime.getUTCMilliseconds(); //transcriber time will always be earlier if (offset < 0) { offset = 0; //presume 0 if it somehow not earlier } var array = "["; answer.wordArray.forEach(function(wordObject) { wordObject.begin += offset; wordObject.end += offset; array += wordObject.word+","; }); array += "]"; console.log(array); //give a name value to the Array object so that the merging can access //the name value without having to use the whole recordingResult object //in the algorithm answer.wordArray.name = answer.name; } //then store the array and decrease the counter this.results.push(answer.wordArray); this.counter--; console.log("current counter: " + this.counter); //and check if all results have been received. this.maybeMerge(); }; /** * this method will check if the counter is zero. If it is, it will call * the merging method */ transcriber.prototype.maybeMerge = function(){ if(this.state === TRANSCRIBING_STATE && this.counter === 0){ //make sure to include the events in the result arrays before //merging starts this.merge(); } }; /** * This method will merge all speech-to-text arrays together in one * readable transcription string */ transcriber.prototype.merge = function() { console.log("starting merge process!\n The length of the array: " + this.results.length); this.transcription = ""; //the merging algorithm will look over all Word objects who are at pos 0 in //every array. It will then select the one closest in time to the //previously placed word, while removing the selected word from its array //note: words can be skipped the skipped word's begin and end time somehow //end up between the closest word start and end time var arrays = this.results; //arrays of Word objects var potentialWords = []; //array of the first Word objects //check if any arrays are already empty and remove them hasPopulatedArrays(arrays); //populate all the potential Words for a first time arrays.forEach(function (array){ pushWordToSortedArray(potentialWords, array); }); //keep adding words to transcription until all arrays are exhausted var lowestWordArray; var wordToAdd; var foundSmaller; while(hasPopulatedArrays(arrays)){ //first select the lowest array; lowestWordArray = arrays[0]; arrays.forEach(function(wordArray){ if(wordArray[0].begin < lowestWordArray[0].begin){ lowestWordArray = wordArray; } }); //put the word in the transcription wordToAdd = lowestWordArray.shift(); this.updateTranscription(wordToAdd,lowestWordArray.name); //keep going until a word in another array has a smaller time //or the array is empty while(!foundSmaller && lowestWordArray.length > 0){ arrays.forEach(function(wordArray){ if(wordArray[0].begin < lowestWordArray[0].begin){ foundSmaller = true; } }); //add next word if no smaller time has been found if(!foundSmaller){ wordToAdd = lowestWordArray.shift(); this.updateTranscription(wordToAdd, null); } } } //set the state to finished and do the necessary left-over tasks this.state = FINISHED_STATE; if(this.callback){ this.callback(this.transcription); } }; /** * Appends a word object to the transcription. It will make a new line with a * name if a name is specified * @param {Word} word the Word object holding the word to append * @param {String|null} name the name of a new speaker. Null if not applicable */ transcriber.prototype.updateTranscription = function(word, name){ if(name !== undefined && name !== null){ this.transcription += "\n" + name + ":"; this.lineLength = name.length + 1; //+1 for the semi-colon } if(this.lineLength + word.word.length > MAXIMUM_SENTENCE_LENGTH){ this.transcription += "\n "; this.lineLength = 4; //because of the 4 spaces after the new line } this.transcription += " " + word.word; this.lineLength += word.word.length + 1; //+1 for the space }; /** * Check if the given 2 dimensional array has any non-zero Word-arrays in them. * All zero-element arrays inside will be removed * If any non-zero-element arrays are found, the method will return true. * otherwise it will return false * @param {Array} twoDimensionalArray the array to check * @returns {boolean} true if any non-zero arrays inside, otherwise false */ var hasPopulatedArrays = function(twoDimensionalArray){ var i; for(i = 0; i < twoDimensionalArray.length; i++){ if(twoDimensionalArray[i].length === 0){ twoDimensionalArray.splice(i, 1); } } return twoDimensionalArray.length > 0; }; /** * Push a word to the right location in a sorted array. The array is sorted * from lowest to highest start time. Every word is stored in an object which * includes the name of the person saying the word. * * @param {Array} array the sorted array to push to * @param {Word} word the word to push into the array */ var pushWordToSortedArray = function(array, word){ if(array.length === 0) { array.push(word); } else{ if(array[array.length - 1].begin <= word.begin){ array.push(word); return; } var i; for(i = 0; i < array.length; i++){ if(word.begin < array[i].begin){ array.splice(i, 0, word); return; } } array.push(word); //fail safe } }; /** * Gives the transcriber a JitsiTrack holding an audioStream to transcribe. * The JitsiTrack is given to the audioRecorder. If it doesn't hold an * audiostream, it will not be added by the audioRecorder * @param {JitsiTrack} track the track to give to the audioRecorder */ transcriber.prototype.addTrack = function(track){ this.audioRecorder.addTrack(track); }; /** * Remove the given track from the auioRecorder * @param track */ transcriber.prototype.removeTrack = function(track){ this.audioRecorder.removeTrack(track); }; /** * Will return the created transcription if it's avialable or throw an error * when it's not done yet * @returns {String} the transcription as a String */ transcriber.prototype.getTranscription = function(){ if(this.state !== FINISHED_STATE){ throw new Error("The transcription can only be retrieved when it's in" + " the\"" + FINISHED_STATE + "\" state. It's currently in the " + "\"" + this.state + "\" state"); } return this.transcription; }; /** * Returns the current state of the transcription process */ transcriber.prototype.getState = function(){ return this.state; }; /** * Resets the state to the "before" state, such that it's again possible to * call the start method */ transcriber.prototype.reset = function() { this.state = BEFORE_STATE; this.counter = null; this.transcription = null; this.startTime = null; this.callback = null; this.results = []; this.lineLength = 0; }; module.exports = transcriber;