Adjustments to make lip-sync work. Stream merging is now done by Jicofo.

modules/RTC/LocalStream.js

@@ -1,7 +1,8 @@
 /* global APP */
-var StreamEventTypes = require("../../service/RTC/StreamEventTypes.js");
+var MediaStreamType = require("../../service/RTC/MediaStreamTypes");
 var RTCEvents = require("../../service/RTC/RTCEvents");
 var RTCBrowserType = require("./RTCBrowserType");
+var StreamEventTypes = require("../../service/RTC/StreamEventTypes.js");
 
 /**
  * This implements 'onended' callback normally fired by WebRTC after the stream
@@ -29,7 +30,7 @@ function LocalStream(stream, type, eventEmitter, videoType, isGUMStream) {
     if(isGUMStream === false)
         this.isGUMStream = isGUMStream;
     var self = this;
-    if(type == "audio") {
+    if (MediaStreamType.AUDIO_TYPE === type) {
         this.getTracks = function () {
             return self.stream.getAudioTracks();
         };
@@ -60,7 +61,11 @@ LocalStream.prototype.getOriginalStream = function()
 };
 
 LocalStream.prototype.isAudioStream = function () {
-    return this.type === "audio";
+    return MediaStreamType.AUDIO_TYPE === this.type;
+};
+
+LocalStream.prototype.isVideoStream = function () {
+    return MediaStreamType.VIDEO_TYPE === this.type;
 };
 
 LocalStream.prototype.setMute = function (mute)

modules/RTC/MediaStream.js

@@ -36,8 +36,16 @@ function MediaStream(data, ssrc, browser, eventEmitter, muted, type) {
     this.eventEmitter = eventEmitter;
 }
 
+// FIXME duplicated with LocalStream methods - extract base class
+MediaStream.prototype.isAudioStream = function () {
+    return MediaStreamType.AUDIO_TYPE === this.type;
+};
+
+MediaStream.prototype.isVideoStream = function () {
+    return MediaStreamType.VIDEO_TYPE === this.type;
+};
 
-MediaStream.prototype.getOriginalStream = function() {
+MediaStream.prototype.getOriginalStream = function () {
     return this.stream;
 };
 

modules/RTC/RTC.js

@@ -76,7 +76,7 @@ var RTC = {
         if(isMuted === true)
             localStream.setMute(true);
 
-        if(type == "audio") {
+        if (MediaStreamType.AUDIO_TYPE === type) {
             this.localAudio = localStream;
         } else {
             this.localVideo = localStream;
@@ -100,12 +100,12 @@ var RTC = {
 
         var self = this;
         [MediaStreamType.AUDIO_TYPE, MediaStreamType.VIDEO_TYPE].forEach(
-            function(type) {
+            function (type) {
             var tracks =
                 type == MediaStreamType.AUDIO_TYPE
-                ? data.stream.getAudioTracks : data.stream.getVideoTracks();
+                ? data.stream.getAudioTracks() : data.stream.getVideoTracks();
             if (!tracks || !Array.isArray(tracks) || !tracks.length) {
-                console.log("Not creating a(n) "+type+" stream: no tracks");
+                console.log("Not creating a(n) " + type + " stream: no tracks");
                 return;
             }
 
@@ -229,7 +229,9 @@ var RTC = {
     changeLocalAudio: function (stream, callback) {
         var oldStream = this.localAudio.getOriginalStream();
         var newStream = this.rtcUtils.createStream(stream);
-        this.localAudio = this.createLocalStream(newStream, "audio", true);
+        this.localAudio
+            = this.createLocalStream(
+                    newStream, MediaStreamType.AUDIO_TYPE, true);
         // Stop the stream
         this.stopMediaStream(oldStream);
         APP.xmpp.switchStreams(newStream, oldStream, callback, true);

modules/RTC/RTCUtils.js

@@ -2,6 +2,7 @@
     RTCPeerConnection, webkitMediaStream, webkitURL, webkitRTCPeerConnection,
     mozRTCIceCandidate, mozRTCSessionDescription, mozRTCPeerConnection */
 /* jshint -W101 */
+var MediaStreamType = require("../../service/RTC/MediaStreamTypes");
 var RTCBrowserType = require("./RTCBrowserType");
 var Resolutions = require("../../service/RTC/Resolutions");
 var AdapterJS = require("./adapter.screenshare");
@@ -523,10 +524,12 @@ RTCUtils.prototype.handleLocalStream = function(stream, usageOptions) {
         videoGUM = (!usageOptions || usageOptions.video !== false);
 
 
-    this.service.createLocalStream(audioStream, "audio", null, null,
+    this.service.createLocalStream(
+        audioStream, MediaStreamType.AUDIO_TYPE, null, null,
         audioMuted, audioGUM);
 
-    this.service.createLocalStream(videoStream, "video", null, 'camera',
+    this.service.createLocalStream(
+        videoStream, MediaStreamType.VIDEO_TYPE, null, 'camera',
         videoMuted, videoGUM);
 };
 

modules/UI/UI.js

@@ -26,6 +26,7 @@ var JitsiPopover = require("./util/JitsiPopover");
 var CQEvents = require("../../service/connectionquality/CQEvents");
 var DesktopSharingEventTypes
     = require("../../service/desktopsharing/DesktopSharingEventTypes");
+var MediaStreamType = require("../../service/RTC/MediaStreamTypes");
 var RTCEvents = require("../../service/RTC/RTCEvents");
 var RTCBrowserType = require("../RTC/RTCBrowserType");
 var StreamEventTypes = require("../../service/RTC/StreamEventTypes");
@@ -110,14 +111,14 @@ function setupToolbars() {
 
 function streamHandler(stream, isMuted) {
     switch (stream.type) {
-        case "audio":
+        case MediaStreamType.AUDIO_TYPE:
             VideoLayout.changeLocalAudio(stream, isMuted);
             break;
-        case "video":
+        case MediaStreamType.VIDEO_TYPE:
             VideoLayout.changeLocalVideo(stream, isMuted);
             break;
-        case "stream":
-            VideoLayout.changeLocalStream(stream, isMuted);
+        default:
+            console.error("Unknown stream type: " + stream.type);
             break;
     }
 }

modules/UI/videolayout/LargeVideo.js

@@ -302,7 +302,8 @@ function createLargeVideoHTML()
                 '<canvas id="activeSpeakerAudioLevel"></canvas>' +
             '</div>' +
             '<div id="largeVideoWrapper">' +
-                '<video id="largeVideo" autoplay oncontextmenu="return false;"></video>' +
+                '<video id="largeVideo" muted="true"' +
+                        'autoplay oncontextmenu="return false;"></video>' +
             '</div id="largeVideoWrapper">' +
             '<span id="videoConnectionMessage"></span>';
     html += '</div>';

modules/UI/videolayout/RemoteVideo.js

@@ -2,6 +2,7 @@
 var ConnectionIndicator = require("./ConnectionIndicator");
 var SmallVideo = require("./SmallVideo");
 var AudioLevels = require("../audio_levels/AudioLevels");
+var MediaStreamType = require("../../../service/RTC/MediaStreamTypes");
 var RTCBrowserType = require("../../RTC/RTCBrowserType");
 var UIUtils = require("../util/UIUtil");
 var XMPPEvents = require("../../../service/xmpp/XMPPEvents");
@@ -178,8 +179,9 @@ RemoteVideo.prototype.remove = function () {
 
 RemoteVideo.prototype.waitForPlayback = function (sel, stream) {
 
-    var isVideo = stream.getVideoTracks().length > 0;
-    if (!isVideo || stream.id === 'mixedmslabel') {
+    var webRtcStream = stream.getOriginalStream();
+    var isVideo = stream.isVideoStream();
+    if (!isVideo || webRtcStream.id === 'mixedmslabel') {
         return;
     }
 
@@ -191,7 +193,7 @@ RemoteVideo.prototype.waitForPlayback = function (sel, stream) {
     var onPlayingHandler = function () {
         // FIXME: why do i have to do this for FF?
         if (RTCBrowserType.isFirefox()) {
-            APP.RTC.attachMediaStream(sel, stream);
+            APP.RTC.attachMediaStream(sel, webRtcStream);
         }
         if (RTCBrowserType.isTemasysPluginUsed()) {
             sel = self.selectVideoElement();
@@ -212,7 +214,8 @@ RemoteVideo.prototype.addRemoteStreamElement = function (stream) {
         return;
 
     var self = this;
-    var isVideo = stream.getVideoTracks().length > 0;
+    var webRtcStream = stream.getOriginalStream();
+    var isVideo = stream.isVideoStream();
     var streamElement = SmallVideo.createStreamElement(stream);
     var newElementId = streamElement.id;
 
@@ -226,14 +229,14 @@ RemoteVideo.prototype.addRemoteStreamElement = function (stream) {
     if (!isVideo || (this.container.offsetParent !== null && isVideo)) {
         this.waitForPlayback(sel, stream);
 
-        APP.RTC.attachMediaStream(sel, stream);
+        APP.RTC.attachMediaStream(sel, webRtcStream);
     }
 
     APP.RTC.addMediaStreamInactiveHandler(
-        stream, function () {
+        webRtcStream, function () {
             console.log('stream ended', this);
 
-            self.removeRemoteStreamElement(stream, isVideo, newElementId);
+            self.removeRemoteStreamElement(webRtcStream, isVideo, newElementId);
     });
 
     // Add click handler.

modules/UI/videolayout/SmallVideo.js

@@ -4,6 +4,7 @@ var Avatar = require("../avatar/Avatar");
 var UIUtil = require("../util/UIUtil");
 var LargeVideo = require("./LargeVideo");
 var RTCBrowserType = require("../../RTC/RTCBrowserType");
+var MediaStreamType = require("../../../service/RTC/MediaStreamTypes");
 
 function SmallVideo() {
     this.isMuted = false;
@@ -105,7 +106,7 @@ SmallVideo.prototype.setPresenceStatus = function (statusMsg) {
  * Creates an audio or video element for a particular MediaStream.
  */
 SmallVideo.createStreamElement = function (stream) {
-    var isVideo = stream.getVideoTracks().length > 0;
+    var isVideo = stream.isVideoStream();
 
     var element = isVideo ? document.createElement('video')
         : document.createElement('audio');
@@ -118,9 +119,9 @@ SmallVideo.createStreamElement = function (stream) {
     }
 
     element.id = (isVideo ? 'remoteVideo_' : 'remoteAudio_') +
-        APP.RTC.getStreamID(stream);
+        APP.RTC.getStreamID(stream.getOriginalStream());
 
-    element.onplay = function() {
+    element.onplay = function () {
         console.log("(TIME) Render " + (isVideo ? 'video' : 'audio') + ":\t",
                     window.performance.now());
     };

modules/UI/videolayout/VideoLayout.js

@@ -55,10 +55,6 @@ var VideoLayout = (function (my) {
                 lastNEndpointsCache.indexOf(resource) !== -1);
     };
 
-    my.changeLocalStream = function (stream, isMuted) {
-        VideoLayout.changeLocalVideo(stream, isMuted);
-    };
-
     my.changeLocalAudio = function(stream, isMuted) {
         if (isMuted)
             APP.UI.setAudioMuted(true, true);
@@ -187,8 +183,7 @@ var VideoLayout = (function (my) {
             VideoLayout.ensurePeerContainerExists(stream.peerjid);
 
             var resourceJid = Strophe.getResourceFromJid(stream.peerjid);
-            remoteVideos[resourceJid].addRemoteStreamElement(
-                stream.getOriginalStream());
+            remoteVideos[resourceJid].addRemoteStreamElement(stream);
         }
     };
 

modules/xmpp/JingleSessionPC.js

@@ -554,50 +554,6 @@ JingleSessionPC.prototype.getSsrcOwner = function (ssrc) {
     return this.ssrcOwners[ssrc];
 };
 
-function copyParamAttr(dstElem, srcElem, name) {
-    var srcSelector = $(srcElem).find('parameter[name="' + name + '"]');
-    var dstSelector = $(dstElem).find('parameter[name="' + name + '"]');
-    if (srcSelector.length && dstSelector.length) {
-        dstSelector[0].setAttribute('value', srcSelector[0].getAttribute('value'));
-    } else {
-        console.error("did not copy " + name + " from|to ", srcElem, dstElem);
-    }
-}
-
-JingleSessionPC.prototype.mungeRemoteMsid = function (jingle) {
-
-    var videoContent = $(jingle).find(">content[name='video']");
-    console.info("Video contents", videoContent);
-    var audioContent = $(jingle).find(">content[name='audio']");
-    console.info("Audio contents", audioContent);
-
-    var videoSSRCInfos = videoContent.find('description>source>ssrc-info');
-    console.info("Video SSRC infos: ", videoSSRCInfos);
-    var videoSSRCs = {};
-    videoSSRCInfos.each(function (idx, ssrcInfo) {
-        var owner = ssrcInfo.getAttribute('owner');
-        if (owner !== 'jvb') {
-            console.info("Got video SSRC owner: " + owner + " of: ", ssrcInfo.parentNode);
-            videoSSRCs[owner] = ssrcInfo.parentNode;
-        }
-    });
-    Object.keys(videoSSRCs).forEach(function (owner) {
-        console.info("Looking for audio SSRC owned by: " + owner);
-        var audioSSRCInfo = audioContent.find('description>source>ssrc-info[owner="' + owner + '"]');
-        if (audioSSRCInfo.length) {
-            var audioSSRC = audioSSRCInfo[0].parentNode;
-            console.info("Found corresponding audio SSRC: ", audioSSRC);
-            var videoSSRC = videoSSRCs[owner];
-            console.info("Will copy fields from: ", videoSSRC);
-            copyParamAttr(audioSSRC, videoSSRC, 'msid');
-            copyParamAttr(audioSSRC, videoSSRC, 'mslabel');
-            copyParamAttr(audioSSRC, videoSSRC, 'label');
-        }
-    });
-
-    return jingle;
-};
-
 JingleSessionPC.prototype.setRemoteDescription = function (elem, desctype) {
     this.remoteSDP = new SDP('');
     if (config.webrtcIceTcpDisable) {
@@ -606,8 +562,6 @@ JingleSessionPC.prototype.setRemoteDescription = function (elem, desctype) {
     if (config.webrtcIceUdpDisable) {
         this.remoteSDP.removeUdpCandidates = true;
     }
-    elem = this.mungeRemoteMsid(elem);
-    console.info("Jingle after munge: ", elem[0]);
     this.remoteSDP.fromJingle(elem);
     this.readSsrcInfo($(elem).find(">content"));
     if (this.peerconnection.remoteDescription !== null) {
@@ -924,12 +878,6 @@ JingleSessionPC.prototype.addSource = function (elem) {
     console.log('addssrc', new Date().getTime());
     console.log('ice', this.peerconnection.iceConnectionState);
 
-    elem = this.mungeRemoteMsid(elem);
-    console.info("SSRC-ADD Jingle after munge: ", elem[0]);
-
-    elem = $(elem).find('>content');
-    console.info("ELEM: ", elem);
-
     this.readSsrcInfo(elem);
 
     var sdp = new SDP(this.peerconnection.remoteDescription.sdp);

modules/xmpp/strophe.jingle.js

@@ -177,7 +177,7 @@ module.exports = function(XMPP, eventEmitter) {
                 case 'addsource': // FIXME: proprietary, un-jingleish
                 case 'source-add': // FIXME: proprietary
                     console.info("source-add", iq);
-                    sess.addSource($(iq).find('>jingle'));
+                    sess.addSource($(iq).find('>jingle>content'));
                     break;
                 case 'removesource': // FIXME: proprietary, un-jingleish
                 case 'source-remove': // FIXME: proprietary

service/RTC/MediaStreamTypes.js

@@ -1,6 +1,6 @@
 var MediaStreamType = {
-    VIDEO_TYPE: "Video",
+    VIDEO_TYPE: "video",
 
-    AUDIO_TYPE: "Audio"
+    AUDIO_TYPE: "audio"
 };
 module.exports = MediaStreamType;