您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符

TPCUtils.js 20KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504
  1. import { getLogger } from 'jitsi-meet-logger';
  2. import transform from 'sdp-transform';
  3. import * as MediaType from '../../service/RTC/MediaType';
  4. import browser from '../browser';
  5. const logger = getLogger(__filename);
  6. const SIM_LAYER_1_RID = '1';
  7. const SIM_LAYER_2_RID = '2';
  8. const SIM_LAYER_3_RID = '3';
  9. const TransceiverDirection = {
  10. INACTIVE: 'inactive',
  11. RECVONLY: 'recvonly',
  12. SENDONLY: 'sendonly',
  13. SENDRECV: 'sendrecv'
  14. };
  15. export const SIM_LAYER_RIDS = [ SIM_LAYER_1_RID, SIM_LAYER_2_RID, SIM_LAYER_3_RID ];
  16. /**
  17. * Handles track related operations on TraceablePeerConnection when browser is
  18. * running in unified plan mode.
  19. */
  20. export class TPCUtils {
  21. /**
  22. * Creates a new instance for a given TraceablePeerConnection
  23. *
  24. * @param peerconnection - the tpc instance for which we have utility functions.
  25. * @param videoBitrates - the bitrates to be configured on the video senders for
  26. * different resolutions both in unicast and simulcast mode.
  27. */
  28. constructor(peerconnection, videoBitrates) {
  29. this.pc = peerconnection;
  30. this.videoBitrates = videoBitrates.VP8 || videoBitrates;
  31. /**
  32. * The startup configuration for the stream encodings that are applicable to
  33. * the video stream when a new sender is created on the peerconnection. The initial
  34. * config takes into account the differences in browser's simulcast implementation.
  35. *
  36. * Encoding parameters:
  37. * active - determine the on/off state of a particular encoding.
  38. * maxBitrate - max. bitrate value to be applied to that particular encoding
  39. * based on the encoding's resolution and config.js videoQuality settings if applicable.
  40. * rid - Rtp Stream ID that is configured for a particular simulcast stream.
  41. * scaleResolutionDownBy - the factor by which the encoding is scaled down from the
  42. * original resolution of the captured video.
  43. */
  44. this.localStreamEncodingsConfig = [
  45. {
  46. active: true,
  47. maxBitrate: browser.isFirefox() ? this.videoBitrates.high : this.videoBitrates.low,
  48. rid: SIM_LAYER_1_RID,
  49. scaleResolutionDownBy: browser.isFirefox() ? 1.0 : 4.0
  50. },
  51. {
  52. active: true,
  53. maxBitrate: this.videoBitrates.standard,
  54. rid: SIM_LAYER_2_RID,
  55. scaleResolutionDownBy: 2.0
  56. },
  57. {
  58. active: true,
  59. maxBitrate: browser.isFirefox() ? this.videoBitrates.low : this.videoBitrates.high,
  60. rid: SIM_LAYER_3_RID,
  61. scaleResolutionDownBy: browser.isFirefox() ? 4.0 : 1.0
  62. }
  63. ];
  64. }
  65. /**
  66. * Returns the transceiver associated with a given RTCRtpSender/RTCRtpReceiver.
  67. *
  68. * @param {string} mediaType - type of track associated with the transceiver 'audio' or 'video'.
  69. * @param {JitsiLocalTrack} localTrack - local track to be used for lookup.
  70. * @returns {RTCRtpTransceiver}
  71. */
  72. _findTransceiver(mediaType, localTrack = null) {
  73. let transceiver = null;
  74. // Check if the local track has been removed from the peerconnection already.
  75. const trackRemoved = !localTrack
  76. || (localTrack
  77. && browser.doesVideoMuteByStreamRemove()
  78. && localTrack.isVideoTrack()
  79. && localTrack.isMuted());
  80. if (trackRemoved) {
  81. transceiver = this.pc.peerconnection.getTransceivers()
  82. .find(t => t.receiver?.track?.kind === mediaType);
  83. } else if (localTrack) {
  84. transceiver = this.pc.peerconnection.getTransceivers()
  85. .find(t => t.sender?.track?.id === localTrack.getTrackId());
  86. }
  87. return transceiver;
  88. }
  89. /**
  90. * Obtains stream encodings that need to be configured on the given track based
  91. * on the track media type and the simulcast setting.
  92. * @param {JitsiLocalTrack} localTrack
  93. */
  94. _getStreamEncodings(localTrack) {
  95. if (this.pc.isSimulcastOn() && localTrack.isVideoTrack()) {
  96. return this.localStreamEncodingsConfig;
  97. }
  98. return localTrack.isVideoTrack()
  99. ? [ {
  100. active: true,
  101. maxBitrate: this.videoBitrates.high
  102. } ]
  103. : [ { active: true } ];
  104. }
  105. /**
  106. * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e.,
  107. * the primary ssrc first and the secondary rtx ssrc later. This is important for unified
  108. * plan since we have only one FID group per media description.
  109. * @param {Object} description the webRTC session description instance for the remote
  110. * description.
  111. * @private
  112. */
  113. ensureCorrectOrderOfSsrcs(description) {
  114. const parsedSdp = transform.parse(description.sdp);
  115. parsedSdp.media.forEach(mLine => {
  116. if (mLine.type === MediaType.AUDIO) {
  117. return;
  118. }
  119. if (!mLine.ssrcGroups || !mLine.ssrcGroups.length) {
  120. return;
  121. }
  122. let reorderedSsrcs = [];
  123. mLine.ssrcGroups[0].ssrcs.split(' ').forEach(ssrc => {
  124. const sources = mLine.ssrcs.filter(source => source.id.toString() === ssrc);
  125. reorderedSsrcs = reorderedSsrcs.concat(sources);
  126. });
  127. mLine.ssrcs = reorderedSsrcs;
  128. });
  129. return new RTCSessionDescription({
  130. type: description.type,
  131. sdp: transform.write(parsedSdp)
  132. });
  133. }
  134. /**
  135. * Takes in a *unified plan* offer and inserts the appropriate
  136. * parameters for adding simulcast receive support.
  137. * @param {Object} desc - A session description object
  138. * @param {String} desc.type - the type (offer/answer)
  139. * @param {String} desc.sdp - the sdp content
  140. *
  141. * @return {Object} A session description (same format as above) object
  142. * with its sdp field modified to advertise simulcast receive support
  143. */
  144. insertUnifiedPlanSimulcastReceive(desc) {
  145. // a=simulcast line is not needed on browsers where we SDP munging is used for enabling on simulcast.
  146. // Remove this check when the client switches to RID/MID based simulcast on all browsers.
  147. if (browser.usesSdpMungingForSimulcast()) {
  148. return desc;
  149. }
  150. const sdp = transform.parse(desc.sdp);
  151. const idx = sdp.media.findIndex(mline => mline.type === MediaType.VIDEO);
  152. if (sdp.media[idx].rids && (sdp.media[idx].simulcast_03 || sdp.media[idx].simulcast)) {
  153. // Make sure we don't have the simulcast recv line on video descriptions other than
  154. // the first video description.
  155. sdp.media.forEach((mline, i) => {
  156. if (mline.type === MediaType.VIDEO && i !== idx) {
  157. sdp.media[i].rids = undefined;
  158. sdp.media[i].simulcast = undefined;
  159. // eslint-disable-next-line camelcase
  160. sdp.media[i].simulcast_03 = undefined;
  161. }
  162. });
  163. return new RTCSessionDescription({
  164. type: desc.type,
  165. sdp: transform.write(sdp)
  166. });
  167. }
  168. // In order of highest to lowest spatial quality
  169. sdp.media[idx].rids = [
  170. {
  171. id: SIM_LAYER_1_RID,
  172. direction: 'recv'
  173. },
  174. {
  175. id: SIM_LAYER_2_RID,
  176. direction: 'recv'
  177. },
  178. {
  179. id: SIM_LAYER_3_RID,
  180. direction: 'recv'
  181. }
  182. ];
  183. // Firefox 72 has stopped parsing the legacy rid= parameters in simulcast attributes.
  184. // eslint-disable-next-line max-len
  185. // https://www.fxsitecompat.dev/en-CA/docs/2019/pt-and-rid-in-webrtc-simulcast-attributes-are-no-longer-supported/
  186. const simulcastLine = browser.isFirefox() && browser.isVersionGreaterThan(71)
  187. ? `recv ${SIM_LAYER_RIDS.join(';')}`
  188. : `recv rid=${SIM_LAYER_RIDS.join(';')}`;
  189. // eslint-disable-next-line camelcase
  190. sdp.media[idx].simulcast_03 = {
  191. value: simulcastLine
  192. };
  193. return new RTCSessionDescription({
  194. type: desc.type,
  195. sdp: transform.write(sdp)
  196. });
  197. }
  198. /**
  199. * Adds {@link JitsiLocalTrack} to the WebRTC peerconnection for the first time.
  200. * @param {JitsiLocalTrack} track - track to be added to the peerconnection.
  201. * @param {boolean} isInitiator - boolean that indicates if the endpoint is offerer in a p2p connection.
  202. * @returns {void}
  203. */
  204. addTrack(localTrack, isInitiator) {
  205. const track = localTrack.getTrack();
  206. if (isInitiator) {
  207. // Use pc.addTransceiver() for the initiator case when local tracks are getting added
  208. // to the peerconnection before a session-initiate is sent over to the peer.
  209. const transceiverInit = {
  210. direction: TransceiverDirection.SENDRECV,
  211. streams: [ localTrack.getOriginalStream() ],
  212. sendEncodings: []
  213. };
  214. if (!browser.isFirefox()) {
  215. transceiverInit.sendEncodings = this._getStreamEncodings(localTrack);
  216. }
  217. this.pc.peerconnection.addTransceiver(track, transceiverInit);
  218. } else {
  219. // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created
  220. // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing
  221. // unused "recv-only" transceiver.
  222. this.pc.peerconnection.addTrack(track);
  223. }
  224. }
  225. /**
  226. * Adds a track on the RTCRtpSender as part of the unmute operation.
  227. * @param {JitsiLocalTrack} localTrack - track to be unmuted.
  228. * @returns {Promise<void>} - resolved when done.
  229. */
  230. addTrackUnmute(localTrack) {
  231. const mediaType = localTrack.getType();
  232. const track = localTrack.getTrack();
  233. const transceiver = this._findTransceiver(mediaType);
  234. if (!transceiver) {
  235. return Promise.reject(new Error(`RTCRtpTransceiver for ${mediaType} not found`));
  236. }
  237. logger.debug(`Adding ${localTrack} on ${this.pc}`);
  238. return transceiver.sender.replaceTrack(track);
  239. }
  240. /**
  241. * Obtains the current local video track's height constraints based on the
  242. * initial stream encodings configuration on the sender and the resolution
  243. * of the current local track added to the peerconnection.
  244. * @param {MediaStreamTrack} localTrack local video track
  245. * @returns {Array[number]} an array containing the resolution heights of
  246. * simulcast streams configured on the video sender.
  247. */
  248. getLocalStreamHeightConstraints(localTrack) {
  249. // React-native hasn't implemented MediaStreamTrack getSettings yet.
  250. if (browser.isReactNative()) {
  251. return null;
  252. }
  253. const localVideoHeightConstraints = [];
  254. // Firefox doesn't return the height of the desktop track, assume a min. height of 720.
  255. const { height = 720 } = localTrack.getSettings();
  256. for (const encoding of this.localStreamEncodingsConfig) {
  257. localVideoHeightConstraints.push(height / encoding.scaleResolutionDownBy);
  258. }
  259. return localVideoHeightConstraints;
  260. }
  261. /**
  262. * Removes the track from the RTCRtpSender as part of the mute operation.
  263. * @param {JitsiLocalTrack} localTrack - track to be removed.
  264. * @returns {Promise<void>} - resolved when done.
  265. */
  266. removeTrackMute(localTrack) {
  267. const mediaType = localTrack.getType();
  268. const transceiver = this._findTransceiver(mediaType, localTrack);
  269. if (!transceiver) {
  270. return Promise.reject(new Error(`RTCRtpTransceiver for ${mediaType} not found`));
  271. }
  272. logger.debug(`Removing ${localTrack} on ${this.pc}`);
  273. return transceiver.sender.replaceTrack(null);
  274. }
  275. /**
  276. * Replaces the existing track on a RTCRtpSender with the given track.
  277. * @param {JitsiLocalTrack} oldTrack - existing track on the sender that needs to be removed.
  278. * @param {JitsiLocalTrack} newTrack - new track that needs to be added to the sender.
  279. * @returns {Promise<void>} - resolved when done.
  280. */
  281. replaceTrack(oldTrack, newTrack) {
  282. if (oldTrack && newTrack) {
  283. const mediaType = newTrack.getType();
  284. const stream = newTrack.getOriginalStream();
  285. // Ignore cases when the track is replaced while the device is in a muted state,like
  286. // replacing camera when video muted or replacing mic when audio muted. These JitsiLocalTracks
  287. // do not have a mediastream attached. Replace track will be called again when the device is
  288. // unmuted and the track will be replaced on the peerconnection then.
  289. if (!stream) {
  290. this.pc.localTracks.delete(oldTrack.rtcId);
  291. this.pc.localTracks.set(newTrack.rtcId, newTrack);
  292. return Promise.resolve();
  293. }
  294. const transceiver = this._findTransceiver(mediaType, oldTrack);
  295. const track = newTrack.getTrack();
  296. if (!transceiver) {
  297. return Promise.reject(new Error('replace track failed'));
  298. }
  299. logger.debug(`Replacing ${oldTrack} with ${newTrack} on ${this.pc}`);
  300. return transceiver.sender.replaceTrack(track)
  301. .then(() => {
  302. const ssrc = this.pc.localSSRCs.get(oldTrack.rtcId);
  303. this.pc.localTracks.delete(oldTrack.rtcId);
  304. this.pc.localSSRCs.delete(oldTrack.rtcId);
  305. this.pc._addedStreams = this.pc._addedStreams.filter(s => s !== stream);
  306. this.pc.localTracks.set(newTrack.rtcId, newTrack);
  307. this.pc._addedStreams.push(stream);
  308. this.pc.localSSRCs.set(newTrack.rtcId, ssrc);
  309. });
  310. } else if (oldTrack && !newTrack) {
  311. return this.removeTrackMute(oldTrack)
  312. .then(() => {
  313. const mediaType = oldTrack.getType();
  314. const transceiver = this._findTransceiver(mediaType);
  315. // Change the direction on the transceiver to 'recvonly' so that a 'removetrack'
  316. // is fired on the associated media stream on the remote peer.
  317. if (transceiver) {
  318. transceiver.direction = TransceiverDirection.RECVONLY;
  319. }
  320. // Remove the old track from the list of local tracks.
  321. this.pc.localTracks.delete(oldTrack.rtcId);
  322. this.pc.localSSRCs.delete(oldTrack.rtcId);
  323. });
  324. } else if (newTrack && !oldTrack) {
  325. return this.addTrackUnmute(newTrack)
  326. .then(() => {
  327. const mediaType = newTrack.getType();
  328. const transceiver = this._findTransceiver(mediaType, newTrack);
  329. // Change the direction on the transceiver back to 'sendrecv' so that a 'track'
  330. // event is fired on the remote peer.
  331. if (transceiver) {
  332. transceiver.direction = TransceiverDirection.SENDRECV;
  333. }
  334. // Avoid configuring the encodings on Chromium/Safari until simulcast is configured
  335. // for the newly added track using SDP munging which happens during the renegotiation.
  336. const promise = browser.usesSdpMungingForSimulcast()
  337. ? Promise.resolve()
  338. : this.setEncodings(newTrack);
  339. return promise
  340. .then(() => {
  341. // Add the new track to the list of local tracks.
  342. this.pc.localTracks.set(newTrack.rtcId, newTrack);
  343. });
  344. });
  345. }
  346. logger.info('TPCUtils.replaceTrack called with no new track and no old track');
  347. return Promise.resolve();
  348. }
  349. /**
  350. * Enables/disables audio transmission on the peer connection. When
  351. * disabled the audio transceiver direction will be set to 'inactive'
  352. * which means that no data will be sent nor accepted, but
  353. * the connection should be kept alive.
  354. * @param {boolean} active - true to enable audio media transmission or
  355. * false to disable.
  356. * @returns {void}
  357. */
  358. setAudioTransferActive(active) {
  359. this.setMediaTransferActive(MediaType.AUDIO, active);
  360. }
  361. /**
  362. * Set the simulcast stream encoding properties on the RTCRtpSender.
  363. * @param {JitsiLocalTrack} track - the current track in use for which
  364. * the encodings are to be set.
  365. * @returns {Promise<void>} - resolved when done.
  366. */
  367. setEncodings(track) {
  368. const mediaType = track.getType();
  369. const transceiver = this._findTransceiver(mediaType, track);
  370. const parameters = transceiver?.sender?.getParameters();
  371. // Resolve if the encodings are not available yet. This happens immediately after the track is added to the
  372. // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the
  373. // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that.
  374. if (!parameters?.encodings?.length) {
  375. return Promise.resolve();
  376. }
  377. parameters.encodings = this._getStreamEncodings(track);
  378. return transceiver.sender.setParameters(parameters);
  379. }
  380. /**
  381. * Enables/disables media transmission on the peerconnection by changing the direction
  382. * on the transceiver for the specified media type.
  383. * @param {String} mediaType - 'audio' or 'video'
  384. * @param {boolean} active - true to enable media transmission or false
  385. * to disable.
  386. * @returns {void}
  387. */
  388. setMediaTransferActive(mediaType, active) {
  389. const transceivers = this.pc.peerconnection.getTransceivers()
  390. .filter(t => t.receiver && t.receiver.track && t.receiver.track.kind === mediaType);
  391. const localTracks = this.pc.getLocalTracks(mediaType);
  392. logger.info(`${active ? 'Enabling' : 'Suspending'} ${mediaType} media transfer on ${this.pc}`);
  393. transceivers.forEach((transceiver, idx) => {
  394. if (active) {
  395. // The first transceiver is for the local track and only this one can be set to 'sendrecv'
  396. if (idx === 0 && localTracks.length) {
  397. transceiver.direction = TransceiverDirection.SENDRECV;
  398. } else {
  399. transceiver.direction = TransceiverDirection.RECVONLY;
  400. }
  401. } else {
  402. transceiver.direction = TransceiverDirection.INACTIVE;
  403. }
  404. });
  405. }
  406. /**
  407. * Enables/disables video media transmission on the peer connection. When
  408. * disabled the SDP video media direction in the local SDP will be adjusted to
  409. * 'inactive' which means that no data will be sent nor accepted, but
  410. * the connection should be kept alive.
  411. * @param {boolean} active - true to enable video media transmission or
  412. * false to disable.
  413. * @returns {void}
  414. */
  415. setVideoTransferActive(active) {
  416. this.setMediaTransferActive(MediaType.VIDEO, active);
  417. }
  418. /**
  419. * Ensures that the resolution of the stream encodings are consistent with the values
  420. * that were configured on the RTCRtpSender when the source was added to the peerconnection.
  421. * This should prevent us from overriding the default values if the browser returns
  422. * erroneous values when RTCRtpSender.getParameters is used for getting the encodings info.
  423. * @param {Object} parameters - the RTCRtpEncodingParameters obtained from the browser.
  424. * @returns {void}
  425. */
  426. updateEncodingsResolution(parameters) {
  427. if (!(browser.isWebKitBased() && parameters.encodings && Array.isArray(parameters.encodings))) {
  428. return;
  429. }
  430. const allEqualEncodings
  431. = encodings => encodings.every(encoding => typeof encoding.scaleResolutionDownBy !== 'undefined'
  432. && encoding.scaleResolutionDownBy === encodings[0].scaleResolutionDownBy);
  433. // Implement the workaround only when all the encodings report the same resolution.
  434. if (allEqualEncodings(parameters.encodings)) {
  435. parameters.encodings.forEach((encoding, idx) => {
  436. encoding.scaleResolutionDownBy = this.localStreamEncodingsConfig[idx].scaleResolutionDownBy;
  437. });
  438. }
  439. }
  440. }