Source/WebCore/ChangeLog

 12018-11-13 Thibault Saunier <tsaunier@igalia.com>
 2
 3 [GStreamer][MediaStream] Handle track addition and removal
 4 https://bugs.webkit.org/show_bug.cgi?id=191599
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 Test: fast/mediastream/MediaStream-video-element-remove-track.html
 9
 10 * platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:
 11 (WebCore::WebKitMediaStreamObserver::~WebKitMediaStreamObserver):
 12 (WebCore::WebKitMediaStreamObserver::WebKitMediaStreamObserver):
 13 (WebCore::webkitMediaStreamSrcFinalize):
 14 (WebCore::webkitMediaStreamSrcChangeState):
 15 (WebCore::webkit_media_stream_src_init):
 16 (WebCore::webkitMediaStreamSrcSetupSrc):
 17 (WebCore::webkitMediaStreamSrcAddTrack):
 18 (WebCore::webkitMediaStreamSrcRemoveTrackByType):
 19 (WebCore::webkitMediaStreamSrcSetStream):
 20
1212018-11-13 Thibault Saunier <tsaunier@igalia.com>
222
323 [GStreamer] Enhance debugging by making sure to print the pipeline in MediaPlayerPrivateGStreamer

Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp

@@namespace WebCore {
4141static void webkitMediaStreamSrcPushVideoSample(WebKitMediaStreamSrc* self, GstSample* gstsample);
4242static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc* self, GstSample* gstsample);
4343static void webkitMediaStreamSrcTrackEnded(WebKitMediaStreamSrc* self, MediaStreamTrackPrivate&);
 44static void webkitMediaStreamSrcAddTrack(WebKitMediaStreamSrc* self, MediaStreamTrackPrivate*);
 45static void webkitMediaStreamSrcRemoveTrackByType(WebKitMediaStreamSrc* self, RealtimeMediaSource::Type trackType);
4446
4547static GstStaticPadTemplate videoSrcTemplate = GST_STATIC_PAD_TEMPLATE("video_src",
4648 GST_PAD_SRC,

@@private:
142144 WebKitMediaStreamSrc* m_mediaStreamSrc;
143145};
144146
 147class WebKitMediaStreamObserver
 148 : public MediaStreamPrivate::Observer {
 149public:
 150 virtual ~WebKitMediaStreamObserver() { };
 151 WebKitMediaStreamObserver(WebKitMediaStreamSrc* src)
 152 : m_mediaStreamSrc(src) { }
 153
 154 void characteristicsChanged() final { GST_DEBUG_OBJECT(m_mediaStreamSrc.get(), "renegotiation should happen."); }
 155 void activeStatusChanged() final { }
 156
 157 void didAddTrack(MediaStreamTrackPrivate& track) final
 158 {
 159 webkitMediaStreamSrcAddTrack(m_mediaStreamSrc.get(), &track);
 160 }
 161
 162 void didRemoveTrack(MediaStreamTrackPrivate& track) final
 163 {
 164 webkitMediaStreamSrcRemoveTrackByType(m_mediaStreamSrc.get(), track.type());
 165 }
 166
 167private:
 168 GRefPtr<WebKitMediaStreamSrc> m_mediaStreamSrc;
 169};
 170
145171typedef struct _WebKitMediaStreamSrcClass WebKitMediaStreamSrcClass;
146172struct _WebKitMediaStreamSrc {
147173 GstBin parent_instance;

@@struct _WebKitMediaStreamSrc {
153179 GstElement* videoSrc;
154180 GstClockTime firstFramePts;
155181
156  std::unique_ptr<WebKitMediaStreamTrackObserver> observer;
 182 std::unique_ptr<WebKitMediaStreamTrackObserver> mediaStreamTrackObserver;
 183 std::unique_ptr<WebKitMediaStreamObserver> mediaStreamObserver;
157184 volatile gint npads;
158185 gulong probeid;
159186 RefPtr<MediaStreamPrivate> stream;

@@static void webkitMediaStreamSrcFinalize(GObject* object)
266293 WebKitMediaStreamSrc* self = WEBKIT_MEDIA_STREAM_SRC(object);
267294
268295 GST_OBJECT_LOCK(self);
269  for (auto& track : self->stream->tracks())
270  track->removeObserver(*self->observer.get());
 296 if (self->stream) {
 297 for (auto& track : self->stream->tracks())
 298 track->removeObserver(*self->mediaStreamTrackObserver.get());
 299
 300 self->stream->removeObserver(*self->mediaStreamObserver);
 301 self->stream = nullptr;
 302 }
271303 GST_OBJECT_UNLOCK(self);
272304
273305 g_clear_pointer(&self->uri, g_free);

@@static GstStateChangeReturn webkitMediaStreamSrcChangeState(GstElement* element,
283315
284316 GST_OBJECT_LOCK(self);
285317 for (auto& track : self->stream->tracks())
286  track->removeObserver(*self->observer.get());
 318 track->removeObserver(*self->mediaStreamTrackObserver.get());
287319 GST_OBJECT_UNLOCK(self);
288320 }
289321

@@static void webkit_media_stream_src_class_init(WebKitMediaStreamSrcClass* klass)
319351
320352static void webkit_media_stream_src_init(WebKitMediaStreamSrc* self)
321353{
322  self->observer = std::make_unique<WebKitMediaStreamTrackObserver>(self);
 354 self->mediaStreamTrackObserver = std::make_unique<WebKitMediaStreamTrackObserver>(self);
 355 self->mediaStreamObserver = std::make_unique<WebKitMediaStreamObserver>(self);
323356 self->flowCombiner = gst_flow_combiner_new();
324357 self->firstAudioBufferPts = GST_CLOCK_TIME_NONE;
325358 self->firstFramePts = GST_CLOCK_TIME_NONE;

@@static gboolean webkitMediaStreamSrcSetupSrc(WebKitMediaStreamSrc* self,
420453 });
421454
422455 if (observe_track)
423  track->addObserver(*self->observer.get());
 456 track->addObserver(*self->mediaStreamTrackObserver.get());
424457
425458 gst_element_sync_state_with_parent(element);
426459 return TRUE;

@@static void webkitMediaStreamSrcPostStreamCollection(WebKitMediaStreamSrc* self,
451484 gst_message_new_stream_collection(GST_OBJECT(self), self->streamCollection.get()));
452485}
453486
454 gboolean webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream)
 487static void
 488webkitMediaStreamSrcAddTrack(WebKitMediaStreamSrc* self, MediaStreamTrackPrivate* track)
455489{
456  g_return_val_if_fail(WEBKIT_IS_MEDIA_STREAM_SRC(self), FALSE);
457 
458  if (self->audioSrc) {
459  gst_element_set_state(self->audioSrc, GST_STATE_NULL);
460  gst_bin_remove(GST_BIN(self), self->audioSrc);
461  self->audioSrc = nullptr;
 490 if (track->type() == RealtimeMediaSource::Type::Audio) {
 491 webkitMediaStreamSrcSetupAppSrc(self, track, &self->audioSrc,
 492 &audioSrcTemplate);
 493 } else if (track->type() == RealtimeMediaSource::Type::Video) {
 494 webkitMediaStreamSrcSetupAppSrc(self, track, &self->videoSrc,
 495 &videoSrcTemplate);
 496 } else {
 497 GST_INFO("Unsuported track type: %d", (gint) track->type());
462498 }
 499}
463500
464  if (self->videoSrc) {
465  gst_element_set_state(self->videoSrc, GST_STATE_NULL);
466  gst_bin_remove(GST_BIN(self), self->videoSrc);
467  self->videoSrc = nullptr;
 501static void
 502webkitMediaStreamSrcRemoveTrackByType(WebKitMediaStreamSrc* self, RealtimeMediaSource::Type trackType)
 503{
 504 if (trackType == RealtimeMediaSource::Type::Audio) {
 505 if (self->audioSrc) {
 506 gst_element_set_state(self->audioSrc, GST_STATE_NULL);
 507 gst_bin_remove(GST_BIN(self), self->audioSrc);
 508 self->audioSrc = nullptr;
 509 }
 510
 511 } else if (trackType == RealtimeMediaSource::Type::Video) {
 512 if (self->videoSrc) {
 513 gst_element_set_state(self->videoSrc, GST_STATE_NULL);
 514 gst_bin_remove(GST_BIN(self), self->videoSrc);
 515 self->videoSrc = nullptr;
 516 }
 517 } else {
 518 GST_INFO("Unsuported track type: %d", (gint) trackType);
468519 }
 520}
 521
 522gboolean webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream)
 523{
 524 g_return_val_if_fail(WEBKIT_IS_MEDIA_STREAM_SRC(self), FALSE);
 525
 526 webkitMediaStreamSrcRemoveTrackByType(self, RealtimeMediaSource::Type::Audio);
 527 webkitMediaStreamSrcRemoveTrackByType(self, RealtimeMediaSource::Type::Video);
469528
470529 webkitMediaStreamSrcPostStreamCollection(self, stream);
471530
472531 self->stream = stream;
473  for (auto& track : stream->tracks()) {
474  if (track->type() == RealtimeMediaSource::Type::Audio) {
475  webkitMediaStreamSrcSetupAppSrc(self, track.get(), &self->audioSrc,
476  &audioSrcTemplate);
477  } else if (track->type() == RealtimeMediaSource::Type::Video) {
478  webkitMediaStreamSrcSetupAppSrc(self, track.get(), &self->videoSrc,
479  &videoSrcTemplate);
480  } else {
481  GST_INFO("Unsuported track type: %d", (gint) track->type());
482  continue;
483  }
484  }
 532 self->stream->addObserver(*self->mediaStreamObserver.get());
 533 for (auto& track : stream->tracks())
 534 webkitMediaStreamSrcAddTrack(self, track.get());
485535
486536 return TRUE;
487537}

LayoutTests/ChangeLog

 12018-11-13 Thibault Saunier <tsaunier@igalia.com>
 2
 3 [GStreamer][MediaStream] Handle track addition and removal
 4 https://bugs.webkit.org/show_bug.cgi?id=191599
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 * fast/mediastream/MediaStream-video-element-remove-track-expected.txt: Added.
 9 * fast/mediastream/MediaStream-video-element-remove-track.html: Added.
 10
1112018-11-12 Youenn Fablet <youenn@apple.com>
212
313 Support setting stream ids when adding a transceiver

LayoutTests/fast/mediastream/MediaStream-video-element-remove-track-expected.txt

 1Tests checking removing MediaStream track applies to the video element.
 2
 3On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
 4
 5
 6
 7
 8**** calling mediaDevices.getUserMedia() ****
 9PASS mediaDevices.getUserMedia succeeded.
 10
 11**** setup video element ****
 12video.srcObject = mediaStream
 13Event 'canplay'
 14
 15*** start playback ****
 16video.play()
 17video.pause()
 18
 19**** check video element ****
 20
 21**** check video tracks ****
 22PASS video.videoTracks.length is 1
 23PASS video.videoTracks[0].id is mediaStream.getVideoTracks()[0].id
 24
 25**** check audio tracks ****
 26PASS video.audioTracks.length is 1
 27PASS video.audioTracks[0].id is mediaStream.getAudioTracks()[0].id
 28
 29**** removing audio track ****
 30
 31**** check video element ****
 32PASS video.videoWidth is mediaStream.getVideoTracks()[0].getSettings().width
 33PASS video.videoHeight is mediaStream.getVideoTracks()[0].getSettings().height
 34
 35**** check video tracks ****
 36PASS video.videoTracks.length is 1
 37PASS video.videoTracks[0].id is mediaStream.getVideoTracks()[0].id
 38PASS video.videoTracks[0].language is ""
 39PASS video.videoTracks[0].kind is "main"
 40
 41**** check no audio track ****
 42PASS video.audioTracks.length is 0
 43PASS mediaStream.getAudioTracks().length is 0
 44PASS successfullyParsed is true
 45
 46TEST COMPLETE
 47

LayoutTests/fast/mediastream/MediaStream-video-element-remove-track.html

 1<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
 2<html>
 3 <head>
 4 <script src="../../resources/js-test-pre.js"></script>
 5 <script>
 6 var video;
 7 var mediaStream;
 8
 9 function logEvent(element, eventName, func)
 10 {
 11 function _eventCallback(evt)
 12 {
 13 if (window.wasFinishJSTestCalled)
 14 return;
 15
 16 debug(`Event <em>'${evt.type}'</em>`);
 17 if (func)
 18 func(evt);
 19 }
 20 element.addEventListener(eventName, _eventCallback, true);
 21 }
 22
 23 function checkVideoElement()
 24 {
 25 evalAndLog("video.pause()");
 26
 27 debug("<br>**** check video element ****");
 28 debug("<br>**** check video tracks ****");
 29 shouldBe('video.videoTracks.length', '1');
 30 shouldBe('video.videoTracks[0].id', 'mediaStream.getVideoTracks()[0].id');
 31
 32 debug("<br>**** check audio tracks ****");
 33 shouldBe('video.audioTracks.length', '1');
 34 shouldBe('video.audioTracks[0].id', 'mediaStream.getAudioTracks()[0].id');
 35
 36 setTimeout(removeAudioTrack, 100);
 37 }
 38
 39 function checkVideoElement2()
 40 {
 41 debug("<br>**** check video element ****");
 42 shouldBe('video.videoWidth', 'mediaStream.getVideoTracks()[0].getSettings().width');
 43 shouldBe('video.videoHeight', 'mediaStream.getVideoTracks()[0].getSettings().height');
 44
 45 debug("<br>**** check video tracks ****");
 46 shouldBe('video.videoTracks.length', '1');
 47 shouldBe('video.videoTracks[0].id', 'mediaStream.getVideoTracks()[0].id');
 48 shouldBeEqualToString('video.videoTracks[0].language', '');
 49 shouldBeEqualToString('video.videoTracks[0].kind', 'main');
 50
 51 debug("<br>**** check no audio track ****");
 52 shouldBe('video.audioTracks.length', '0');
 53 shouldBe('mediaStream.getAudioTracks().length', '0');
 54
 55 finishJSTest();
 56 }
 57
 58 function canplay()
 59 {
 60 debug("<br>*** start playback ****");
 61 evalAndLog("video.play()");
 62 setTimeout(checkVideoElement, 100);
 63 }
 64
 65 function removeAudioTrack() {
 66 track = mediaStream.getAudioTracks()[0];
 67 debug("<br>**** removing audio track ****");
 68 try {
 69 mediaStream.removeTrack(track);
 70 } catch (exception) {
 71 testFailed("removeTrack threw an exception.");
 72 finishJSTest();
 73 }
 74 setTimeout(checkVideoElement2, 100);
 75 }
 76
 77 function setupStream(stream)
 78 {
 79 mediaStream = stream;
 80 testPassed('mediaDevices.getUserMedia succeeded.');
 81
 82 debug("<br>**** setup video element ****");
 83 evalAndLog("video.srcObject = mediaStream");
 84 }
 85
 86 function start()
 87 {
 88 description("Tests checking removing MediaStream track applies to the video element.");
 89 video = document.querySelector('video');
 90 logEvent(video, 'canplay', canplay)
 91
 92 debug("<br>**** calling mediaDevices.getUserMedia() ****");
 93 if (window.testRunner)
 94 testRunner.setUserMediaPermission(true);
 95 navigator.mediaDevices.getUserMedia( {video: true, audio: true} )
 96 .then(setupStream)
 97 .catch(function(reason) {
 98 debug(`Stream generation failed with error: ${reason}`);
 99 });
 100 }
 101
 102 window.jsTestIsAsync = true;
 103 window.successfullyParsed = true;
 104 </script>
 105 </head>
 106 <body onload="start()">
 107 <p id="description"></p>
 108 <video controls width="680" height="360"></video>
 109 <div id="console"></div>
 110 <script src="../../resources/js-test-post.js"></script>
 111 </body>
 112</html>