Source/WebCore/ChangeLog

 12016-07-27 George Ruan <gruan@apple.com>
 2
 3 HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
 4 https://bugs.webkit.org/show_bug.cgi?id=160222
 5 <rdar://problem/27557313>
 6
 7 Reviewed by NOBODY (OOPS!).
 8
 9 Tests: fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html
 10 fast/mediastream/MediaStream-video-element-video-tracks-disabled.html
 11
 12 * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
 13 * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
 14 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Change criteria to enqueue a
 15 Sample Buffer to the AVSampleBufferDisplayLayer to allow an initial frame to be shown.
 16 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer): Allow an initial frame to be shown.
 17 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers): Removes all buffers from the
 18 AVSampleBufferDisplayLayer.
 19 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer): Make the AVSampleBufferDisplayLayer's background black.
 20 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode): Remove all buffers from the AVSampleBufferDisplayLayer
 21 when state of MediaPlayerPrivateMediaStreamAVFObjC is changed to None of PaintItBlack.
 22 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage): Updates paused image.
 23 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::pause): Calls updatePausedImage.
 24 (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext): Allow an initial frame to be painted to canvas.
 25
1262016-07-26 George Ruan <gruan@apple.com>
227
328 HTMLVideoElement frames do not update on iOS when src is a MediaStream blob

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h

@@private:
124124
125125 void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
126126 void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
 127 bool shouldEnqueueVideoSampleBuffer() const;
 128 void flushAndRemoveVideoSampleBuffers();
127129
128130 void paint(GraphicsContext&, const FloatRect&) override;
129131 void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;

@@private:
161163 };
162164 DisplayMode currentDisplayMode() const;
163165 void updateDisplayMode();
 166 void updatePausedImage();
164167
165168 // MediaStreamPrivate::Observer
166169 void activeStatusChanged() override;

@@private:
201204 bool m_ended { false };
202205 bool m_hasEverEnqueuedVideoFrame { false };
203206 bool m_hasReceivedMedia { false };
 207 bool m_isFrameDisplayed { false };
204208
205209#if PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE)
206210 std::unique_ptr<VideoFullscreenLayerManager> m_videoFullscreenLayerManager;

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm

@@void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(Med
129129 if (&track != m_mediaStreamPrivate->activeVideoTrack())
130130 return;
131131
132  if (m_displayMode == LivePreview && [m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
 132 if (shouldEnqueueVideoSampleBuffer()) {
133133 [m_sampleBufferDisplayLayer enqueueSampleBuffer:platformSample.sample.cmSampleBuffer];
 134 m_isFrameDisplayed = true;
134135
135136 if (!m_hasEverEnqueuedVideoFrame) {
136137 m_hasEverEnqueuedVideoFrame = true;
137138 m_player->firstVideoFrameAvailable();
 139
 140 updatePausedImage();
138141 }
139142 }
140143}
141144
 145bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
 146{
 147 if ([m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
 148 if (m_displayMode == LivePreview)
 149 return true;
 150
 151 if (m_displayMode == PausedImage && !m_isFrameDisplayed)
 152 return true;
 153 }
 154
 155 return false;
 156}
 157
 158void MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers()
 159{
 160 [m_sampleBufferDisplayLayer flushAndRemoveImage];
 161 m_isFrameDisplayed = false;
 162}
 163
142164void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
143165{
144166 if (m_sampleBufferDisplayLayer)

@@void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
148170#ifndef NDEBUG
149171 [m_sampleBufferDisplayLayer setName:@"MediaPlayerPrivateMediaStreamAVFObjC AVSampleBufferDisplayLayer"];
150172#endif
 173 m_sampleBufferDisplayLayer.get().backgroundColor = cachedCGColor(Color::black);
151174
152175 renderingModeChanged();
153176

@@void MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode()
257280 return;
258281 m_displayMode = displayMode;
259282
260  if (m_displayMode == None)
 283 if (m_displayMode < PausedImage && m_sampleBufferDisplayLayer)
 284 flushAndRemoveVideoSampleBuffers();
 285}
 286
 287void MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage()
 288{
 289 ASSERT(m_displayMode == currentDisplayMode());
 290
 291 if (m_displayMode < PausedImage)
261292 return;
 293
 294 RefPtr<Image> image = m_mediaStreamPrivate->currentFrameImage();
 295
 296 ASSERT(image);
 297
 298 m_pausedImage = image->getCGImageRef();
 299 if (!m_pausedImage)
 300 m_displayMode = PaintItBlack;
262301}
263302
264303void MediaPlayerPrivateMediaStreamAVFObjC::play()

@@void MediaPlayerPrivateMediaStreamAVFObjC::pause()
286325 m_clock->stop();
287326 m_playing = false;
288327 updateDisplayMode();
 328 updatePausedImage();
289329}
290330
291331bool MediaPlayerPrivateMediaStreamAVFObjC::paused() const

@@void MediaPlayerPrivateMediaStreamAVFObjC::paint(GraphicsContext& context, const
577617
578618void MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext(GraphicsContext& context, const FloatRect& rect)
579619{
580  if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled() || !m_haveEverPlayed)
 620 if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled())
581621 return;
582622
583 
584623 if (m_displayMode == LivePreview)
585624 m_mediaStreamPrivate->paintCurrentFrameInContext(context, rect);
586625 else {

LayoutTests/ChangeLog

 12016-07-27 George Ruan <gruan@apple.com>
 2
 3 HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
 4 https://bugs.webkit.org/show_bug.cgi?id=160222
 5 <rdar://problem/27557313>
 6
 7 Reviewed by NOBODY (OOPS!).
 8
 9 * fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html: Added.
 10 * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt: Added.
 11 * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html: Added. Checks
 12 that the video frames display captured media if all video tracks were disabled and then a single
 13 video track is re-enabled. This test also checks that an initial frame is painted to
 14 canvas if the video has not yet been played.
 15 * fast/mediastream/MediaStream-video-element-video-tracks-disabled.html: Added. Reference tests the
 16 frames of the video to be black, since the canvas is painted black regardless of the state of the video frames
 17 if displayMode of MediaPlayerPrivateMediaStreamAVFObjC is PaintItBlack.
 18
1192016-07-26 George Ruan <gruan@apple.com>
220
321 HTMLVideoElement frames do not update on iOS when src is a MediaStream blob

LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html

 1<!DOCTYPE html>
 2
 3<html>
 4<head>
 5 <style>
 6 .video {
 7 position: absolute;
 8 left: 10px;
 9 top: 50px;
 10 height: 360px;
 11 width: 680px;
 12 background-color: black;
 13 will-change: transform;
 14 }
 15
 16 .masker {
 17 position: absolute;
 18 left: 10px;
 19 top: 50px;
 20 height: 360px;
 21 width: 680px;
 22 border-top: 50px solid white;
 23 border-right: 300px solid white;
 24 border-bottom: 50px solid white;
 25 border-left: 300px solid white;
 26 box-sizing: border-box;
 27 }
 28 </style>
 29</head>
 30<body>
 31<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
 32<div class="video"></div>
 33<div class="masker"></div>
 34
 35</body>
 36</html>

LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt

 1Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.
 2
 3On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
 4
 5
 6PASS mediaDevices.getUserMedia generated a stream successfully.
 7video.src = window.URL.createObjectURL(mediaStream)
 8
 9 === beginning round of pixel tests ===
 10PASS pixel was white
 11
 12 === all video tracks disabled ===
 13PASS pixel was black.
 14
 15 === video track reenabled ===
 16PASS pixel was white.
 17
 18 ===== play video =====
 19video.play()
 20
 21 === beginning round of pixel tests ===
 22PASS pixel was white
 23
 24 === all video tracks disabled ===
 25PASS pixel was black.
 26
 27 === video track reenabled ===
 28PASS pixel was white.
 29PASS successfullyParsed is true
 30
 31TEST COMPLETE
 32

LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html

 1<!DOCTYPE html>
 2<html>
 3<head>
 4 <script src="../../resources/js-test-pre.js"></script>
 5 <script src="./resources/getUserMedia-helper.js"></script>
 6</head>
 7<body onload="start()">
 8<p id="description"></p>
 9<div id="console"></div>
 10<video controls width="680" height="360"></video>
 11<canvas width="680" height="360"></canvas>
 12<script>
 13 let canvas;
 14 let context;
 15 let mediaStream;
 16 let video;
 17
 18 let buffer;
 19
 20 function isPixelBlack(pixel)
 21 {
 22 return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 255;
 23 }
 24
 25 function isPixelTransparent(pixel)
 26 {
 27 return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 0;
 28 }
 29
 30 function isPixelWhite(pixel)
 31 {
 32 return pixel[0] === 255 && pixel[1] === 255 && pixel[2] === 255 && pixel[3] === 255;
 33 }
 34
 35 function attempt(numberOfTries, call, callback, successMessage)
 36 {
 37 if (numberOfTries <= 0) {
 38 testFailed('Pixel check did not succeed after multiple tries.');
 39 return;
 40 }
 41
 42 let attemptSucceeded = call();
 43 if (attemptSucceeded) {
 44 testPassed(successMessage);
 45 callback();
 46
 47 return;
 48 }
 49
 50 setTimeout(function() {
 51 attempt(--numberOfTries, call, callback, successMessage);
 52 }, 50);
 53 }
 54
 55 function repeatWithVideoPlayingAndFinishTest()
 56 {
 57 if (video.paused) {
 58 debug('<br> ===== play video =====');
 59 evalAndLog('video.play()');
 60 beginTestRound();
 61 } else
 62 finishJSTest();
 63 }
 64
 65 function reenableTrack()
 66 {
 67 mediaStream.getVideoTracks()[0].enabled = true;
 68 debug('<br> === video track reenabled ===');
 69
 70 // The video is not guaranteed to render non-black frames before the canvas is drawn to and the pixels are checked.
 71 // A timeout is used to ensure that the pixel check is done after the video renders non-black frames.
 72 attempt(10, checkPixels, repeatWithVideoPlayingAndFinishTest, 'pixel was white.');
 73 }
 74
 75 function checkPixels()
 76 {
 77 context.clearRect(0, 0, canvas.width, canvas.height);
 78 buffer = context.getImageData(30, 242, 1, 1).data;
 79 if(!isPixelTransparent(buffer)) {
 80 testFailed('pixel was not transparent after clearing canvas.');
 81 }
 82
 83 context.drawImage(video, 0, 0, canvas.width, canvas.height);
 84 buffer = context.getImageData(30, 242, 1, 1).data;
 85
 86 if (mediaStream.getVideoTracks()[0].enabled)
 87 return isPixelWhite(buffer);
 88 else
 89 return isPixelBlack(buffer);
 90 }
 91
 92 function disableAllTracks()
 93 {
 94 mediaStream.getVideoTracks()[0].enabled = false;
 95 debug('<br> === all video tracks disabled ===');
 96
 97 // The video is not guaranteed to render black frames before the canvas is drawn to and the pixels are checked.
 98 // A timeout is used to ensure that the pixel check is done after the video renders black frames.
 99 attempt(10, checkPixels, reenableTrack, 'pixel was black.');
 100 }
 101
 102 function beginTestRound()
 103 {
 104 debug('<br> === beginning round of pixel tests ===');
 105 attempt(1, checkPixels, disableAllTracks, 'pixel was white');
 106 }
 107
 108 function canplay()
 109 {
 110 canvas = document.querySelector('canvas');
 111 context = canvas.getContext('2d');
 112
 113 beginTestRound();
 114 }
 115
 116 function start()
 117 {
 118 description("Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.");
 119
 120 video = document.querySelector('video');
 121 video.addEventListener('canplay', canplay);
 122
 123 getUserMedia("allow", {video:true}, setupVideoElementWithStream);
 124 }
 125
 126 window.jsTestIsAsync = true;
 127</script>
 128<script src="../../resources/js-test-post.js"></script>
 129</body>
 130</html>

LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html

 1<!DOCTYPE html>
 2<html>
 3<head>
 4 <style>
 5 video {
 6 position: absolute;
 7 left: 10px;
 8 top: 50px;
 9 }
 10
 11 .masker {
 12 position: absolute;
 13 left: 10px;
 14 top: 50px;
 15 height: 360px;
 16 width: 680px;
 17 border-top: 50px solid white;
 18 border-right: 300px solid white;
 19 border-bottom: 50px solid white;
 20 border-left: 300px solid white;
 21 box-sizing: border-box;
 22 }
 23 </style>
 24</head>
 25
 26<body>
 27<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
 28<video controls width="680" height="360"></video>
 29<div class="masker"></div>
 30
 31<script>
 32 let mediaStream;
 33 let video;
 34
 35 function debug(msg)
 36 {
 37 let span = document.createElement('span');
 38 document.body.appendChild(span);
 39 span.innerHTML = `${msg} <br />`;
 40 }
 41
 42 function canplaythrough()
 43 {
 44 mediaStream.getVideoTracks()[0].enabled = false;
 45 window.testRunner.notifyDone();
 46 }
 47
 48 function canplay()
 49 {
 50 video.play();
 51 }
 52
 53 function setupStream(stream)
 54 {
 55 mediaStream = stream;
 56 video.src = window.URL.createObjectURL(mediaStream);
 57 }
 58
 59 function failedToSetupStream()
 60 {
 61 debug('Failed to setup stream');
 62 }
 63
 64 function start()
 65 {
 66 video = document.querySelector('video');
 67 video.addEventListener('canplay', canplay, false);
 68 video.addEventListener('canplaythrough', canplaythrough, false);
 69 navigator.mediaDevices.getUserMedia({video:true})
 70 .then(setupStream)
 71 .catch(failedToSetupStream);
 72 }
 73
 74 if (window.testRunner) {
 75 window.testRunner.waitUntilDone();
 76 window.testRunner.setUserMediaPermission(true);
 77 start();
 78 }
 79</script>
 80</body>
 81</html>