Attachment 284734 Details for Bug 160222 – Patch

[patch] Patch

bug-160222-20160727151413.patch (text/plain), 17.92 KB, created by George Ruan on 2016-07-27 15:15:23 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: George Ruan

Created: 2016-07-27 15:15:23 PDT

Size: 17.92 KB

patch

obsolete

>Subversion Revision: 203384
>diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
>index 1456c4b8a00522e10dd21f717f6eced9e00062de..2fb97c7ab05b1c7b6bf7f41b230768aa052d4eaa 100644
>--- a/Source/WebCore/ChangeLog
>+++ b/Source/WebCore/ChangeLog
>@@ -1,3 +1,28 @@
>+2016-07-27  George Ruan  <gruan@apple.com>
>+
>+        HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
>+        https://bugs.webkit.org/show_bug.cgi?id=160222
>+        <rdar://problem/27557313>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        Tests: fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html
>+               fast/mediastream/MediaStream-video-element-video-tracks-disabled.html
>+
>+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
>+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Change criteria to enqueue a
>+        Sample Buffer to the AVSampleBufferDisplayLayer to allow an initial frame to be shown.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer): Allow an initial frame to be shown.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers): Removes all buffers from the
>+        AVSampleBufferDisplayLayer.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer): Make the AVSampleBufferDisplayLayer's background black.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode): Remove all buffers from the AVSampleBufferDisplayLayer
>+        when state of MediaPlayerPrivateMediaStreamAVFObjC is changed to None of PaintItBlack.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage): Updates paused image.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::pause): Calls updatePausedImage.
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext): Allow an initial frame to be painted to canvas.
>+
> 2016-07-26  George Ruan  <gruan@apple.com>
> 
>         HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
>diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>index 7360f24e60e3c06a2c13f0ede7fff6492f0ea75b..69d3b2ed5223e20734874886a7449ca10ce9d1ce 100644
>--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>@@ -124,6 +124,8 @@ private:
> 
>     void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
>     void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
>+    bool shouldEnqueueVideoSampleBuffer() const;
>+    void flushAndRemoveVideoSampleBuffers();
> 
>     void paint(GraphicsContext&, const FloatRect&) override;
>     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
>@@ -161,6 +163,7 @@ private:
>     };
>     DisplayMode currentDisplayMode() const;
>     void updateDisplayMode();
>+    void updatePausedImage();
> 
>     // MediaStreamPrivate::Observer
>     void activeStatusChanged() override;
>@@ -201,6 +204,7 @@ private:
>     bool m_ended { false };
>     bool m_hasEverEnqueuedVideoFrame { false };
>     bool m_hasReceivedMedia { false };
>+    bool m_isFrameDisplayed { false };
> 
> #if PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE)
>     std::unique_ptr<VideoFullscreenLayerManager> m_videoFullscreenLayerManager;
>diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>index 9693825419e825540e885f9c91853babd020cfdf..539ae85c01a50680e185e1ac92c03a053e54f634 100644
>--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>@@ -129,16 +129,39 @@ void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(Med
>     if (&track != m_mediaStreamPrivate->activeVideoTrack())
>         return;
> 
>-    if (m_displayMode == LivePreview && [m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
>+    if (shouldEnqueueVideoSampleBuffer()) {
>         [m_sampleBufferDisplayLayer enqueueSampleBuffer:platformSample.sample.cmSampleBuffer];
>+        m_isFrameDisplayed = true;
>         
>         if (!m_hasEverEnqueuedVideoFrame) {
>             m_hasEverEnqueuedVideoFrame = true;
>             m_player->firstVideoFrameAvailable();
>+
>+            updatePausedImage();
>         }
>     }
> }
> 
>+bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
>+{
>+    if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData])
>+        return false;
>+
>+    if (m_displayMode == LivePreview)
>+        return true;
>+
>+    if (m_displayMode == PausedImage && !m_isFrameDisplayed)
>+        return true;
>+
>+    return false;
>+}
>+
>+void MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers()
>+{
>+    [m_sampleBufferDisplayLayer flushAndRemoveImage];
>+    m_isFrameDisplayed = false;
>+}
>+
> void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
> {
>     if (m_sampleBufferDisplayLayer)
>@@ -148,6 +171,7 @@ void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
> #ifndef NDEBUG
>     [m_sampleBufferDisplayLayer setName:@"MediaPlayerPrivateMediaStreamAVFObjC AVSampleBufferDisplayLayer"];
> #endif
>+    m_sampleBufferDisplayLayer.get().backgroundColor = cachedCGColor(Color::black);
>     
>     renderingModeChanged();
>     
>@@ -257,8 +281,24 @@ void MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode()
>         return;
>     m_displayMode = displayMode;
> 
>-    if (m_displayMode == None)
>+    if (m_displayMode < PausedImage && m_sampleBufferDisplayLayer)
>+        flushAndRemoveVideoSampleBuffers();
>+}
>+
>+void MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage()
>+{
>+    ASSERT(m_displayMode == currentDisplayMode());
>+
>+    if (m_displayMode < PausedImage)
>+        return;
>+
>+    RefPtr<Image> image = m_mediaStreamPrivate->currentFrameImage();
>+    ASSERT(image);
>+    if (!image)
>         return;
>+
>+    m_pausedImage = image->getCGImageRef();
>+    ASSERT(m_pausedImage);
> }
> 
> void MediaPlayerPrivateMediaStreamAVFObjC::play()
>@@ -286,6 +326,7 @@ void MediaPlayerPrivateMediaStreamAVFObjC::pause()
>     m_clock->stop();
>     m_playing = false;
>     updateDisplayMode();
>+    updatePausedImage();
> }
> 
> bool MediaPlayerPrivateMediaStreamAVFObjC::paused() const
>@@ -577,10 +618,9 @@ void MediaPlayerPrivateMediaStreamAVFObjC::paint(GraphicsContext& context, const
> 
> void MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext(GraphicsContext& context, const FloatRect& rect)
> {
>-    if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled() || !m_haveEverPlayed)
>+    if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled())
>         return;
> 
>-
>     if (m_displayMode == LivePreview)
>         m_mediaStreamPrivate->paintCurrentFrameInContext(context, rect);
>     else {
>diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog
>index 251164fceeca8a5cce5d1caaf259a7170bfedc3c..4ff8241c6c724fe5f1ec00cce0855baa8d2a6bbb 100644
>--- a/LayoutTests/ChangeLog
>+++ b/LayoutTests/ChangeLog
>@@ -1,3 +1,21 @@
>+2016-07-27  George Ruan  <gruan@apple.com>
>+
>+        HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
>+        https://bugs.webkit.org/show_bug.cgi?id=160222
>+        <rdar://problem/27557313>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html: Added.
>+        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt: Added.
>+        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html: Added. Checks
>+        that the video frames display captured media if all video tracks were disabled and then a single
>+        video track is re-enabled. This test also checks that an initial frame is painted to
>+        canvas if the video has not yet been played.
>+        * fast/mediastream/MediaStream-video-element-video-tracks-disabled.html: Added. Reference tests the
>+        frames of the video to be black, since the canvas is painted black regardless of the state of the video frames
>+        if displayMode of MediaPlayerPrivateMediaStreamAVFObjC is PaintItBlack.
>+
> 2016-07-26  George Ruan  <gruan@apple.com>
> 
>         HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
>diff --git a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html
>new file mode 100644
>index 0000000000000000000000000000000000000000..5c556982247c1853b7806727b2b900457621eae0
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html
>@@ -0,0 +1,36 @@
>+<!DOCTYPE html>
>+
>+<html>
>+<head>
>+    <style>
>+        .video {
>+            position: absolute;
>+            left: 10px;
>+            top: 50px;
>+            height: 360px;
>+            width: 680px;
>+            background-color: black;
>+            will-change: transform;
>+        }
>+        
>+        .masker {
>+            position: absolute;
>+            left: 10px;
>+            top: 50px;
>+            height: 360px;
>+            width: 680px;
>+            border-top: 50px solid white;
>+            border-right: 300px solid white;
>+            border-bottom: 50px solid white;
>+            border-left: 300px solid white;
>+            box-sizing: border-box;
>+        }
>+    </style>
>+</head>
>+<body>
>+<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
>+<div class="video"></div>
>+<div class="masker"></div>
>+
>+</body>
>+</html>
>diff --git a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt
>new file mode 100644
>index 0000000000000000000000000000000000000000..daf074d832e220c2c2d791452e8f0eafd3e7976d
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt
>@@ -0,0 +1,32 @@
>+Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.
>+
>+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
>+
>+
>+PASS mediaDevices.getUserMedia generated a stream successfully.
>+video.src = window.URL.createObjectURL(mediaStream)
>+
>+ === beginning round of pixel tests ===
>+PASS pixel was white
>+
>+ === all video tracks disabled ===
>+PASS pixel was black.
>+
>+ === video track reenabled ===
>+PASS pixel was white.
>+
>+ ===== play video =====
>+video.play()
>+
>+ === beginning round of pixel tests ===
>+PASS pixel was white
>+
>+ === all video tracks disabled ===
>+PASS pixel was black.
>+
>+ === video track reenabled ===
>+PASS pixel was white.
>+PASS successfullyParsed is true
>+
>+TEST COMPLETE
>+ 
>diff --git a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html
>new file mode 100644
>index 0000000000000000000000000000000000000000..f61530cf7408ce51ac604e3894a70bac33a89676
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html
>@@ -0,0 +1,128 @@
>+<!DOCTYPE html>
>+<html>
>+<head>
>+    <script src="../../resources/js-test-pre.js"></script>
>+    <script src="./resources/getUserMedia-helper.js"></script>
>+</head>
>+<body onload="start()">
>+<p id="description"></p>
>+<div id="console"></div>
>+<video controls width="680" height="360"></video>
>+<canvas width="680" height="360"></canvas>
>+<script>
>+    let canvas;
>+    let context;
>+    let mediaStream;
>+    let video;
>+    
>+    let buffer;
>+
>+    function isPixelBlack(pixel)
>+    {
>+        return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 255;
>+    }
>+
>+    function isPixelTransparent(pixel)
>+    {
>+        return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 0;
>+    }
>+
>+    function isPixelWhite(pixel)
>+    {
>+        return pixel[0] === 255 && pixel[1] === 255 && pixel[2] === 255 && pixel[3] === 255;
>+    }
>+
>+    function attempt(numberOfTries, call, callback, successMessage)
>+    {
>+        if (numberOfTries <= 0) {
>+            testFailed('Pixel check did not succeed after multiple tries.');
>+            return;
>+        }
>+
>+        let attemptSucceeded = call();
>+        if (attemptSucceeded) {
>+            testPassed(successMessage);
>+            callback();
>+
>+            return;
>+        }
>+        
>+        setTimeout(() => { attempt(--numberOfTries, call, callback, successMessage); }, 50);
>+    }
>+
>+    function repeatWithVideoPlayingAndFinishTest()
>+    {
>+        if (video.paused) {
>+            debug('<br> ===== play video =====');
>+            evalAndLog('video.play()');
>+            beginTestRound();
>+        } else
>+            finishJSTest();
>+    }
>+
>+    function reenableTrack()
>+    {
>+        mediaStream.getVideoTracks()[0].enabled = true;
>+        debug('<br> === video track reenabled ===');
>+
>+        // The video is not guaranteed to render non-black frames before the canvas is drawn to and the pixels are checked.
>+        // A timeout is used to ensure that the pixel check is done after the video renders non-black frames.
>+        attempt(10, checkPixels, repeatWithVideoPlayingAndFinishTest, 'pixel was white.');
>+    }
>+
>+    function checkPixels()
>+    {
>+        context.clearRect(0, 0, canvas.width, canvas.height);
>+        buffer = context.getImageData(30, 242, 1, 1).data;
>+        if(!isPixelTransparent(buffer)) {
>+            testFailed('pixel was not transparent after clearing canvas.');
>+        }
>+
>+        context.drawImage(video, 0, 0, canvas.width, canvas.height);
>+        buffer = context.getImageData(30, 242, 1, 1).data;
>+
>+        if (mediaStream.getVideoTracks()[0].enabled)
>+            return isPixelWhite(buffer);
>+        else
>+            return isPixelBlack(buffer);
>+    }
>+
>+    function disableAllTracks()
>+    {
>+        mediaStream.getVideoTracks()[0].enabled = false;
>+        debug('<br> === all video tracks disabled ===');
>+        
>+        // The video is not guaranteed to render black frames before the canvas is drawn to and the pixels are checked.
>+        // A timeout is used to ensure that the pixel check is done after the video renders black frames.
>+        attempt(10, checkPixels, reenableTrack, 'pixel was black.');
>+    }
>+
>+    function beginTestRound()
>+    {
>+        debug('<br> === beginning round of pixel tests ===');
>+        attempt(1, checkPixels, disableAllTracks, 'pixel was white');
>+    }
>+
>+    function canplay()
>+    {
>+        canvas = document.querySelector('canvas');
>+        context = canvas.getContext('2d');
>+
>+        beginTestRound();
>+    }
>+
>+    function start()
>+    {
>+        description("Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.");
>+
>+        video = document.querySelector('video');
>+        video.addEventListener('canplay', canplay);
>+
>+        getUserMedia("allow", {video:true}, setupVideoElementWithStream);
>+    }
>+
>+    window.jsTestIsAsync = true;
>+</script>
>+<script src="../../resources/js-test-post.js"></script>
>+</body>
>+</html>
>\ No newline at end of file
>diff --git a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html
>new file mode 100644
>index 0000000000000000000000000000000000000000..106bd7f64e49b50e07dffee8ffd6abd31a78d8ed
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html
>@@ -0,0 +1,81 @@
>+<!DOCTYPE html>
>+<html>
>+<head>
>+    <style>
>+        video {
>+            position: absolute;
>+            left: 10px;
>+            top: 50px;
>+        }
>+        
>+        .masker {
>+            position: absolute;
>+            left: 10px;
>+            top: 50px;
>+            height: 360px;
>+            width: 680px;
>+            border-top: 50px solid white;
>+            border-right: 300px solid white;
>+            border-bottom: 50px solid white;
>+            border-left: 300px solid white;
>+            box-sizing: border-box;
>+        }
>+    </style>
>+</head>
>+
>+<body>
>+<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
>+<video controls width="680" height="360"></video>
>+<div class="masker"></div>
>+
>+<script>
>+    let mediaStream;
>+    let video;
>+
>+    function debug(msg)
>+    {
>+        let span = document.createElement('span');
>+        document.body.appendChild(span);
>+        span.innerHTML = `${msg} <br />`;
>+    }
>+
>+    function canplaythrough()
>+    {    
>+        mediaStream.getVideoTracks()[0].enabled = false;
>+        window.testRunner.notifyDone();
>+    }
>+
>+    function canplay()
>+    {
>+        video.play();
>+    }
>+
>+    function setupStream(stream)
>+    {
>+        mediaStream = stream;
>+        video.src = window.URL.createObjectURL(mediaStream);
>+    }
>+
>+    function failedToSetupStream()
>+    {
>+        debug('Failed to setup stream');
>+    }
>+
>+    function start()
>+    {
>+        video = document.querySelector('video');
>+        video.addEventListener('canplay', canplay, false);
>+        video.addEventListener('canplaythrough', canplaythrough, false);
>+        navigator.mediaDevices.getUserMedia({video:true})
>+            .then(setupStream)
>+            .catch(failedToSetupStream);
>+    }
>+
>+    if (window.testRunner) {
>+        window.testRunner.waitUntilDone();
>+        window.testRunner.setUserMediaPermission(true);
>+        start();
>+    }
>+</script>
>+</body>
>+</html>

Actions: View | Formatted Diff | Diff

Attachments on bug 160222: 284725 | 284734 | 284839