| Differences between
and this patch
- a/Source/WebCore/ChangeLog +25 lines
Lines 1-3 a/Source/WebCore/ChangeLog_sec1
1
2016-07-27  George Ruan  <gruan@apple.com>
2
3
        HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
4
        https://bugs.webkit.org/show_bug.cgi?id=160222
5
        <rdar://problem/27557313>
6
7
        Reviewed by NOBODY (OOPS!).
8
9
        Tests: fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html
10
               fast/mediastream/MediaStream-video-element-video-tracks-disabled.html
11
12
        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
13
        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
14
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Change criteria to enqueue a 
15
        Sample Buffer to the AVSampleBufferDisplayLayer to allow an initial frame to be shown.
16
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer): Allow an initial frame to be shown.
17
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers): Removes all buffers from the 
18
        AVSampleBufferDisplayLayer.
19
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer): Make the AVSampleBufferDisplayLayer's background black.
20
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode): Remove all buffers from the AVSampleBufferDisplayLayer
21
        when state of MediaPlayerPrivateMediaStreamAVFObjC is changed to None of PaintItBlack.
22
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage): Updates paused image.
23
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::pause): Calls updatePausedImage.
24
        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext): Allow an initial frame to be painted to canvas.
25
1
2016-07-26  George Ruan  <gruan@apple.com>
26
2016-07-26  George Ruan  <gruan@apple.com>
2
27
3
        HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
28
        HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h +4 lines
Lines 124-129 private: a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h_sec1
124
124
125
    void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
125
    void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
126
    void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
126
    void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, PlatformSample);
127
    bool shouldEnqueueVideoSampleBuffer() const;
128
    void flushAndRemoveVideoSampleBuffers();
127
129
128
    void paint(GraphicsContext&, const FloatRect&) override;
130
    void paint(GraphicsContext&, const FloatRect&) override;
129
    void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
131
    void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
Lines 161-166 private: a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h_sec2
161
    };
163
    };
162
    DisplayMode currentDisplayMode() const;
164
    DisplayMode currentDisplayMode() const;
163
    void updateDisplayMode();
165
    void updateDisplayMode();
166
    void updatePausedImage();
164
167
165
    // MediaStreamPrivate::Observer
168
    // MediaStreamPrivate::Observer
166
    void activeStatusChanged() override;
169
    void activeStatusChanged() override;
Lines 201-206 private: a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h_sec3
201
    bool m_ended { false };
204
    bool m_ended { false };
202
    bool m_hasEverEnqueuedVideoFrame { false };
205
    bool m_hasEverEnqueuedVideoFrame { false };
203
    bool m_hasReceivedMedia { false };
206
    bool m_hasReceivedMedia { false };
207
    bool m_isFrameDisplayed { false };
204
208
205
#if PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE)
209
#if PLATFORM(MAC) && ENABLE(VIDEO_PRESENTATION_MODE)
206
    std::unique_ptr<VideoFullscreenLayerManager> m_videoFullscreenLayerManager;
210
    std::unique_ptr<VideoFullscreenLayerManager> m_videoFullscreenLayerManager;
- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm -4 / +43 lines
Lines 129-144 void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(Med a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm_sec1
129
    if (&track != m_mediaStreamPrivate->activeVideoTrack())
129
    if (&track != m_mediaStreamPrivate->activeVideoTrack())
130
        return;
130
        return;
131
131
132
    if (m_displayMode == LivePreview && [m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
132
    if (shouldEnqueueVideoSampleBuffer()) {
133
        [m_sampleBufferDisplayLayer enqueueSampleBuffer:platformSample.sample.cmSampleBuffer];
133
        [m_sampleBufferDisplayLayer enqueueSampleBuffer:platformSample.sample.cmSampleBuffer];
134
        m_isFrameDisplayed = true;
134
        
135
        
135
        if (!m_hasEverEnqueuedVideoFrame) {
136
        if (!m_hasEverEnqueuedVideoFrame) {
136
            m_hasEverEnqueuedVideoFrame = true;
137
            m_hasEverEnqueuedVideoFrame = true;
137
            m_player->firstVideoFrameAvailable();
138
            m_player->firstVideoFrameAvailable();
139
140
            updatePausedImage();
138
        }
141
        }
139
    }
142
    }
140
}
143
}
141
144
145
bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
146
{
147
    if ([m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
148
        if (m_displayMode == LivePreview)
149
            return true;
150
151
        if (m_displayMode == PausedImage && !m_isFrameDisplayed)
152
            return true;
153
    }
154
155
    return false;
156
}
157
158
void MediaPlayerPrivateMediaStreamAVFObjC::flushAndRemoveVideoSampleBuffers()
159
{
160
    [m_sampleBufferDisplayLayer flushAndRemoveImage];
161
    m_isFrameDisplayed = false;
162
}
163
142
void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
164
void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer()
143
{
165
{
144
    if (m_sampleBufferDisplayLayer)
166
    if (m_sampleBufferDisplayLayer)
Lines 148-153 void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayer() a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm_sec2
148
#ifndef NDEBUG
170
#ifndef NDEBUG
149
    [m_sampleBufferDisplayLayer setName:@"MediaPlayerPrivateMediaStreamAVFObjC AVSampleBufferDisplayLayer"];
171
    [m_sampleBufferDisplayLayer setName:@"MediaPlayerPrivateMediaStreamAVFObjC AVSampleBufferDisplayLayer"];
150
#endif
172
#endif
173
    m_sampleBufferDisplayLayer.get().backgroundColor = cachedCGColor(Color::black);
151
    
174
    
152
    renderingModeChanged();
175
    renderingModeChanged();
153
    
176
    
Lines 257-264 void MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode() a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm_sec3
257
        return;
280
        return;
258
    m_displayMode = displayMode;
281
    m_displayMode = displayMode;
259
282
260
    if (m_displayMode == None)
283
    if (m_displayMode < PausedImage && m_sampleBufferDisplayLayer)
284
        flushAndRemoveVideoSampleBuffers();
285
}
286
287
void MediaPlayerPrivateMediaStreamAVFObjC::updatePausedImage()
288
{
289
    ASSERT(m_displayMode == currentDisplayMode());
290
291
    if (m_displayMode < PausedImage)
261
        return;
292
        return;
293
294
    RefPtr<Image> image = m_mediaStreamPrivate->currentFrameImage();
295
296
    ASSERT(image);
297
298
    m_pausedImage = image->getCGImageRef();
299
    if (!m_pausedImage)
300
        m_displayMode = PaintItBlack;
262
}
301
}
263
302
264
void MediaPlayerPrivateMediaStreamAVFObjC::play()
303
void MediaPlayerPrivateMediaStreamAVFObjC::play()
Lines 286-291 void MediaPlayerPrivateMediaStreamAVFObjC::pause() a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm_sec4
286
    m_clock->stop();
325
    m_clock->stop();
287
    m_playing = false;
326
    m_playing = false;
288
    updateDisplayMode();
327
    updateDisplayMode();
328
    updatePausedImage();
289
}
329
}
290
330
291
bool MediaPlayerPrivateMediaStreamAVFObjC::paused() const
331
bool MediaPlayerPrivateMediaStreamAVFObjC::paused() const
Lines 577-586 void MediaPlayerPrivateMediaStreamAVFObjC::paint(GraphicsContext& context, const a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm_sec5
577
617
578
void MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext(GraphicsContext& context, const FloatRect& rect)
618
void MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext(GraphicsContext& context, const FloatRect& rect)
579
{
619
{
580
    if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled() || !m_haveEverPlayed)
620
    if (m_displayMode == None || !metaDataAvailable() || context.paintingDisabled())
581
        return;
621
        return;
582
622
583
584
    if (m_displayMode == LivePreview)
623
    if (m_displayMode == LivePreview)
585
        m_mediaStreamPrivate->paintCurrentFrameInContext(context, rect);
624
        m_mediaStreamPrivate->paintCurrentFrameInContext(context, rect);
586
    else {
625
    else {
- a/LayoutTests/ChangeLog +18 lines
Lines 1-3 a/LayoutTests/ChangeLog_sec1
1
2016-07-27  George Ruan  <gruan@apple.com>
2
3
        HTMLVideoElement with MediaStream src shows paused image when all video tracks are disabled
4
        https://bugs.webkit.org/show_bug.cgi?id=160222
5
        <rdar://problem/27557313>
6
7
        Reviewed by NOBODY (OOPS!).
8
9
        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html: Added.
10
        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt: Added.
11
        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html: Added. Checks 
12
        that the video frames display captured media if all video tracks were disabled and then a single
13
        video track is re-enabled. This test also checks that an initial frame is painted to 
14
        canvas if the video has not yet been played.
15
        * fast/mediastream/MediaStream-video-element-video-tracks-disabled.html: Added. Reference tests the 
16
        frames of the video to be black, since the canvas is painted black regardless of the state of the video frames
17
        if displayMode of MediaPlayerPrivateMediaStreamAVFObjC is PaintItBlack.
18
1
2016-07-26  George Ruan  <gruan@apple.com>
19
2016-07-26  George Ruan  <gruan@apple.com>
2
20
3
        HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
21
        HTMLVideoElement frames do not update on iOS when src is a MediaStream blob
- a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html +36 lines
Line 0 a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-expected.html_sec1
1
<!DOCTYPE html>
2
3
<html>
4
<head>
5
    <style>
6
        .video {
7
            position: absolute;
8
            left: 10px;
9
            top: 50px;
10
            height: 360px;
11
            width: 680px;
12
            background-color: black;
13
            will-change: transform;
14
        }
15
        
16
        .masker {
17
            position: absolute;
18
            left: 10px;
19
            top: 50px;
20
            height: 360px;
21
            width: 680px;
22
            border-top: 50px solid white;
23
            border-right: 300px solid white;
24
            border-bottom: 50px solid white;
25
            border-left: 300px solid white;
26
            box-sizing: border-box;
27
        }
28
    </style>
29
</head>
30
<body>
31
<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
32
<div class="video"></div>
33
<div class="masker"></div>
34
35
</body>
36
</html>
- a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt +32 lines
Line 0 a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt_sec1
1
Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.
2
3
On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
4
5
6
PASS mediaDevices.getUserMedia generated a stream successfully.
7
video.src = window.URL.createObjectURL(mediaStream)
8
9
 === beginning round of pixel tests ===
10
PASS pixel was white
11
12
 === all video tracks disabled ===
13
PASS pixel was black.
14
15
 === video track reenabled ===
16
PASS pixel was white.
17
18
 ===== play video =====
19
video.play()
20
21
 === beginning round of pixel tests ===
22
PASS pixel was white
23
24
 === all video tracks disabled ===
25
PASS pixel was black.
26
27
 === video track reenabled ===
28
PASS pixel was white.
29
PASS successfullyParsed is true
30
31
TEST COMPLETE
32
 
- a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html +130 lines
Line 0 a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html_sec1
1
<!DOCTYPE html>
2
<html>
3
<head>
4
    <script src="../../resources/js-test-pre.js"></script>
5
    <script src="./resources/getUserMedia-helper.js"></script>
6
</head>
7
<body onload="start()">
8
<p id="description"></p>
9
<div id="console"></div>
10
<video controls width="680" height="360"></video>
11
<canvas width="680" height="360"></canvas>
12
<script>
13
    let canvas;
14
    let context;
15
    let mediaStream;
16
    let video;
17
    
18
    let buffer;
19
20
    function isPixelBlack(pixel)
21
    {
22
        return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 255;
23
    }
24
25
    function isPixelTransparent(pixel)
26
    {
27
        return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 0;
28
    }
29
30
    function isPixelWhite(pixel)
31
    {
32
        return pixel[0] === 255 && pixel[1] === 255 && pixel[2] === 255 && pixel[3] === 255;
33
    }
34
35
    function attempt(numberOfTries, call, callback, successMessage)
36
    {
37
        if (numberOfTries <= 0) {
38
            testFailed('Pixel check did not succeed after multiple tries.');
39
            return;
40
        }
41
42
        let attemptSucceeded = call();
43
        if (attemptSucceeded) {
44
            testPassed(successMessage);
45
            callback();
46
47
            return;
48
        }
49
        
50
        setTimeout(function() {
51
            attempt(--numberOfTries, call, callback, successMessage);
52
        }, 50);
53
    }
54
55
    function repeatWithVideoPlayingAndFinishTest()
56
    {
57
        if (video.paused) {
58
            debug('<br> ===== play video =====');
59
            evalAndLog('video.play()');
60
            beginTestRound();
61
        } else
62
            finishJSTest();
63
    }
64
65
    function reenableTrack()
66
    {
67
        mediaStream.getVideoTracks()[0].enabled = true;
68
        debug('<br> === video track reenabled ===');
69
70
        // The video is not guaranteed to render non-black frames before the canvas is drawn to and the pixels are checked.
71
        // A timeout is used to ensure that the pixel check is done after the video renders non-black frames.
72
        attempt(10, checkPixels, repeatWithVideoPlayingAndFinishTest, 'pixel was white.');
73
    }
74
75
    function checkPixels()
76
    {
77
        context.clearRect(0, 0, canvas.width, canvas.height);
78
        buffer = context.getImageData(30, 242, 1, 1).data;
79
        if(!isPixelTransparent(buffer)) {
80
            testFailed('pixel was not transparent after clearing canvas.');
81
        }
82
83
        context.drawImage(video, 0, 0, canvas.width, canvas.height);
84
        buffer = context.getImageData(30, 242, 1, 1).data;
85
86
        if (mediaStream.getVideoTracks()[0].enabled)
87
            return isPixelWhite(buffer);
88
        else
89
            return isPixelBlack(buffer);
90
    }
91
92
    function disableAllTracks()
93
    {
94
        mediaStream.getVideoTracks()[0].enabled = false;
95
        debug('<br> === all video tracks disabled ===');
96
        
97
        // The video is not guaranteed to render black frames before the canvas is drawn to and the pixels are checked.
98
        // A timeout is used to ensure that the pixel check is done after the video renders black frames.
99
        attempt(10, checkPixels, reenableTrack, 'pixel was black.');
100
    }
101
102
    function beginTestRound()
103
    {
104
        debug('<br> === beginning round of pixel tests ===');
105
        attempt(1, checkPixels, disableAllTracks, 'pixel was white');
106
    }
107
108
    function canplay()
109
    {
110
        canvas = document.querySelector('canvas');
111
        context = canvas.getContext('2d');
112
113
        beginTestRound();
114
    }
115
116
    function start()
117
    {
118
        description("Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.");
119
120
        video = document.querySelector('video');
121
        video.addEventListener('canplay', canplay);
122
123
        getUserMedia("allow", {video:true}, setupVideoElementWithStream);
124
    }
125
126
    window.jsTestIsAsync = true;
127
</script>
128
<script src="../../resources/js-test-post.js"></script>
129
</body>
130
</html>
- a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html +81 lines
Line 0 a/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled.html_sec1
1
<!DOCTYPE html>
2
<html>
3
<head>
4
    <style>
5
        video {
6
            position: absolute;
7
            left: 10px;
8
            top: 50px;
9
        }
10
        
11
        .masker {
12
            position: absolute;
13
            left: 10px;
14
            top: 50px;
15
            height: 360px;
16
            width: 680px;
17
            border-top: 50px solid white;
18
            border-right: 300px solid white;
19
            border-bottom: 50px solid white;
20
            border-left: 300px solid white;
21
            box-sizing: border-box;
22
        }
23
    </style>
24
</head>
25
26
<body>
27
<p>Tests that the video frames of an HTMLVideoElement are black if no video MediaStreamTrack is enabled.</p>
28
<video controls width="680" height="360"></video>
29
<div class="masker"></div>
30
31
<script>
32
    let mediaStream;
33
    let video;
34
35
    function debug(msg)
36
    {
37
        let span = document.createElement('span');
38
        document.body.appendChild(span);
39
        span.innerHTML = `${msg} <br />`;
40
    }
41
42
    function canplaythrough()
43
    {    
44
        mediaStream.getVideoTracks()[0].enabled = false;
45
        window.testRunner.notifyDone();
46
    }
47
48
    function canplay()
49
    {
50
        video.play();
51
    }
52
53
    function setupStream(stream)
54
    {
55
        mediaStream = stream;
56
        video.src = window.URL.createObjectURL(mediaStream);
57
    }
58
59
    function failedToSetupStream()
60
    {
61
        debug('Failed to setup stream');
62
    }
63
64
    function start()
65
    {
66
        video = document.querySelector('video');
67
        video.addEventListener('canplay', canplay, false);
68
        video.addEventListener('canplaythrough', canplaythrough, false);
69
        navigator.mediaDevices.getUserMedia({video:true})
70
            .then(setupStream)
71
            .catch(failedToSetupStream);
72
    }
73
74
    if (window.testRunner) {
75
        window.testRunner.waitUntilDone();
76
        window.testRunner.setUserMediaPermission(true);
77
        start();
78
    }
79
</script>
80
</body>
81
</html>

Return to Bug 160222