Attachment #454196 for bug #237468

View Details Raw Unified Return to Bug 237468

| Differences between



2022-03-08  Youenn Fablet  <youenn@apple.com>

        WebRTC decoded frames are not correctly rotated in case GPU Process DOM rendering flag is set to true
        https://bugs.webkit.org/show_bug.cgi?id=237468
        <rdar://problem/89807876>

        Reviewed by Eric Carlson.

        We were creating remote video frames at webrtc decoder level but at that level, we do not know yet the rotation and timestamps of the frame.
        We need to set those values when the frame is exposed to RealtimeIncomingVideoSource.
        Previous tests did not catch the regression as we were correctly computing the size of the video using the webrtc rotation and not the rotation from the frame.
        We should probably migrate to RemoteVideoFrameProxy as a buffer wrapper so that we can easily create remote video frames from a RemoteVideoFrameProxy buffer.
        In the meantime, we use const_cast in VideoFrame::initializeCharacteristics.

        Covered by updated test.

        * platform/VideoFrame.cpp:
        * platform/VideoFrame.h:
        * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
        * testing/Internals.cpp:
        * testing/Internals.h:
        * testing/Internals.idl:

2022-03-08  Michael Catanzaro  <mcatanzaro@gnome.org>

        [GTK] Sync gtk-overlay-scrolling setting to web process

Lines 131-136 void VideoFrame::dump(PrintStream&) const a/Source/WebCore/platform/VideoFrame.cpp_sec1
- a/Source/WebCore/platform/VideoFrame.cpp +7 lines
131	{	131	{
132	}	132	}
133		133
		134	void VideoFrame::initializeCharacteristics(MediaTime presentationTime, bool isMirrored, VideoRotation rotation)
		135	{
		136	const_cast<MediaTime&>(m_presentationTime) = presentationTime;
		137	const_cast<bool&>(m_isMirrored) = isMirrored;
		138	const_cast<VideoRotation&>(m_rotation) = rotation;
		139	}
		140
134	}	141	}
135		142
136	#endif	143	#endif

Lines 57-62 public: a/Source/WebCore/platform/VideoFrame.h_sec1
- a/Source/WebCore/platform/VideoFrame.h +2 lines
57	virtual RefPtr<WebCore::VideoFrameCV> asVideoFrameCV() = 0;	57	virtual RefPtr<WebCore::VideoFrameCV> asVideoFrameCV() = 0;
58	#endif	58	#endif
59		59
		60	void initializeCharacteristics(MediaTime presentationTime, bool isMirrored, VideoRotation);
		61
60	protected:	62	protected:
61	WEBCORE_EXPORT VideoFrame(MediaTime presentationTime, bool isMirrored, VideoRotation);	63	WEBCORE_EXPORT VideoFrame(MediaTime presentationTime, bool isMirrored, VideoRotation);
62	const MediaTime m_presentationTime;	64	const MediaTime m_presentationTime;

Lines 130-136 RefPtr<MediaSample> RealtimeIncomingVideoSourceCocoa::toVideoFrame(const webrtc: a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm_sec1
- a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm -1 / +3 lines
130		130
131	if (auto* provider = videoFrameBufferProvider(frame)) {	131	if (auto* provider = videoFrameBufferProvider(frame)) {
132	// The only supported provider is VideoFrame.	132	// The only supported provider is VideoFrame.
133	return static_cast<VideoFrame*>(provider);	133	auto* videoFrame = static_cast<VideoFrame*>(provider);
		134	videoFrame->initializeCharacteristics(MediaTime { frame.timestamp_us(), 1000000 }, false, rotation);
		135	return videoFrame;
134	}	136	}
135		137
136	// In case of in memory samples, we have non interleaved YUV data while CVPixelBuffers prefer interleaved YUV data.	138	// In case of in memory samples, we have non interleaved YUV data while CVPixelBuffers prefer interleaved YUV data.



{
    stopObservingRealtimeMediaSource();

    m_trackVideoRotation = -1;
    m_trackSource = &track.source();
    m_trackSource->addObserver(*this);
    switch (m_trackSource->type()) {

    m_nextTrackFramePromise = makeUnique<TrackFramePromise>(WTFMove(promise));
}

void Internals::mediaStreamTrackVideoFrameRotation(DOMPromiseDeferred<IDLShort>&& promise)
{
    promise.resolve(m_trackVideoRotation);
}

void Internals::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
{
    callOnMainThread([this, weakThis = WeakPtr { *this }, sample = Ref { sample }] {
        if (!weakThis)
            return;
        m_trackVideoSampleCount++;
        m_trackVideoRotation = static_cast<int>(sample->videoRotation());
        if (!m_nextTrackFramePromise)
            return;




    void observeMediaStreamTrack(MediaStreamTrack&);
    using TrackFramePromise = DOMPromiseDeferred<IDLInterface<ImageData>>;
    void grabNextMediaStreamTrackFrame(TrackFramePromise&&);
    void mediaStreamTrackVideoFrameRotation(DOMPromiseDeferred<IDLShort>&&);
    void delayMediaStreamTrackSamples(MediaStreamTrack&, float);
    void setMediaStreamTrackMuted(MediaStreamTrack&, bool);
    void removeMediaStreamTrack(MediaStream&, MediaStreamTrack&);

    unsigned long m_trackAudioSampleCount { 0 };
    RefPtr<RealtimeMediaSource> m_trackSource;
    std::unique_ptr<TrackFramePromise> m_nextTrackFramePromise;
    int m_trackVideoRotation { 0 };
#endif
#if ENABLE(MEDIA_SESSION)
    std::unique_ptr<ArtworkImageLoader> m_artworkLoader;

Lines 918-923 typedef (FetchRequest or FetchResponse) FetchObject; a/Source/WebCore/testing/Internals.idl_sec1
- a/Source/WebCore/testing/Internals.idl +1 lines
918	[Conditional=MEDIA_STREAM] undefined setCameraMediaStreamTrackOrientation(MediaStreamTrack track, short orientation);	918	[Conditional=MEDIA_STREAM] undefined setCameraMediaStreamTrackOrientation(MediaStreamTrack track, short orientation);
919	[Conditional=MEDIA_STREAM] undefined observeMediaStreamTrack(MediaStreamTrack track);	919	[Conditional=MEDIA_STREAM] undefined observeMediaStreamTrack(MediaStreamTrack track);
920	[Conditional=MEDIA_STREAM] Promise<ImageData> grabNextMediaStreamTrackFrame();	920	[Conditional=MEDIA_STREAM] Promise<ImageData> grabNextMediaStreamTrackFrame();
		921	[Conditional=MEDIA_STREAM] Promise<short> mediaStreamTrackVideoFrameRotation();
921	[Conditional=MEDIA_STREAM] readonly attribute unsigned long trackAudioSampleCount;	922	[Conditional=MEDIA_STREAM] readonly attribute unsigned long trackAudioSampleCount;
922	[Conditional=MEDIA_STREAM] readonly attribute unsigned long trackVideoSampleCount;	923	[Conditional=MEDIA_STREAM] readonly attribute unsigned long trackVideoSampleCount;
923	[Conditional=MEDIA_STREAM] undefined delayMediaStreamTrackSamples(MediaStreamTrack track, float delay);	924	[Conditional=MEDIA_STREAM] undefined delayMediaStreamTrackSamples(MediaStreamTrack track, float delay);



2022-03-08  Youenn Fablet  <youenn@apple.com>

        WebRTC decoded frames are not correctly rotated in case GPU Process DOM rendering flag is set to true
        https://bugs.webkit.org/show_bug.cgi?id=237468
        <rdar://problem/89807876>

        Reviewed by Eric Carlson.

        * webrtc/video-rotation.html:
        Observe actual video frame rotation value.

2022-03-08  J Pascoe  <j_pascoe@apple.com>

        [ iOS ] 2X http/wpt/webauthn/public-key-credential-create-failure-local (layout-tests) are constant text failures



    });
}

async function waitForTrackRotation(track, angle, counter)
{
    if (!window.internals)
        return;

    if (!counter) {
        internals.observeMediaStreamTrack(track);
        counter = 1;
    }

    await new Promise(resolve => setTimeout(resolve, 50));
    const value = await internals.mediaStreamTrackVideoFrameRotation()
    if (angle === value)
        return;    

    if (++counter > 50)
        return Promise.reject("waitForTrackRotation timed out with value " + value + " while expecting " + angle);

    return waitForTrackRotation(track, angle, counter);
}

var track;
promise_test((test) => {
    if (window.testRunner)


            createConnections((firstConnection) => {
                firstConnection.addTrack(track, localStream);
                if (window.internals)
                    internals.applyRotationForOutgoingVideoSources(firstConnection);
            }, (secondConnection) => {
                secondConnection.ontrack = (trackEvent) => {
                    resolve(trackEvent.streams[0]);

    await waitForVideoSize(localVideo, 240, 320);
}, "Track is enabled and rotated, local video should not be black and should change size");

promise_test(async (test) => {
    if (window.internals)
        window.internals.setCameraMediaStreamTrackOrientation(track, 90);
    if (window.testRunner)
        testRunner.setMockCameraOrientation(90);

    await checkVideoBlack(false, remoteVideo, "canvas2").then(() => {
        return waitForVideoSize(remoteVideo, 240, 320);
    });

    await waitForTrackRotation(remoteVideo.srcObject.getVideoTracks()[0], 90);
}, "Track is enabled and rotated, remote video should not be black and should change size");

promise_test(async (test) => {
    if (window.internals)
        window.internals.setCameraMediaStreamTrackOrientation(track, 180);
    if (window.testRunner)
        testRunner.setMockCameraOrientation(180);

    await checkVideoBlack(false, remoteVideo, "canvas3").then(() => {
        return waitForVideoSize(remoteVideo, 320, 240);
    });

    await waitForTrackRotation(remoteVideo.srcObject.getVideoTracks()[0], 180);
}, "Track is enabled and rotated again, video should not be black and should change size");
        </script>
    </body>

Return to Bug 237468

Lines 5510-5515 void Internals::observeMediaStreamTrack(MediaStreamTrack& track) a/Source/WebCore/testing/Internals.cpp_sec1
- a/Source/WebCore/testing/Internals.cpp +7 lines
5510	{	5510	{
5511	stopObservingRealtimeMediaSource();	5511	stopObservingRealtimeMediaSource();
5512		5512
		5513	m_trackVideoRotation = -1;
5513	m_trackSource = &track.source();	5514	m_trackSource = &track.source();
5514	m_trackSource->addObserver(*this);	5515	m_trackSource->addObserver(*this);
5515	switch (m_trackSource->type()) {	5516	switch (m_trackSource->type()) {
Lines 5532-5543 void Internals::grabNextMediaStreamTrackFrame(TrackFramePromise&& promise) a/Source/WebCore/testing/Internals.cpp_sec2
5532	m_nextTrackFramePromise = makeUnique<TrackFramePromise>(WTFMove(promise));	5533	m_nextTrackFramePromise = makeUnique<TrackFramePromise>(WTFMove(promise));
5533	}	5534	}
5534		5535
		5536	void Internals::mediaStreamTrackVideoFrameRotation(DOMPromiseDeferred<IDLShort>&& promise)
		5537	{
		5538	promise.resolve(m_trackVideoRotation);
		5539	}
		5540
5535	void Internals::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)	5541	void Internals::videoSampleAvailable(MediaSample& sample, VideoSampleMetadata)
5536	{	5542	{
5537	callOnMainThread([this, weakThis = WeakPtr { *this }, sample = Ref { sample }] {	5543	callOnMainThread([this, weakThis = WeakPtr { *this }, sample = Ref { sample }] {
5538	if (!weakThis)	5544	if (!weakThis)
5539	return;	5545	return;
5540	m_trackVideoSampleCount++;	5546	m_trackVideoSampleCount++;
		5547	m_trackVideoRotation = static_cast<int>(sample->videoRotation());
5541	if (!m_nextTrackFramePromise)	5548	if (!m_nextTrackFramePromise)
5542	return;	5549	return;
5543		5550

Lines 1-3 a/Source/WebCore/ChangeLog_sec1
- a/Source/WebCore/ChangeLog +23 lines
		1	2022-03-08 Youenn Fablet <youenn@apple.com>
		2
		3	WebRTC decoded frames are not correctly rotated in case GPU Process DOM rendering flag is set to true
		4	https://bugs.webkit.org/show_bug.cgi?id=237468
		5	<rdar://problem/89807876>
		6
		7	Reviewed by Eric Carlson.
		8
		9	We were creating remote video frames at webrtc decoder level but at that level, we do not know yet the rotation and timestamps of the frame.
		10	We need to set those values when the frame is exposed to RealtimeIncomingVideoSource.
		11	Previous tests did not catch the regression as we were correctly computing the size of the video using the webrtc rotation and not the rotation from the frame.
		12	We should probably migrate to RemoteVideoFrameProxy as a buffer wrapper so that we can easily create remote video frames from a RemoteVideoFrameProxy buffer.
		13	In the meantime, we use const_cast in VideoFrame::initializeCharacteristics.
		14
		15	Covered by updated test.
		16
		17	* platform/VideoFrame.cpp:
		18	* platform/VideoFrame.h:
		19	* platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
		20	* testing/Internals.cpp:
		21	* testing/Internals.h:
		22	* testing/Internals.idl:
		23
1	2022-03-08 Michael Catanzaro <mcatanzaro@gnome.org>	24	2022-03-08 Michael Catanzaro <mcatanzaro@gnome.org>
2		25
3	[GTK] Sync gtk-overlay-scrolling setting to web process	26	[GTK] Sync gtk-overlay-scrolling setting to web process

Lines 873-878 public: a/Source/WebCore/testing/Internals.h_sec1
- a/Source/WebCore/testing/Internals.h +2 lines
873	void observeMediaStreamTrack(MediaStreamTrack&);	873	void observeMediaStreamTrack(MediaStreamTrack&);
874	using TrackFramePromise = DOMPromiseDeferred<IDLInterface<ImageData>>;	874	using TrackFramePromise = DOMPromiseDeferred<IDLInterface<ImageData>>;
875	void grabNextMediaStreamTrackFrame(TrackFramePromise&&);	875	void grabNextMediaStreamTrackFrame(TrackFramePromise&&);
		876	void mediaStreamTrackVideoFrameRotation(DOMPromiseDeferred<IDLShort>&&);
876	void delayMediaStreamTrackSamples(MediaStreamTrack&, float);	877	void delayMediaStreamTrackSamples(MediaStreamTrack&, float);
877	void setMediaStreamTrackMuted(MediaStreamTrack&, bool);	878	void setMediaStreamTrackMuted(MediaStreamTrack&, bool);
878	void removeMediaStreamTrack(MediaStream&, MediaStreamTrack&);	879	void removeMediaStreamTrack(MediaStream&, MediaStreamTrack&);
Lines 1301-1306 private: a/Source/WebCore/testing/Internals.h_sec2
1301	unsigned long m_trackAudioSampleCount { 0 };	1302	unsigned long m_trackAudioSampleCount { 0 };
1302	RefPtr<RealtimeMediaSource> m_trackSource;	1303	RefPtr<RealtimeMediaSource> m_trackSource;
1303	std::unique_ptr<TrackFramePromise> m_nextTrackFramePromise;	1304	std::unique_ptr<TrackFramePromise> m_nextTrackFramePromise;
		1305	int m_trackVideoRotation { 0 };
1304	#endif	1306	#endif
1305	#if ENABLE(MEDIA_SESSION)	1307	#if ENABLE(MEDIA_SESSION)
1306	std::unique_ptr<ArtworkImageLoader> m_artworkLoader;	1308	std::unique_ptr<ArtworkImageLoader> m_artworkLoader;

Lines 1-3 a/LayoutTests/ChangeLog_sec1
- a/LayoutTests/ChangeLog +11 lines
		1	2022-03-08 Youenn Fablet <youenn@apple.com>
		2
		3	WebRTC decoded frames are not correctly rotated in case GPU Process DOM rendering flag is set to true
		4	https://bugs.webkit.org/show_bug.cgi?id=237468
		5	<rdar://problem/89807876>
		6
		7	Reviewed by Eric Carlson.
		8
		9	* webrtc/video-rotation.html:
		10	Observe actual video frame rotation value.
		11
1	2022-03-08 J Pascoe <j_pascoe@apple.com>	12	2022-03-08 J Pascoe <j_pascoe@apple.com>
2		13
3	[ iOS ] 2X http/wpt/webauthn/public-key-credential-create-failure-local (layout-tests) are constant text failures	14	[ iOS ] 2X http/wpt/webauthn/public-key-credential-create-failure-local (layout-tests) are constant text failures

Lines 50-55 function checkVideoBlack(expected, video, canvasId) a/LayoutTests/webrtc/video-rotation.html_sec1
- a/LayoutTests/webrtc/video-rotation.html -6 / +29 lines
50	});	50	});
51	}	51	}
52		52
		53	async function waitForTrackRotation(track, angle, counter)
		54	{
		55	if (!window.internals)
		56	return;
		57
		58	if (!counter) {
		59	internals.observeMediaStreamTrack(track);
		60	counter = 1;
		61	}
		62
		63	await new Promise(resolve => setTimeout(resolve, 50));
		64	const value = await internals.mediaStreamTrackVideoFrameRotation()
		65	if (angle === value)
		66	return;
		67
		68	if (++counter > 50)
		69	return Promise.reject("waitForTrackRotation timed out with value " + value + " while expecting " + angle);
		70
		71	return waitForTrackRotation(track, angle, counter);
		72	}
		73
53	var track;	74	var track;
54	promise_test((test) => {	75	promise_test((test) => {
55	if (window.testRunner)	76	if (window.testRunner)
Lines 62-69 promise_test((test) => { a/LayoutTests/webrtc/video-rotation.html_sec2
62		83
63	createConnections((firstConnection) => {	84	createConnections((firstConnection) => {
64	firstConnection.addTrack(track, localStream);	85	firstConnection.addTrack(track, localStream);
65	if (window.internals)
66	internals.applyRotationForOutgoingVideoSources(firstConnection);
67	}, (secondConnection) => {	86	}, (secondConnection) => {
68	secondConnection.ontrack = (trackEvent) => {	87	secondConnection.ontrack = (trackEvent) => {
69	resolve(trackEvent.streams[0]);	88	resolve(trackEvent.streams[0]);
Lines 95-120 promise_test(async (test) => { a/LayoutTests/webrtc/video-rotation.html_sec3
95	await waitForVideoSize(localVideo, 240, 320);	114	await waitForVideoSize(localVideo, 240, 320);
96	}, "Track is enabled and rotated, local video should not be black and should change size");	115	}, "Track is enabled and rotated, local video should not be black and should change size");
97		116
98	promise_test((test) => {	117	promise_test(async (test) => {
99	if (window.internals)	118	if (window.internals)
100	window.internals.setCameraMediaStreamTrackOrientation(track, 90);	119	window.internals.setCameraMediaStreamTrackOrientation(track, 90);
101	if (window.testRunner)	120	if (window.testRunner)
102	testRunner.setMockCameraOrientation(90);	121	testRunner.setMockCameraOrientation(90);
103		122
104	return checkVideoBlack(false, remoteVideo, "canvas2").then(() => {	123	await checkVideoBlack(false, remoteVideo, "canvas2").then(() => {
105	return waitForVideoSize(remoteVideo, 240, 320);	124	return waitForVideoSize(remoteVideo, 240, 320);
106	});	125	});
		126
		127	await waitForTrackRotation(remoteVideo.srcObject.getVideoTracks()[0], 90);
107	}, "Track is enabled and rotated, remote video should not be black and should change size");	128	}, "Track is enabled and rotated, remote video should not be black and should change size");
108		129
109	promise_test((test) => {	130	promise_test(async (test) => {
110	if (window.internals)	131	if (window.internals)
111	window.internals.setCameraMediaStreamTrackOrientation(track, 180);	132	window.internals.setCameraMediaStreamTrackOrientation(track, 180);
112	if (window.testRunner)	133	if (window.testRunner)
113	testRunner.setMockCameraOrientation(180);	134	testRunner.setMockCameraOrientation(180);
114		135
115	return checkVideoBlack(false, remoteVideo, "canvas3").then(() => {	136	await checkVideoBlack(false, remoteVideo, "canvas3").then(() => {
116	return waitForVideoSize(remoteVideo, 320, 240);	137	return waitForVideoSize(remoteVideo, 320, 240);
117	});	138	});
		139
		140	await waitForTrackRotation(remoteVideo.srcObject.getVideoTracks()[0], 180);
118	}, "Track is enabled and rotated again, video should not be black and should change size");	141	}, "Track is enabled and rotated again, video should not be black and should change size");
119	</script>	142	</script>
120	</body>	143	</body>