1/*
2 * Copyright (C) 2020 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#import "config.h"
27#import "LocalSampleBufferDisplayLayer.h"
28
29#if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31#import "Color.h"
32#import "IntSize.h"
33#import "MediaSample.h"
34
35#import <AVFoundation/AVSampleBufferDisplayLayer.h>
36#import <QuartzCore/CALayer.h>
37#import <QuartzCore/CATransaction.h>
38
39#import <wtf/MainThread.h>
40
41#import <pal/cocoa/AVFoundationSoftLink.h>
42
43using namespace WebCore;
44
45@interface WebAVSampleBufferStatusChangeListener : NSObject {
46 LocalSampleBufferDisplayLayer* _parent;
47}
48
49- (id)initWithParent:(LocalSampleBufferDisplayLayer*)callback;
50- (void)invalidate;
51- (void)beginObservingLayers;
52- (void)stopObservingLayers;
53@end
54
55@implementation WebAVSampleBufferStatusChangeListener
56
57- (id)initWithParent:(LocalSampleBufferDisplayLayer*)parent
58{
59 if (!(self = [super init]))
60 return nil;
61
62 _parent = parent;
63
64 return self;
65}
66
67- (void)dealloc
68{
69 [self invalidate];
70 [super dealloc];
71}
72
73- (void)invalidate
74{
75 [self stopObservingLayers];
76 _parent = nullptr;
77}
78
79- (void)beginObservingLayers
80{
81 ASSERT(_parent);
82 ASSERT(_parent->displayLayer());
83 ASSERT(_parent->rootLayer());
84
85 [_parent->displayLayer() addObserver:self forKeyPath:@"status" options:NSKeyValueObservingOptionNew context:nil];
86 [_parent->displayLayer() addObserver:self forKeyPath:@"error" options:NSKeyValueObservingOptionNew context:nil];
87 [_parent->rootLayer() addObserver:self forKeyPath:@"bounds" options:NSKeyValueObservingOptionNew context:nil];
88}
89
90- (void)stopObservingLayers
91{
92 if (!_parent)
93 return;
94
95 if (_parent->displayLayer()) {
96 [_parent->displayLayer() removeObserver:self forKeyPath:@"status"];
97 [_parent->displayLayer() removeObserver:self forKeyPath:@"error"];
98 }
99 if (_parent->rootLayer())
100 [_parent->rootLayer() removeObserver:self forKeyPath:@"bounds"];
101}
102
103- (void)observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object change:(NSDictionary *)change context:(void *)context
104{
105 UNUSED_PARAM(context);
106 UNUSED_PARAM(keyPath);
107 ASSERT(_parent);
108
109 if (!_parent)
110 return;
111
112 if ([object isKindOfClass:PAL::getAVSampleBufferDisplayLayerClass()]) {
113 RetainPtr<AVSampleBufferDisplayLayer> layer = (AVSampleBufferDisplayLayer *)object;
114 ASSERT(layer.get() == _parent->displayLayer());
115
116 if ([keyPath isEqualToString:@"status"]) {
117 RetainPtr<NSNumber> status = [change valueForKey:NSKeyValueChangeNewKey];
118 callOnMainThread([protectedSelf = RetainPtr<WebAVSampleBufferStatusChangeListener>(self), layer = WTFMove(layer), status = WTFMove(status)] {
119 if (!protectedSelf->_parent)
120 return;
121
122 protectedSelf->_parent->layerStatusDidChange();
123 });
124 return;
125 }
126
127 if ([keyPath isEqualToString:@"error"]) {
128 RetainPtr<NSNumber> status = [change valueForKey:NSKeyValueChangeNewKey];
129 callOnMainThread([protectedSelf = RetainPtr<WebAVSampleBufferStatusChangeListener>(self), layer = WTFMove(layer), status = WTFMove(status)] {
130 if (!protectedSelf->_parent)
131 return;
132
133 protectedSelf->_parent->layerErrorDidChange();
134 });
135 return;
136 }
137 }
138
139 if ([[change valueForKey:NSKeyValueChangeNotificationIsPriorKey] boolValue])
140 return;
141
142 if ((CALayer *)object == _parent->rootLayer()) {
143 if ([keyPath isEqualToString:@"bounds"]) {
144 if (!_parent)
145 return;
146
147 if (isMainThread()) {
148 _parent->rootLayerBoundsDidChange();
149 return;
150 }
151
152 callOnMainThread([protectedSelf = RetainPtr<WebAVSampleBufferStatusChangeListener>(self)] {
153 if (!protectedSelf->_parent)
154 return;
155
156 protectedSelf->_parent->rootLayerBoundsDidChange();
157 });
158 }
159 }
160
161}
162@end
163
164namespace WebCore {
165
166static void runWithoutAnimations(const WTF::Function<void()>& function)
167{
168 [CATransaction begin];
169 [CATransaction setAnimationDuration:0];
170 [CATransaction setDisableActions:YES];
171 function();
172 [CATransaction commit];
173}
174
175std::unique_ptr<SampleBufferDisplayLayer> LocalSampleBufferDisplayLayer::create(Client& client, bool hideRootLayer, IntSize size)
176{
177 auto sampleBufferDisplayLayer = adoptNS([PAL::allocAVSampleBufferDisplayLayerInstance() init]);
178 if (!sampleBufferDisplayLayer)
179 return nullptr;
180
181 return makeUnique<LocalSampleBufferDisplayLayer>(WTFMove(sampleBufferDisplayLayer), client, hideRootLayer, size);
182}
183
184LocalSampleBufferDisplayLayer::LocalSampleBufferDisplayLayer(RetainPtr<AVSampleBufferDisplayLayer>&& sampleBufferDisplayLayer, Client& client, bool hideRootLayer, IntSize size)
185 : SampleBufferDisplayLayer(client)
186 , m_statusChangeListener(adoptNS([[WebAVSampleBufferStatusChangeListener alloc] initWithParent:this]))
187 , m_sampleBufferDisplayLayer(WTFMove(sampleBufferDisplayLayer))
188{
189 m_sampleBufferDisplayLayer.get().backgroundColor = cachedCGColor(Color::black);
190 m_sampleBufferDisplayLayer.get().anchorPoint = { .5, .5 };
191 m_sampleBufferDisplayLayer.get().needsDisplayOnBoundsChange = YES;
192 m_sampleBufferDisplayLayer.get().videoGravity = AVLayerVideoGravityResizeAspectFill;
193
194 m_rootLayer = adoptNS([[CALayer alloc] init]);
195 m_rootLayer.get().hidden = hideRootLayer;
196
197 m_rootLayer.get().backgroundColor = cachedCGColor(Color::black);
198 m_rootLayer.get().needsDisplayOnBoundsChange = YES;
199
200 m_rootLayer.get().bounds = CGRectMake(0, 0, size.width(), size.height());
201
202 [m_statusChangeListener beginObservingLayers];
203
204 [m_rootLayer addSublayer:m_sampleBufferDisplayLayer.get()];
205
206#ifndef NDEBUG
207 [m_sampleBufferDisplayLayer setName:@"LocalSampleBufferDisplayLayer AVSampleBufferDisplayLayer"];
208 [m_rootLayer setName:@"LocalSampleBufferDisplayLayer AVSampleBufferDisplayLayer parent"];
209#endif
210}
211
212LocalSampleBufferDisplayLayer::~LocalSampleBufferDisplayLayer()
213{
214 [m_statusChangeListener stopObservingLayers];
215
216 m_pendingVideoSampleQueue.clear();
217
218 [m_sampleBufferDisplayLayer stopRequestingMediaData];
219 [m_sampleBufferDisplayLayer flush];
220 m_sampleBufferDisplayLayer = nullptr;
221
222 m_rootLayer = nullptr;
223}
224
225void LocalSampleBufferDisplayLayer::layerStatusDidChange()
226{
227 ASSERT(isMainThread());
228 if (m_sampleBufferDisplayLayer.get().status != AVQueuedSampleBufferRenderingStatusRendering)
229 return;
230 if (!m_client)
231 return;
232 m_client->sampleBufferDisplayLayerStatusDidChange(*this);
233}
234
235void LocalSampleBufferDisplayLayer::layerErrorDidChange()
236{
237 ASSERT(isMainThread());
238 // FIXME: Log error.
239}
240
241void LocalSampleBufferDisplayLayer::rootLayerBoundsDidChange()
242{
243 ASSERT(isMainThread());
244 if (!m_client)
245 return;
246 m_client->sampleBufferDisplayLayerBoundsDidChange(*this);
247}
248
249PlatformLayer* LocalSampleBufferDisplayLayer::displayLayer()
250{
251 return m_sampleBufferDisplayLayer.get();
252}
253
254PlatformLayer* LocalSampleBufferDisplayLayer::rootLayer()
255{
256 return m_rootLayer.get();
257}
258
259bool LocalSampleBufferDisplayLayer::didFail() const
260{
261 return [m_sampleBufferDisplayLayer status] == AVQueuedSampleBufferRenderingStatusFailed;
262}
263
264void LocalSampleBufferDisplayLayer::updateDisplayMode(bool hideDisplayLayer, bool hideRootLayer)
265{
266 if (m_rootLayer.get().hidden == hideRootLayer && m_sampleBufferDisplayLayer.get().hidden == hideDisplayLayer)
267 return;
268
269 runWithoutAnimations([&] {
270 m_sampleBufferDisplayLayer.get().hidden = hideDisplayLayer;
271 m_rootLayer.get().hidden = hideRootLayer;
272 });
273}
274
275CGRect LocalSampleBufferDisplayLayer::bounds() const
276{
277 return m_rootLayer.get().bounds;
278}
279
280void LocalSampleBufferDisplayLayer::updateAffineTransform(CGAffineTransform transform)
281{
282 runWithoutAnimations([&] {
283 m_sampleBufferDisplayLayer.get().affineTransform = transform;
284 });
285}
286
287void LocalSampleBufferDisplayLayer::updateBoundsAndPosition(CGRect videoBounds, CGPoint position)
288{
289 runWithoutAnimations([&] {
290 m_sampleBufferDisplayLayer.get().bounds = videoBounds;
291 m_sampleBufferDisplayLayer.get().position = position;
292 });
293}
294
295void LocalSampleBufferDisplayLayer::flush()
296{
297 [m_sampleBufferDisplayLayer flush];
298}
299
300void LocalSampleBufferDisplayLayer::flushAndRemoveImage()
301{
302 [m_sampleBufferDisplayLayer flushAndRemoveImage];
303}
304
305void LocalSampleBufferDisplayLayer::enqueueSample(MediaSample& sample)
306{
307 if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
308 addSampleToPendingQueue(sample);
309 requestNotificationWhenReadyForVideoData();
310 return;
311 }
312
313 [m_sampleBufferDisplayLayer enqueueSampleBuffer:sample.platformSample().sample.cmSampleBuffer];
314}
315
316void LocalSampleBufferDisplayLayer::removeOldSamplesFromPendingQueue()
317{
318 if (m_pendingVideoSampleQueue.isEmpty() || !m_client)
319 return;
320
321 auto decodeTime = m_pendingVideoSampleQueue.first()->decodeTime();
322 if (!decodeTime.isValid() || decodeTime < MediaTime::zeroTime()) {
323 while (m_pendingVideoSampleQueue.size() > 5)
324 m_pendingVideoSampleQueue.removeFirst();
325
326 return;
327 }
328
329 MediaTime now = m_client->streamTime();
330 while (!m_pendingVideoSampleQueue.isEmpty()) {
331 if (m_pendingVideoSampleQueue.first()->decodeTime() > now)
332 break;
333 m_pendingVideoSampleQueue.removeFirst();
334 }
335}
336
337void LocalSampleBufferDisplayLayer::addSampleToPendingQueue(MediaSample& sample)
338{
339 removeOldSamplesFromPendingQueue();
340 m_pendingVideoSampleQueue.append(sample);
341}
342
343void LocalSampleBufferDisplayLayer::clearEnqueuedSamples()
344{
345 m_pendingVideoSampleQueue.clear();
346}
347
348void LocalSampleBufferDisplayLayer::requestNotificationWhenReadyForVideoData()
349{
350 auto weakThis = makeWeakPtr(*this);
351 [m_sampleBufferDisplayLayer requestMediaDataWhenReadyOnQueue:dispatch_get_main_queue() usingBlock:^{
352 if (!weakThis)
353 return;
354
355 [m_sampleBufferDisplayLayer stopRequestingMediaData];
356
357 while (!m_pendingVideoSampleQueue.isEmpty()) {
358 if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
359 requestNotificationWhenReadyForVideoData();
360 return;
361 }
362
363 // FIXME: we should reenqueue in client.
364 auto sample = m_pendingVideoSampleQueue.takeFirst();
365 enqueueSample(sample.get());
366 }
367 }];
368}
369
370}
371
372#endif