1/*
2 * Copyright (C) 2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "MediaRecorderPrivateWriterCocoa.h"
28
29#if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31#include "AudioStreamDescription.h"
32#include "FileSystem.h"
33#include "Logging.h"
34#include "WebAudioBufferList.h"
35#include <AVFoundation/AVAssetWriter.h>
36#include <AVFoundation/AVAssetWriterInput.h>
37#include <pal/cf/CoreMediaSoftLink.h>
38
39typedef AVAssetWriter AVAssetWriterType;
40typedef AVAssetWriterInput AVAssetWriterInputType;
41
42SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
43
44SOFT_LINK_CLASS(AVFoundation, AVAssetWriter)
45SOFT_LINK_CLASS(AVFoundation, AVAssetWriterInput)
46
47SOFT_LINK_CONSTANT(AVFoundation, AVFileTypeMPEG4, NSString *)
48SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecKey, NSString *)
49SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecH264, NSString *)
50SOFT_LINK_CONSTANT(AVFoundation, AVVideoWidthKey, NSString *)
51SOFT_LINK_CONSTANT(AVFoundation, AVVideoHeightKey, NSString *)
52SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
53SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeAudio, NSString *)
54SOFT_LINK_CONSTANT(AVFoundation, AVEncoderBitRatePerChannelKey, NSString *)
55SOFT_LINK_CONSTANT(AVFoundation, AVFormatIDKey, NSString *)
56SOFT_LINK_CONSTANT(AVFoundation, AVNumberOfChannelsKey, NSString *)
57SOFT_LINK_CONSTANT(AVFoundation, AVSampleRateKey, NSString *)
58
59#define AVFileTypeMPEG4 getAVFileTypeMPEG4()
60#define AVMediaTypeAudio getAVMediaTypeAudio()
61#define AVMediaTypeVideo getAVMediaTypeVideo()
62#define AVVideoCodecKey getAVVideoCodecKey()
63#define AVVideoCodecH264 getAVVideoCodecH264()
64#define AVVideoWidthKey getAVVideoWidthKey()
65#define AVVideoHeightKey getAVVideoHeightKey()
66#define AVEncoderBitRatePerChannelKey getAVEncoderBitRatePerChannelKey()
67#define AVFormatIDKey getAVFormatIDKey()
68#define AVNumberOfChannelsKey getAVNumberOfChannelsKey()
69#define AVSampleRateKey getAVSampleRateKey()
70
71using namespace WebCore;
72
73namespace WebCore {
74
75using namespace PAL;
76
77bool MediaRecorderPrivateWriter::setupWriter()
78{
79 ASSERT(!m_writer);
80
81 NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
82 NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
83 NSString *path = [directory stringByAppendingString:filename];
84
85 NSURL *outputURL = [NSURL fileURLWithPath:path];
86 m_path = [path UTF8String];
87 NSError *error = nil;
88 m_writer = adoptNS([allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
89 if (error) {
90 RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
91 m_writer = nullptr;
92 return false;
93 }
94 return true;
95}
96
97bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
98{
99 ASSERT(!m_videoInput);
100
101 NSDictionary *videoSettings = @{ AVVideoCodecKey: AVVideoCodecH264, AVVideoWidthKey: [NSNumber numberWithInt:width], AVVideoHeightKey: [NSNumber numberWithInt:height] };
102 m_videoInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
103 [m_videoInput setExpectsMediaDataInRealTime:true];
104
105 if (![m_writer canAddInput:m_videoInput.get()]) {
106 m_videoInput = nullptr;
107 RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
108 return false;
109 }
110 [m_writer addInput:m_videoInput.get()];
111 m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
112 return true;
113}
114
115bool MediaRecorderPrivateWriter::setAudioInput()
116{
117 ASSERT(!m_audioInput);
118
119 NSDictionary *audioSettings = @{ AVEncoderBitRatePerChannelKey : @(28000), AVFormatIDKey : @(kAudioFormatMPEG4AAC), AVNumberOfChannelsKey : @(1), AVSampleRateKey : @(22050) };
120
121 m_audioInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
122 [m_audioInput setExpectsMediaDataInRealTime:true];
123
124 if (![m_writer canAddInput:m_audioInput.get()]) {
125 m_audioInput = nullptr;
126 RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
127 return false;
128 }
129 [m_writer addInput:m_audioInput.get()];
130 m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
131 return true;
132}
133
134static inline CMSampleBufferRef copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
135{
136 CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
137 CMItemCount count = 0;
138 CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
139
140 Vector<CMSampleTimingInfo> timeInfo(count);
141 CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
142
143 for (CMItemCount i = 0; i < count; i++) {
144 timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
145 timeInfo[i].presentationTimeStamp = startTime;
146 }
147
148 CMSampleBufferRef newBuffer;
149 CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
150 return newBuffer;
151}
152
153void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
154{
155 ASSERT(m_videoInput);
156 if (m_isStopped)
157 return;
158
159 if (!m_hasStartedWriting) {
160 if (![m_writer startWriting]) {
161 m_isStopped = true;
162 RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
163 return;
164 }
165 [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
166 m_hasStartedWriting = true;
167 RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
168 [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis] {
169 do {
170 if (![m_videoInput isReadyForMoreMediaData])
171 break;
172 auto locker = holdLock(m_videoLock);
173 if (m_videoBufferPool.isEmpty())
174 break;
175 auto buffer = m_videoBufferPool.takeFirst();
176 locker.unlockEarly();
177 if (![m_videoInput appendSampleBuffer:buffer.get()])
178 break;
179 } while (true);
180 if (m_isStopped && m_videoBufferPool.isEmpty()) {
181 [m_videoInput markAsFinished];
182 m_finishWritingVideoSemaphore.signal();
183 }
184 }];
185 return;
186 }
187 CMSampleBufferRef bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
188 auto locker = holdLock(m_videoLock);
189 m_videoBufferPool.append(retainPtr(bufferWithCurrentTime));
190}
191
192void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
193{
194 ASSERT(m_audioInput);
195 if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
196 return;
197 CMSampleBufferRef sampleBuffer;
198 CMFormatDescriptionRef format;
199 OSStatus error;
200 auto& basicDescription = *WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
201 error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &format);
202 if (m_isFirstAudioSample) {
203 if (!m_videoInput) {
204 // audio-only recording.
205 if (![m_writer startWriting]) {
206 m_isStopped = true;
207 return;
208 }
209 [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
210 m_hasStartedWriting = true;
211 }
212 m_isFirstAudioSample = false;
213 RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
214 [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis] {
215 do {
216 if (![m_audioInput isReadyForMoreMediaData])
217 break;
218 auto locker = holdLock(m_audioLock);
219 if (m_audioBufferPool.isEmpty())
220 break;
221 auto buffer = m_audioBufferPool.takeFirst();
222 locker.unlockEarly();
223 [m_audioInput appendSampleBuffer:buffer.get()];
224 } while (true);
225 if (m_isStopped && m_audioBufferPool.isEmpty()) {
226 [m_audioInput markAsFinished];
227 m_finishWritingAudioSemaphore.signal();
228 }
229 }];
230 }
231 CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
232
233 error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
234 if (error)
235 return;
236 error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
237 if (error)
238 return;
239
240 auto locker = holdLock(m_audioLock);
241 m_audioBufferPool.append(retainPtr(sampleBuffer));
242}
243
244void MediaRecorderPrivateWriter::stopRecording()
245{
246 m_isStopped = true;
247 if (!m_hasStartedWriting)
248 return;
249 ASSERT([m_writer status] == AVAssetWriterStatusWriting);
250 if (m_videoInput)
251 m_finishWritingVideoSemaphore.wait();
252
253 if (m_audioInput)
254 m_finishWritingAudioSemaphore.wait();
255
256 [m_writer finishWritingWithCompletionHandler:^{
257 m_isStopped = false;
258 m_hasStartedWriting = false;
259 m_isFirstAudioSample = true;
260 if (m_videoInput) {
261 m_videoInput.clear();
262 m_videoInput = nullptr;
263 dispatch_release(m_videoPullQueue);
264 }
265 if (m_audioInput) {
266 m_audioInput.clear();
267 m_audioInput = nullptr;
268 dispatch_release(m_audioPullQueue);
269 }
270 m_writer.clear();
271 m_writer = nullptr;
272 m_finishWritingSemaphore.signal();
273 }];
274}
275
276RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
277{
278 if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
279 return nullptr;
280
281 m_finishWritingSemaphore.wait();
282 return SharedBuffer::createWithContentsOfFile(m_path);
283}
284
285} // namespace WebCore
286
287#endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)