Attachment #371345 for bug #195124

View Details Raw Unified Return to Bug 195124

| Differences between



2019-06-04  Youenn Fablet  <youenn@apple.com>

        Call was negotiated with H264 Base Profile 42e01f but encoded in High Profile
        https://bugs.webkit.org/show_bug.cgi?id=195124
        <rdar://problem/48453085>

        Reviewed by NOBODY (OOPS!).

        Use VTB directly instead of VCP when baseline is requested.
        For platforms supporting the VCP-in-VTB API, use VCP for high profile, VTB for baseline.
        For platforms not supporting the VCP-in-VTB API, use regular VTB for both baseline and high profile.
        On MacOS, if VTB session creation fails, use VCP as a fallback.

        * Source/webrtc/sdk/WebKit/EncoderUtilities.h: Removed.
        * Source/webrtc/sdk/WebKit/VideoProcessingSoftLink.h:
        * Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
        (-[RTCSingleVideoEncoderH264 initWithCodecInfo:simulcastIndex:]):
        (-[RTCSingleVideoEncoderH264 hasCompressionSession]):
        (-[RTCSingleVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
        (-[RTCSingleVideoEncoderH264 resetCompressionSessionIfNeededWithFrame:]):
        (-[RTCSingleVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
        (-[RTCSingleVideoEncoderH264 configureCompressionSession]):
        (-[RTCSingleVideoEncoderH264 destroyCompressionSession]):
        (-[RTCSingleVideoEncoderH264 setEncoderBitrateBps:]):
        * Source/webrtc/sdk/objc/components/video_codec/helpers.cc:
        * Source/webrtc/sdk/objc/components/video_codec/helpers.h:

2019-05-28  Youenn Fablet  <youenn@apple.com>

        createAnswer() SDP Rejected by setLocalDescription()



/*
 * Copyright (C) 2018 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#pragma once

#include "VideoProcessingSoftLink.h"

#if ENABLE_VCP_ENCODER

#define CompressionSessionRef VCPCompressionSessionRef
#define CompressionSessionSetProperty webrtc::VCPCompressionSessionSetProperty
#define CompressionSessionGetPixelBufferPool webrtc::VCPCompressionSessionGetPixelBufferPool
#define CompressionSessionEncodeFrame webrtc::VCPCompressionSessionEncodeFrame
#define CompressionSessionCreate webrtc::VCPCompressionSessionCreate
#define kCodecTypeH264 kVCPCodecType4CC_H264
#define CompressionSessionInvalidate webrtc::VCPCompressionSessionInvalidate

#else

#define CompressionSessionRef VTCompressionSessionRef
#define CompressionSessionSetProperty VTSessionSetProperty
#define CompressionSessionGetPixelBufferPool VTCompressionSessionGetPixelBufferPool
#define CompressionSessionEncodeFrame VTCompressionSessionEncodeFrame
#define CompressionSessionCreate VTCompressionSessionCreate
#define kCodecTypeH264 kCMVideoCodecType_H264
#define CompressionSessionInvalidate VTCompressionSessionInvalidate

#endif

Lines 37-46 a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/VideoProcessingSoftLink.h_sec1
- a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/VideoProcessingSoftLink.h -2 / +4 lines
37	#define ENABLE_VCP_ENCODER 0	37	#define ENABLE_VCP_ENCODER 0
38	#define ENABLE_VCP_VTB_ENCODER 0	38	#define ENABLE_VCP_VTB_ENCODER 0
39	#elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)	39	#elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)
40	#define ENABLE_VCP_ENCODER __MAC_OS_X_VERSION_MAX_ALLOWED < 101500	40	#define ENABLE_VCP_ENCODER 1
41	#define ENABLE_VCP_VTB_ENCODER __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500	41	#define ENABLE_VCP_VTB_ENCODER __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500
42	#elif (defined(TARGET_OS_MAC) && TARGET_OS_MAC)	42	#elif (defined(TARGET_OS_MAC) && TARGET_OS_MAC)
43	#define ENABLE_VCP_ENCODER (__MAC_OS_X_VERSION_MIN_REQUIRED >= 101300 && __MAC_OS_X_VERSION_MAX_ALLOWED < 101500)	43	#define ENABLE_VCP_ENCODER __MAC_OS_X_VERSION_MIN_REQUIRED >= 101300
44	#define ENABLE_VCP_VTB_ENCODER __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500	44	#define ENABLE_VCP_VTB_ENCODER __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500
45	#endif	45	#endif
46		46
Lines 145-150 SOFT_LINK_FUNCTION_FOR_HEADER(webrtc, VideoProcessing, VCPCompressionSessionCrea a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/VideoProcessingSoftLink.h_sec2
145	SOFT_LINK_FUNCTION_FOR_HEADER(webrtc, VideoProcessing, VCPCompressionSessionInvalidate, void, (VCPCompressionSessionRef session), (session))	145	SOFT_LINK_FUNCTION_FOR_HEADER(webrtc, VideoProcessing, VCPCompressionSessionInvalidate, void, (VCPCompressionSessionRef session), (session))
146	#define VCPCompressionSessionInvalidate softLink_VideoProcessing_VCPCompressionSessionInvalidate	146	#define VCPCompressionSessionInvalidate softLink_VideoProcessing_VCPCompressionSessionInvalidate
147		147
		148	#else
		149	using VCPCompressionSessionRef = void*;
148	#endif // ENABLE_VCP_ENCODER	150	#endif // ENABLE_VCP_ENCODER
149		151
150	#endif // __APPLE__	152	#endif // __APPLE__



#include "system_wrappers/include/clock.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"

#include "sdk/WebKit/EncoderUtilities.h"
#include "sdk/WebKit/WebKitUtilities.h"

#import <dlfcn.h>

VT_EXPORT const CFStringRef kVTVideoEncoderSpecification_Usage;
VT_EXPORT const CFStringRef kVTCompressionPropertyKey_Usage;

#if !ENABLE_VCP_ENCODER && !defined(WEBRTC_IOS) && !ENABLE_VCP_VTB_ENCODER
static inline bool isStandardFrameSize(int32_t width, int32_t height)
{
    // FIXME: Envision relaxing this rule, something like width and height dividable by 4 or 8 should be good enough.
    if (width == 1280)
        return height == 720;
    if (width == 720)
        return height == 1280;
    if (width == 960)
        return height == 540;
    if (width == 540)
        return height == 960;
    if (width == 640)
        return height == 480;
    if (width == 480)
        return height == 640;
    if (width == 288)
        return height == 352;
    if (width == 352)
        return height == 288;
    if (width == 320)
        return height == 240;
    if (width == 240)
        return height == 320;
    return false;
}
#endif

__attribute__((objc_runtime_name("WK_RTCSingleVideoEncoderH264")))
@interface RTCSingleVideoEncoderH264 : NSObject <RTCVideoEncoder>
- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo simulcastIndex: (int)index;

  RTCVideoEncoderCallback _callback;
  int32_t _width;
  int32_t _height;
  bool _useVCP;
  VTCompressionSessionRef _compressionSession;
  VCPCompressionSessionRef _vcpCompressionSession;
  CVPixelBufferPoolRef _pixelBufferPool;
  RTCVideoCodecMode _mode;


    _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
    _packetizationMode = RTCH264PacketizationModeNonInterleaved;
    _profile = ExtractProfile([codecInfo nativeSdpVideoFormat]);
#if ENABLE_VCP_VTB_ENCODER
    _useVCP = [(__bridge NSString *)_profile containsString: @"High"];
#else
    _useVCP = false;
#endif
    _simulcastIndex = index;
    RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);
    RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]);

  return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat];
}

- (bool)hasCompressionSession
{
    return _compressionSession || _vcpCompressionSession;
}

- (NSInteger)encode:(RTCVideoFrame *)frame
    codecSpecificInfo:(nullable id<RTCCodecSpecificInfo>)codecSpecificInfo
           frameTypes:(NSArray<NSNumber *> *)frameTypes {
  RTC_DCHECK_EQ(frame.width, _width);
  RTC_DCHECK_EQ(frame.height, _height);
  if (!_callback || ![self hasCompressionSession]) {
    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  }
  BOOL isKeyframeRequired = NO;

  // Update the bitrate if needed.
  [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];

  OSStatus status;
  if (_compressionSession) {
    status = VTCompressionSessionEncodeFrame(_compressionSession,
                                                    pixelBuffer,
                                                    presentationTimeStamp,
                                                    kCMTimeInvalid,
                                                    frameProperties,
                                                    encodeParams.release(),
                                                    nullptr);
  } else {
#if ENABLE_VCP_ENCODER
    status = webrtc::VCPCompressionSessionEncodeFrame(_vcpCompressionSession,
                                                    pixelBuffer,
                                                    presentationTimeStamp,
                                                    kCMTimeInvalid,
                                                    frameProperties,
                                                    encodeParams.release(),
                                                    nullptr);
#else
    status = 1;
#endif
  }
  if (frameProperties) {
    CFRelease(frameProperties);
  }

  // configured with, make sure the compression session is reset using the correct pixel format.
  OSType framePixelFormat = [self pixelFormatOfFrame:frame];

  if ([self hasCompressionSession]) {
#if defined(WEBRTC_WEBKIT_BUILD)
    if (!_pixelBufferPool) {
      return NO;

#endif
  CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);

#if ENABLE_VCP_ENCODER
  if (_useVCP) {
    int usageValue = 1;
    auto usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);
    CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_Usage, usage);
    CFRelease(usage);
  }
#endif
#if ENABLE_VCP_VTB_ENCODER
  if (_useVCP) {
    CFDictionarySetValue(encoderSpecs, kVTVideoEncoderList_EncoderID, CFSTR("com.apple.videotoolbox.videoencoder.h264.rtvc"));
  }
#endif
  OSStatus status =
      VTCompressionSessionCreate(nullptr,  // use default allocator
                                 _width,
                                 _height,
                                 kCMVideoCodecType_H264,
                                 encoderSpecs,  // use hardware accelerated encoder if available
                                 sourceAttributes,
                                 nullptr,  // use default compressed data allocator
                                 compressionOutputCallback,
                                 nullptr,
                                 &_compressionSession);
  if (sourceAttributes) {
    CFRelease(sourceAttributes);
    sourceAttributes = nullptr;
  }
  if (encoderSpecs) {
    CFRelease(encoderSpecs);
    encoderSpecs = nullptr;
  }

#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && ENABLE_VCP_ENCODER
  if (!_useVCP) {
    CFBooleanRef hwaccl_enabled = nullptr;
    if (status == noErr) {
      status = VTSessionCopyProperty(_compressionSession,
#endif
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
  CFBooleanRef hwaccl_enabled = nullptr;
  if (status == noErr) {
    status = VTSessionCopyProperty(_compressionSession,
                                 kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
                                 nullptr,
                                 &hwaccl_enabled);
  }
  if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
    RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
  } else {
    RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";

#if !ENABLE_VCP_ENCODER && !ENABLE_VCP_VTB_ENCODER && !defined(WEBRTC_IOS)
    if (!isStandardFrameSize(_width, _height)) {
      _disableEncoding = true;
      RTC_LOG(LS_ERROR) << "Using H264 software encoder with non standard size is not supported";
      return WEBRTC_VIDEO_CODEC_ERROR;
    }

    CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
    int64_t pixelFormatType = framePixelFormat;
    CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType);

    const size_t attributesSize = 3;
    CFTypeRef keys[attributesSize] = {
      kCVPixelBufferOpenGLCompatibilityKey,
      kCVPixelBufferIOSurfacePropertiesKey,
      kCVPixelBufferPixelFormatTypeKey
    };
    CFTypeRef values[attributesSize] = {
      kCFBooleanTrue,
      ioSurfaceValue,
      pixelFormat};
    CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize);

    if (ioSurfaceValue) {
      CFRelease(ioSurfaceValue);
      ioSurfaceValue = nullptr;
    }
    if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
      RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
    } else {
      if (_compressionSession) {
        VTCompressionSessionInvalidate(_compressionSession);
        CFRelease(_compressionSession);
        _compressionSession = nullptr;
      }
      // Use VCP instead.
      int usageValue = 1;
      auto usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);
      CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_Usage, usage);
      CFRelease(usage);
      usage = nullptr;
    }

    [self destroyCompressionSession];

      RTC_LOG(LS_INFO) << "Compression session created with VCP";
      status =
          webrtc::VCPCompressionSessionCreate(nullptr,  // use default allocator
                                 _width,
                                 _height,
                                 kVCPCodecType4CC_H264,
                                 encoderSpecs,
                                 sourceAttributes,
                                 nullptr,  // use default compressed data allocator
                                 compressionOutputCallback,
                                 nullptr,
                                 &_vcpCompressionSession);
    if (sourceAttributes) {
      CFRelease(sourceAttributes);
      sourceAttributes = nullptr;
    }
    if (encoderSpecs) {
      CFRelease(encoderSpecs);
      encoderSpecs = nullptr;
    }
    if (status != noErr) {
      return WEBRTC_VIDEO_CODEC_ERROR;
    }
#endif
  }
#endif
  if (sourceAttributes) {
    CFRelease(sourceAttributes);
    sourceAttributes = nullptr;
  }
  if (encoderSpecs) {
    CFRelease(encoderSpecs);
    encoderSpecs = nullptr;
  }

  if (status != noErr) {
    RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
    return WEBRTC_VIDEO_CODEC_ERROR;
  }

  [self configureCompressionSession];

#if !defined(WEBRTC_WEBKIT_BUILD)
  // The pixel buffer pool is dependent on the compression session so if the session is reset, the
  // pool should be reset as well.
  if (_compressionSession)
    _pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
#if ENABLE_VCP_ENCODER
  else
    _pixelBufferPool = webrtc::VCPCompressionSessionGetPixelBufferPool(_compressionSession);
#endif
#endif
  return WEBRTC_VIDEO_CODEC_OK;
}

- (void)configureCompressionSession {
  RTC_DCHECK([self hasCompressionSession]);
  SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_RealTime, true);
  SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile);
  SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
#if ENABLE_VCP_ENCODER
  if (_useVCP) {
    SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_Usage, 1);
  }
#endif
  [self setEncoderBitrateBps:_targetBitrateBps];
  // TODO(tkchin): Look at entropy mode and colorspace matrices.

  //     1);

  // Set a relatively large value for keyframe emission (7200 frames or 4 minutes).
  SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
  SetVTSessionProperty(
      _compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
}

- (void)destroyCompressionSession {
  if (_compressionSession) {
    VTCompressionSessionInvalidate(_compressionSession);
    CFRelease(_compressionSession);
    _compressionSession = nullptr;
    _pixelBufferPool = nullptr;
  }
#if ENABLE_VCP_ENCODER
  if (_vcpCompressionSession) {
    webrtc::VCPCompressionSessionInvalidate(_vcpCompressionSession);
    CFRelease(_vcpCompressionSession);
    _vcpCompressionSession = nullptr;
  }
#endif
  _pixelBufferPool = nullptr;
}

- (NSString *)implementationName {

}

- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
  if ([self hasCompressionSession]) {
    SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);

    // TODO(tkchin): Add a helper method to set array value.
    int64_t dataLimitBytesPerSecondValue =

        CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
    const void *nums[2] = {bytesPerSecond, oneSecond};
    CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);

    OSStatus status = noErr;
    if (_compressionSession)
      VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
#if ENABLE_VCP_ENCODER
    else
      webrtc::VCPCompressionSessionSetProperty(_vcpCompressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
#endif
    if (status != noErr) {
      RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << kVTCompressionPropertyKey_DataRateLimits
        << ": " << status;
    }

    if (bytesPerSecond) {
      CFRelease(bytesPerSecond);
    }

    if (dataRateLimits) {
      CFRelease(dataRateLimits);
    }
    if (status != noErr) {
      RTC_LOG(LS_ERROR) << "Failed to set data rate limit";
    }

    _encoderBitrateBps = bitrateBps;
  }
}



}

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
                          CFStringRef key,
                          int32_t value) {
  CFNumberRef cfNum =
      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
  OSStatus status = noErr;
  if (session)
    status = VTSessionSetProperty(session, key, cfNum);
#if ENABLE_VCP_ENCODER
  else
    status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cfNum);
#endif
  CFRelease(cfNum);
  if (status != noErr) {
    std::string key_string = CFStringToString(key);

}

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
                          CFStringRef key,
                          uint32_t value) {
  int64_t value_64 = value;
  CFNumberRef cfNum =
      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
  OSStatus status = noErr;
  if (session)
    status = VTSessionSetProperty(session, key, cfNum);
#if ENABLE_VCP_ENCODER
  else
    status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cfNum);
#endif
  CFRelease(cfNum);
  if (status != noErr) {
    std::string key_string = CFStringToString(key);

}

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession, CFStringRef key, bool value) {
  CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
  OSStatus status = noErr;
  if (session)
    status = VTSessionSetProperty(session, key, cf_bool);
#if ENABLE_VCP_ENCODER
  else
    status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cf_bool);
#endif
  if (status != noErr) {
    std::string key_string = CFStringToString(key);
    RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string

}

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
                          CFStringRef key,
                          CFStringRef value) {
  OSStatus status = noErr;
  if (session)
    status = VTSessionSetProperty(session, key, value);
#if ENABLE_VCP_ENCODER
  else
    status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, value);
#endif
  if (status != noErr) {
    std::string key_string = CFStringToString(key);
    std::string val_string = CFStringToString(value);



#include <VideoToolbox/VideoToolbox.h>
#include <string>

#include "sdk/WebKit/VideoProcessingSoftLink.h"

// Convenience function for creating a dictionary.
inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys,

std::string CFStringToString(const CFStringRef cf_string);

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,CFStringRef key, int32_t value);

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
                          CFStringRef key,
                          uint32_t value);

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession, CFStringRef key, bool value);

// Convenience function for setting a VT property.
void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
                          CFStringRef key,
                          CFStringRef value);




2019-06-04  Youenn Fablet  <youenn@apple.com>

        Call was negotiated with H264 Base Profile 42e01f but encoded in High Profile
        https://bugs.webkit.org/show_bug.cgi?id=195124
        <rdar://problem/48453085>

        Reviewed by NOBODY (OOPS!).

        * webrtc/video-h264-expected.txt: Added.
        * webrtc/video-h264.html: Added.

2019-06-02  Devin Rousso  <drousso@apple.com>

        Flaky Test: inspector/canvas/recording.html

Line 0 a/LayoutTests/webrtc/video-h264-expected.txt_sec1
- a/LayoutTests/webrtc/video-h264-expected.txt +5 lines
	1
	2
	3	PASS Baseline H264
	4	PASS High H264
	5



<!doctype html>
<html>
    <head>
        <meta charset="utf-8">
        <title>Testing H264 baseline and high profile</title>
        <script src="../resources/testharness.js"></script>
        <script src="../resources/testharnessreport.js"></script>
    </head>
    <body>
        <video id="video" autoplay=""></video>
        <canvas id="canvas" width="640" height="480"></canvas>
        <script src ="routines.js"></script>
        <script>
function grabFrameData(x, y, w, h)
{
    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    canvas.getContext('2d').drawImage(video, x, y, w, h, x, y, w, h);
    return canvas.getContext('2d').getImageData(x, y, w, h).data;
}

function testImage()
{
    const data = grabFrameData(10, 325, 250, 1);

    var index = 20;
    assert_true(data[index] < 100);
    assert_true(data[index + 1] < 100);
    assert_true(data[index + 2] < 100);

    index = 80;
    assert_true(data[index] > 200);
    assert_true(data[index + 1] > 200);
    assert_true(data[index + 2] > 200);

    index += 80;
    assert_true(data[index] > 200);
    assert_true(data[index + 1] > 200);
    assert_true(data[index + 2] < 100);
}

promise_test(async (test) => {
    if (window.testRunner)
        testRunner.setUserMediaPermission(true);

    const localStream = await navigator.mediaDevices.getUserMedia({video: true});
    const stream = await new Promise((resolve, reject) => {
        createConnections((firstConnection) => {
            firstConnection.addTrack(localStream.getVideoTracks()[0], localStream);
        }, (secondConnection) => {
            secondConnection.ontrack = (trackEvent) => {
                resolve(trackEvent.streams[0]);
            };
        }, { observeOffer : (offer) => {
            offer.sdp = offer.sdp.replace("640c1f", "42e01f");
            return offer;
        }
        });
        setTimeout(() => reject("Test timed out"), 5000);
    });

    video.srcObject = stream;
    await video.play();

    testImage();
}, "Baseline H264");

promise_test(async (test) => {
    if (window.testRunner)
        testRunner.setUserMediaPermission(true);

    const localStream = await navigator.mediaDevices.getUserMedia({video: true});
    const stream = await new Promise((resolve, reject) => {
        createConnections((firstConnection) => {
            firstConnection.addTrack(localStream.getVideoTracks()[0], localStream);
        }, (secondConnection) => {
            secondConnection.ontrack = (trackEvent) => {
                resolve(trackEvent.streams[0]);
            };
        }, { observeOffer : (offer) => {
            offer.sdp = offer.sdp.replace("42e01f", "640c1f");
            return offer;
        }
        });
        setTimeout(() => reject("Test timed out"), 5000);
    });

    video.srcObject = stream;
    await video.play();

    testImage();
}, "High H264");
        </script>
    </body>
</html>

Return to Bug 195124

Lines 1-3 a/Source/ThirdParty/libwebrtc/ChangeLog_sec1
- a/Source/ThirdParty/libwebrtc/ChangeLog +27 lines
		1	2019-06-04 Youenn Fablet <youenn@apple.com>
		2
		3	Call was negotiated with H264 Base Profile 42e01f but encoded in High Profile
		4	https://bugs.webkit.org/show_bug.cgi?id=195124
		5	<rdar://problem/48453085>
		6
		7	Reviewed by NOBODY (OOPS!).
		8
		9	Use VTB directly instead of VCP when baseline is requested.
		10	For platforms supporting the VCP-in-VTB API, use VCP for high profile, VTB for baseline.
		11	For platforms not supporting the VCP-in-VTB API, use regular VTB for both baseline and high profile.
		12	On MacOS, if VTB session creation fails, use VCP as a fallback.
		13
		14	* Source/webrtc/sdk/WebKit/EncoderUtilities.h: Removed.
		15	* Source/webrtc/sdk/WebKit/VideoProcessingSoftLink.h:
		16	* Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
		17	(-[RTCSingleVideoEncoderH264 initWithCodecInfo:simulcastIndex:]):
		18	(-[RTCSingleVideoEncoderH264 hasCompressionSession]):
		19	(-[RTCSingleVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
		20	(-[RTCSingleVideoEncoderH264 resetCompressionSessionIfNeededWithFrame:]):
		21	(-[RTCSingleVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
		22	(-[RTCSingleVideoEncoderH264 configureCompressionSession]):
		23	(-[RTCSingleVideoEncoderH264 destroyCompressionSession]):
		24	(-[RTCSingleVideoEncoderH264 setEncoderBitrateBps:]):
		25	* Source/webrtc/sdk/objc/components/video_codec/helpers.cc:
		26	* Source/webrtc/sdk/objc/components/video_codec/helpers.h:
		27
1	2019-05-28 Youenn Fablet <youenn@apple.com>	28	2019-05-28 Youenn Fablet <youenn@apple.com>
2		29
3	createAnswer() SDP Rejected by setLocalDescription()	30	createAnswer() SDP Rejected by setLocalDescription()

Lines 1-50 a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/EncoderUtilities.h_sec1
- a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/EncoderUtilities.h -50 lines
1	/*
2	* Copyright (C) 2018 Apple Inc. All rights reserved.
3	*
4	* Redistribution and use in source and binary forms, with or without
5	* modification, are permitted provided that the following conditions
6	* are met:
7	* 1. Redistributions of source code must retain the above copyright
8	* notice, this list of conditions and the following disclaimer.
9	* 2. Redistributions in binary form must reproduce the above copyright
10	* notice, this list of conditions and the following disclaimer in the
11	* documentation and/or other materials provided with the distribution.
12	*
13	* THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15	* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17	* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23	* THE POSSIBILITY OF SUCH DAMAGE.
24	*/
25
26	#pragma once
27
28	#include "VideoProcessingSoftLink.h"
29
30	#if ENABLE_VCP_ENCODER
31
32	#define CompressionSessionRef VCPCompressionSessionRef
33	#define CompressionSessionSetProperty webrtc::VCPCompressionSessionSetProperty
34	#define CompressionSessionGetPixelBufferPool webrtc::VCPCompressionSessionGetPixelBufferPool
35	#define CompressionSessionEncodeFrame webrtc::VCPCompressionSessionEncodeFrame
36	#define CompressionSessionCreate webrtc::VCPCompressionSessionCreate
37	#define kCodecTypeH264 kVCPCodecType4CC_H264
38	#define CompressionSessionInvalidate webrtc::VCPCompressionSessionInvalidate
39
40	#else
41
42	#define CompressionSessionRef VTCompressionSessionRef
43	#define CompressionSessionSetProperty VTSessionSetProperty
44	#define CompressionSessionGetPixelBufferPool VTCompressionSessionGetPixelBufferPool
45	#define CompressionSessionEncodeFrame VTCompressionSessionEncodeFrame
46	#define CompressionSessionCreate VTCompressionSessionCreate
47	#define kCodecTypeH264 kCMVideoCodecType_H264
48	#define CompressionSessionInvalidate VTCompressionSessionInvalidate
49
50	#endif

Lines 31-37 a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec1
- a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm -139 / +117 lines
31	#include "system_wrappers/include/clock.h"	31	#include "system_wrappers/include/clock.h"
32	#include "third_party/libyuv/include/libyuv/convert_from.h"	32	#include "third_party/libyuv/include/libyuv/convert_from.h"
33		33
34	#include "sdk/WebKit/EncoderUtilities.h"
35	#include "sdk/WebKit/WebKitUtilities.h"	34	#include "sdk/WebKit/WebKitUtilities.h"
36		35
37	#import <dlfcn.h>	36	#import <dlfcn.h>
Lines 40-73 a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec2
40	VT_EXPORT const CFStringRef kVTVideoEncoderSpecification_Usage;	39	VT_EXPORT const CFStringRef kVTVideoEncoderSpecification_Usage;
41	VT_EXPORT const CFStringRef kVTCompressionPropertyKey_Usage;	40	VT_EXPORT const CFStringRef kVTCompressionPropertyKey_Usage;
42		41
43	#if !ENABLE_VCP_ENCODER && !defined(WEBRTC_IOS) && !ENABLE_VCP_VTB_ENCODER
44	static inline bool isStandardFrameSize(int32_t width, int32_t height)
45	{
46	// FIXME: Envision relaxing this rule, something like width and height dividable by 4 or 8 should be good enough.
47	if (width == 1280)
48	return height == 720;
49	if (width == 720)
50	return height == 1280;
51	if (width == 960)
52	return height == 540;
53	if (width == 540)
54	return height == 960;
55	if (width == 640)
56	return height == 480;
57	if (width == 480)
58	return height == 640;
59	if (width == 288)
60	return height == 352;
61	if (width == 352)
62	return height == 288;
63	if (width == 320)
64	return height == 240;
65	if (width == 240)
66	return height == 320;
67	return false;
68	}
69	#endif
70
71	__attribute__((objc_runtime_name("WK_RTCSingleVideoEncoderH264")))	42	__attribute__((objc_runtime_name("WK_RTCSingleVideoEncoderH264")))
72	@interface RTCSingleVideoEncoderH264 : NSObject <RTCVideoEncoder>	43	@interface RTCSingleVideoEncoderH264 : NSObject <RTCVideoEncoder>
73	- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo simulcastIndex: (int)index;	44	- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo simulcastIndex: (int)index;
Lines 318-324 @implementation RTCSingleVideoEncoderH264 { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec3
318	RTCVideoEncoderCallback _callback;	289	RTCVideoEncoderCallback _callback;
319	int32_t _width;	290	int32_t _width;
320	int32_t _height;	291	int32_t _height;
321	CompressionSessionRef _compressionSession;	292	bool _useVCP;
		293	VTCompressionSessionRef _compressionSession;
		294	VCPCompressionSessionRef _vcpCompressionSession;
322	CVPixelBufferPoolRef _pixelBufferPool;	295	CVPixelBufferPoolRef _pixelBufferPool;
323	RTCVideoCodecMode _mode;	296	RTCVideoCodecMode _mode;
324		297
Lines 343-348 - (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo simulcastIndex: a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec4
343	_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));	316	_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
344	_packetizationMode = RTCH264PacketizationModeNonInterleaved;	317	_packetizationMode = RTCH264PacketizationModeNonInterleaved;
345	_profile = ExtractProfile([codecInfo nativeSdpVideoFormat]);	318	_profile = ExtractProfile([codecInfo nativeSdpVideoFormat]);
		319	#if ENABLE_VCP_VTB_ENCODER
		320	_useVCP = [(__bridge NSString *)_profile containsString: @"High"];
		321	#else
		322	_useVCP = false;
		323	#endif
346	_simulcastIndex = index;	324	_simulcastIndex = index;
347	RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);	325	RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);
348	RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]);	326	RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]);
Lines 376-387 - (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec5
376	return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat];	354	return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat];
377	}	355	}
378		356
		357	- (bool)hasCompressionSession
		358	{
		359	return _compressionSession \|\| _vcpCompressionSession;
		360	}
		361
379	- (NSInteger)encode:(RTCVideoFrame *)frame	362	- (NSInteger)encode:(RTCVideoFrame *)frame
380	codecSpecificInfo:(nullable id<RTCCodecSpecificInfo>)codecSpecificInfo	363	codecSpecificInfo:(nullable id<RTCCodecSpecificInfo>)codecSpecificInfo
381	frameTypes:(NSArray<NSNumber > )frameTypes {	364	frameTypes:(NSArray<NSNumber > )frameTypes {
382	RTC_DCHECK_EQ(frame.width, _width);	365	RTC_DCHECK_EQ(frame.width, _width);
383	RTC_DCHECK_EQ(frame.height, _height);	366	RTC_DCHECK_EQ(frame.height, _height);
384	if (!_callback \|\| !_compressionSession) {	367	if (!_callback \|\| ![self hasCompressionSession]) {
385	return WEBRTC_VIDEO_CODEC_UNINITIALIZED;	368	return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
386	}	369	}
387	BOOL isKeyframeRequired = NO;	370	BOOL isKeyframeRequired = NO;
Lines 474-486 - (NSInteger)encode:(RTCVideoFrame *)frame a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec6
474	// Update the bitrate if needed.	457	// Update the bitrate if needed.
475	[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];	458	[self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
476		459
477	OSStatus status = CompressionSessionEncodeFrame(_compressionSession,	460	OSStatus status;
		461	if (_compressionSession) {
		462	status = VTCompressionSessionEncodeFrame(_compressionSession,
478	pixelBuffer,	463	pixelBuffer,
479	presentationTimeStamp,	464	presentationTimeStamp,
480	kCMTimeInvalid,	465	kCMTimeInvalid,
481	frameProperties,	466	frameProperties,
482	encodeParams.release(),	467	encodeParams.release(),
483	nullptr);	468	nullptr);
		469	} else {
		470	#if ENABLE_VCP_ENCODER
		471	status = webrtc::VCPCompressionSessionEncodeFrame(_vcpCompressionSession,
		472	pixelBuffer,
		473	presentationTimeStamp,
		474	kCMTimeInvalid,
		475	frameProperties,
		476	encodeParams.release(),
		477	nullptr);
		478	#else
		479	status = 1;
		480	#endif
		481	}
484	if (frameProperties) {	482	if (frameProperties) {
485	CFRelease(frameProperties);	483	CFRelease(frameProperties);
486	}	484	}
Lines 540-546 - (BOOL)resetCompressionSessionIfNeededWithFrame:(RTCVideoFrame *)frame { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec7
540	// configured with, make sure the compression session is reset using the correct pixel format.	538	// configured with, make sure the compression session is reset using the correct pixel format.
541	OSType framePixelFormat = [self pixelFormatOfFrame:frame];	539	OSType framePixelFormat = [self pixelFormatOfFrame:frame];
542		540
543	if (_compressionSession) {	541	if ([self hasCompressionSession]) {
544	#if defined(WEBRTC_WEBKIT_BUILD)	542	#if defined(WEBRTC_WEBKIT_BUILD)
545	if (!_pixelBufferPool) {	543	if (!_pixelBufferPool) {
546	return NO;	544	return NO;
Lines 618-758 - (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec8
618	#endif	616	#endif
619	CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);	617	CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
620		618
621	#if ENABLE_VCP_ENCODER \|\| ENABLE_VCP_VTB_ENCODER	619	#if ENABLE_VCP_ENCODER
622	int usageValue = 1;	620	if (_useVCP) {
623	auto usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);	621	int usageValue = 1;
624	CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_Usage, usage);	622	auto usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);
625	CFRelease(usage);	623	CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_Usage, usage);
		624	CFRelease(usage);
		625	}
626	#endif	626	#endif
627	#if ENABLE_VCP_VTB_ENCODER	627	#if ENABLE_VCP_VTB_ENCODER
		628	if (_useVCP) {
628	CFDictionarySetValue(encoderSpecs, kVTVideoEncoderList_EncoderID, CFSTR("com.apple.videotoolbox.videoencoder.h264.rtvc"));	629	CFDictionarySetValue(encoderSpecs, kVTVideoEncoderList_EncoderID, CFSTR("com.apple.videotoolbox.videoencoder.h264.rtvc"));
		630	}
629	#endif	631	#endif
630	OSStatus status =	632	OSStatus status =
631	CompressionSessionCreate(nullptr, // use default allocator	633	VTCompressionSessionCreate(nullptr, // use default allocator
632	_width,	634	_width,
633	_height,	635	_height,
634	kCodecTypeH264,	636	kCMVideoCodecType_H264,
635	encoderSpecs, // use hardware accelerated encoder if available	637	encoderSpecs, // use hardware accelerated encoder if available
636	sourceAttributes,	638	sourceAttributes,
637	nullptr, // use default compressed data allocator	639	nullptr, // use default compressed data allocator
638	compressionOutputCallback,	640	compressionOutputCallback,
639	nullptr,	641	nullptr,
640	&_compressionSession);	642	&_compressionSession);
641	if (sourceAttributes) {
642	CFRelease(sourceAttributes);
643	sourceAttributes = nullptr;
644	}
645	if (encoderSpecs) {
646	CFRelease(encoderSpecs);
647	encoderSpecs = nullptr;
648	}
649		643
650	#if ENABLE_VCP_ENCODER \|\| defined(WEBRTC_IOS)	644	#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && ENABLE_VCP_ENCODER
651	if (status != noErr) {	645	if (!_useVCP) {
652	RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;	646	CFBooleanRef hwaccl_enabled = nullptr;
653	return WEBRTC_VIDEO_CODEC_ERROR;	647	if (status == noErr) {
654	}	648	status = VTSessionCopyProperty(_compressionSession,
655	#endif
656	#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
657	CFBooleanRef hwaccl_enabled = nullptr;
658	if (status == noErr) {
659	status = VTSessionCopyProperty(_compressionSession,
660	kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,	649	kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
661	nullptr,	650	nullptr,
662	&hwaccl_enabled);	651	&hwaccl_enabled);
663	}
664	if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
665	RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
666	} else {
667	RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled";
668
669	#if !ENABLE_VCP_ENCODER && !ENABLE_VCP_VTB_ENCODER && !defined(WEBRTC_IOS)
670	if (!isStandardFrameSize(_width, _height)) {
671	_disableEncoding = true;
672	RTC_LOG(LS_ERROR) << "Using H264 software encoder with non standard size is not supported";
673	return WEBRTC_VIDEO_CODEC_ERROR;
674	}
675
676	CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
677	int64_t pixelFormatType = framePixelFormat;
678	CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType);
679
680	const size_t attributesSize = 3;
681	CFTypeRef keys[attributesSize] = {
682	kCVPixelBufferOpenGLCompatibilityKey,
683	kCVPixelBufferIOSurfacePropertiesKey,
684	kCVPixelBufferPixelFormatTypeKey
685	};
686	CFTypeRef values[attributesSize] = {
687	kCFBooleanTrue,
688	ioSurfaceValue,
689	pixelFormat};
690	CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize);
691
692	if (ioSurfaceValue) {
693	CFRelease(ioSurfaceValue);
694	ioSurfaceValue = nullptr;
695	}	652	}
696	if (pixelFormat) {	653	if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) {
697	CFRelease(pixelFormat);	654	RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled";
698	pixelFormat = nullptr;	655	} else {
699	}	656	if (_compressionSession) {
700		657	VTCompressionSessionInvalidate(_compressionSession);
701	CFMutableDictionaryRef encoderSpecs = CFDictionaryCreateMutable(nullptr, 2, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);	658	CFRelease(_compressionSession);
702	CFDictionarySetValue(encoderSpecs, kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, kCFBooleanFalse);	659	_compressionSession = nullptr;
703	int usageValue = 1;	660	}
704	CFNumberRef usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);	661	// Use VCP instead.
705	CFDictionarySetValue(encoderSpecs, kVTVideoEncoderSpecification_Usage, usage);	662	int usageValue = 1;
706		663	auto usage = CFNumberCreate(nullptr, kCFNumberIntType, &usageValue);
707	if (usage) {	664	CFDictionarySetValue(encoderSpecs, kVTCompressionPropertyKey_Usage, usage);
708	CFRelease(usage);	665	CFRelease(usage);
709	usage = nullptr;
710	}
711
712	[self destroyCompressionSession];
713		666
714	OSStatus status =	667	RTC_LOG(LS_INFO) << "Compression session created with VCP";
715	CompressionSessionCreate(nullptr, // use default allocator	668	status =
		669	webrtc::VCPCompressionSessionCreate(nullptr, // use default allocator
716	_width,	670	_width,
717	_height,	671	_height,
718	kCodecTypeH264,	672	kVCPCodecType4CC_H264,
719	encoderSpecs,	673	encoderSpecs,
720	sourceAttributes,	674	sourceAttributes,
721	nullptr, // use default compressed data allocator	675	nullptr, // use default compressed data allocator
722	compressionOutputCallback,	676	compressionOutputCallback,
723	nullptr,	677	nullptr,
724	&_compressionSession);	678	&_vcpCompressionSession);
725	if (sourceAttributes) {
726	CFRelease(sourceAttributes);
727	sourceAttributes = nullptr;
728	}
729	if (encoderSpecs) {
730	CFRelease(encoderSpecs);
731	encoderSpecs = nullptr;
732	}
733	if (status != noErr) {
734	return WEBRTC_VIDEO_CODEC_ERROR;
735	}	679	}
736	#endif
737	}	680	}
738	#endif	681	#endif
		682	if (sourceAttributes) {
		683	CFRelease(sourceAttributes);
		684	sourceAttributes = nullptr;
		685	}
		686	if (encoderSpecs) {
		687	CFRelease(encoderSpecs);
		688	encoderSpecs = nullptr;
		689	}
		690
		691	if (status != noErr) {
		692	RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status;
		693	return WEBRTC_VIDEO_CODEC_ERROR;
		694	}
		695
739	[self configureCompressionSession];	696	[self configureCompressionSession];
740		697
741	#if !defined(WEBRTC_WEBKIT_BUILD)	698	#if !defined(WEBRTC_WEBKIT_BUILD)
742	// The pixel buffer pool is dependent on the compression session so if the session is reset, the	699	// The pixel buffer pool is dependent on the compression session so if the session is reset, the
743	// pool should be reset as well.	700	// pool should be reset as well.
744	_pixelBufferPool = CompressionSessionGetPixelBufferPool(_compressionSession);	701	if (_compressionSession)
		702	_pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
		703	#if ENABLE_VCP_ENCODER
		704	else
		705	_pixelBufferPool = webrtc::VCPCompressionSessionGetPixelBufferPool(_compressionSession);
		706	#endif
745	#endif	707	#endif
746	return WEBRTC_VIDEO_CODEC_OK;	708	return WEBRTC_VIDEO_CODEC_OK;
747	}	709	}
748		710
749	- (void)configureCompressionSession {	711	- (void)configureCompressionSession {
750	RTC_DCHECK(_compressionSession);	712	RTC_DCHECK([self hasCompressionSession]);
751	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true);	713	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_RealTime, true);
752	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile);	714	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile);
753	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);	715	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
754	#if ENABLE_VCP_ENCODER	716	#if ENABLE_VCP_ENCODER
755	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_Usage, 1);	717	if (_useVCP) {
		718	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_Usage, 1);
		719	}
756	#endif	720	#endif
757	[self setEncoderBitrateBps:_targetBitrateBps];	721	[self setEncoderBitrateBps:_targetBitrateBps];
758	// TODO(tkchin): Look at entropy mode and colorspace matrices.	722	// TODO(tkchin): Look at entropy mode and colorspace matrices.
Lines 764-781 - (void)configureCompressionSession { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec9
764	// 1);	728	// 1);
765		729
766	// Set a relatively large value for keyframe emission (7200 frames or 4 minutes).	730	// Set a relatively large value for keyframe emission (7200 frames or 4 minutes).
767	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);	731	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
768	SetVTSessionProperty(	732	SetVTSessionProperty(
769	_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);	733	_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
770	}	734	}
771		735
772	- (void)destroyCompressionSession {	736	- (void)destroyCompressionSession {
773	if (_compressionSession) {	737	if (_compressionSession) {
774	CompressionSessionInvalidate(_compressionSession);	738	VTCompressionSessionInvalidate(_compressionSession);
775	CFRelease(_compressionSession);	739	CFRelease(_compressionSession);
776	_compressionSession = nullptr;	740	_compressionSession = nullptr;
777	_pixelBufferPool = nullptr;
778	}	741	}
		742	#if ENABLE_VCP_ENCODER
		743	if (_vcpCompressionSession) {
		744	webrtc::VCPCompressionSessionInvalidate(_vcpCompressionSession);
		745	CFRelease(_vcpCompressionSession);
		746	_vcpCompressionSession = nullptr;
		747	}
		748	#endif
		749	_pixelBufferPool = nullptr;
779	}	750	}
780		751
781	- (NSString *)implementationName {	752	- (NSString *)implementationName {
Lines 789-796 - (void)setBitrateBps:(uint32_t)bitrateBps { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec10
789	}	760	}
790		761
791	- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {	762	- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
792	if (_compressionSession) {	763	if ([self hasCompressionSession]) {
793	SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);	764	SetVTSessionProperty(_compressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
794		765
795	// TODO(tkchin): Add a helper method to set array value.	766	// TODO(tkchin): Add a helper method to set array value.
796	int64_t dataLimitBytesPerSecondValue =	767	int64_t dataLimitBytesPerSecondValue =
Lines 802-809 - (void)setEncoderBitrateBps:(uint32_t)bitrateBps { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec11
802	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);	773	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
803	const void *nums[2] = {bytesPerSecond, oneSecond};	774	const void *nums[2] = {bytesPerSecond, oneSecond};
804	CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);	775	CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
805	OSStatus status = CompressionSessionSetProperty(	776
806	_compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);	777	OSStatus status = noErr;
		778	if (_compressionSession)
		779	VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
		780	#if ENABLE_VCP_ENCODER
		781	else
		782	webrtc::VCPCompressionSessionSetProperty(_vcpCompressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
		783	#endif
		784	if (status != noErr) {
		785	RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << kVTCompressionPropertyKey_DataRateLimits
		786	<< ": " << status;
		787	}
		788
807	if (bytesPerSecond) {	789	if (bytesPerSecond) {
808	CFRelease(bytesPerSecond);	790	CFRelease(bytesPerSecond);
809	}	791	}
Lines 813-822 - (void)setEncoderBitrateBps:(uint32_t)bitrateBps { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm_sec12
813	if (dataRateLimits) {	795	if (dataRateLimits) {
814	CFRelease(dataRateLimits);	796	CFRelease(dataRateLimits);
815	}	797	}
816	if (status != noErr) {
817	RTC_LOG(LS_ERROR) << "Failed to set data rate limit";
818	}
819
820	_encoderBitrateBps = bitrateBps;	798	_encoderBitrateBps = bitrateBps;
821	}	799	}
822	}	800	}

Lines 35-46 std::string CFStringToString(const CFStringRef cf_string) { a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.cc_sec1
- a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.cc -8 / +32 lines
35	}	35	}
36		36
37	// Convenience function for setting a VT property.	37	// Convenience function for setting a VT property.
38	void SetVTSessionProperty(CompressionSessionRef session,	38	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
39	CFStringRef key,	39	CFStringRef key,
40	int32_t value) {	40	int32_t value) {
41	CFNumberRef cfNum =	41	CFNumberRef cfNum =
42	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);	42	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
43	OSStatus status = CompressionSessionSetProperty(session, key, cfNum);	43	OSStatus status = noErr;
		44	if (session)
		45	status = VTSessionSetProperty(session, key, cfNum);
		46	#if ENABLE_VCP_ENCODER
		47	else
		48	status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cfNum);
		49	#endif
44	CFRelease(cfNum);	50	CFRelease(cfNum);
45	if (status != noErr) {	51	if (status != noErr) {
46	std::string key_string = CFStringToString(key);	52	std::string key_string = CFStringToString(key);
Lines 50-62 void SetVTSessionProperty(CompressionSessionRef session, a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.cc_sec2
50	}	56	}
51		57
52	// Convenience function for setting a VT property.	58	// Convenience function for setting a VT property.
53	void SetVTSessionProperty(CompressionSessionRef session,	59	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
54	CFStringRef key,	60	CFStringRef key,
55	uint32_t value) {	61	uint32_t value) {
56	int64_t value_64 = value;	62	int64_t value_64 = value;
57	CFNumberRef cfNum =	63	CFNumberRef cfNum =
58	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);	64	CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
59	OSStatus status = CompressionSessionSetProperty(session, key, cfNum);	65	OSStatus status = noErr;
		66	if (session)
		67	status = VTSessionSetProperty(session, key, cfNum);
		68	#if ENABLE_VCP_ENCODER
		69	else
		70	status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cfNum);
		71	#endif
60	CFRelease(cfNum);	72	CFRelease(cfNum);
61	if (status != noErr) {	73	if (status != noErr) {
62	std::string key_string = CFStringToString(key);	74	std::string key_string = CFStringToString(key);
Lines 66-74 void SetVTSessionProperty(CompressionSessionRef session, a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.cc_sec3
66	}	78	}
67		79
68	// Convenience function for setting a VT property.	80	// Convenience function for setting a VT property.
69	void SetVTSessionProperty(CompressionSessionRef session, CFStringRef key, bool value) {	81	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession, CFStringRef key, bool value) {
70	CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;	82	CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
71	OSStatus status = CompressionSessionSetProperty(session, key, cf_bool);	83	OSStatus status = noErr;
		84	if (session)
		85	status = VTSessionSetProperty(session, key, cf_bool);
		86	#if ENABLE_VCP_ENCODER
		87	else
		88	status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, cf_bool);
		89	#endif
72	if (status != noErr) {	90	if (status != noErr) {
73	std::string key_string = CFStringToString(key);	91	std::string key_string = CFStringToString(key);
74	RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string	92	RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
Lines 77-86 void SetVTSessionProperty(CompressionSessionRef session, CFStringRef key, bool v a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.cc_sec4
77	}	95	}
78		96
79	// Convenience function for setting a VT property.	97	// Convenience function for setting a VT property.
80	void SetVTSessionProperty(CompressionSessionRef session,	98	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
81	CFStringRef key,	99	CFStringRef key,
82	CFStringRef value) {	100	CFStringRef value) {
83	OSStatus status = CompressionSessionSetProperty(session, key, value);	101	OSStatus status = noErr;
		102	if (session)
		103	status = VTSessionSetProperty(session, key, value);
		104	#if ENABLE_VCP_ENCODER
		105	else
		106	status = webrtc::VCPCompressionSessionSetProperty(vcpSession, key, value);
		107	#endif
84	if (status != noErr) {	108	if (status != noErr) {
85	std::string key_string = CFStringToString(key);	109	std::string key_string = CFStringToString(key);
86	std::string val_string = CFStringToString(value);	110	std::string val_string = CFStringToString(value);

Lines 16-22 a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.h_sec1
- a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.h -5 / +5 lines
16	#include <VideoToolbox/VideoToolbox.h>	16	#include <VideoToolbox/VideoToolbox.h>
17	#include <string>	17	#include <string>
18		18
19	#include "sdk/WebKit/EncoderUtilities.h"	19	#include "sdk/WebKit/VideoProcessingSoftLink.h"
20		20
21	// Convenience function for creating a dictionary.	21	// Convenience function for creating a dictionary.
22	inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys,	22	inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys,
Lines 31-48 inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys, a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/helpers.h_sec2
31	std::string CFStringToString(const CFStringRef cf_string);	31	std::string CFStringToString(const CFStringRef cf_string);
32		32
33	// Convenience function for setting a VT property.	33	// Convenience function for setting a VT property.
34	void SetVTSessionProperty(CompressionSessionRef session, CFStringRef key, int32_t value);	34	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,CFStringRef key, int32_t value);
35		35
36	// Convenience function for setting a VT property.	36	// Convenience function for setting a VT property.
37	void SetVTSessionProperty(CompressionSessionRef session,	37	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
38	CFStringRef key,	38	CFStringRef key,
39	uint32_t value);	39	uint32_t value);
40		40
41	// Convenience function for setting a VT property.	41	// Convenience function for setting a VT property.
42	void SetVTSessionProperty(CompressionSessionRef session, CFStringRef key, bool value);	42	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession, CFStringRef key, bool value);
43		43
44	// Convenience function for setting a VT property.	44	// Convenience function for setting a VT property.
45	void SetVTSessionProperty(CompressionSessionRef session,	45	void SetVTSessionProperty(VTCompressionSessionRef session, VCPCompressionSessionRef vcpSession,
46	CFStringRef key,	46	CFStringRef key,
47	CFStringRef value);	47	CFStringRef value);
48		48

Lines 1-3 a/LayoutTests/ChangeLog_sec1
- a/LayoutTests/ChangeLog +11 lines
		1	2019-06-04 Youenn Fablet <youenn@apple.com>
		2
		3	Call was negotiated with H264 Base Profile 42e01f but encoded in High Profile
		4	https://bugs.webkit.org/show_bug.cgi?id=195124
		5	<rdar://problem/48453085>
		6
		7	Reviewed by NOBODY (OOPS!).
		8
		9	* webrtc/video-h264-expected.txt: Added.
		10	* webrtc/video-h264.html: Added.
		11
1	2019-06-02 Devin Rousso <drousso@apple.com>	12	2019-06-02 Devin Rousso <drousso@apple.com>
2		13
3	Flaky Test: inspector/canvas/recording.html	14	Flaky Test: inspector/canvas/recording.html

Line 0 a/LayoutTests/webrtc/video-h264.html_sec1
- a/LayoutTests/webrtc/video-h264.html +96 lines
		1	<!doctype html>
		2	<html>
		3	<head>
		4	<meta charset="utf-8">
		5	<title>Testing H264 baseline and high profile</title>
		6	<script src="../resources/testharness.js"></script>
		7	<script src="../resources/testharnessreport.js"></script>
		8	</head>
		9	<body>
		10	<video id="video" autoplay=""></video>
		11	<canvas id="canvas" width="640" height="480"></canvas>
		12	<script src ="routines.js"></script>
		13	<script>
		14	function grabFrameData(x, y, w, h)
		15	{
		16	canvas.width = video.videoWidth;
		17	canvas.height = video.videoHeight;
		18
		19	canvas.getContext('2d').drawImage(video, x, y, w, h, x, y, w, h);
		20	return canvas.getContext('2d').getImageData(x, y, w, h).data;
		21	}
		22
		23	function testImage()
		24	{
		25	const data = grabFrameData(10, 325, 250, 1);
		26
		27	var index = 20;
		28	assert_true(data[index] < 100);
		29	assert_true(data[index + 1] < 100);
		30	assert_true(data[index + 2] < 100);
		31
		32	index = 80;
		33	assert_true(data[index] > 200);
		34	assert_true(data[index + 1] > 200);
		35	assert_true(data[index + 2] > 200);
		36
		37	index += 80;
		38	assert_true(data[index] > 200);
		39	assert_true(data[index + 1] > 200);
		40	assert_true(data[index + 2] < 100);
		41	}
		42
		43	promise_test(async (test) => {
		44	if (window.testRunner)
		45	testRunner.setUserMediaPermission(true);
		46
		47	const localStream = await navigator.mediaDevices.getUserMedia({video: true});
		48	const stream = await new Promise((resolve, reject) => {
		49	createConnections((firstConnection) => {
		50	firstConnection.addTrack(localStream.getVideoTracks()[0], localStream);
		51	}, (secondConnection) => {
		52	secondConnection.ontrack = (trackEvent) => {
		53	resolve(trackEvent.streams[0]);
		54	};
		55	}, { observeOffer : (offer) => {
		56	offer.sdp = offer.sdp.replace("640c1f", "42e01f");
		57	return offer;
		58	}
		59	});
		60	setTimeout(() => reject("Test timed out"), 5000);
		61	});
		62
		63	video.srcObject = stream;
		64	await video.play();
65
66	testImage();
67	}, "Baseline H264");
68
69	promise_test(async (test) => {
70	if (window.testRunner)
71	testRunner.setUserMediaPermission(true);
72
73	const localStream = await navigator.mediaDevices.getUserMedia({video: true});
74	const stream = await new Promise((resolve, reject) => {
75	createConnections((firstConnection) => {
76	firstConnection.addTrack(localStream.getVideoTracks()[0], localStream);
77	}, (secondConnection) => {
78	secondConnection.ontrack = (trackEvent) => {
79	resolve(trackEvent.streams[0]);
80	};
81	}, { observeOffer : (offer) => {
82	offer.sdp = offer.sdp.replace("42e01f", "640c1f");
83	return offer;
84	}
85	});
86	setTimeout(() => reject("Test timed out"), 5000);
87	});
88
89	video.srcObject = stream;
90	await video.play();
91
92	testImage();
93	}, "High H264");
94	</script>
95	</body>
96	</html>