24 #include "config_components.h"
40 #include <Availability.h>
41 #include <AvailabilityMacros.h>
42 #include <TargetConditionals.h>
44 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
45 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
47 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
48 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
51 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
55 #if !HAVE_KCMVIDEOCODECTYPE_VP9
59 #if !HAVE_KCMVIDEOCODECTYPE_AV1
63 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
74 CVPixelBufferRelease(
ref->pixbuf);
130 frame->crop_right = 0;
131 frame->crop_left = 0;
133 frame->crop_bottom = 0;
138 frame->data[3] = (uint8_t*)
ref->pixbuf;
140 if (
ref->hw_frames_ctx) {
143 if (!
frame->hw_frames_ctx)
180 #define AV_W8(p, v) *(p) = (v)
188 for (
i = 0;
i < src_size;
i++) {
189 if (
i + 2 < src_size &&
191 src[
i + 1] == 0x00 &&
192 src[
i + 2] <= 0x03) {
219 int vt_extradata_size;
220 uint8_t *vt_extradata;
222 vt_extradata_size = 6 + 2 + sps_size + 3 + pps_size;
223 vt_extradata =
av_malloc(vt_extradata_size);
231 AV_W8(p + 1,
h->ps.sps->data[1]);
232 AV_W8(p + 2,
h->ps.sps->data[2]);
233 AV_W8(p + 3,
h->ps.sps->data[3]);
238 p +=
escape_ps(p,
h->ps.sps->data,
h->ps.sps->data_size);
242 p +=
escape_ps(p,
h->ps.pps->data,
h->ps.pps->data_size);
244 av_assert0(p - vt_extradata == vt_extradata_size);
249 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
251 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
259 int i, num_vps = 0, num_sps = 0, num_pps = 0;
265 uint8_t parallelismType;
268 int vt_extradata_size = 23 + 3 + 3 + 3;
269 uint8_t *vt_extradata;
271 #define COUNT_SIZE_PS(T, t) \
272 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
273 if (h->ps.t##ps_list[i]) { \
274 const HEVC##T##PS *lps = h->ps.t##ps_list[i]; \
275 vt_extradata_size += 2 + escape_ps(NULL, lps->data, lps->data_size); \
284 vt_extradata =
av_malloc(vt_extradata_size);
302 for (
i = 0;
i < 4;
i++) {
337 else if (
pps->entropy_coding_sync_enabled_flag &&
pps->tiles_enabled_flag)
339 else if (
pps->entropy_coding_sync_enabled_flag)
341 else if (
pps->tiles_enabled_flag)
345 AV_W8(p + 15, 0xfc | parallelismType);
351 AV_W8(p + 16,
sps->chroma_format_idc | 0xfc);
357 AV_W8(p + 17, (
sps->bit_depth - 8) | 0xf8);
363 AV_W8(p + 18, (
sps->bit_depth_chroma - 8) | 0xf8);
374 AV_W8(p + 21, 0 << 6 |
375 sps->max_sub_layers << 3 |
376 sps->temporal_id_nesting << 2 |
384 #define APPEND_PS(T, t) \
391 HEVC_NAL_##T##PS & 0x3f); \
393 AV_WB16(p + 1, num_##t##ps); \
395 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
396 if (h->ps.t##ps_list[i]) { \
397 const HEVC##T##PS *lps = h->ps.t##ps_list[i]; \
398 int size = escape_ps(p + 2, lps->data, lps->data_size); \
410 av_assert0(p - vt_extradata == vt_extradata_size);
412 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
424 if (
h->is_avc == 1) {
441 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
489 #if CONFIG_VIDEOTOOLBOX
516 VTDecompressionSessionInvalidate(videotoolbox->
session);
517 CFRelease(videotoolbox->
session);
530 CVPixelBufferRelease(vtctx->
frame);
533 videotoolbox_stop(avctx);
544 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->
frame;
545 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
547 int width = CVPixelBufferGetWidth(pixbuf);
548 int height = CVPixelBufferGetHeight(pixbuf);
562 CVPixelBufferRelease(
ref->pixbuf);
586 hw_ctx = hw_frames->
hwctx;
601 if (!
ref->hw_frames_ctx)
607 static void videotoolbox_write_mp4_descr_length(
PutByteContext *pb,
int length)
612 for (
i = 3;
i >= 0;
i--) {
613 b = (length >> (
i * 7)) & 0x7F;
617 bytestream2_put_byteu(pb,
b);
621 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
624 uint8_t *rw_extradata;
635 bytestream2_put_byteu(&pb, 0);
639 bytestream2_put_byteu(&pb, 0x03);
640 videotoolbox_write_mp4_descr_length(&pb, full_size);
642 bytestream2_put_byteu(&pb, 0);
645 bytestream2_put_byteu(&pb, 0x04);
646 videotoolbox_write_mp4_descr_length(&pb, config_size);
647 bytestream2_put_byteu(&pb, 32);
648 bytestream2_put_byteu(&pb, 0x11);
654 bytestream2_put_byteu(&pb, 0x05);
660 bytestream2_put_byteu(&pb, 0x06);
661 bytestream2_put_byteu(&pb, 0x01);
662 bytestream2_put_byteu(&pb, 0x02);
666 data = CFDataCreate(kCFAllocatorDefault, rw_extradata,
s);
672 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
677 CMBlockBufferRef block_buf;
678 CMSampleBufferRef sample_buf;
683 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
694 status = CMSampleBufferCreate(kCFAllocatorDefault,
709 CFRelease(block_buf);
714 static void videotoolbox_decoder_callback(
void *opaque,
715 void *sourceFrameRefCon,
717 VTDecodeInfoFlags
flags,
718 CVImageBufferRef image_buffer,
725 CVPixelBufferRelease(vtctx->
frame);
731 "vt decoder cb: output image buffer is null: %i\n",
status);
735 vtctx->
frame = CVPixelBufferRetain(image_buffer);
738 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
741 CMSampleBufferRef sample_buf;
745 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
752 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
758 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
760 CFRelease(sample_buf);
765 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
766 CFDictionaryRef decoder_spec,
770 CMFormatDescriptionRef cm_fmt_desc;
773 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
786 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
790 CFMutableDictionaryRef buffer_attributes;
791 CFMutableDictionaryRef io_surface_properties;
792 CFNumberRef cv_pix_fmt;
796 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
797 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
798 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
pix_fmt);
800 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
802 &kCFTypeDictionaryKeyCallBacks,
803 &kCFTypeDictionaryValueCallBacks);
804 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
806 &kCFTypeDictionaryKeyCallBacks,
807 &kCFTypeDictionaryValueCallBacks);
810 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
811 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
812 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey,
w);
813 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey,
h);
815 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
817 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
820 CFRelease(io_surface_properties);
821 CFRelease(cv_pix_fmt);
825 return buffer_attributes;
828 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
831 CFMutableDictionaryRef avc_info;
834 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
836 &kCFTypeDictionaryKeyCallBacks,
837 &kCFTypeDictionaryValueCallBacks);
839 CFDictionarySetValue(config_info,
845 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
847 &kCFTypeDictionaryKeyCallBacks,
848 &kCFTypeDictionaryValueCallBacks);
851 case kCMVideoCodecType_MPEG4Video :
853 data = videotoolbox_esds_extradata_create(avctx);
855 CFDictionarySetValue(avc_info, CFSTR(
"esds"),
data);
857 case kCMVideoCodecType_H264 :
860 CFDictionarySetValue(avc_info, CFSTR(
"avcC"),
data);
865 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"),
data);
867 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
871 CFDictionarySetValue(avc_info, CFSTR(
"vpcC"),
data);
874 #if CONFIG_AV1_VIDEOTOOLBOX_HWACCEL
878 CFDictionarySetValue(avc_info, CFSTR(
"av1C"),
data);
885 CFDictionarySetValue(config_info,
886 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
900 VTDecompressionOutputCallbackRecord decoder_cb;
901 CFDictionaryRef decoder_spec;
902 CFDictionaryRef buf_attr;
933 case MKTAG(
'a',
'p',
'c',
'o'):
934 case
MKTAG(
'a',
'p',
'c',
's'):
935 case
MKTAG(
'a',
'p',
'c',
'n'):
936 case
MKTAG(
'a',
'p',
'c',
'h'):
937 case
MKTAG(
'a',
'p',
'4',
'h'):
938 case
MKTAG(
'a',
'p',
'4',
'x'):
939 videotoolbox->cm_codec_type =
av_bswap32(avctx->codec_tag);
953 #if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) && AV_HAS_BUILTIN(__builtin_available)
955 if (__builtin_available(macOS 10.9, *)) {
956 VTRegisterProfessionalVideoWorkflowVideoDecoders();
961 #if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) && AV_HAS_BUILTIN(__builtin_available)
962 if (__builtin_available(macOS 11.0, *)) {
963 VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->
cm_codec_type);
967 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
980 CFRelease(decoder_spec);
986 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
990 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
1001 CFRelease(decoder_spec);
1003 CFRelease(buf_attr);
1006 case kVTVideoDecoderNotAvailableNowErr:
1009 case kVTVideoDecoderUnsupportedDataFormatErr:
1012 case kVTCouldNotFindVideoDecoderErr:
1015 case kVTVideoDecoderMalfunctionErr:
1018 case kVTVideoDecoderBadDataErr:
1029 static const char *videotoolbox_error_string(OSStatus
status)
1032 case kVTVideoDecoderBadDataErr:
1034 case kVTVideoDecoderMalfunctionErr:
1035 return "decoder malfunction";
1036 case kVTInvalidSessionErr:
1037 return "invalid session";
1051 videotoolbox_stop(avctx);
1052 if (videotoolbox_start(avctx) != 0) {
1060 status = videotoolbox_session_decode_frame(avctx);
1062 if (
status == kVTVideoDecoderMalfunctionErr ||
status == kVTInvalidSessionErr)
1068 if (!vtctx->
frame) {
1073 return videotoolbox_buffer_create(avctx,
frame);
1093 frame->crop_right = 0;
1094 frame->crop_left = 0;
1095 frame->crop_top = 0;
1096 frame->crop_bottom = 0;
1109 static int videotoolbox_hevc_decode_params(
AVCodecContext *avctx,
1153 static int videotoolbox_prores_start_frame(
AVCodecContext *avctx,
1160 static int videotoolbox_prores_decode_slice(
AVCodecContext *avctx,
1188 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
1193 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
1195 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
1202 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
1204 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
1211 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
1227 if (cv_pix_fmt_type == 0) {
1228 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1230 ret->cv_pix_fmt_type = cv_pix_fmt_type;
1248 return videotoolbox_start(avctx);
1252 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1273 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1276 hw_ctx = hw_frames->
hwctx;
1299 "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1300 "a VideoToolbox format!\n",
1301 attempted_format ? attempted_format->
name :
"<unknown>",
1307 err = videotoolbox_start(avctx);
1326 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1332 .
p.
name =
"h263_videotoolbox",
1337 .start_frame = videotoolbox_mpeg_start_frame,
1338 .decode_slice = videotoolbox_mpeg_decode_slice,
1339 .end_frame = videotoolbox_mpeg_end_frame,
1347 .
p.
name =
"hevc_videotoolbox",
1352 .start_frame = videotoolbox_hevc_start_frame,
1353 .decode_slice = videotoolbox_hevc_decode_slice,
1354 .decode_params = videotoolbox_hevc_decode_params,
1355 .end_frame = videotoolbox_hevc_end_frame,
1363 .
p.
name =
"h264_videotoolbox",
1371 .end_frame = videotoolbox_h264_end_frame,
1379 .
p.
name =
"mpeg1_videotoolbox",
1384 .start_frame = videotoolbox_mpeg_start_frame,
1385 .decode_slice = videotoolbox_mpeg_decode_slice,
1386 .end_frame = videotoolbox_mpeg_end_frame,
1394 .
p.
name =
"mpeg2_videotoolbox",
1399 .start_frame = videotoolbox_mpeg_start_frame,
1400 .decode_slice = videotoolbox_mpeg_decode_slice,
1401 .end_frame = videotoolbox_mpeg_end_frame,
1409 .
p.
name =
"mpeg4_videotoolbox",
1414 .start_frame = videotoolbox_mpeg_start_frame,
1415 .decode_slice = videotoolbox_mpeg_decode_slice,
1416 .end_frame = videotoolbox_mpeg_end_frame,
1424 .
p.
name =
"prores_videotoolbox",
1429 .start_frame = videotoolbox_prores_start_frame,
1430 .decode_slice = videotoolbox_prores_decode_slice,
1431 .end_frame = videotoolbox_prores_end_frame,