24 #include "config_components.h"
40 #include <Availability.h>
41 #include <AvailabilityMacros.h>
42 #include <TargetConditionals.h>
44 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
45 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
47 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
48 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
51 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
55 #if !HAVE_KCMVIDEOCODECTYPE_VP9
59 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
70 CVPixelBufferRelease(
ref->pixbuf);
106 frame->crop_right = 0;
107 frame->crop_left = 0;
109 frame->crop_bottom = 0;
114 frame->data[3] = (uint8_t*)
ref->pixbuf;
116 if (
ref->hw_frames_ctx) {
119 if (!
frame->hw_frames_ctx)
156 #define AV_W8(p, v) *(p) = (v)
164 for (
i = 0;
i < src_size;
i++) {
165 if (
i + 2 < src_size &&
167 src[
i + 1] == 0x00 &&
168 src[
i + 2] <= 0x03) {
195 int vt_extradata_size;
196 uint8_t *vt_extradata;
198 vt_extradata_size = 6 + 2 + sps_size + 3 + pps_size;
199 vt_extradata =
av_malloc(vt_extradata_size);
207 AV_W8(p + 1,
h->ps.sps->data[1]);
208 AV_W8(p + 2,
h->ps.sps->data[2]);
209 AV_W8(p + 3,
h->ps.sps->data[3]);
214 p +=
escape_ps(p,
h->ps.sps->data,
h->ps.sps->data_size);
218 p +=
escape_ps(p,
h->ps.pps->data,
h->ps.pps->data_size);
220 av_assert0(p - vt_extradata == vt_extradata_size);
225 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
227 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
235 int i, num_vps = 0, num_sps = 0, num_pps = 0;
241 uint8_t parallelismType;
244 int vt_extradata_size = 23 + 3 + 3 + 3;
245 uint8_t *vt_extradata;
247 #define COUNT_SIZE_PS(T, t) \
248 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
249 if (h->ps.t##ps_list[i]) { \
250 const HEVC##T##PS *lps = h->ps.t##ps_list[i]; \
251 vt_extradata_size += 2 + escape_ps(NULL, lps->data, lps->data_size); \
260 vt_extradata =
av_malloc(vt_extradata_size);
278 for (
i = 0;
i < 4;
i++) {
313 else if (
pps->entropy_coding_sync_enabled_flag &&
pps->tiles_enabled_flag)
315 else if (
pps->entropy_coding_sync_enabled_flag)
317 else if (
pps->tiles_enabled_flag)
321 AV_W8(p + 15, 0xfc | parallelismType);
327 AV_W8(p + 16,
sps->chroma_format_idc | 0xfc);
333 AV_W8(p + 17, (
sps->bit_depth - 8) | 0xf8);
339 AV_W8(p + 18, (
sps->bit_depth_chroma - 8) | 0xf8);
350 AV_W8(p + 21, 0 << 6 |
351 sps->max_sub_layers << 3 |
352 sps->temporal_id_nesting << 2 |
360 #define APPEND_PS(T, t) \
367 HEVC_NAL_##T##PS & 0x3f); \
369 AV_WB16(p + 1, num_##t##ps); \
371 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
372 if (h->ps.t##ps_list[i]) { \
373 const HEVC##T##PS *lps = h->ps.t##ps_list[i]; \
374 int size = escape_ps(p + 2, lps->data, lps->data_size); \
386 av_assert0(p - vt_extradata == vt_extradata_size);
388 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
400 if (
h->is_avc == 1) {
417 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
465 #if CONFIG_VIDEOTOOLBOX
492 VTDecompressionSessionInvalidate(videotoolbox->
session);
493 CFRelease(videotoolbox->
session);
506 CVPixelBufferRelease(vtctx->
frame);
509 videotoolbox_stop(avctx);
520 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->
frame;
521 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
523 int width = CVPixelBufferGetWidth(pixbuf);
524 int height = CVPixelBufferGetHeight(pixbuf);
538 CVPixelBufferRelease(
ref->pixbuf);
562 hw_ctx = hw_frames->
hwctx;
577 if (!
ref->hw_frames_ctx)
583 static void videotoolbox_write_mp4_descr_length(
PutByteContext *pb,
int length)
588 for (
i = 3;
i >= 0;
i--) {
589 b = (length >> (
i * 7)) & 0x7F;
593 bytestream2_put_byteu(pb,
b);
597 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
600 uint8_t *rw_extradata;
611 bytestream2_put_byteu(&pb, 0);
615 bytestream2_put_byteu(&pb, 0x03);
616 videotoolbox_write_mp4_descr_length(&pb, full_size);
618 bytestream2_put_byteu(&pb, 0);
621 bytestream2_put_byteu(&pb, 0x04);
622 videotoolbox_write_mp4_descr_length(&pb, config_size);
623 bytestream2_put_byteu(&pb, 32);
624 bytestream2_put_byteu(&pb, 0x11);
630 bytestream2_put_byteu(&pb, 0x05);
636 bytestream2_put_byteu(&pb, 0x06);
637 bytestream2_put_byteu(&pb, 0x01);
638 bytestream2_put_byteu(&pb, 0x02);
642 data = CFDataCreate(kCFAllocatorDefault, rw_extradata,
s);
648 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
653 CMBlockBufferRef block_buf;
654 CMSampleBufferRef sample_buf;
659 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
670 status = CMSampleBufferCreate(kCFAllocatorDefault,
685 CFRelease(block_buf);
690 static void videotoolbox_decoder_callback(
void *opaque,
691 void *sourceFrameRefCon,
693 VTDecodeInfoFlags
flags,
694 CVImageBufferRef image_buffer,
701 CVPixelBufferRelease(vtctx->
frame);
707 "vt decoder cb: output image buffer is null: %i\n",
status);
711 vtctx->
frame = CVPixelBufferRetain(image_buffer);
714 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
717 CMSampleBufferRef sample_buf;
721 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
728 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
734 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
736 CFRelease(sample_buf);
741 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
742 CFDictionaryRef decoder_spec,
746 CMFormatDescriptionRef cm_fmt_desc;
749 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
762 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
766 CFMutableDictionaryRef buffer_attributes;
767 CFMutableDictionaryRef io_surface_properties;
768 CFNumberRef cv_pix_fmt;
772 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
773 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
774 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
pix_fmt);
776 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
778 &kCFTypeDictionaryKeyCallBacks,
779 &kCFTypeDictionaryValueCallBacks);
780 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
782 &kCFTypeDictionaryKeyCallBacks,
783 &kCFTypeDictionaryValueCallBacks);
786 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
787 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
788 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey,
w);
789 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey,
h);
791 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
793 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
796 CFRelease(io_surface_properties);
797 CFRelease(cv_pix_fmt);
801 return buffer_attributes;
804 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
807 CFMutableDictionaryRef avc_info;
810 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
812 &kCFTypeDictionaryKeyCallBacks,
813 &kCFTypeDictionaryValueCallBacks);
815 CFDictionarySetValue(config_info,
821 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
823 &kCFTypeDictionaryKeyCallBacks,
824 &kCFTypeDictionaryValueCallBacks);
827 case kCMVideoCodecType_MPEG4Video :
829 data = videotoolbox_esds_extradata_create(avctx);
831 CFDictionarySetValue(avc_info, CFSTR(
"esds"),
data);
833 case kCMVideoCodecType_H264 :
836 CFDictionarySetValue(avc_info, CFSTR(
"avcC"),
data);
841 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"),
data);
843 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
847 CFDictionarySetValue(avc_info, CFSTR(
"vpcC"),
data);
854 CFDictionarySetValue(config_info,
855 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
869 VTDecompressionOutputCallbackRecord decoder_cb;
870 CFDictionaryRef decoder_spec;
871 CFDictionaryRef buf_attr;
902 case MKTAG(
'a',
'p',
'c',
'o'):
903 case
MKTAG(
'a',
'p',
'c',
's'):
904 case
MKTAG(
'a',
'p',
'c',
'n'):
905 case
MKTAG(
'a',
'p',
'c',
'h'):
906 case
MKTAG(
'a',
'p',
'4',
'h'):
907 case
MKTAG(
'a',
'p',
'4',
'x'):
908 videotoolbox->cm_codec_type =
av_bswap32(avctx->codec_tag);
919 #if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) && AV_HAS_BUILTIN(__builtin_available)
921 if (__builtin_available(macOS 10.9, *)) {
922 VTRegisterProfessionalVideoWorkflowVideoDecoders();
927 #if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) && AV_HAS_BUILTIN(__builtin_available)
928 if (__builtin_available(macOS 11.0, *)) {
929 VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->
cm_codec_type);
933 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
946 CFRelease(decoder_spec);
952 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
956 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
967 CFRelease(decoder_spec);
972 case kVTVideoDecoderNotAvailableNowErr:
975 case kVTVideoDecoderUnsupportedDataFormatErr:
978 case kVTCouldNotFindVideoDecoderErr:
981 case kVTVideoDecoderMalfunctionErr:
984 case kVTVideoDecoderBadDataErr:
995 static const char *videotoolbox_error_string(OSStatus
status)
998 case kVTVideoDecoderBadDataErr:
1000 case kVTVideoDecoderMalfunctionErr:
1001 return "decoder malfunction";
1002 case kVTInvalidSessionErr:
1003 return "invalid session";
1017 videotoolbox_stop(avctx);
1018 if (videotoolbox_start(avctx) != 0) {
1026 status = videotoolbox_session_decode_frame(avctx);
1028 if (
status == kVTVideoDecoderMalfunctionErr ||
status == kVTInvalidSessionErr)
1034 if (!vtctx->
frame) {
1039 return videotoolbox_buffer_create(avctx,
frame);
1059 frame->crop_right = 0;
1060 frame->crop_left = 0;
1061 frame->crop_top = 0;
1062 frame->crop_bottom = 0;
1075 static int videotoolbox_hevc_decode_params(
AVCodecContext *avctx,
1119 static int videotoolbox_prores_start_frame(
AVCodecContext *avctx,
1126 static int videotoolbox_prores_decode_slice(
AVCodecContext *avctx,
1155 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
1160 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
1162 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
1169 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
1171 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
1178 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
1194 if (cv_pix_fmt_type == 0) {
1195 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1197 ret->cv_pix_fmt_type = cv_pix_fmt_type;
1215 return videotoolbox_start(avctx);
1219 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1240 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1243 hw_ctx = hw_frames->
hwctx;
1266 "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1267 "a VideoToolbox format!\n",
1268 attempted_format ? attempted_format->
name :
"<unknown>",
1274 err = videotoolbox_start(avctx);
1293 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1299 .
p.
name =
"h263_videotoolbox",
1304 .start_frame = videotoolbox_mpeg_start_frame,
1305 .decode_slice = videotoolbox_mpeg_decode_slice,
1306 .end_frame = videotoolbox_mpeg_end_frame,
1314 .
p.
name =
"hevc_videotoolbox",
1319 .start_frame = videotoolbox_hevc_start_frame,
1320 .decode_slice = videotoolbox_hevc_decode_slice,
1321 .decode_params = videotoolbox_hevc_decode_params,
1322 .end_frame = videotoolbox_hevc_end_frame,
1330 .
p.
name =
"h264_videotoolbox",
1338 .end_frame = videotoolbox_h264_end_frame,
1346 .
p.
name =
"mpeg1_videotoolbox",
1351 .start_frame = videotoolbox_mpeg_start_frame,
1352 .decode_slice = videotoolbox_mpeg_decode_slice,
1353 .end_frame = videotoolbox_mpeg_end_frame,
1361 .
p.
name =
"mpeg2_videotoolbox",
1366 .start_frame = videotoolbox_mpeg_start_frame,
1367 .decode_slice = videotoolbox_mpeg_decode_slice,
1368 .end_frame = videotoolbox_mpeg_end_frame,
1376 .
p.
name =
"mpeg4_videotoolbox",
1381 .start_frame = videotoolbox_mpeg_start_frame,
1382 .decode_slice = videotoolbox_mpeg_decode_slice,
1383 .end_frame = videotoolbox_mpeg_end_frame,
1391 .
p.
name =
"prores_videotoolbox",
1396 .start_frame = videotoolbox_prores_start_frame,
1397 .decode_slice = videotoolbox_prores_decode_slice,
1398 .end_frame = videotoolbox_prores_end_frame,