24 #include "config_components.h"
38 #include <Availability.h>
39 #include <AvailabilityMacros.h>
40 #include <TargetConditionals.h>
42 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
43 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
45 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
46 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
49 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
53 #if !HAVE_KCMVIDEOCODECTYPE_VP9
57 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
68 CVPixelBufferRelease(
ref->pixbuf);
104 frame->crop_right = 0;
105 frame->crop_left = 0;
107 frame->crop_bottom = 0;
112 frame->data[3] = (uint8_t*)
ref->pixbuf;
114 if (
ref->hw_frames_ctx) {
117 if (!
frame->hw_frames_ctx)
154 #define AV_W8(p, v) *(p) = (v)
162 for (
i = 0;
i < src_size;
i++) {
163 if (
i + 2 < src_size &&
165 src[
i + 1] == 0x00 &&
166 src[
i + 2] <= 0x03) {
193 int vt_extradata_size;
194 uint8_t *vt_extradata;
196 vt_extradata_size = 6 + 2 + sps_size + 3 + pps_size;
197 vt_extradata =
av_malloc(vt_extradata_size);
205 AV_W8(p + 1,
h->ps.sps->data[1]);
206 AV_W8(p + 2,
h->ps.sps->data[2]);
207 AV_W8(p + 3,
h->ps.sps->data[3]);
212 p +=
escape_ps(p,
h->ps.sps->data,
h->ps.sps->data_size);
216 p +=
escape_ps(p,
h->ps.pps->data,
h->ps.pps->data_size);
218 av_assert0(p - vt_extradata == vt_extradata_size);
223 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
225 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
233 int i, num_vps = 0, num_sps = 0, num_pps = 0;
239 uint8_t parallelismType;
242 int vt_extradata_size = 23 + 3 + 3 + 3;
243 uint8_t *vt_extradata;
245 #define COUNT_SIZE_PS(T, t) \
246 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
247 if (h->ps.t##ps_list[i]) { \
248 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
249 vt_extradata_size += 2 + escape_ps(NULL, lps->data, lps->data_size); \
258 vt_extradata =
av_malloc(vt_extradata_size);
276 for (
i = 0;
i < 4;
i++) {
311 else if (
pps->entropy_coding_sync_enabled_flag &&
pps->tiles_enabled_flag)
313 else if (
pps->entropy_coding_sync_enabled_flag)
315 else if (
pps->tiles_enabled_flag)
319 AV_W8(p + 15, 0xfc | parallelismType);
325 AV_W8(p + 16,
sps->chroma_format_idc | 0xfc);
331 AV_W8(p + 17, (
sps->bit_depth - 8) | 0xf8);
337 AV_W8(p + 18, (
sps->bit_depth_chroma - 8) | 0xf8);
348 AV_W8(p + 21, 0 << 6 |
349 sps->max_sub_layers << 3 |
350 sps->temporal_id_nesting_flag << 2 |
358 #define APPEND_PS(T, t) \
365 HEVC_NAL_##T##PS & 0x3f); \
367 AV_WB16(p + 1, num_##t##ps); \
369 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
370 if (h->ps.t##ps_list[i]) { \
371 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
372 int size = escape_ps(p + 2, lps->data, lps->data_size); \
384 av_assert0(p - vt_extradata == vt_extradata_size);
386 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
398 if (
h->is_avc == 1) {
415 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
463 #if CONFIG_VIDEOTOOLBOX
490 VTDecompressionSessionInvalidate(videotoolbox->
session);
491 CFRelease(videotoolbox->
session);
504 CVPixelBufferRelease(vtctx->
frame);
507 videotoolbox_stop(avctx);
518 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->
frame;
519 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
521 int width = CVPixelBufferGetWidth(pixbuf);
522 int height = CVPixelBufferGetHeight(pixbuf);
536 CVPixelBufferRelease(
ref->pixbuf);
572 if (!
ref->hw_frames_ctx)
578 static void videotoolbox_write_mp4_descr_length(
PutByteContext *pb,
int length)
583 for (
i = 3;
i >= 0;
i--) {
584 b = (length >> (
i * 7)) & 0x7F;
588 bytestream2_put_byteu(pb,
b);
592 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
595 uint8_t *rw_extradata;
606 bytestream2_put_byteu(&pb, 0);
610 bytestream2_put_byteu(&pb, 0x03);
611 videotoolbox_write_mp4_descr_length(&pb, full_size);
613 bytestream2_put_byteu(&pb, 0);
616 bytestream2_put_byteu(&pb, 0x04);
617 videotoolbox_write_mp4_descr_length(&pb, config_size);
618 bytestream2_put_byteu(&pb, 32);
619 bytestream2_put_byteu(&pb, 0x11);
625 bytestream2_put_byteu(&pb, 0x05);
631 bytestream2_put_byteu(&pb, 0x06);
632 bytestream2_put_byteu(&pb, 0x01);
633 bytestream2_put_byteu(&pb, 0x02);
637 data = CFDataCreate(kCFAllocatorDefault, rw_extradata,
s);
643 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
648 CMBlockBufferRef block_buf;
649 CMSampleBufferRef sample_buf;
654 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
665 status = CMSampleBufferCreate(kCFAllocatorDefault,
680 CFRelease(block_buf);
685 static void videotoolbox_decoder_callback(
void *opaque,
686 void *sourceFrameRefCon,
688 VTDecodeInfoFlags
flags,
689 CVImageBufferRef image_buffer,
696 CVPixelBufferRelease(vtctx->
frame);
702 "vt decoder cb: output image buffer is null: %i\n",
status);
706 vtctx->
frame = CVPixelBufferRetain(image_buffer);
709 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
712 CMSampleBufferRef sample_buf;
716 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
723 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
729 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
731 CFRelease(sample_buf);
736 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
737 CFDictionaryRef decoder_spec,
741 CMFormatDescriptionRef cm_fmt_desc;
744 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
757 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
761 CFMutableDictionaryRef buffer_attributes;
762 CFMutableDictionaryRef io_surface_properties;
763 CFNumberRef cv_pix_fmt;
767 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
768 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
769 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
pix_fmt);
771 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
773 &kCFTypeDictionaryKeyCallBacks,
774 &kCFTypeDictionaryValueCallBacks);
775 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
777 &kCFTypeDictionaryKeyCallBacks,
778 &kCFTypeDictionaryValueCallBacks);
781 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
782 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
783 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey,
w);
784 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey,
h);
786 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
788 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
791 CFRelease(io_surface_properties);
792 CFRelease(cv_pix_fmt);
796 return buffer_attributes;
799 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
802 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
804 &kCFTypeDictionaryKeyCallBacks,
805 &kCFTypeDictionaryValueCallBacks);
807 CFDictionarySetValue(config_info,
813 CFMutableDictionaryRef avc_info;
816 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
818 &kCFTypeDictionaryKeyCallBacks,
819 &kCFTypeDictionaryValueCallBacks);
822 case kCMVideoCodecType_MPEG4Video :
824 data = videotoolbox_esds_extradata_create(avctx);
826 CFDictionarySetValue(avc_info, CFSTR(
"esds"),
data);
828 case kCMVideoCodecType_H264 :
831 CFDictionarySetValue(avc_info, CFSTR(
"avcC"),
data);
836 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"),
data);
838 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
842 CFDictionarySetValue(avc_info, CFSTR(
"vpcC"),
data);
849 CFDictionarySetValue(config_info,
850 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
864 VTDecompressionOutputCallbackRecord decoder_cb;
865 CFDictionaryRef decoder_spec;
866 CFDictionaryRef buf_attr;
897 case MKTAG(
'a',
'p',
'c',
'o'):
898 case
MKTAG(
'a',
'p',
'c',
's'):
899 case
MKTAG(
'a',
'p',
'c',
'n'):
900 case
MKTAG(
'a',
'p',
'c',
'h'):
901 case
MKTAG(
'a',
'p',
'4',
'h'):
902 case
MKTAG(
'a',
'p',
'4',
'x'):
903 videotoolbox->cm_codec_type =
av_bswap32(avctx->codec_tag);
914 #if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) && AV_HAS_BUILTIN(__builtin_available)
916 if (__builtin_available(macOS 10.9, *)) {
917 VTRegisterProfessionalVideoWorkflowVideoDecoders();
922 #if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) && AV_HAS_BUILTIN(__builtin_available)
923 if (__builtin_available(macOS 11.0, *)) {
924 VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->
cm_codec_type);
928 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
941 CFRelease(decoder_spec);
947 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
951 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
962 CFRelease(decoder_spec);
967 case kVTVideoDecoderNotAvailableNowErr:
970 case kVTVideoDecoderUnsupportedDataFormatErr:
973 case kVTCouldNotFindVideoDecoderErr:
976 case kVTVideoDecoderMalfunctionErr:
979 case kVTVideoDecoderBadDataErr:
990 static const char *videotoolbox_error_string(OSStatus
status)
993 case kVTVideoDecoderBadDataErr:
995 case kVTVideoDecoderMalfunctionErr:
996 return "decoder malfunction";
997 case kVTInvalidSessionErr:
998 return "invalid session";
1012 videotoolbox_stop(avctx);
1013 if (videotoolbox_start(avctx) != 0) {
1021 status = videotoolbox_session_decode_frame(avctx);
1023 if (
status == kVTVideoDecoderMalfunctionErr ||
status == kVTInvalidSessionErr)
1029 if (!vtctx->
frame) {
1034 return videotoolbox_buffer_create(avctx,
frame);
1062 static int videotoolbox_hevc_decode_params(
AVCodecContext *avctx,
1076 h->output_frame->crop_right = 0;
1077 h->output_frame->crop_left = 0;
1078 h->output_frame->crop_top = 0;
1079 h->output_frame->crop_bottom = 0;
1110 static int videotoolbox_prores_start_frame(
AVCodecContext *avctx,
1117 static int videotoolbox_prores_decode_slice(
AVCodecContext *avctx,
1144 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
1149 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
1151 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
1158 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
1160 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
1167 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
1183 if (cv_pix_fmt_type == 0) {
1184 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1186 ret->cv_pix_fmt_type = cv_pix_fmt_type;
1202 return videotoolbox_start(avctx);
1206 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1227 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1251 "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1252 "a VideoToolbox format!\n",
1253 attempted_format ? attempted_format->
name :
"<unknown>",
1259 err = videotoolbox_start(avctx);
1278 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1284 .
name =
"h263_videotoolbox",
1289 .start_frame = videotoolbox_mpeg_start_frame,
1290 .decode_slice = videotoolbox_mpeg_decode_slice,
1291 .end_frame = videotoolbox_mpeg_end_frame,
1299 .
name =
"hevc_videotoolbox",
1304 .start_frame = videotoolbox_hevc_start_frame,
1305 .decode_slice = videotoolbox_hevc_decode_slice,
1306 .decode_params = videotoolbox_hevc_decode_params,
1307 .end_frame = videotoolbox_hevc_end_frame,
1315 .
name =
"h264_videotoolbox",
1323 .end_frame = videotoolbox_h264_end_frame,
1331 .
name =
"mpeg1_videotoolbox",
1336 .start_frame = videotoolbox_mpeg_start_frame,
1337 .decode_slice = videotoolbox_mpeg_decode_slice,
1338 .end_frame = videotoolbox_mpeg_end_frame,
1346 .
name =
"mpeg2_videotoolbox",
1351 .start_frame = videotoolbox_mpeg_start_frame,
1352 .decode_slice = videotoolbox_mpeg_decode_slice,
1353 .end_frame = videotoolbox_mpeg_end_frame,
1361 .
name =
"mpeg4_videotoolbox",
1366 .start_frame = videotoolbox_mpeg_start_frame,
1367 .decode_slice = videotoolbox_mpeg_decode_slice,
1368 .end_frame = videotoolbox_mpeg_end_frame,
1376 .
name =
"prores_videotoolbox",
1381 .start_frame = videotoolbox_prores_start_frame,
1382 .decode_slice = videotoolbox_prores_decode_slice,
1383 .end_frame = videotoolbox_prores_end_frame,
1392 #if FF_API_VT_HWACCEL_CONTEXT
1395 return videotoolbox_alloc_context_with_pix_fmt(
AV_PIX_FMT_NONE,
false);
1400 return av_videotoolbox_default_init2(avctx,
NULL);
1416 videotoolbox_stop(avctx);