[FFmpeg-devel] [PATCH v2 01/10] Added VideoToolbox H.264 encoder.

Michael Niedermayer michaelni at gmx.at
Wed Nov 25 19:24:14 CET 2015


On Wed, Nov 18, 2015 at 01:49:49PM -0500, kernrj at gmail.com wrote:
> From: Rick Kern <kernrj at gmail.com>
> 
> Enable with configure --enable-vtenc and encode using -codec:v vtenc_h264.
> 
> Signed-off-by: Rick Kern <kernrj at gmail.com>
> ---
>  MAINTAINERS            |    1 +
>  configure              |   14 +
>  libavcodec/Makefile    |    1 +
>  libavcodec/allcodecs.c |    1 +
>  libavcodec/vtenc.c     | 1089 ++++++++++++++++++++++++++++++++++++++++++++++++
>  5 files changed, 1106 insertions(+)
>  create mode 100644 libavcodec/vtenc.c
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 3735742..28782d2 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -286,6 +286,7 @@ Codecs:
>    vp8                                   David Conrad, Jason Garrett-Glaser, Ronald Bultje
>    vp9                                   Ronald Bultje, Clément Bœsch
>    vqavideo.c                            Mike Melanson
> +  vtenc.c                               Rick Kern
>    wavpack.c                             Kostya Shishkov
>    wmaprodec.c                           Sascha Sommer
>    wmavoice.c                            Ronald S. Bultje
> diff --git a/configure b/configure
> index 9a736ce..e465424 100755
> --- a/configure
> +++ b/configure
> @@ -286,6 +286,7 @@ External library support:
>    --disable-sdl            disable sdl [autodetect]
>    --disable-securetransport disable Secure Transport, needed for TLS support
>                             on OSX if openssl and gnutls are not used [autodetect]
> +  --enable-vtenc           enable VideoToolbox encoding support [no]
>    --enable-x11grab         enable X11 grabbing (legacy) [no]
>    --disable-xlib           disable xlib [autodetect]
>    --disable-zlib           disable zlib [autodetect]
> @@ -1473,6 +1474,7 @@ EXTERNAL_LIBRARY_LIST="
>      schannel
>      sdl
>      securetransport
> +    vtenc
>      x11grab
>      xlib
>      zlib
> @@ -2609,6 +2611,7 @@ libzvbi_teletext_decoder_deps="libzvbi"
>  nvenc_encoder_deps="nvenc"
>  nvenc_h264_encoder_deps="nvenc"
>  nvenc_hevc_encoder_deps="nvenc"
> +vtenc_h264_encoder_deps="vtenc"
>  
>  # demuxers / muxers
>  ac3_demuxer_select="ac3_parser"
> @@ -5471,6 +5474,17 @@ enabled openssl           && { check_lib openssl/ssl.h SSL_library_init -lssl -l
>                                 check_lib openssl/ssl.h SSL_library_init -lssl32 -leay32 ||
>                                 check_lib openssl/ssl.h SSL_library_init -lssl -lcrypto -lws2_32 -lgdi32 ||
>                                 die "ERROR: openssl not found"; }
> +enabled vtenc             && { { check_header VideoToolbox/VideoToolbox.h ||
> +                                 die "ERROR: VideoToolbox/VideoToolbox.h not found."
> +                               } &&
> +                               { check_header "Availability.h" &&
> +                                 { check_cpp_condition "Availability.h" "defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0" ||
> +                                   check_cpp_condition "Availability.h" "defined(__MAC_10_9) && __MAC_OS_X_VERSION_MAX_ALLOWED >= __MAC_10_9"
> +                                  } ||
> +                                  die "VideoToolbox requires Mac OSX 10.9+ or iOS 8.0+"
> +                               }
> +                             }
> +
>  enabled qtkit_indev      && { check_header_oc QTKit/QTKit.h || disable qtkit_indev; }
>  
>  if enabled gnutls; then
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 68a573f..afb38e4 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -116,6 +116,7 @@ OBJS-$(CONFIG_TEXTUREDSP)              += texturedsp.o
>  OBJS-$(CONFIG_TEXTUREDSPENC)           += texturedspenc.o
>  OBJS-$(CONFIG_TPELDSP)                 += tpeldsp.o
>  OBJS-$(CONFIG_VIDEODSP)                += videodsp.o
> +OBJS-$(CONFIG_VTENC)                   += vtenc.o
>  OBJS-$(CONFIG_VP3DSP)                  += vp3dsp.o
>  OBJS-$(CONFIG_VP56DSP)                 += vp56dsp.o
>  OBJS-$(CONFIG_VP8DSP)                  += vp8dsp.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index 9f60d7c..6465df0 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -605,6 +605,7 @@ void avcodec_register_all(void)
>      REGISTER_ENCODER(HEVC_QSV,          hevc_qsv);
>      REGISTER_ENCODER(LIBKVAZAAR,        libkvazaar);
>      REGISTER_ENCODER(MPEG2_QSV,         mpeg2_qsv);
> +    REGISTER_ENCODER(VTENC_H264,        vtenc_h264);
>  
>      /* parsers */
>      REGISTER_PARSER(AAC,                aac);
> diff --git a/libavcodec/vtenc.c b/libavcodec/vtenc.c
> new file mode 100644
> index 0000000..06c5360
> --- /dev/null
> +++ b/libavcodec/vtenc.c
> @@ -0,0 +1,1089 @@
> +/*
> + * copyright (c) 2015 Rick Kern <kernrj at gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include <VideoToolbox/VideoToolbox.h>
> +#include <CoreVideo/CoreVideo.h>
> +#include <CoreMedia/CoreMedia.h>
> +#include <TargetConditionals.h>
> +#include <Availability.h>
> +#include "avcodec.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/atomic.h"
> +#include "libavcodec/avcodec.h"
> +#include "internal.h"
> +#include "golomb.h"
> +#include <pthread.h>
> +
> +

> +const uint8_t start_code[] = {0, 0, 0, 1};

static or some prefix is needed


> +
> +typedef struct BufNode{
> +    CMSampleBufferRef cm_buffer;
> +    struct BufNode* next;
> +    int error;
> +} BufNode;
> +
> +typedef struct VTEncContext{
> +    AVClass* class;
> +    VTCompressionSessionRef session;
> +    
> +    pthread_mutex_t lock;
> +    pthread_cond_t cv_sample_sent;
> +    int async_error;
> +    
> +    BufNode* q_head;
> +    BufNode* q_tail;
> +    
> +    int64_t frame_ct_out;
> +    int64_t frame_ct_in;
> +    
> +    int64_t first_pts;
> +    int64_t dts_delta;
> +    
> +    char* profile;
> +    char* level;
> +
> +    bool flushing;
> +    bool has_b_frames;
> +    bool warned_color_range;
> +} VTEncContext;
> +
> +static void set_async_error(VTEncContext* vtctx, int err){
> +    pthread_mutex_lock(&vtctx->lock);
> +    
> +    vtctx->async_error = err;
> +    
> +    BufNode* info = vtctx->q_head;
> +    vtctx->q_head = vtctx->q_tail = NULL;
> +    
> +    while(info){
> +        CFRelease(info->cm_buffer);
> +        
> +        BufNode* next = info->next;
> +        free(info);
> +        info = next;
> +    }
> +    
> +    pthread_mutex_unlock(&vtctx->lock);
> +}
> +
> +static int vtenc_q_pop(VTEncContext* vtctx, bool wait, CMSampleBufferRef* buf){
> +    pthread_mutex_lock(&vtctx->lock);
> +    
> +    if(vtctx->async_error){
> +        pthread_mutex_unlock(&vtctx->lock);
> +        return vtctx->async_error;
> +    }
> +    
> +    if(vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out){
> +        *buf = NULL;
> +        
> +        pthread_mutex_unlock(&vtctx->lock);
> +        return 0;
> +    }
> +    
> +    if(!vtctx->q_head && wait){
> +        pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
> +    }
> +    
> +    if(!vtctx->q_head){
> +        pthread_mutex_unlock(&vtctx->lock);
> +        *buf = NULL;
> +        return 0;
> +    }
> +    
> +    BufNode* info = vtctx->q_head;
> +    vtctx->q_head = vtctx->q_head->next;
> +    if(!vtctx->q_head){
> +        vtctx->q_tail = NULL;
> +    }
> +    
> +    pthread_mutex_unlock(&vtctx->lock);
> +    
> +    CMSampleBufferRef buffer = info->cm_buffer;
> +    free(info);
> +    
> +    vtctx->frame_ct_out++;
> +    *buf = buffer;
> +    
> +    return 0;
> +}
> +
> +static void vtenc_q_push(VTEncContext* vtctx, CMSampleBufferRef buffer){
> +    BufNode* info = (BufNode*)malloc(sizeof(BufNode));
> +
> +    CFRetain(buffer);
> +    info->cm_buffer = buffer;
> +    info->next = NULL;
> +    
> +    pthread_mutex_lock(&vtctx->lock);
> +    pthread_cond_signal(&vtctx->cv_sample_sent);
> +    
> +    if(!vtctx->q_head){
> +        vtctx->q_head = vtctx->q_tail = info;
> +        pthread_mutex_unlock(&vtctx->lock);
> +        return;
> +    }
> +    
> +    vtctx->q_tail->next = info;
> +    vtctx->q_tail = info;
> +    
> +    pthread_mutex_unlock(&vtctx->lock);
> +}
> +
> +static av_cold CMVideoCodecType get_cm_codec_type(enum AVCodecID id){
> +    switch(id){
> +        case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
> +        default:               return 0;
> +    }
> +}
> +
> +static void vtenc_free_block(void* opaque, uint8_t* data){
> +    CMBlockBufferRef block = (CMBlockBufferRef)opaque;
> +    CFRelease(block);
> +}
> +
> +/**
> + * Get the parameter sets from a CMSampleBufferRef.
> + * @param dst If *dst isn't NULL, the parameters are copied into existing
> + *            memory. *dst_size must be set accordingly when *dst != NULL.
> + *            If *dst is NULL, it will be allocated.
> + *            In all cases, *dst_size is set to the number of bytes used starting
> + *            at *dst.
> + */
> +
> +static int get_params_info(
> +    AVCodecContext* avctx,
> +    CMVideoFormatDescriptionRef vid_fmt,
> +    size_t* size)
> +{
> +    size_t total_size = 0;
> +    size_t ps_count;
> +    
> +    OSStatus status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL, &ps_count, NULL);
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Error getting parameter set count: %d\n", status);
> +        return AVERROR_EXTERNAL;
> +    }
> +    
> +    for(size_t i = 0; i < ps_count; i++){
> +        const uint8_t* ps;
> +        size_t ps_size;
> +        status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, i, &ps, &ps_size, NULL, NULL);
> +        if(status){
> +            av_log(avctx, AV_LOG_ERROR, "Error getting parameter set size for index %zd: %d\n", i, status);
> +            return AVERROR_EXTERNAL;
> +        }
> +        
> +        total_size += ps_size + sizeof(start_code);
> +    }
> +
> +    *size = total_size;
> +    return 0;
> +}
> +
> +static int copy_param_sets(
> +    AVCodecContext* avctx,
> +    CMVideoFormatDescriptionRef vid_fmt,
> +    uint8_t* dst,
> +    size_t dst_size)
> +{
> +    size_t ps_count;
> +    OSStatus status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL, &ps_count, NULL);
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Error getting parameter set count for copying: %d\n", status);
> +        return AVERROR_EXTERNAL;
> +    }
> +    
> +    size_t offset = 0;
> +    for(size_t i = 0; i < ps_count; i++){
> +        const uint8_t* ps;
> +        size_t ps_size;
> +        status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, i, &ps, &ps_size, NULL, NULL);
> +        if(status){
> +            av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data for index %zd: %d\n", i, status);
> +            return AVERROR_EXTERNAL;
> +        }
> +        
> +        size_t next_offset = offset + sizeof(start_code) + ps_size;
> +        if(dst_size < next_offset){
> +            av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
> +            return AVERROR_BUFFER_TOO_SMALL;
> +        }
> +        
> +        memcpy(dst + offset, start_code, sizeof(start_code));
> +        offset += sizeof(start_code);
> +        
> +        memcpy(dst + offset, ps, ps_size);
> +        offset = next_offset;
> +    }
> +    
> +    return 0;
> +}
> +
> +static int set_extradata(AVCodecContext* avctx, CMSampleBufferRef sample_buffer){
> +    CMVideoFormatDescriptionRef vid_fmt;
> +    vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
> +    if(!vid_fmt){
> +        av_log(avctx, AV_LOG_ERROR, "No video format.\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +    
> +    size_t total_size;
> +    int status;
> +    status = get_params_info(avctx, vid_fmt, &total_size);
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
> +        return status;
> +    }


> +    avctx->extradata = malloc(total_size);
> +    if(!avctx->extradata){
> +        return AVERROR(ENOMEM);
> +    }
> +    avctx->extradata_size = total_size;

code in ffmpeg should use av_malloc() unless it needs to interface
to external stuff


> +    
> +    status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
> +        return status;
> +    }
> +    
> +    return 0;
> +}
> +
> +static av_cold void vtenc_output_callback(
> +    void* CM_NULLABLE ctx,
> +    void* sourceFrameCtx,
> +    OSStatus status,
> +    VTEncodeInfoFlags flags,
> +    CM_NULLABLE CMSampleBufferRef sample_buffer)
> +{
> +    av_assert0(ctx);
> +    AVCodecContext* avctx = (AVCodecContext*)ctx;

> +    VTEncContext*   vtctx = (VTEncContext*)avctx->priv_data;

void* doesnt need casts


> +    
> +    if(vtctx->async_error){
> +        CFRelease(sample_buffer);
> +        return;
> +    }
> +    
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", status);
> +        set_async_error(vtctx, AVERROR_EXTERNAL);
> +        return;
> +    }
> +    
> +    if(!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)){
> +        int set_status = set_extradata(avctx, sample_buffer);
> +        if(set_status){
> +            set_async_error(vtctx, set_status);
> +            return;
> +        }
> +    }
> +    
> +    vtenc_q_push(vtctx, sample_buffer);
> +}
> +
> +static int get_length_code_size(
> +    AVCodecContext* avctx,
> +    CMSampleBufferRef sample_buffer,
> +    size_t* size)
> +{
> +    CMVideoFormatDescriptionRef vid_fmt;
> +    vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
> +    if(!vid_fmt){
> +        av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +    
> +    int isize;
> +    OSStatus status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL, NULL, &isize);
> +    if(status){
> +        av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
> +        return AVERROR_EXTERNAL;
> +    }
> +    
> +    *size = isize;
> +    return 0;
> +}
> +
> +static bool get_h264_profile(AVCodecContext* avctx, int* profile_num){
> +    VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +    
> +    const char* profile = vtctx->profile;
> +    if(!profile){
> +        *profile_num = FF_PROFILE_UNKNOWN;
> +    }

> +    else if(!strcasecmp("baseline", profile)){

av_strcasecmp()

[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

No human being will ever know the Truth, for even if they happen to say it
by chance, they would not even known they had done so. -- Xenophanes
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20151125/49d3a2b3/attachment.sig>


More information about the ffmpeg-devel mailing list