[Libav-user] AVFilter with hardware transcoding

[Luis Felipe Domínguez Vega]

Hi, I trying to apply "yadif_cuda,scale_cuda" filter to a transcoder 
app. The problem is with the Pixel Format.

I'm using 'h264' as decoder and 'h264_nvenc' as encoder, selected CUDA 
device as hardware context and this is working great, the problem is 
when i "insert in the middle" the filter process.

The filter that i trying to use is 
"hwupload_cuda,yadif_cuda,scale_cuda=w=100:h=100" and the code for init 
filters is at the end.

One of the thing that I dont underestand well is that when you init the 
filter (the "in") you must pass the pix_fmt, but the problem is, the 
image is on yuv420p format, but at real the hardware decoding process 
put the decoded frame with cuda format, so, how work with this 2 formats.

static int init_filters(const char *filtersDescr) {

     int ret = 0;

     const AVFilter *bufferSrc  = avfilter_get_by_name("buffer");
     const AVFilter *bufferSink = avfilter_get_by_name("buffersink");
     AVFilterInOut *outputs = avfilter_inout_alloc();
     AVFilterInOut *inputs  = avfilter_inout_alloc();

     AVRational time_base = dFmtCtx->streams[videoStream]->time_base;

     filterGraph = avfilter_graph_alloc();

     if (!outputs || !inputs || !filterGraph) {
         ret = AVERROR(ENOMEM);

         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }

     spdlog::debug("Init filter with input format: {} {}", 
dvCodecCtx->pix_fmt, dvCodecCtx->sw_pix_fmt);

     char args[512];
     snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
             dvCodecCtx->width, dvCodecCtx->height, AV_PIX_FMT_CUDA,
             time_base.num, time_base.den,
             dvCodecCtx->sample_aspect_ratio.num, 
dvCodecCtx->sample_aspect_ratio.den);


     spdlog::debug("Making 'in' filter");

     ret = avfilter_graph_create_filter(&bufferSrcCtx, bufferSrc, "in",
                                        args, NULL, filterGraph);

     if (ret < 0) {
         av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");

         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }

     spdlog::debug("Making 'out' filter");

      /* buffer video sink: to terminate the filter chain. */
     ret = avfilter_graph_create_filter(&bufferSinkCtx, bufferSink, "out",
                                        NULL, NULL, filterGraph);
     if (ret < 0) {
         av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");

         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }

    /* ret = av_opt_set_int_list(bufferSinkCtx, "pix_fmts", pixFmts,
                               AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
     if (ret < 0) {
         spdlog::error("Cannot set output pixel format");

         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }*/

     outputs->name       = av_strdup("in");
     outputs->filter_ctx = bufferSrcCtx;
     outputs->pad_idx    = 0;
     outputs->next       = NULL;

     inputs->name       = av_strdup("out");
     inputs->filter_ctx = bufferSinkCtx;
     inputs->pad_idx    = 0;
     inputs->next       = NULL;

     spdlog::debug("Parsing custom filter");

     if ((ret = avfilter_graph_parse_ptr(filterGraph, filtersDescr,
                                     &inputs, &outputs, NULL)) < 0){
         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }

     spdlog::debug("Configuring filter graph");

     if ((ret = avfilter_graph_config(filterGraph, NULL)) < 0) {

         avfilter_inout_free(&inputs);
         avfilter_inout_free(&outputs);

         return ret;
     }

     avfilter_inout_free(&inputs);
     avfilter_inout_free(&outputs);

     return ret;
}