[FFmpeg-devel] [RFC] libavfilter audio API and related issues

Stefano Sabatini stefano.sabatini-lala
Sun Jun 6 01:07:34 CEST 2010


On date Saturday 2010-06-05 13:24:35 -0700, S.N. Hemanth Meenakshisundaram encoded:
[...]
> Hi All,
> 
> Attached is a rough draft of the audio API. I am now writing a
> simple volume scaling filter using this API. Please review and
> comment. Some observations:
>
> 1. I am temporarily retaining the AVFilterPic struct as a separate struct
> for video and have introduced the AVFilterBuffer struct. The plan is to
> merge these two. AVFilterSamplesRef will reference this structure for
> audio.
> 
> 2. AVFilterSamplesRef has a lot of redundancy. E.g. 'size' is same as
> samples_nb*bytes_per_sample*num_channels and sampstep is the distance to
> next sample of same channel (for both planar and packed cases). This is to
> avoid recomputing sizes. Please let me know if its too redundant.

No strong opinion. I believe this should be eventually benchmarked
before to be committed to main SVN, if the gain is too little then I'd
prefer to avoid redundancy.
 
> 3. data[8] will have the start addresses of the data for that buffer for
> up to 8 channels in case of planar. If packed, the entire buffer ptr is at
> data[0] and the rest are NULL.
> 
> 4. AVFilterFormats and formats.c were modified to support audio (sample)
> formats as well. At the moment, I am skipping channel layouts here. Need
> to handle differences in supported channel layouts of different filters in
> some other way.
> 
> 5. 'filter_samples' does the job of start_frame, draw_slice and end_frame
> all at once.
>
> 6. Whenever audio variables like aformat were introduced, I plan to change
> the corresponding video variables like format to vformat etc. But at the
> moment I haven't done this so as not to break video.
>
> I am not sure if this is a right use of this RFC thread. Please let me
> know if I need to move this to a separate thread on ffmpeg-soc instead.

That's fine, RFC and PATCH threads tend to confuse, as for this list
or the soc one to me is completely indifferent, if we're going to
commit on SVN then this is the right list.

BTW let us know if you want to work on a SOC SVN repo or with
github/gitorius/whatever, the last option should be the more flexible
for you.

> 
> Regards,
> Hemanth

> Index: defaults.c
> ===================================================================
> --- defaults.c	(revision 23145)
> +++ defaults.c	(working copy)
> @@ -21,6 +21,7 @@
>  
>  #include "libavcodec/imgconvert.h"
>  #include "avfilter.h"
> +#include "libavcodec/audioconvert.h"
>  
>  /* TODO: buffer pool.  see comment for avfilter_default_get_video_buffer() */
>  static void avfilter_default_free_video_buffer(AVFilterPic *pic)
> @@ -29,6 +30,12 @@
>      av_free(pic);
>  }
>  
> +static void avfilter_default_free_audio_buffer(AVFilterBuffer *buffer)
> +{
> +    av_free(buffer->data[0]);
> +    av_free(buffer);
> +}
> +
>  /* TODO: set the buffer's priv member to a context structure for the whole
>   * filter chain.  This will allow for a buffer pool instead of the constant
>   * alloc & free cycle currently implemented. */
> @@ -65,6 +72,65 @@
>      return ref;
>  }
>  
> +AVFilterSamplesRef *avfilter_default_get_audio_buffer(AVFilterLink *link, int perms, int samples_nb,
> +                                                      int64_t channel_layout,
> +                                                      enum SampleFormat sample_fmt, int planar)
> +{
> +    AVFilterBuffer *buffer = av_mallocz(sizeof(AVFilterBuffer));
> +    AVFilterSamplesRef *ref = av_mallocz(sizeof(AVFilterSamplesRef));
> +    int i, sampsize, numchan, bufsize, stepsize = 0;
> +    char *buf;
> +
> +    ref->buffer         = buffer;
> +    ref->channel_layout = channel_layout;
> +    ref->sample_fmt     = sample_fmt;
> +    ref->samples_nb     = samples_nb;
> +
> +    /* make sure the buffer gets read permission or it's useless for output */
> +    ref->perms = perms | AV_PERM_READ;
> +
> +    buffer->refcount   = 1;
> +    buffer->free       = avfilter_default_free_audio_buffer;
> +
> +    sampsize = (av_get_bits_per_sample_format(ref->sample_fmt))>>3;
> +    numchan = avcodec_channel_layout_num_channels(ref->channel_layout);
> +
> +    /* Set the number of bytes to traverse to reach next sample of a particular channel:
> +     * For planar, this is simply the sample size.
> +     * For packed, this is the number of samples * sample_size.
> +     */
> +    for(i = 0; i < numchan; i++)
> +        buffer->sampstep[i] = (planar > 0)?(samples_nb*sampsize):sampsize;

nit: K&R => for (...)

> +    for(i = numchan+1; i < 8; i++)
> +        buffer->sampstep[i] = 0;
> +
> +    /* Calculate total buffer size, round to multiple of 16 to be SIMD friendly */
> +    bufsize = (samples_nb*numchan*sampsize + 15)&~15;
> +    buf = av_malloc(bufsize);
> +
> +    /* For planar, set the start point of each channel's data within the buffer
> +     * For packed, set the start point of the entire buffer only
> +     */
> +    buffer->data[0] = buf;
> +    if(planar > 0) {
> +        for(i = 1; i < numchan; i++) {
> +            stepsize += samples_nb*sampsize;
> +            buffer->data[i] = buf + stepsize;
> +        }

> +        for(i = numchan+1; i < 8; i++)
> +            buffer->data[i] = NULL;
> +
> +    } else {
> +        for(i = 1; i < 8; i++)
> +            buffer->data[i] = NULL;
> +    }

Simpler:
if (planar > 0) {
   for (...) {
   }
}
memset(buffer[1], ...);

> +
> +    memcpy(ref->data,     buffer->data,     sizeof(buffer->data));
> +    memcpy(ref->sampstep, buffer->sampstep, sizeof(buffer->sampstep));
> +
> +    return ref;
> +}
> +
>  void avfilter_default_start_frame(AVFilterLink *link, AVFilterPicRef *picref)
>  {
>      AVFilterLink *out = NULL;
> @@ -113,6 +179,27 @@
>      }
>  }
>  
> +void avfilter_default_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref)
> +{
> +    AVFilterLink *out = NULL;
> +
> +    if(link->dst->output_count)
> +        out = link->dst->outputs[0];
> +
> +    if(out) {
> +        out->outsamples = avfilter_default_get_audio_buffer(link, AV_PERM_WRITE, samplesref->samples_nb,
> +                                                            samplesref->channel_layout,
> +                                                            samplesref->sample_fmt, samplesref->planar);
> +        out->outsamples->samples_nb = samplesref->samples_nb;
> +        out->outsamples->channel_layout = samplesref->channel_layout;
> +        out->outsamples->sample_rate = samplesref->sample_rate;
> +        out->outsamples->sample_fmt = samplesref->sample_fmt;
> +        out->outsamples->planar = samplesref->planar;
> +        out->outsamples->sample_rate = samplesref->sample_rate;
> +        avfilter_filter_samples(out, avfilter_ref_samples(out->outsamples, ~0));

Nit: vertical align.

> +    }
> +}
> +
>  /**
>   * default config_link() implementation for output video links to simplify
>   * the implementation of one input one output video filters */
> @@ -183,8 +270,21 @@
>      avfilter_end_frame(link->dst->outputs[0]);
>  }
>  
> +void avfilter_null_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref)
> +{
> +    avfilter_filter_samples(link->dst->outputs[0], samplesref);
> +}
> +
>  AVFilterPicRef *avfilter_null_get_video_buffer(AVFilterLink *link, int perms, int w, int h)
>  {
>      return avfilter_get_video_buffer(link->dst->outputs[0], perms, w, h);
>  }
>  
> +AVFilterSamplesRef *avfilter_null_get_audio_buffer(AVFilterLink *link, int perms, int samples_nb,
> +                                                   int64_t channel_layout,
> +                                                   enum SampleFormat sample_fmt, int packed)
> +{
> +    return avfilter_get_audio_buffer(link->dst->outputs[0], perms, samples_nb,
> +                                     channel_layout, sample_fmt, packed);
> +}
> +
> Index: formats.c
> ===================================================================
> --- formats.c	(revision 23145)
> +++ formats.c	(working copy)
> @@ -46,13 +46,23 @@
>  
>      ret = av_mallocz(sizeof(AVFilterFormats));
>  
> -    /* merge list of formats */
> -    ret->formats = av_malloc(sizeof(*ret->formats) * FFMIN(a->format_count,

> +    if(a->type == AVMEDIA_TYPE_VIDEO) {
> +        /* merge list of formats */
> +        ret->formats = av_malloc(sizeof(*ret->formats) * FFMIN(a->format_count,

cosmetics, avoid to reindent this, makes the patch more readable

>                                                             b->format_count));
> -    for(i = 0; i < a->format_count; i ++)
> -        for(j = 0; j < b->format_count; j ++)
> -            if(a->formats[i] == b->formats[j])
> -                ret->formats[k++] = a->formats[i];
> +        for(i = 0; i < a->format_count; i ++)
> +            for(j = 0; j < b->format_count; j ++)
> +                if(a->formats[i] == b->formats[j])
> +                    ret->formats[k++] = a->formats[i];
> +    } else if(a->type == AVMEDIA_TYPE_AUDIO) {
> +        /* merge list of formats */
> +        ret->aformats = av_malloc(sizeof(*ret->aformats) * FFMIN(a->format_count,
> +                                                           b->format_count));
> +        for(i = 0; i < a->format_count; i ++)
> +            for(j = 0; j < b->format_count; j ++)
> +                if(a->aformats[i] == b->aformats[j])
> +                    ret->aformats[k++] = a->aformats[i];
> +    }

Also maybe this can be factorized:

aformats = type == AUDIO ? a->aformats : a->formats;
bformats = type == AUDIO ? a->aformats : a->formats;
...

>  
>      ret->format_count = k;
>      /* check that there was at least one common format */
> @@ -115,6 +125,50 @@
>      return ret;
>  }
>  
> +AVFilterFormats *avfilter_make_aformat_list(const enum SampleFormat *sample_fmts)
> +{
> +    AVFilterFormats *formats;
> +    int count;
> +
> +    for (count = 0; sample_fmts[count] != SAMPLE_FMT_NONE; count++)
> +        ;
> +
> +    formats               = av_mallocz(sizeof(AVFilterFormats));
> +    formats->aformats     = av_malloc(sizeof(*formats->aformats) * count);
> +    formats->format_count = count;
> +    memcpy(formats->aformats, sample_fmts, sizeof(*formats->aformats) * count);
> +
> +    return formats;
> +}

This is missing various checks, yes I know also make_format_list has
the same problem. Also maybe we could use a macro to avoid to repeat
the same code twice (that would be a separate commit though).

> +
> +int avfilter_add_sampleformat(AVFilterFormats **avff, enum SampleFormat sample_fmt)
> +{
> +    enum SampleFormat *sample_fmts;
> +
> +    if (!(*avff) && !(*avff = av_mallocz(sizeof(AVFilterFormats))))
> +        return AVERROR(ENOMEM);
> +
> +    sample_fmts = av_realloc((*avff)->aformats,
> +                          sizeof((*avff)->aformats) * ((*avff)->format_count+1));
> +    if (!sample_fmts)
> +        return AVERROR(ENOMEM);
> +
> +    (*avff)->aformats = sample_fmts;
> +    (*avff)->aformats[(*avff)->format_count++] = sample_fmt;
> +    return 0;
> +}
> +
> +AVFilterFormats *avfilter_all_sampleformats(void)
> +{
> +    AVFilterFormats *ret = NULL;
> +    enum SampleFormat sample_fmt;
> +
> +    for (sample_fmt = 0; sample_fmt < SAMPLE_FMT_NB; sample_fmt++)
> +        avfilter_add_sampleformat(&ret, sample_fmt);
> +
> +    return ret;
> +}
> +
>  void avfilter_formats_ref(AVFilterFormats *f, AVFilterFormats **ref)
>  {
>      *ref = f;
> Index: avfilter.c
> ===================================================================
> --- avfilter.c	(revision 23145)
> +++ avfilter.c	(working copy)
> @@ -60,6 +60,22 @@
>      av_free(ref);
>  }
>  
> +AVFilterSamplesRef *avfilter_ref_samples(AVFilterSamplesRef *ref, int pmask)
> +{
> +    AVFilterSamplesRef *ret = av_malloc(sizeof(AVFilterSamplesRef));
> +    *ret = *ref;
> +    ret->perms &= pmask;

> +    ret->buffer->refcount ++;

nit: refcount++

> +    return ret;
> +}
> +
> +void avfilter_unref_samples(AVFilterSamplesRef *ref)
> +{
> +    if(!(--ref->buffer->refcount))
> +        ref->buffer->free(ref->buffer);
> +    av_free(ref);
> +}
> +
>  void avfilter_insert_pad(unsigned idx, unsigned *count, size_t padidx_off,
>                           AVFilterPad **pads, AVFilterLink ***links,
>                           AVFilterPad *newpad)
> @@ -97,7 +113,9 @@
>      link->dst     = dst;
>      link->srcpad  = srcpad;
>      link->dstpad  = dstpad;
> +    link->type    = src->output_pads[srcpad].type;
>      link->format  = PIX_FMT_NONE;
> +    link->aformat = SAMPLE_FMT_NONE;
>  
>      return 0;
>  }
> @@ -210,6 +228,20 @@
>      return ret;
>  }
>  
> +AVFilterSamplesRef *avfilter_get_audio_buffer(AVFilterLink *link, int perms, int samples_nb,
> +                                              int64_t channel_layout, enum SampleFormat sample_fmt, int planar)
> +{
> +    AVFilterSamplesRef *ret = NULL;
> +
> +    if(link_dpad(link).get_audio_buffer)
> +        ret = link_dpad(link).get_audio_buffer(link, perms, samples_nb, channel_layout, sample_fmt, planar);
> +
> +    if(!ret)
> +        ret = avfilter_default_get_audio_buffer(link, perms, samples_nb, channel_layout, sample_fmt, planar);
> +
> +    return ret;
> +}
> +
>  int avfilter_request_frame(AVFilterLink *link)
>  {
>      DPRINTF_START(NULL, request_frame); dprintf_link(NULL, link, 1);
> @@ -221,6 +253,14 @@
>      else return -1;
>  }
>  
> +int avfilter_request_samples(AVFilterLink *link)
> +{
> +    if(link_spad(link).request_samples)
> +        return link_spad(link).request_samples(link);
> +    else if(link->src->inputs[0])
> +        return avfilter_request_samples(link->src->inputs[0]);

> +    else return -1;

AVERROR(EINVAL)

> +}
>  int avfilter_poll_frame(AVFilterLink *link)
>  {
>      int i, min=INT_MAX;
> @@ -334,6 +374,35 @@
>      draw_slice(link, y, h, slice_dir);
>  }
>  
> +void avfilter_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref)
> +{
> +    void (*filter_samples)(AVFilterLink *, AVFilterSamplesRef *);
> +    AVFilterPad *dst = &link_dpad(link);
> +
> +    if(!(filter_samples = dst->filter_samples))
> +        filter_samples = avfilter_default_filter_samples;
> +
> +    /* prepare to copy the samples if the buffer has insufficient permissions */
> +    if((dst->min_perms & samplesref->perms) != dst->min_perms ||
> +        dst->rej_perms & samplesref->perms) {
> +
> +        link->cur_samples = avfilter_default_get_audio_buffer(link, dst->min_perms, samplesref->samples_nb,
> +                                                              samplesref->channel_layout,
> +                                                              samplesref->sample_fmt, samplesref->planar);
> +        link->cur_samples->samples_nb = samplesref->samples_nb;
> +        link->cur_samples->channel_layout = samplesref->channel_layout;
> +        link->cur_samples->sample_rate = samplesref->sample_rate;
> +        link->cur_samples->sample_fmt = samplesref->sample_fmt;
> +        link->cur_samples->planar = samplesref->planar;
> +        link->cur_samples->sample_rate = samplesref->sample_rate;
> +        avfilter_unref_samples(samplesref);
> +    }
> +    else
> +        link->cur_samples = samplesref;
> +
> +    filter_samples(link, link->cur_samples);
> +}
> +
>  #define MAX_REGISTERED_AVFILTERS_NB 64
>  
>  static AVFilter *registered_avfilters[MAX_REGISTERED_AVFILTERS_NB + 1];
> Index: avfilter.h
> ===================================================================
> --- avfilter.h	(revision 23145)
> +++ avfilter.h	(working copy)
> @@ -36,6 +36,12 @@
>                                             LIBAVFILTER_VERSION_MICRO)
>  #define LIBAVFILTER_BUILD       LIBAVFILTER_VERSION_INT
>  
> +#define LIBAVFILTER_AUDIO_BUF_SIZE(ref) { \
> +    int num_channels = avcodec_channel_layout_num_channels(ref->channel_layout); \
> +    int bytes_per_sample = (av_get_bits_per_sample_format(ref->sample_fmt))>>3; \
> +    ref->size = ref->samples_nb * num_channels * bytes_per_sample; \
> +}
> +
>  #include <stddef.h>
>  #include "libavcodec/avcodec.h"
>  
> @@ -88,6 +94,29 @@
>      int w, h;                  ///< width and height of the allocated buffer
>  } AVFilterPic;
>  
> +/*
> + * Temporary structure for audio data used by the filter system. Later to be
> + * merged with FilterPic above and generalized.
> + */
> +typedef struct AVFilterBuffer
> +{
> +    uint8_t *data[8];           ///< audio data for each channel
> +    int sampstep[8];            ///< number of bytes to next sample

Shouldn't this simply called "step" or "linesize"?

> +
> +    unsigned refcount;          ///< number of references to this image

s/image/buffer/                                  

> +
> +    /** private data to be used by a custom free function */
> +    void *priv;
> +    /**

> +     * A pointer to the function to deallocate this image if the default

ditto

> +     * function is not sufficient. This could, for example, add the memory
> +     * back into a memory pool to be reused later without the overhead of
> +     * reallocating it from scratch.
> +     */
> +    void (*free)(struct AVFilterBuffer *buffer);
> +
> +} AVFilterBuffer;
> +
>  /**
>   * A reference to an AVFilterPic. Since filters can manipulate the origin of
>   * a picture to, for example, crop image without any memcpy, the picture origin
> @@ -96,6 +125,11 @@
>   *
>   * TODO: add anything necessary for frame reordering
>   */
> +#define AV_PERM_READ     0x01   ///< can read from the buffer
> +#define AV_PERM_WRITE    0x02   ///< can write to the buffer
> +#define AV_PERM_PRESERVE 0x04   ///< nobody else can overwrite the buffer
> +#define AV_PERM_REUSE    0x08   ///< can output the buffer multiple times, with the same contents each time
> +#define AV_PERM_REUSE2   0x10   ///< can output the buffer multiple times, modified each time
>  typedef struct AVFilterPicRef
>  {
>      AVFilterPic *pic;           ///< the picture that this is a reference to
> @@ -110,17 +144,37 @@
>      AVRational pixel_aspect;    ///< pixel aspect ratio
>  
>      int perms;                  ///< permissions
> -#define AV_PERM_READ     0x01   ///< can read from the buffer
> -#define AV_PERM_WRITE    0x02   ///< can write to the buffer
> -#define AV_PERM_PRESERVE 0x04   ///< nobody else can overwrite the buffer
> -#define AV_PERM_REUSE    0x08   ///< can output the buffer multiple times, with the same contents each time
> -#define AV_PERM_REUSE2   0x10   ///< can output the buffer multiple times, modified each time
>  
>      int interlaced;             ///< is frame interlaced
>      int top_field_first;
>  } AVFilterPicRef;
>  

>  /**
> + * A reference to an AVFilterBuffer for audio. Since filters can manipulate the
> + * origin of an audio buffer to, for example, reduce precision without any memcpy,
> + * sample format and channel_layout are per-reference properties. Sample step is
> + * also useful when reducing no. of channels, etc, and so is also per-reference.
> + */
> +typedef struct AVFilterSamplesRef
> +{
> +    AVFilterBuffer *buffer;       ///< the audio buffer that this is a reference to

> +    uint8_t *data[8];             ///< audio data for each channel
> +    int sampstep[8];              ///< number of bytes to next sample

There is a specific reason for which this is repeated? Shouldn't the
the data/linesize in buffer be sufficient?

> +
> +    int64_t channel_layout;       ///< channel layout of current buffer (see avcodec.h)
> +    int64_t sample_rate;          ///< samples per second
> +    enum SampleFormat sample_fmt; ///< sample format (see avcodec.h)
> +
> +    int samples_nb;               ///< number of samples in this buffer
> +    /* Should this go here or in the AVFilterBuffer struct? */
> +    int size;                     ///< size of buffer
> +
> +    int perms;                    ///< permissions
> +
> +    int planar;                   ///< is buffer planar or packed
> +} AVFilterSamplesRef;
> +
> +/**
>   * Adds a new reference to a picture.
>   * @param ref   an existing reference to the picture
>   * @param pmask a bitmask containing the allowable permissions in the new
> @@ -138,6 +192,23 @@
>  void avfilter_unref_pic(AVFilterPicRef *ref);
>  
>  /**
> + * Adds a new reference to an audio samples buffer.
> + * @param ref   an existing reference to the buffer
> + * @param pmask a bitmask containing the allowable permissions in the new
> + *              reference
> + * @return      a new reference to the buffer with the same properties as the
> + *              old, excluding any permissions denied by pmask
> + */
> +AVFilterSamplesRef *avfilter_ref_samples(AVFilterSamplesRef *ref, int pmask);
> +
> +/**
> + * Removes a reference to a buffer of audio samples. If this is the last reference
> + * to the buffer, the buffer itself is also automatically freed.
> + * @param ref reference to the buffer
> + */
> +void avfilter_unref_samples(AVFilterSamplesRef *ref);
> +
> +/**
>   * A list of supported formats for one end of a filter link. This is used
>   * during the format negotiation process to try to pick the best format to
>   * use to minimize the number of necessary conversions. Each filter gives a
> @@ -180,16 +251,18 @@
>  typedef struct AVFilterFormats AVFilterFormats;
>  struct AVFilterFormats

>  {
> -    unsigned format_count;      ///< number of formats
> -    enum PixelFormat *formats;  ///< list of pixel formats
> +    unsigned format_count;        ///< number of formats
> +    enum AVMediaType type;        ///< filter type
> +    enum PixelFormat *formats;    ///< list of pixel formats for video
> +    enum SampleFormat *aformats; ///< list of sample formats for audio
>  
> -    unsigned refcount;          ///< number of references to this list
> -    AVFilterFormats ***refs;    ///< references to this list
> +    unsigned refcount;            ///< number of references to this list
> +    AVFilterFormats ***refs;      ///< references to this list

Avoid to reindent lines which are not changed.

>  };
>  
>  /**
> - * Creates a list of supported formats. This is intended for use in
> - * AVFilter->query_formats().
> + * Creates a list of supported pixel formats. This is intended for use in
> + * AVFilter->query_formats() for video filters.
>   * @param pix_fmt list of pixel formats, terminated by PIX_FMT_NONE
>   * @return the format list, with no existing references
>   */
> @@ -211,6 +284,29 @@
>  AVFilterFormats *avfilter_all_colorspaces(void);
>  
>  /**
> + * Creates a list of supported sample formats. This is intended for use in
> + * AVFilter->query_formats() for audio filters.
> + * @param sample_fmt list of sample formats, terminated by SAMPLE_FMT_NONE
> + * @return the format list, with no existing references
> + */
> +AVFilterFormats *avfilter_make_aformat_list(const enum SampleFormat *sample_fmts);
> +
> +/**
> + * Adds sample_fmt to the list of sample formats contained in *avff.
> + * If *avff is NULL the function allocates the filter formats struct
> + * and puts its pointer in *avff.
> + *
> + * @return a non negative value in case of success, or a negative
> + * value corresponding to an AVERROR code in case of error
> + */
> +int avfilter_add_sampleformat(AVFilterFormats **avff, enum SampleFormat sample_fmt);
> +
> +/**
> + * Returns a list of all sampleformats supported by FFmpeg.
> + */
> +AVFilterFormats *avfilter_all_sampleformats(void);
> +
> +/**
>   * Returns a format list which contains the intersection of the formats of
>   * a and b. Also, all the references of a, all the references of b, and
>   * a and b themselves will be deallocated.
> @@ -280,8 +376,7 @@
>      const char *name;
>  
>      /**
> -     * AVFilterPad type. Only video supported now, hopefully someone will
> -     * add audio in the future.
> +     * AVFilterPad type. Audio support still in progress.
>       */
>      enum AVMediaType type;
>  
> @@ -315,7 +410,7 @@
>      void (*start_frame)(AVFilterLink *link, AVFilterPicRef *picref);
>  
>      /**
> -     * Callback function to get a buffer. If NULL, the filter system will
> +     * Callback function to get a video buffer. If NULL, the filter system will
>       * use avfilter_default_get_video_buffer().
>       *
>       * Input video pads only.
> @@ -323,6 +418,16 @@
>      AVFilterPicRef *(*get_video_buffer)(AVFilterLink *link, int perms, int w, int h);
>  
>      /**
> +     * Callback function to get an audio buffer. If NULL, the filter system will
> +     * use avfilter_default_get_audio_buffer().
> +     *
> +     * Input audio pads only.
> +     */
> +    AVFilterSamplesRef *(*get_audio_buffer)(AVFilterLink *link, int perms,
> +                                            int samples_nb, int64_t channel_layout,
> +                                            enum SampleFormat sample_fmt, int planar);
> +
> +    /**
>       * Callback called after the slices of a frame are completely sent. If
>       * NULL, the filter layer will default to releasing the reference stored
>       * in the link structure during start_frame().
> @@ -340,6 +445,14 @@
>      void (*draw_slice)(AVFilterLink *link, int y, int height, int slice_dir);
>  
>      /**
> +     * Samples filtering callback. This is where a filter receives audio data
> +     * and should do its processing.
> +     *
> +     * Input audio pads only.
> +     */
> +    void (*filter_samples)(AVFilterLink *link, AVFilterSamplesRef *samplesref);
> +
> +    /**
>       * Frame poll callback. This returns the number of immediately available
>       * frames. It should return a positive value if the next request_frame()
>       * is guaranteed to return one frame (with no delay).
> @@ -360,6 +473,15 @@
>      int (*request_frame)(AVFilterLink *link);
>  
>      /**
> +     * Samples request callback. A call to this should result in at least one
> +     * sample being output over the given link. This should return zero on
> +     * success, and another value on error.
> +     *
> +     * Output audio pads only.
> +     */
> +    int (*request_samples)(AVFilterLink *link);
> +
> +    /**
>       * Link configuration callback.
>       *
>       * For output pads, this should set the link properties such as
> @@ -382,13 +504,19 @@
>  void avfilter_default_draw_slice(AVFilterLink *link, int y, int h, int slice_dir);
>  /** default handler for end_frame() for video inputs */
>  void avfilter_default_end_frame(AVFilterLink *link);
> -/** default handler for config_props() for video outputs */
> +/** default handler for filter_samples() for audio inputs */
> +void avfilter_default_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref);
> +/** default handler for config_props() for audio/video outputs */
>  int avfilter_default_config_output_link(AVFilterLink *link);
> -/** default handler for config_props() for video inputs */
> +/** default handler for config_props() for audio/video inputs */
>  int avfilter_default_config_input_link (AVFilterLink *link);
>  /** default handler for get_video_buffer() for video inputs */
>  AVFilterPicRef *avfilter_default_get_video_buffer(AVFilterLink *link,
>                                                    int perms, int w, int h);
> +/** default handler for get_audio_buffer() for audio inputs */
> +AVFilterSamplesRef *avfilter_default_get_audio_buffer(AVFilterLink *link, int perms,
> +                                                      int samples_nb, int64_t channel_layout,
> +                                                      enum SampleFormat sample_fmt, int planar);
>  /**
>   * A helper for query_formats() which sets all links to the same list of
>   * formats. If there are no links hooked to this filter, the list of formats is
> @@ -407,10 +535,18 @@
>  /** end_frame() handler for filters which simply pass video along */
>  void avfilter_null_end_frame(AVFilterLink *link);

  
> +/** filter_samples() handler for filters which simply pass audio along */
> +void avfilter_null_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref);
> +
>  /** get_video_buffer() handler for filters which simply pass video along */
>  AVFilterPicRef *avfilter_null_get_video_buffer(AVFilterLink *link,
>                                                    int perms, int w, int h);
>  
> +/** get_audio_buffer() handler for filters which simply pass audio along */
> +AVFilterSamplesRef *avfilter_null_get_audio_buffer(AVFilterLink *link, int perms,
> +                                               int samples_nb, int64_t channel_layout,
> +                                               enum SampleFormat sample_fmt, int planar);

weird indent

> +
>  /**
>   * Filter definition. This defines the pads a filter contains, and all the
>   * callback functions used to interact with the filter.
> @@ -498,10 +634,22 @@
>          AVLINK_INIT             ///< complete
>      } init_state;
>  
> +    /**
> +     * AVFilterPad type. Audio support still in progress.
> +     */
> +    enum AVMediaType type;
> +
> +    /* These three parameters apply only to video */
>      int w;                      ///< agreed upon image width
>      int h;                      ///< agreed upon image height
>      enum PixelFormat format;    ///< agreed upon image colorspace
>  
> +    /* These four parameters apply only to audio */
> +    int samples_nb;             ///< number of samples in this buffer
> +    int64_t channel_layout;     ///< channel layout of current buffer (see avcodec.h)
> +    int64_t sample_rate;        ///< samples per second
> +    enum SampleFormat aformat;  ///< sample format (see avcodec.h)
> + 
>      /**
>       * Lists of formats supported by the input and output filters respectively.
>       * These lists are used for negotiating the format to actually be used,
> @@ -511,16 +659,21 @@
>      AVFilterFormats *out_formats;
>  
>      /**
> -     * The picture reference currently being sent across the link by the source
> -     * filter. This is used internally by the filter system to allow
> -     * automatic copying of pictures which do not have sufficient permissions
> -     * for the destination. This should not be accessed directly by the
> -     * filters.
> +     * The picture (for video) or samples (for audio) reference currently being
> +     * sent across the link by the source filter. This is used internally by the
> +     * filter system to allow automatic copying of pictures/samples which do not
> +     * have sufficient permissions for the destination. This should not be accessed
> +     * directly by the filters.
>       */
>      AVFilterPicRef *srcpic;
>  
>      AVFilterPicRef *cur_pic;
>      AVFilterPicRef *outpic;
> +
> +    AVFilterSamplesRef *srcsamples;
> +
> +    AVFilterSamplesRef *cur_samples;
> +    AVFilterSamplesRef *outsamples;
>  };
>  
>  /**
> @@ -555,6 +708,21 @@
>                                            int w, int h);
>  
>  /**
> + * Requests an audio samples buffer with a specific set of permissions.
> + * @param link           the output link to the filter from which the buffer will
> + *                       be requested

Put an empty line before the first @param, here and below.

> + * @param perms          the required access permissions
> + * @param samples_nb     the no. of samples in the buffer to allocate
> + * @param channel_layout the no. & type of channels per sample in the buffer to allocate
> + * @param sample_fmt     the format of each sample in the buffer to allocate
> + * @return               A reference to the samples. This must be unreferenced with
> + *                       avfilter_unref_samples when you are finished with it.
> + */
> +AVFilterSamplesRef *avfilter_get_audio_buffer(AVFilterLink *link, int perms,
> +                                             int samples_nb, int64_t channel_layout,
> +                                             enum SampleFormat sample_fmt, int planar);
> +
> +/**
>   * Requests an input frame from the filter at the other end of the link.
>   * @param link the input link
>   * @return     zero on success
> @@ -562,6 +730,13 @@
>  int avfilter_request_frame(AVFilterLink *link);
>  
>  /**
> + * Requests input audio samples from the filter at the other end of the link.
> + * @param  link the input link
> + * @return zero on success
> + */
> +int avfilter_request_samples(AVFilterLink *link);
> +
> +/**
>   * Polls a frame from the filter chain.
>   * @param  link the input link
>   * @return the number of immediately available frames, a negative
> @@ -602,6 +777,14 @@
>   */
>  void avfilter_draw_slice(AVFilterLink *link, int y, int h, int slice_dir);
>  
> +/**
> + * Sends a buffer of audio samples to the next filter.
> + *
> + * @param link   the output link over which the audio samples are being sent
> + * @param planar samples are packed if 0 or planar if 1
> + */
> +void avfilter_filter_samples(AVFilterLink *link, AVFilterSamplesRef *samplesref);
> +
>  /** Initializes the filter system. Registers all builtin filters. */
>  void avfilter_register_all(void);

Regards.
-- 
FFmpeg = Fundamental & Furious Meaningless Proud Ermetic Governor



More information about the ffmpeg-devel mailing list