[FFmpeg-devel] [PATCH] aacenc: replace VLA with malloced buffer in context

[Måns Rullgård]

Alex Converse <alex.converse at gmail.com> writes:

> On Sat, Jul 3, 2010 at 8:59 PM, Mans Rullgard <mans at mansr.com> wrote:
>> ---
>> ?libavcodec/aacenc.c | ? ?6 +++++-
>> ?libavcodec/aacenc.h | ? ?1 +
>> ?2 files changed, 6 insertions(+), 1 deletions(-)
>>
>> diff --git a/libavcodec/aacenc.c b/libavcodec/aacenc.c
>> index 1aaaf3a..78eb7d2 100644
>> --- a/libavcodec/aacenc.c
>> +++ b/libavcodec/aacenc.c
>> @@ -202,6 +202,9 @@ static av_cold int aac_encode_init(AVCodecContext *avctx)
>> ? ? ff_psy_init(&s->psy, avctx, 2, sizes, lengths);
>> ? ? s->psypp = ff_psy_preprocess_init(avctx);
>> ? ? s->coder = &ff_aac_coders[2];
>> + ? ?s->psywindows = av_malloc(avctx->channels * sizeof(*s->psywindows));
>> + ? ?if (!s->psywindows)
>> + ? ? ? ?return AVERROR(ENOMEM);
>>
>> ? ? s->lambda = avctx->global_quality ? avctx->global_quality : 120;
>>
>> @@ -486,7 +489,7 @@ static int aac_encode_frame(AVCodecContext *avctx,
>> ? ? int i, j, chans, tag, start_ch;
>> ? ? const uint8_t *chan_map = aac_chan_configs[avctx->channels-1];
>> ? ? int chan_el_counter[4];
>> - ? ?FFPsyWindowInfo windows[avctx->channels];
>> + ? ?FFPsyWindowInfo *windows = s->psywindows;
>>
>> ? ? if (s->last_frame)
>> ? ? ? ? return 0;
>> @@ -625,6 +628,7 @@ static av_cold int aac_encode_end(AVCodecContext *avctx)
>> ? ? ff_mdct_end(&s->mdct128);
>> ? ? ff_psy_end(&s->psy);
>> ? ? ff_psy_preprocess_end(s->psypp);
>> + ? ?av_freep(&s->psywindows);
>> ? ? av_freep(&s->samples);
>> ? ? av_freep(&s->cpe);
>> ? ? return 0;
>> diff --git a/libavcodec/aacenc.h b/libavcodec/aacenc.h
>> index 86c68d3..1f730c0 100644
>> --- a/libavcodec/aacenc.h
>> +++ b/libavcodec/aacenc.h
>> @@ -60,6 +60,7 @@ typedef struct AACEncContext {
>> ? ? ChannelElement *cpe; ? ? ? ? ? ? ? ? ? ? ? ? ///< channel elements
>> ? ? FFPsyContext psy;
>> ? ? struct FFPsyPreprocessContext* psypp;
>> + ? ?FFPsyWindowInfo *psywindows;
>> ? ? AACCoefficientsEncoder *coder;
>> ? ? int cur_channel;
>> ? ? int last_frame;
>> --
>
> Is there a reason you didn't just make this fixed sized on the stack?

The struct is rather large, and I had the idea AAC allows quite a few
channels.

> aacenc only supports 6 channels.

Is that the spec or just us?

>  169     if (avctx->channels > 6) {
>  170         av_log(avctx, AV_LOG_ERROR, "Unsupported number of
> channels: %d\n", avctx->channels);
>  171         return -1;
>  172     }

I must have missed that one.  All I saw was a warning for channels > 5.

-- 
M?ns Rullg?rd
mans at mansr.com