[FFmpeg-devel] [PATCH]Avoid an assertion failure in ff_init_vlc_sparse

[Derek Buitenhuis]

On 7/12/2013 3:13 PM, Carl Eugen Hoyos wrote:
> diff --git a/libavcodec/bitstream.c b/libavcodec/bitstream.c
> index bbd9491..70801a5 100644
> --- a/libavcodec/bitstream.c
> +++ b/libavcodec/bitstream.c
> @@ -282,7 +282,8 @@ int ff_init_vlc_sparse(VLC *vlc, int nb_bits, int nb_codes,
>                                   codes, codes_wrap, codes_size,
>                                   symbols, symbols_wrap, symbols_size,
>                                   flags & ~INIT_VLC_USE_NEW_STATIC);
> -        av_assert0(ret >= 0);
> +        if (ret < 0)
> +            return ret;
>          av_assert0(dyn_vlc.table_size <= vlc->table_allocated);
>          if (dyn_vlc.table_size < vlc->table_allocated)
>              av_log(NULL, AV_LOG_ERROR, "needed %d had %d\n", dyn_vlc.table_size, vlc->table_allocated);

This functional change should be in a separate commit. So
should each change to other generic ff_ functions. 

> -        ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
> +        ret = ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
>                             ccitt_codes_lens[i], 1, 1,
>                             ccitt_codes_bits[i], 1, 1,
>                             ccitt_syms, 2, 2,
>                             INIT_VLC_USE_NEW_STATIC);
> +        if (ret < 0)
> +            return ret;
>      }

Please properly re-indent things like these after, in a separate commit.
e.g.:

>      if(!aic_top_vlc.bits)
> -        rv40_init_tables();
> +        ret = rv40_init_tables();
> +    if (ret < 0)
> +        return ret;

This is wrong. ret is only set if !aic_top_vlc.bits.

>  static av_cold int tiff_init(AVCodecContext *avctx)
>  {
>      TiffContext *s = avctx->priv_data;
> +    int ret;
>  
>      s->width  = 0;
>      s->height = 0;
>      s->avctx  = avctx;
>      ff_lzw_decode_open(&s->lzw);
> -    ff_ccitt_unpack_init();
> +    ret = ff_ccitt_unpack_init();
>  
> -    return 0;
> +    return ret;
>  }

return ff_ccitt_unpack_init()

- Derek