[FFmpeg-devel] [PATCH 4/5] mlpenc: improve lpc filtering

[Jai Luthra]

* fix a possible memory leak (apply_filter returned before freeing)
* use apply_filters in process_major_frame
* revert back to checking bounds with 24 bitdepth, as huff offset takes
care of it

Signed-off-by: Jai Luthra <me at jailuthra.in>
---
 libavcodec/mlpenc.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/libavcodec/mlpenc.c b/libavcodec/mlpenc.c
index 41030f6f07..9805e7ff23 100644
--- a/libavcodec/mlpenc.c
+++ b/libavcodec/mlpenc.c
@@ -1799,7 +1799,7 @@ static void determine_bits(MLPEncodeContext *ctx)
 
 /** Applies the filter to the current samples, and saves the residual back
  *  into the samples buffer. If the filter is too bad and overflows the
- *  maximum amount of bits allowed (16 or 24), the samples buffer is left as is and
+ *  maximum amount of bits allowed (24), the samples buffer is left as is and
  *  the function returns -1.
  */
 static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
@@ -1812,7 +1812,7 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
     unsigned int number_of_samples = ctx->number_of_samples;
     unsigned int filter_shift = fp[FIR]->shift;
     int filter;
-    int i;
+    int i, ret = 0;
 
     for (i = 0; i < NUM_FILTERS; i++) {
         unsigned int size = ctx->number_of_samples;
@@ -1835,7 +1835,7 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
         int32_t sample = *sample_buffer;
         unsigned int order;
         int64_t accum = 0;
-        int32_t residual;
+        int64_t residual;
 
         for (filter = 0; filter < NUM_FILTERS; filter++) {
             int32_t *fcoeff = ctx->cur_channel_params[channel].coeff[filter];
@@ -1847,11 +1847,13 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
         accum  >>= filter_shift;
         residual = sample - (accum & mask);
 
-        if (residual < SAMPLE_MIN(ctx->wordlength) || residual > SAMPLE_MAX(ctx->wordlength))
-            return -1;
+        if (residual < SAMPLE_MIN(24) || residual > SAMPLE_MAX(24)) {
+            ret = -1;
+            goto free_and_return;
+        }
 
         filter_state_buffer[FIR][i] = sample;
-        filter_state_buffer[IIR][i] = residual;
+        filter_state_buffer[IIR][i] = (int32_t) residual;
 
         sample_buffer += ctx->num_channels;
     }
@@ -1863,11 +1865,12 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
         sample_buffer += ctx->num_channels;
     }
 
+free_and_return:
     for (i = 0; i < NUM_FILTERS; i++) {
         av_freep(&filter_state_buffer[i]);
     }
 
-    return 0;
+    return ret;
 }
 
 static void apply_filters(MLPEncodeContext *ctx)
@@ -2209,8 +2212,7 @@ static void process_major_frame(MLPEncodeContext *ctx)
         generate_2_noise_channels(ctx);
         rematrix_channels        (ctx);
 
-        for (channel = rh->min_channel; channel <= rh->max_channel; channel++)
-            apply_filter(ctx, channel);
+        apply_filters(ctx);
     }
 }
 
-- 
2.22.0