[FFmpeg-devel] [PATCH] lavfi/vf_deshake: fix segfaults #2443

[Carl Eugen Hoyos]

On Monday 15 April 2013 02:46:44 pm Michael Niedermayer wrote:
> > -        "psadbw (%2), %%xmm0            \n\t"
> > -        "psadbw (%2, %4), %%xmm1        \n\t"
> > +        "movdqu (%2), %%xmm2            \n\t"
> > +        "movdqu (%2, %4), %%xmm3        \n\t"
> > +        "psadbw %%xmm2, %%xmm0          \n\t"
> > +        "psadbw %%xmm3, %%xmm1          \n\t"
>
> The input to this function must be aligned to the blocksize of 8 or 16
> the caller is buggy if it calls this function on misaligned data,
> the alignment requirement exists to maximize speed and to simplify
> SIMD implementions

Is attached an acceptable solution (with realloc_fast if size changes are 
allowed)?
(Contains the edge fix that is in github)

Carl Eugen
-------------- next part --------------
diff --git a/libavfilter/deshake.h b/libavfilter/deshake.h
index c24090e..f2dad5e 100644
--- a/libavfilter/deshake.h
+++ b/libavfilter/deshake.h
@@ -75,9 +75,10 @@ typedef struct {
 typedef struct {
     const AVClass *class;
     AVFrame *ref;              ///< Previous frame
+    uint8_t *src_copy;         ///< Holds an aligned copy of the motion search block
     int rx;                    ///< Maximum horizontal shift
     int ry;                    ///< Maximum vertical shift
-    int edge;                  ///< Edge fill method
+    enum FillMethod edge;      ///< Edge fill method
     int blocksize;             ///< Size of blocks to compare
     int contrast;              ///< Contrast threshold
     int search;                ///< Motion search method
diff --git a/libavfilter/vf_deshake.c b/libavfilter/vf_deshake.c
index 01ef721..334735a 100644
--- a/libavfilter/vf_deshake.c
+++ b/libavfilter/vf_deshake.c
@@ -259,6 +259,12 @@ static void find_motion(DeshakeContext *deshake, uint8_t *src1, uint8_t *src2,
             counts[x][y] = 0;
         }
     }
+    if (deshake->rx & 0xF) {
+        if (!deshake->src_copy)
+            deshake->src_copy = av_malloc(height * stride + MAX_R + FF_INPUT_BUFFER_PADDING_SIZE);
+        memcpy(deshake->src_copy + MAX_R - deshake->rx, src1, height * stride);
+        src1 = deshake->src_copy + MAX_R - deshake->rx;
+    }
 
     pos = 0;
     // Find motion for every block and store the motion vector in the counts
@@ -426,6 +432,7 @@ static av_cold void uninit(AVFilterContext *ctx)
     if (deshake->avctx)
         avcodec_close(deshake->avctx);
     av_freep(&deshake->avctx);
+    av_freep(&deshake->src_copy);
 }
 
 static int filter_frame(AVFilterLink *link, AVFrame *in)