[FFmpeg-cvslog] hqdn3d: Fix out of array read in LOWPASS

[Loren Merritt]

ffmpeg | branch: master | Loren Merritt <lorenm at u.washington.edu> | Sat Sep 22 01:43:16 2012 +0200| [5b3c1aecb253828d09fa9825c5a4aed97badf086] | committer: Anton Khirnov

hqdn3d: Fix out of array read in LOWPASS

CC:libav-stable at libav.org
Signed-off-by: Anton Khirnov <anton at khirnov.net>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=5b3c1aecb253828d09fa9825c5a4aed97badf086
---

 libavfilter/vf_hqdn3d.c       |    8 ++++----
 libavfilter/x86/vf_hqdn3d.asm |    2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c
index 50e904b..eb0ea65 100644
--- a/libavfilter/vf_hqdn3d.c
+++ b/libavfilter/vf_hqdn3d.c
@@ -37,10 +37,10 @@
 #include "vf_hqdn3d.h"
 
 #define LUT_BITS (depth==16 ? 8 : 4)
-#define RIGHTSHIFT(a,b) (((a)+(((1<<(b))-1)>>1))>>(b))
-#define LOAD(x) ((depth==8 ? src[x] : AV_RN16A(src+(x)*2)) << (16-depth))
-#define STORE(x,val) (depth==8 ? dst[x] = RIGHTSHIFT(val, 16-depth)\
-                    : AV_WN16A(dst+(x)*2, RIGHTSHIFT(val, 16-depth)))
+#define LOAD(x) (((depth == 8 ? src[x] : AV_RN16A(src + (x) * 2)) << (16 - depth))\
+                 + (((1 << (16 - depth)) - 1) >> 1))
+#define STORE(x,val) (depth == 8 ? dst[x] = (val) >> (16 - depth) : \
+                                   AV_WN16A(dst + (x) * 2, (val) >> (16 - depth)))
 
 av_always_inline
 static uint32_t lowpass(int prev, int cur, int16_t *coef, int depth)
diff --git a/libavfilter/x86/vf_hqdn3d.asm b/libavfilter/x86/vf_hqdn3d.asm
index dee2c96..02632a1 100644
--- a/libavfilter/x86/vf_hqdn3d.asm
+++ b/libavfilter/x86/vf_hqdn3d.asm
@@ -39,6 +39,7 @@ SECTION .text
 %endif
 %if %3 != 16
     shl    %1, 16-%3
+    add    %1, (1<<(15-%3))-1
 %endif
 %endmacro
 
@@ -86,7 +87,6 @@ ALIGN 16
     mov       [frameantq+xq*2], t0w
     movifnidn dstq, dstmp
 %if %1 != 16
-    add    t0d, (1<<(15-%1))-1
     shr    t0d, 16-%1 ; could eliminate this by storing from t0h, but only with some contraints on register allocation
 %endif
 %if %1 == 8