[FFmpeg-devel] [PATCH 1/2] avutil/mathematics: return INT64_MIN (=AV_NOPTS_VALUE) from av_rescale_rnd() for overflows
Michael Niedermayer
michaelni at gmx.at
Tue Dec 1 13:50:13 CET 2015
From: Michael Niedermayer <michael at niedermayer.cc>
Fixes integer overflow
Fixes: mozilla bug 1229167
Found-by: Tyson Smith
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
libavutil/mathematics.c | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/libavutil/mathematics.c b/libavutil/mathematics.c
index 689325f..c12c73e 100644
--- a/libavutil/mathematics.c
+++ b/libavutil/mathematics.c
@@ -72,7 +72,7 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
}
if (a < 0)
- return -av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1));
+ return -(uint64_t)av_rescale_rnd(-FFMAX(a, -INT64_MAX), b, c, rnd ^ ((rnd >> 1) & 1));
if (rnd == AV_ROUND_NEAR_INF)
r = c / 2;
@@ -82,8 +82,13 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
if (b <= INT_MAX && c <= INT_MAX) {
if (a <= INT_MAX)
return (a * b + r) / c;
- else
- return a / c * b + (a % c * b + r) / c;
+ else {
+ int64_t ad = a / c;
+ int64_t a2 = (a % c * b + r) / c;
+ if (ad >= INT32_MAX && ad > (INT64_MAX - a2) / b)
+ return INT64_MIN;
+ return ad * b + a2;
+ }
} else {
#if 1
uint64_t a0 = a & 0xFFFFFFFF;
@@ -107,6 +112,8 @@ int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd)
t1++;
}
}
+ if (t1 > INT64_MAX)
+ return INT64_MIN;
return t1;
}
#else
--
1.7.9.5
More information about the ffmpeg-devel
mailing list