[FFmpeg-cvslog] avcodec/libx264: Check for overflow if necessary

[Andreas Rheinhardt]

ffmpeg | branch: master | Andreas Rheinhardt <andreas.rheinhardt at outlook.com> | Sun Nov  7 14:50:27 2021 +0100| [6f1130be7880dc9e5e7c812f377dc3e480580704] | committer: Andreas Rheinhardt

avcodec/libx264: Check for overflow if necessary

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=6f1130be7880dc9e5e7c812f377dc3e480580704
---

 libavcodec/libx264.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 0766b4a950..8711c72131 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -138,13 +138,23 @@ static int encode_nals(AVCodecContext *ctx, AVPacket *pkt,
 {
     X264Context *x4 = ctx->priv_data;
     uint8_t *p;
-    int i, size = x4->sei_size, ret;
+    uint64_t size = x4->sei_size;
+    int i;
+    int ret;
 
     if (!nnal)
         return 0;
 
-    for (i = 0; i < nnal; i++)
+    for (int i = 0; i < nnal; i++) {
         size += nals[i].i_payload;
+        /* ff_get_encode_buffer() accepts an int64_t and
+         * so we need to make sure that no overflow happens before
+         * that. With 32bit ints this is automatically true. */
+#if INT_MAX > INT64_MAX / INT_MAX - 1
+        if ((int64_t)size < 0)
+            return AVERROR(ERANGE);
+#endif
+    }
 
     if ((ret = ff_get_encode_buffer(ctx, pkt, size, 0)) < 0)
         return ret;