[FFmpeg-cvslog] avdevice/xcbgrab: check if frame size fits in INT_MAX

[Marton Balint]

ffmpeg | branch: master | Marton Balint <cus at passwd.hu> | Tue Dec 31 01:24:32 2019 +0100| [e931119a41d0c48d1c544af89768b119b13feb4d] | committer: Marton Balint

avdevice/xcbgrab: check if frame size fits in INT_MAX

Also fixes a possible overflow and sets stream bitrate.

Signed-off-by: Marton Balint <cus at passwd.hu>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e931119a41d0c48d1c544af89768b119b13feb4d
---

 libavdevice/xcbgrab.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/libavdevice/xcbgrab.c b/libavdevice/xcbgrab.c
index 06b486a536..6f6b2dbf15 100644
--- a/libavdevice/xcbgrab.c
+++ b/libavdevice/xcbgrab.c
@@ -542,6 +542,7 @@ static int create_stream(AVFormatContext *s)
     AVStream *st      = avformat_new_stream(s, NULL);
     xcb_get_geometry_cookie_t gc;
     xcb_get_geometry_reply_t *geo;
+    int64_t frame_size_bits;
     int ret;
 
     if (!st)
@@ -580,17 +581,18 @@ static int create_stream(AVFormatContext *s)
     c->frame_duration = av_rescale_q(1, c->time_base, AV_TIME_BASE_Q);
     c->time_frame = av_gettime();
 
-    st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
-    st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
-    st->codecpar->width      = c->width;
-    st->codecpar->height     = c->height;
-
     ret = pixfmt_from_pixmap_format(s, geo->depth, &st->codecpar->format, &c->bpp);
     free(geo);
     if (ret < 0)
         return ret;
 
-    c->frame_size = c->width * c->height * c->bpp / 8;
+    frame_size_bits = (int64_t)c->width * c->height * c->bpp;
+    if (frame_size_bits / 8 + AV_INPUT_BUFFER_PADDING_SIZE > INT_MAX) {
+        av_log(s, AV_LOG_ERROR, "Captured area is too large\n");
+        return AVERROR_PATCHWELCOME;
+    }
+    c->frame_size = frame_size_bits / 8;
+
 #if CONFIG_LIBXCB_SHM
     c->shm_pool = av_buffer_pool_init2(c->frame_size + AV_INPUT_BUFFER_PADDING_SIZE,
                                            c->conn, allocate_shm_buffer, NULL);
@@ -598,6 +600,12 @@ static int create_stream(AVFormatContext *s)
         return AVERROR(ENOMEM);
 #endif
 
+    st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
+    st->codecpar->width      = c->width;
+    st->codecpar->height     = c->height;
+    st->codecpar->bit_rate   = av_rescale(frame_size_bits, st->avg_frame_rate.num, st->avg_frame_rate.den);
+
     return ret;
 }