[FFmpeg-cvslog] pngenc: make max_packet_size 64bit check check it.

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Thu Mar 22 21:09:03 2012 +0100| [afc0cc22e17e26b99d3d662b52352945e6e7f52a] | committer: Michael Niedermayer

pngenc: make max_packet_size 64bit check check it.

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=afc0cc22e17e26b99d3d662b52352945e6e7f52a
---

 libavcodec/pngenc.c |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/libavcodec/pngenc.c b/libavcodec/pngenc.c
index 5b2c2e4..7ec80de 100644
--- a/libavcodec/pngenc.c
+++ b/libavcodec/pngenc.c
@@ -219,7 +219,8 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     PNGEncContext *s = avctx->priv_data;
     AVFrame * const p= &s->picture;
     int bit_depth, color_type, y, len, row_size, ret, is_progressive;
-    int bits_per_pixel, pass_row_size, enc_row_size, max_packet_size;
+    int bits_per_pixel, pass_row_size, enc_row_size;
+    int64_t max_packet_size;
     int compression_level;
     uint8_t *ptr, *top;
     uint8_t *crow_base = NULL, *crow_buf, *crow;
@@ -286,9 +287,11 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         return -1;
 
     enc_row_size    = deflateBound(&s->zstream, row_size);
-    max_packet_size = avctx->height * (enc_row_size +
+    max_packet_size = avctx->height * (int64_t)(enc_row_size +
                                        ((enc_row_size + IOBUF_SIZE - 1) / IOBUF_SIZE) * 12)
                       + FF_MIN_BUFFER_SIZE;
+    if (max_packet_size > INT_MAX)
+        return AVERROR(ENOMEM);
     if ((ret = ff_alloc_packet2(avctx, pkt, max_packet_size)) < 0) {
         return ret;
     }