[FFmpeg-devel] [PATCH 2/2] avcodec/h2645_parse: Don't report negative NAL unit sizes

[Andreas Rheinhardt]

This could happen if a four byte NAL unit size is encountered that is
bigger than INT_MAX when read as an uint32_t. This has been changed:
The size is now treated as uint32_t (so that no cast is needed any more
to prevent undefined behaviour when shifting) throughout the code.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at gmail.com>
---
 libavcodec/h2645_parse.h | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libavcodec/h2645_parse.h b/libavcodec/h2645_parse.h
index 3e47f86c53..fd2c945c54 100644
--- a/libavcodec/h2645_parse.h
+++ b/libavcodec/h2645_parse.h
@@ -118,18 +118,19 @@ void ff_h2645_packet_uninit(H2645Packet *pkt);
 static inline int get_nalsize(int nal_length_size, const uint8_t *buf,
                               int buf_size, int *buf_index, void *logctx)
 {
-    int i, nalsize = 0;
+    uint32_t nalsize = 0;
 
     if (*buf_index >= buf_size - nal_length_size) {
         // the end of the buffer is reached, refill it
         return AVERROR(EAGAIN);
     }
 
-    for (i = 0; i < nal_length_size; i++)
-        nalsize = ((unsigned)nalsize << 8) | buf[(*buf_index)++];
-    if (nalsize <= 0 || nalsize > buf_size - *buf_index) {
+    for (int i = 0; i < nal_length_size; i++)
+        nalsize = (nalsize << 8) | buf[(*buf_index)++];
+    if (!nalsize || nalsize > buf_size - *buf_index) {
         av_log(logctx, AV_LOG_ERROR,
-               "Invalid NAL unit size (%d > %d).\n", nalsize, buf_size - *buf_index);
+               "Invalid NAL unit size (%"PRIu32" > %d).\n",
+               nalsize, buf_size - *buf_index);
         return AVERROR_INVALIDDATA;
     }
     return nalsize;
-- 
2.20.1