[FFmpeg-cvslog] h264: Skip odd NALs in extradata, prevent undefined behavior

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Sun Nov 18 14:23:06 2012 +0100| [4fecc3cf0998927456a9f8d8334587dd64154ec5] | committer: Michael Niedermayer

h264: Skip odd NALs in extradata, prevent undefined behavior

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4fecc3cf0998927456a9f8d8334587dd64154ec5
---

 libavcodec/h264.c |   25 ++++++++++++++++++++++++-
 libavcodec/h264.h |    1 +
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index fbc94b8..7d348d2 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1002,7 +1002,7 @@ static av_cold void common_init(H264Context *h)
     memset(h->pps.scaling_matrix8, 16, 2 * 64 * sizeof(uint8_t));
 }
 
-int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size)
+static int ff_h264_decode_extradata_internal(H264Context *h, const uint8_t *buf, int size)
 {
     AVCodecContext *avctx = h->s.avctx;
 
@@ -1059,6 +1059,15 @@ int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size)
     return size;
 }
 
+int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size)
+{
+    int ret;
+    h->decoding_extradata = 1;
+    ret = ff_h264_decode_extradata_internal(h, buf, size);
+    h->decoding_extradata = 0;
+    return ret;
+}
+
 av_cold int ff_h264_decode_init(AVCodecContext *avctx)
 {
     H264Context *h = avctx->priv_data;
@@ -3849,6 +3858,20 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size)
 
 again:
             err = 0;
+
+            if (h->decoding_extradata) {
+                switch (hx->nal_unit_type) {
+                case NAL_IDR_SLICE:
+                case NAL_SLICE:
+                case NAL_DPA:
+                case NAL_DPB:
+                case NAL_DPC:
+                case NAL_AUXILIARY_SLICE:
+                    av_log(h->s.avctx, AV_LOG_WARNING, "Ignoring NAL %d in global header\n", hx->nal_unit_type);
+                    hx->nal_unit_type = NAL_FILLER_DATA;
+                }
+            }
+
             switch (hx->nal_unit_type) {
             case NAL_IDR_SLICE:
                 if (h->nal_unit_type != NAL_IDR_SLICE) {
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 298f983..00a1fc3 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -447,6 +447,7 @@ typedef struct H264Context {
     int nal_unit_type;
     uint8_t *rbsp_buffer[2];
     unsigned int rbsp_buffer_size[2];
+    int decoding_extradata;
 
     /**
      * Used to parse AVC variant of h264