[FFmpeg-devel] [PATCH v2] libavformat/vvc: Make probe more conservative

[post at frankplowman.com]

From: Frank Plowman <post at frankplowman.com>

Reduce false positives for VVC files by adding additional checks in
`vvc_probe`. Specifically, `nuh_temporal_id_plus1` is tested for valid
values in extra cases depending on the NAL unit type, as per ITU-T H.266
section 7.4.2.2.

Resolves trac #10703.

Signed-off-by: Frank Plowman <post at frankplowman.com>
---
 libavformat/vvcdec.c | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/libavformat/vvcdec.c b/libavformat/vvcdec.c
index 31c5ae1f14..0a0e24b71c 100644
--- a/libavformat/vvcdec.c
+++ b/libavformat/vvcdec.c
@@ -24,6 +24,22 @@
 #include "avformat.h"
 #include "rawdec.h"
 
+static int check_temporal_id(uint8_t nuh_temporal_id_plus1, int type)
+{
+    if (nuh_temporal_id_plus1 == 0)
+        return 0;
+
+    if (nuh_temporal_id_plus1 != 1) {
+        if (type >= VVC_IDR_W_RADL && type <= VVC_RSV_IRAP_11
+                || type == VVC_DCI_NUT || type == VVC_OPI_NUT
+                || type == VVC_VPS_NUT || type == VVC_SPS_NUT
+                || type == VVC_EOS_NUT || type == VVC_EOB_NUT)
+            return 0;
+    }
+
+    return 1;
+}
+
 static int vvc_probe(const AVProbeData *p)
 {
     uint32_t code = -1;
@@ -39,7 +55,7 @@ static int vvc_probe(const AVProbeData *p)
             if (code & 0x80) // forbidden_zero_bit
                 return 0;
 
-            if ((nal2 & 0x7) == 0) // nuh_temporal_id_plus1
+            if (!check_temporal_id(nal2 & 0x7, type))
                 return 0;
 
             switch (type) {
-- 
2.43.0