[FFmpeg-cvslog] avcodec/avcodec: Check for more invalid channel layouts

[Andreas Rheinhardt]

ffmpeg | branch: master | Andreas Rheinhardt <andreas.rheinhardt at outlook.com> | Fri Sep 16 02:27:25 2022 +0200| [a06a2d89434abd8feb760a943924f3cfacedfda6] | committer: Andreas Rheinhardt

avcodec/avcodec: Check for more invalid channel layouts

In particular, check the provided channel layout for encoders
without AVCodec.ch_layouts set. This fixes an infinite loop
in the WavPack encoder (and maybe other issues in other encoders
as well) in case the channel count is zero.

Reviewed-by: James Almer <jamrial at gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a06a2d89434abd8feb760a943924f3cfacedfda6
---

 libavcodec/avcodec.c | 14 ++++++++++++++
 libavcodec/decode.c  |  5 -----
 libavcodec/encode.c  |  5 -----
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index 81f30a4a58..737c4bcd04 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -246,6 +246,20 @@ FF_DISABLE_DEPRECATION_WARNINGS
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
 
+    /* AV_CODEC_CAP_CHANNEL_CONF is a decoder-only flag; so the code below
+     * in particular checks that nb_channels is set for all audio encoders. */
+    if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && !avctx->ch_layout.nb_channels
+        && !(codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)) {
+        av_log(avctx, AV_LOG_ERROR, "%s requires channel layout to be set\n",
+               av_codec_is_decoder(codec) ? "Decoder" : "Encoder");
+        ret = AVERROR(EINVAL);
+        goto free_and_end;
+    }
+    if (avctx->ch_layout.nb_channels && !av_channel_layout_check(&avctx->ch_layout)) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid channel layout\n");
+        ret = AVERROR(EINVAL);
+        goto free_and_end;
+    }
     if (avctx->ch_layout.nb_channels > FF_SANE_NB_CHANNELS) {
         av_log(avctx, AV_LOG_ERROR, "Too many channels: %d\n", avctx->ch_layout.nb_channels);
         ret = AVERROR(EINVAL);
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 2961705c9d..6be2d3d6ed 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -1595,11 +1595,6 @@ FF_DISABLE_DEPRECATION_WARNINGS
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
 
-    if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && avctx->ch_layout.nb_channels == 0 &&
-        !(avctx->codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)) {
-        av_log(avctx, AV_LOG_ERROR, "Decoder requires channel count but channels not set\n");
-        return AVERROR(EINVAL);
-    }
     if (avctx->codec->max_lowres < avctx->lowres || avctx->lowres < 0) {
         av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n",
                avctx->codec->max_lowres);
diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index ffac2c1ff3..9f4804cd69 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -602,11 +602,6 @@ static int encode_preinit_audio(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
     if (avctx->codec->ch_layouts) {
-        if (!av_channel_layout_check(&avctx->ch_layout)) {
-            av_log(avctx, AV_LOG_WARNING, "Channel layout not specified correctly\n");
-            return AVERROR(EINVAL);
-        }
-
         for (i = 0; avctx->codec->ch_layouts[i].nb_channels; i++) {
             if (!av_channel_layout_compare(&avctx->ch_layout, &avctx->codec->ch_layouts[i]))
                 break;