[FFmpeg-cvslog] avcodec/ffv1enc: Try to choose slice count so that slice packet sizes are within the supported size

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Thu Jun 15 20:46:32 2017 +0200| [4147bb905346570cab4c3d106bb94e08e4bb5a15] | committer: Michael Niedermayer

avcodec/ffv1enc: Try to choose slice count so that slice packet sizes are within the supported size

Fixes assertion failure

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4147bb905346570cab4c3d106bb94e08e4bb5a15
---

 libavcodec/ffv1enc.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/libavcodec/ffv1enc.c b/libavcodec/ffv1enc.c
index 7f31606775..e59d540737 100644
--- a/libavcodec/ffv1enc.c
+++ b/libavcodec/ffv1enc.c
@@ -848,10 +848,17 @@ FF_ENABLE_DEPRECATION_WARNINGS
     }
 
     if (s->version > 1) {
+        int plane_count = 1 + 2*s->chroma_planes + s->transparency;
         s->num_v_slices = (avctx->width > 352 || avctx->height > 288 || !avctx->slices) ? 2 : 1;
-        for (; s->num_v_slices < 9; s->num_v_slices++) {
+        for (; s->num_v_slices < 32; s->num_v_slices++) {
             for (s->num_h_slices = s->num_v_slices; s->num_h_slices < 2*s->num_v_slices; s->num_h_slices++) {
-                if (avctx->slices == s->num_h_slices * s->num_v_slices && avctx->slices <= 64 || !avctx->slices)
+                int maxw = (avctx->width  + s->num_h_slices - 1) / s->num_h_slices;
+                int maxh = (avctx->height + s->num_v_slices - 1) / s->num_v_slices;
+                if (s->num_h_slices > avctx->width || s->num_v_slices > avctx->height)
+                    continue;
+                if (maxw * maxh * (int64_t)(s->bits_per_raw_sample+1) * plane_count > 8<<24)
+                    continue;
+                if (avctx->slices == s->num_h_slices * s->num_v_slices && avctx->slices <= MAX_SLICES || !avctx->slices)
                     goto slices_ok;
             }
         }