[FFmpeg-devel] [PATCH] avfilter/vidstab: set bytesPerPixel only for packed formats.

[Gyan Doshi]

Patch for ticket #6736.

Regards,
Gyan
-------------- next part --------------
From a3161609f7e9a583506869df3da7a19e37629f32 Mon Sep 17 00:00:00 2001
From: Gyan Doshi <gyandoshi at gmail.com>
Date: Sat, 23 Dec 2017 16:14:25 +0530
Subject: [PATCH] avfilter/vidstab: set bytesPerPixel only for packed formats.

libvidstab introduced this variable only for packed formats but in
vf_vidstab*.c, it's set for all inputs. So the filter fails for YUV422/444P streams.

Fixes #6736.
---
 libavfilter/vf_vidstabdetect.c    | 3 ++-
 libavfilter/vf_vidstabtransform.c | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavfilter/vf_vidstabdetect.c b/libavfilter/vf_vidstabdetect.c
index 63a178a0c2..fd7ff3be24 100644
--- a/libavfilter/vf_vidstabdetect.c
+++ b/libavfilter/vf_vidstabdetect.c
@@ -107,10 +107,11 @@ static int config_input(AVFilterLink *inlink)
     VSMotionDetect* md = &(s->md);
     VSFrameInfo fi;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
 
     vsFrameInfoInit(&fi, inlink->w, inlink->h,
                     ff_av2vs_pixfmt(ctx, inlink->format));
-    if (fi.bytesPerPixel != av_get_bits_per_pixel(desc)/8) {
+    if (!is_planar && fi.bytesPerPixel != av_get_bits_per_pixel(desc)/8) {
         av_log(ctx, AV_LOG_ERROR, "pixel-format error: wrong bits/per/pixel, please report a BUG");
         return AVERROR(EINVAL);
     }
diff --git a/libavfilter/vf_vidstabtransform.c b/libavfilter/vf_vidstabtransform.c
index 11a0e3d512..d1ec1391cb 100644
--- a/libavfilter/vf_vidstabtransform.c
+++ b/libavfilter/vf_vidstabtransform.c
@@ -146,6 +146,7 @@ static int config_input(AVFilterLink *inlink)
     FILE *f;
 
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
 
     VSTransformData *td = &(tc->td);
 
@@ -161,7 +162,7 @@ static int config_input(AVFilterLink *inlink)
         return AVERROR(EINVAL);
     }
 
-    if (fi_src.bytesPerPixel != av_get_bits_per_pixel(desc)/8 ||
+    if ((!is_planar && fi_src.bytesPerPixel != av_get_bits_per_pixel(desc)/8) ||
         fi_src.log2ChromaW != desc->log2_chroma_w ||
         fi_src.log2ChromaH != desc->log2_chroma_h) {
         av_log(ctx, AV_LOG_ERROR, "pixel-format error: bpp %i<>%i  ",
-- 
2.11.1.windows.1