[FFmpeg-devel] [PATCH] avfilter/vf_overlay: add premultiplied alpha mode

Marton Balint cus at passwd.hu
Sat Dec 2 12:19:10 EET 2017



On Sat, 2 Dec 2017, Paul B Mahol wrote:

> On 12/2/17, Marton Balint <cus at passwd.hu> wrote:
>>
>>
>> On Fri, 1 Dec 2017, Paul B Mahol wrote:
>>
>>> Signed-off-by: Paul B Mahol <onemda at gmail.com>
>>> ---
>>> doc/filters.texi         |   4 ++
>>> libavfilter/vf_overlay.c | 160
>>> ++++++++++++++++++++++++++++++++++++++++-------
>>> 2 files changed, 140 insertions(+), 24 deletions(-)
>>>
>>> diff --git a/doc/filters.texi b/doc/filters.texi
>>> index f7c371592f..0699728b7e 100644
>>> --- a/doc/filters.texi
>>> +++ b/doc/filters.texi
>>> @@ -11328,6 +11328,10 @@ Default value is @samp{yuv420}.
>>>
>>> @item repeatlast
>>> See @ref{framesync}.
>>> +
>>> + at item alpha
>>> +Set format of alpha, it can be @var{straight} or @var{premultiplied}.
>>> +Default is @var{straight}.
>>> @end table
>>
>> Does this mean that not only overlay, but input and output is also
>> considered as premultiplied? Maybe better to clarify this in the docs.
>
> I done it for overlay only alpha. For rest I'm little confused what needs doing.

Ok, then just write in the docs:

"Set format of alpha of the overlaid video."

>
>>
>> Have you measured the slowdown caused by the extra condition in the pixel
>> blending code? If it is more than 1-2%, then some DEFINE magic (or
>> making the blend_image functions inline?) would be better IMHO.
>
> It should already be inlined. Thats why there are so many functions.

Not all of them, blend_image_packed_rgb misses an av_always_inline.

Regards,
Marton


>>>
>>> The @option{x}, and @option{y} expressions can contain the following
>>> diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
>>> index 5bf3d66cf1..8206dae454 100644
>>> --- a/libavfilter/vf_overlay.c
>>> +++ b/libavfilter/vf_overlay.c
>>> @@ -109,6 +109,7 @@ typedef struct OverlayContext {
>>>     uint8_t overlay_rgba_map[4];
>>>     uint8_t overlay_has_alpha;
>>>     int format;                 ///< OverlayFormat
>>> +    int alpha_format;
>>>     int eval_mode;              ///< EvalMode
>>>
>>>     FFFrameSync fs;
>>> @@ -403,7 +404,8 @@ static int config_output(AVFilterLink *outlink)
>>>
>>> static void blend_image_packed_rgb(AVFilterContext *ctx,
>>>                                    AVFrame *dst, const AVFrame *src,
>>> -                                   int main_has_alpha, int x, int y)
>>> +                                   int main_has_alpha, int x, int y,
>>> +                                   int is_straight)
>>> {
>>>     OverlayContext *s = ctx->priv;
>>>     int i, imax, j, jmax;
>>> @@ -454,9 +456,9 @@ static void blend_image_packed_rgb(AVFilterContext
>>> *ctx,
>>>             default:
>>>                 // main_value = main_value * (1 - alpha) + overlay_value *
>>> alpha
>>>                 // since alpha is in the range 0-255, the result must
>>> divided by 255
>>> -                d[dr] = FAST_DIV255(d[dr] * (255 - alpha) + S[sr] *
>>> alpha);
>>> -                d[dg] = FAST_DIV255(d[dg] * (255 - alpha) + S[sg] *
>>> alpha);
>>> -                d[db] = FAST_DIV255(d[db] * (255 - alpha) + S[sb] *
>>> alpha);
>>> +                d[dr] = is_straight ? FAST_DIV255(d[dr] * (255 - alpha) +
>>> S[sr] * alpha) : FAST_DIV255(d[dr] * (255 - alpha) + S[sr]);
>>> +                d[dg] = is_straight ? FAST_DIV255(d[dg] * (255 - alpha) +
>>> S[sg] * alpha) : FAST_DIV255(d[dr] * (255 - alpha) + S[sr]);
>>> +                d[db] = is_straight ? FAST_DIV255(d[db] * (255 - alpha) +
>>> S[sb] * alpha) : FAST_DIV255(d[dr] * (255 - alpha) + S[sr]);
>>>             }
>>>             if (main_has_alpha) {
>>>                 switch (alpha) {
>>> @@ -487,7 +489,9 @@ static av_always_inline void
>>> blend_plane(AVFilterContext *ctx,
>>>                                          int main_has_alpha,
>>>                                          int dst_plane,
>>>                                          int dst_offset,
>>> -                                         int dst_step)
>>> +                                         int dst_step,
>>> +                                         int straight,
>>> +                                         int yuv)
>>> {
>>>     int src_wp = AV_CEIL_RSHIFT(src_w, hsub);
>>>     int src_hp = AV_CEIL_RSHIFT(src_h, vsub);
>>> @@ -546,7 +550,14 @@ static av_always_inline void
>>> blend_plane(AVFilterContext *ctx,
>>>                     alpha_d = da[0];
>>>                 alpha = UNPREMULTIPLY_ALPHA(alpha, alpha_d);
>>>             }
>>> -            *d = FAST_DIV255(*d * (255 - alpha) + *s * alpha);
>>> +            if (straight) {
>>> +                *d = FAST_DIV255(*d * (255 - alpha) + *s * alpha);
>>> +            } else {
>>> +                if (i && yuv)
>>> +                    *d = av_clip(FAST_DIV255((*d - 128) * (255 - alpha))
>>> + *s - 128, -128, 128) + 128;
>>> +                else
>>> +                    *d = FFMIN(FAST_DIV255(*d * (255 - alpha)) + *s,
>>> 255);
>>> +            }
>>>             s++;
>>>             d += dst_step;
>>>             da += 1 << hsub;
>>> @@ -605,7 +616,8 @@ static av_always_inline void
>>> blend_image_yuv(AVFilterContext *ctx,
>>>                                              AVFrame *dst, const AVFrame
>>> *src,
>>>                                              int hsub, int vsub,
>>>                                              int main_has_alpha,
>>> -                                             int x, int y)
>>> +                                             int x, int y,
>>> +                                             int is_straight)
>>> {
>>>     OverlayContext *s = ctx->priv;
>>>     const int src_w = src->width;
>>> @@ -614,11 +626,11 @@ static av_always_inline void
>>> blend_image_yuv(AVFilterContext *ctx,
>>>     const int dst_h = dst->height;
>>>
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 0, 0,       0,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[0].plane,
>>> s->main_desc->comp[0].offset, s->main_desc->comp[0].step);
>>> +                s->main_desc->comp[0].plane,
>>> s->main_desc->comp[0].offset, s->main_desc->comp[0].step, is_straight, 1);
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 1, hsub, vsub,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[1].plane,
>>> s->main_desc->comp[1].offset, s->main_desc->comp[1].step);
>>> +                s->main_desc->comp[1].plane,
>>> s->main_desc->comp[1].offset, s->main_desc->comp[1].step, is_straight, 1);
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 2, hsub, vsub,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[2].plane,
>>> s->main_desc->comp[2].offset, s->main_desc->comp[2].step);
>>> +                s->main_desc->comp[2].plane,
>>> s->main_desc->comp[2].offset, s->main_desc->comp[2].step, is_straight, 1);
>>>
>>>     if (main_has_alpha)
>>>         alpha_composite(src, dst, src_w, src_h, dst_w, dst_h, x, y);
>>> @@ -628,7 +640,8 @@ static av_always_inline void
>>> blend_image_planar_rgb(AVFilterContext *ctx,
>>>                                                     AVFrame *dst, const
>>> AVFrame *src,
>>>                                                     int hsub, int vsub,
>>>                                                     int main_has_alpha,
>>> -                                                    int x, int y)
>>> +                                                    int x, int y,
>>> +                                                    int is_straight)
>>> {
>>>     OverlayContext *s = ctx->priv;
>>>     const int src_w = src->width;
>>> @@ -637,11 +650,11 @@ static av_always_inline void
>>> blend_image_planar_rgb(AVFilterContext *ctx,
>>>     const int dst_h = dst->height;
>>>
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 0, 0,       0,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[1].plane,
>>> s->main_desc->comp[1].offset, s->main_desc->comp[1].step);
>>> +                s->main_desc->comp[1].plane,
>>> s->main_desc->comp[1].offset, s->main_desc->comp[1].step, is_straight, 0);
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 1, hsub, vsub,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[2].plane,
>>> s->main_desc->comp[2].offset, s->main_desc->comp[2].step);
>>> +                s->main_desc->comp[2].plane,
>>> s->main_desc->comp[2].offset, s->main_desc->comp[2].step, is_straight, 0);
>>>     blend_plane(ctx, dst, src, src_w, src_h, dst_w, dst_h, 2, hsub, vsub,
>>> x, y, main_has_alpha,
>>> -                s->main_desc->comp[0].plane,
>>> s->main_desc->comp[0].offset, s->main_desc->comp[0].step);
>>> +                s->main_desc->comp[0].plane,
>>> s->main_desc->comp[0].offset, s->main_desc->comp[0].step, is_straight, 0);
>>>
>>>     if (main_has_alpha)
>>>         alpha_composite(src, dst, src_w, src_h, dst_w, dst_h, x, y);
>>> @@ -649,52 +662,102 @@ static av_always_inline void
>>> blend_image_planar_rgb(AVFilterContext *ctx,
>>>
>>> static void blend_image_yuv420(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 1, 1, 0, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 1, 1, 0, x, y, 1);
>>> }
>>>
>>> static void blend_image_yuva420(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 1, 1, 1, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 1, 1, 1, x, y, 1);
>>> }
>>>
>>> static void blend_image_yuv422(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 1, 0, 0, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 1, 0, 0, x, y, 1);
>>> }
>>>
>>> static void blend_image_yuva422(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 1, 0, 1, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 1, 0, 1, x, y, 1);
>>> }
>>>
>>> static void blend_image_yuv444(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 0, 0, 0, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 0, 0, 0, x, y, 1);
>>> }
>>>
>>> static void blend_image_yuva444(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_yuv(ctx, dst, src, 0, 0, 1, x, y);
>>> +    blend_image_yuv(ctx, dst, src, 0, 0, 1, x, y, 1);
>>> }
>>>
>>> static void blend_image_gbrp(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_planar_rgb(ctx, dst, src, 0, 0, 0, x, y);
>>> +    blend_image_planar_rgb(ctx, dst, src, 0, 0, 0, x, y, 1);
>>> }
>>>
>>> static void blend_image_gbrap(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_planar_rgb(ctx, dst, src, 0, 0, 1, x, y);
>>> +    blend_image_planar_rgb(ctx, dst, src, 0, 0, 1, x, y, 1);
>>> +}
>>> +
>>> +static void blend_image_yuv420_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 1, 1, 0, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_yuva420_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 1, 1, 1, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_yuv422_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 1, 0, 0, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_yuva422_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 1, 0, 1, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_yuv444_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 0, 0, 0, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_yuva444_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_yuv(ctx, dst, src, 0, 0, 1, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_gbrp_pm(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_planar_rgb(ctx, dst, src, 0, 0, 0, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_gbrap_pm(AVFilterContext *ctx, AVFrame *dst,
>>> const AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_planar_rgb(ctx, dst, src, 0, 0, 1, x, y, 0);
>>> }
>>>
>>> static void blend_image_rgb(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_packed_rgb(ctx, dst, src, 0, x, y);
>>> +    blend_image_packed_rgb(ctx, dst, src, 0, x, y, 0);
>>> }
>>>
>>> static void blend_image_rgba(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> {
>>> -    blend_image_packed_rgb(ctx, dst, src, 1, x, y);
>>> +    blend_image_packed_rgb(ctx, dst, src, 1, x, y, 0);
>>> +}
>>> +
>>> +static void blend_image_rgb_pm(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_packed_rgb(ctx, dst, src, 0, x, y, 1);
>>> +}
>>> +
>>> +static void blend_image_rgba_pm(AVFilterContext *ctx, AVFrame *dst, const
>>> AVFrame *src, int x, int y)
>>> +{
>>> +    blend_image_packed_rgb(ctx, dst, src, 1, x, y, 1);
>>> }
>>>
>>> static int config_input_main(AVFilterLink *inlink)
>>> @@ -754,6 +817,52 @@ static int config_input_main(AVFilterLink *inlink)
>>>         }
>>>         break;
>>>     }
>>> +
>>> +    if (!s->alpha_format)
>>> +        return 0;
>>> +
>>> +    switch (s->format) {
>>> +    case OVERLAY_FORMAT_YUV420:
>>> +        s->blend_image = s->main_has_alpha ? blend_image_yuva420_pm :
>>> blend_image_yuv420_pm;
>>> +        break;
>>> +    case OVERLAY_FORMAT_YUV422:
>>> +        s->blend_image = s->main_has_alpha ? blend_image_yuva422_pm :
>>> blend_image_yuv422_pm;
>>> +        break;
>>> +    case OVERLAY_FORMAT_YUV444:
>>> +        s->blend_image = s->main_has_alpha ? blend_image_yuva444_pm :
>>> blend_image_yuv444_pm;
>>> +        break;
>>> +    case OVERLAY_FORMAT_RGB:
>>> +        s->blend_image = s->main_has_alpha ? blend_image_rgba_pm :
>>> blend_image_rgb_pm;
>>> +        break;
>>> +    case OVERLAY_FORMAT_GBRP:
>>> +        s->blend_image = s->main_has_alpha ? blend_image_gbrap_pm :
>>> blend_image_gbrp_pm;
>>> +        break;
>>> +    case OVERLAY_FORMAT_AUTO:
>>> +        switch (inlink->format) {
>>> +        case AV_PIX_FMT_YUVA420P:
>>> +            s->blend_image = blend_image_yuva420_pm;
>>> +            break;
>>> +        case AV_PIX_FMT_YUVA422P:
>>> +            s->blend_image = blend_image_yuva422_pm;
>>> +            break;
>>> +        case AV_PIX_FMT_YUVA444P:
>>> +            s->blend_image = blend_image_yuva444_pm;
>>> +            break;
>>> +        case AV_PIX_FMT_ARGB:
>>> +        case AV_PIX_FMT_RGBA:
>>> +        case AV_PIX_FMT_BGRA:
>>> +        case AV_PIX_FMT_ABGR:
>>> +            s->blend_image = blend_image_rgba_pm;
>>> +            break;
>>> +        case AV_PIX_FMT_GBRAP:
>>> +            s->blend_image = blend_image_gbrap_pm;
>>> +            break;
>>> +        default:
>>> +            av_assert0(0);
>>> +            break;
>>> +        }
>>> +        break;
>>> +    }
>>>     return 0;
>>> }
>>>
>>> @@ -835,6 +944,9 @@ static const AVOption overlay_options[] = {
>>>         { "gbrp",   "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_GBRP},
>>>  .flags = FLAGS, .unit = "format" },
>>>         { "auto",   "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_AUTO},
>>>  .flags = FLAGS, .unit = "format" },
>>>     { "repeatlast", "repeat overlay of the last overlay frame",
>>> OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
>>> +    { "alpha", "alpha format", OFFSET(alpha_format), AV_OPT_TYPE_INT,
>>> {.i64=0}, 0, 1, FLAGS, "alpha_format" },
>>> +        { "straight",      "", 0, AV_OPT_TYPE_CONST, {.i64=0}, .flags =
>>> FLAGS, .unit = "alpha_format" },
>>> +        { "premultiplied", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, .flags =
>>> FLAGS, .unit = "alpha_format" },
>>>     { NULL }
>>> };
>>>
>>> --
>>> 2.11.0
>>>
>>> _______________________________________________
>>> ffmpeg-devel mailing list
>>> ffmpeg-devel at ffmpeg.org
>>> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>> _______________________________________________
>> ffmpeg-devel mailing list
>> ffmpeg-devel at ffmpeg.org
>> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel


More information about the ffmpeg-devel mailing list