[FFmpeg-cvslog] libavfilter: add photosensitivity filter
Vladimir Panteleev
git at videolan.org
Mon Sep 30 21:41:20 EEST 2019
ffmpeg | branch: master | Vladimir Panteleev <git at thecybershadow.net> | Fri Mar 8 06:47:42 2019 +0000| [c888adf590eeaf8f854de9067886ecb2f0f08419] | committer: Paul B Mahol
libavfilter: add photosensitivity filter
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c888adf590eeaf8f854de9067886ecb2f0f08419
---
Changelog | 1 +
doc/filters.texi | 20 +++
libavfilter/Makefile | 1 +
libavfilter/allfilters.c | 1 +
libavfilter/version.h | 2 +-
libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++++++++++
6 files changed, 360 insertions(+), 1 deletion(-)
diff --git a/Changelog b/Changelog
index 743612cb91..adecdaaf78 100644
--- a/Changelog
+++ b/Changelog
@@ -13,6 +13,7 @@ version <next>:
- streamhash muxer
- sierpinski video source
- scroll video filter
+- photosensitivity filter
version 4.2:
diff --git a/doc/filters.texi b/doc/filters.texi
index e7b3c8f88e..b00787d2e8 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -14040,6 +14040,26 @@ Filter selects among @samp{t}, @samp{b} and @samp{p} using image analysis only.
@end table
@end table
+ at section photosensitivity
+Reduce various flashes in video, so to help users with epilepsy.
+
+It accepts the following options:
+ at table @option
+ at item frames, f
+Set how many frames to use when filtering. Default is 30.
+
+ at item threshold, t
+Set detection threshold factor. Default is 1.
+Lower is stricter.
+
+ at item skip
+Set how many pixels to skip when sampling frames. Defalt is 1.
+Allowed range is from 1 to 1024.
+
+ at item bypass
+Leave frames unchanged. Default is disabled.
+ at end table
+
@section pixdesctest
Pixel format descriptor test filter, mainly useful for internal
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 124a3496e6..182fe9df4b 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -321,6 +321,7 @@ OBJS-$(CONFIG_PALETTEUSE_FILTER) += vf_paletteuse.o framesync.o
OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o
OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o
OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o
+OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o
OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o
OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o
OBJS-$(CONFIG_PP_FILTER) += vf_pp.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 0408bcd3df..1a26129069 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -305,6 +305,7 @@ extern AVFilter ff_vf_paletteuse;
extern AVFilter ff_vf_perms;
extern AVFilter ff_vf_perspective;
extern AVFilter ff_vf_phase;
+extern AVFilter ff_vf_photosensitivity;
extern AVFilter ff_vf_pixdesctest;
extern AVFilter ff_vf_pixscope;
extern AVFilter ff_vf_pp;
diff --git a/libavfilter/version.h b/libavfilter/version.h
index e51bab8261..ad88845682 100644
--- a/libavfilter/version.h
+++ b/libavfilter/version.h
@@ -30,7 +30,7 @@
#include "libavutil/version.h"
#define LIBAVFILTER_VERSION_MAJOR 7
-#define LIBAVFILTER_VERSION_MINOR 60
+#define LIBAVFILTER_VERSION_MINOR 61
#define LIBAVFILTER_VERSION_MICRO 100
diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c
new file mode 100644
index 0000000000..4559454846
--- /dev/null
+++ b/libavfilter/vf_photosensitivity.c
@@ -0,0 +1,336 @@
+/*
+ * Copyright (c) 2019 Vladimir Panteleev
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <float.h>
+
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "avfilter.h"
+
+#include "formats.h"
+#include "internal.h"
+#include "video.h"
+
+#define MAX_FRAMES 240
+#define GRID_SIZE 8
+#define NUM_CHANNELS 3
+
+typedef struct PhotosensitivityFrame {
+ uint8_t grid[GRID_SIZE][GRID_SIZE][4];
+} PhotosensitivityFrame;
+
+typedef struct PhotosensitivityContext {
+ const AVClass *class;
+
+ int nb_frames;
+ int skip;
+ float threshold_multiplier;
+ int bypass;
+
+ int badness_threshold;
+
+ /* Circular buffer */
+ int history[MAX_FRAMES];
+ int history_pos;
+
+ PhotosensitivityFrame last_frame_e;
+ AVFrame *last_frame_av;
+} PhotosensitivityContext;
+
+#define OFFSET(x) offsetof(PhotosensitivityContext, x)
+#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
+
+static const AVOption photosensitivity_options[] = {
+ { "frames", "set how many frames to use", OFFSET(nb_frames), AV_OPT_TYPE_INT, {.i64=30}, 2, MAX_FRAMES, FLAGS },
+ { "f", "set how many frames to use", OFFSET(nb_frames), AV_OPT_TYPE_INT, {.i64=30}, 2, MAX_FRAMES, FLAGS },
+ { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS },
+ { "t", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS },
+ { "skip", "set pixels to skip when sampling frames", OFFSET(skip), AV_OPT_TYPE_INT, {.i64=1}, 1, 1024, FLAGS },
+ { "bypass", "leave frames unchanged", OFFSET(bypass), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS },
+ { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(photosensitivity);
+
+static int query_formats(AVFilterContext *ctx)
+{
+ static const enum AVPixelFormat pixel_fmts[] = {
+ AV_PIX_FMT_RGB24,
+ AV_PIX_FMT_BGR24,
+ AV_PIX_FMT_NONE
+ };
+ AVFilterFormats *formats = ff_make_format_list(pixel_fmts);
+ if (!formats)
+ return AVERROR(ENOMEM);
+ return ff_set_common_formats(ctx, formats);
+}
+
+typedef struct ThreadData_convert_frame
+{
+ AVFrame *in;
+ PhotosensitivityFrame *out;
+ int skip;
+} ThreadData_convert_frame;
+
+#define NUM_CELLS (GRID_SIZE * GRID_SIZE)
+
+static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+ int cell, gx, gy, x0, x1, y0, y1, x, y, c, area;
+ int sum[NUM_CHANNELS];
+ const uint8_t *p;
+
+ ThreadData_convert_frame *td = arg;
+
+ const int slice_start = (NUM_CELLS * jobnr) / nb_jobs;
+ const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs;
+
+ int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip;
+ const uint8_t *data = td->in->data[0];
+
+ for (cell = slice_start; cell < slice_end; cell++) {
+ gx = cell % GRID_SIZE;
+ gy = cell / GRID_SIZE;
+
+ x0 = width * gx / GRID_SIZE;
+ x1 = width * (gx+1) / GRID_SIZE;
+ y0 = height * gy / GRID_SIZE;
+ y1 = height * (gy+1) / GRID_SIZE;
+
+ for (c = 0; c < NUM_CHANNELS; c++) {
+ sum[c] = 0;
+ }
+ for (y = y0; y < y1; y += skip) {
+ p = data + y * linesize + x0 * NUM_CHANNELS;
+ for (x = x0; x < x1; x += skip) {
+ //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row);
+ sum[0] += p[0];
+ sum[1] += p[1];
+ sum[2] += p[2];
+ p += NUM_CHANNELS * skip;
+ // TODO: variable size
+ }
+ }
+
+ area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip);
+ for (c = 0; c < NUM_CHANNELS; c++) {
+ if (area)
+ sum[c] /= area;
+ td->out->grid[gy][gx][c] = sum[c];
+ }
+ }
+ return 0;
+}
+
+static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame *out, int skip)
+{
+ ThreadData_convert_frame td;
+ td.in = in;
+ td.out = out;
+ td.skip = skip;
+ ctx->internal->execute(ctx, convert_frame_partial, &td, NULL, FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx)));
+}
+
+typedef struct ThreadData_blend_frame
+{
+ AVFrame *target;
+ AVFrame *source;
+ uint16_t s_mul;
+} ThreadData_blend_frame;
+
+static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+ int x, y;
+ uint8_t *t, *s;
+
+ ThreadData_blend_frame *td = arg;
+ const uint16_t s_mul = td->s_mul;
+ const uint16_t t_mul = 0x100 - s_mul;
+ const int slice_start = (td->target->height * jobnr) / nb_jobs;
+ const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs;
+ const int linesize = td->target->linesize[0];
+
+ for (y = slice_start; y < slice_end; y++) {
+ t = td->target->data[0] + y * td->target->linesize[0];
+ s = td->source->data[0] + y * td->source->linesize[0];
+ for (x = 0; x < linesize; x++) {
+ *t = (*t * t_mul + *s * s_mul) >> 8;
+ t++; s++;
+ }
+ }
+ return 0;
+}
+
+static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor)
+{
+ ThreadData_blend_frame td;
+ td.target = target;
+ td.source = source;
+ td.s_mul = (uint16_t)(factor * 0x100);
+ ctx->internal->execute(ctx, blend_frame_partial, &td, NULL, FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx)));
+}
+
+static int get_badness(PhotosensitivityFrame *a, PhotosensitivityFrame *b)
+{
+ int badness, x, y, c;
+ badness = 0;
+ for (c = 0; c < NUM_CHANNELS; c++) {
+ for (y = 0; y < GRID_SIZE; y++) {
+ for (x = 0; x < GRID_SIZE; x++) {
+ badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]);
+ //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness);
+ //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness);
+ }
+ }
+ }
+ return badness;
+}
+
+static int config_input(AVFilterLink *inlink)
+{
+ /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */
+ AVFilterContext *ctx = inlink->dst;
+ PhotosensitivityContext *s = ctx->priv;
+
+ s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128);
+
+ return 0;
+}
+
+static int filter_frame(AVFilterLink *inlink, AVFrame *in)
+{
+ int this_badness, current_badness, fixed_badness, new_badness, i, res;
+ PhotosensitivityFrame ef;
+ AVFrame *src, *out;
+ float factor;
+ AVDictionary **metadata;
+
+ AVFilterContext *ctx = inlink->dst;
+ AVFilterLink *outlink = ctx->outputs[0];
+ PhotosensitivityContext *s = ctx->priv;
+
+ /* weighted moving average */
+ current_badness = 0;
+ for (i = 1; i < s->nb_frames; i++)
+ current_badness += i * s->history[(s->history_pos + i) % s->nb_frames];
+ current_badness /= s->nb_frames;
+
+ convert_frame(ctx, in, &ef, s->skip);
+ this_badness = get_badness(&ef, &s->last_frame_e);
+ new_badness = current_badness + this_badness;
+ av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s)\n",
+ current_badness, new_badness, s->badness_threshold,
+ 100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED");
+
+ fixed_badness = new_badness;
+ if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) {
+ factor = 1; /* for metadata */
+ av_frame_free(&s->last_frame_av);
+ s->last_frame_av = src = in;
+ s->last_frame_e = ef;
+ s->history[s->history_pos] = this_badness;
+ } else {
+ factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness);
+ if (factor <= 0) {
+ /* just duplicate the frame */
+ s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */
+ } else {
+ res = av_frame_make_writable(s->last_frame_av);
+ if (res) {
+ av_frame_free(&in);
+ return res;
+ }
+ blend_frame(ctx, s->last_frame_av, in, factor);
+
+ convert_frame(ctx, s->last_frame_av, &ef, s->skip);
+ this_badness = get_badness(&ef, &s->last_frame_e);
+ fixed_badness = current_badness + this_badness;
+ av_log(s, AV_LOG_VERBOSE, " fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f\n",
+ current_badness, fixed_badness, s->badness_threshold,
+ 100 * new_badness / s->badness_threshold, factor);
+ s->last_frame_e = ef;
+ s->history[s->history_pos] = this_badness;
+ }
+ src = s->last_frame_av;
+ }
+ s->history_pos = (s->history_pos + 1) % s->nb_frames;
+
+ out = ff_get_video_buffer(outlink, in->width, in->height);
+ if (!out) {
+ av_frame_free(&in);
+ return AVERROR(ENOMEM);
+ }
+ av_frame_copy_props(out, in);
+ metadata = &out->metadata;
+ if (metadata) {
+ char value[128];
+
+ snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold);
+ av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0);
+
+ snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold);
+ av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0);
+
+ snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold);
+ av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0);
+
+ snprintf(value, sizeof(value), "%f", factor);
+ av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0);
+ }
+ av_frame_copy(out, src);
+ return ff_filter_frame(outlink, out);
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+ PhotosensitivityContext *s = ctx->priv;
+
+ av_frame_free(&s->last_frame_av);
+}
+
+static const AVFilterPad inputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .filter_frame = filter_frame,
+ .config_props = config_input,
+ },
+ { NULL }
+};
+
+static const AVFilterPad outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ },
+ { NULL }
+};
+
+AVFilter ff_vf_photosensitivity = {
+ .name = "photosensitivity",
+ .description = NULL_IF_CONFIG_SMALL("Filter out photosensitive epilepsy seizure-inducing flashes."),
+ .priv_size = sizeof(PhotosensitivityContext),
+ .priv_class = &photosensitivity_class,
+ .uninit = uninit,
+ .query_formats = query_formats,
+ .inputs = inputs,
+ .outputs = outputs,
+};
More information about the ffmpeg-cvslog
mailing list