FFmpeg
vf_codecview.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (c) 2014 Clément Bœsch <u pkh me>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Codec debug viewer filter.
25  *
26  * All the MV drawing code from Michael Niedermayer is extracted from
27  * libavcodec/mpegvideo.c.
28  *
29  * TODO: segmentation
30  */
31 
32 #include "libavutil/imgutils.h"
34 #include "libavutil/opt.h"
35 #include "avfilter.h"
36 #include "internal.h"
37 
38 #define MV_P_FOR (1<<0)
39 #define MV_B_FOR (1<<1)
40 #define MV_B_BACK (1<<2)
41 #define MV_TYPE_FOR (1<<0)
42 #define MV_TYPE_BACK (1<<1)
43 #define FRAME_TYPE_I (1<<0)
44 #define FRAME_TYPE_P (1<<1)
45 #define FRAME_TYPE_B (1<<2)
46 
47 typedef struct CodecViewContext {
48  const AVClass *class;
49  unsigned mv;
50  unsigned frame_type;
51  unsigned mv_type;
52  int hsub, vsub;
53  int qp;
55 
56 #define OFFSET(x) offsetof(CodecViewContext, x)
57 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
58 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
59 
60 static const AVOption codecview_options[] = {
61  { "mv", "set motion vectors to visualize", OFFSET(mv), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv" },
62  CONST("pf", "forward predicted MVs of P-frames", MV_P_FOR, "mv"),
63  CONST("bf", "forward predicted MVs of B-frames", MV_B_FOR, "mv"),
64  CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
65  { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
66  { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
67  { "mvt", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
68  CONST("fp", "forward predicted MVs", MV_TYPE_FOR, "mv_type"),
69  CONST("bp", "backward predicted MVs", MV_TYPE_BACK, "mv_type"),
70  { "frame_type", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
71  { "ft", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
72  CONST("if", "I-frames", FRAME_TYPE_I, "frame_type"),
73  CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"),
74  CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"),
75  { NULL }
76 };
77 
78 AVFILTER_DEFINE_CLASS(codecview);
79 
81 {
82  // TODO: we can probably add way more pixel formats without any other
83  // changes; anything with 8-bit luma in first plane should be working
85  AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
86  if (!fmts_list)
87  return AVERROR(ENOMEM);
88  return ff_set_common_formats(ctx, fmts_list);
89 }
90 
91 static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
92 {
93  if(*sx > *ex)
94  return clip_line(ex, ey, sx, sy, maxx);
95 
96  if (*sx < 0) {
97  if (*ex < 0)
98  return 1;
99  *sy = *ey + (*sy - *ey) * (int64_t)*ex / (*ex - *sx);
100  *sx = 0;
101  }
102 
103  if (*ex > maxx) {
104  if (*sx > maxx)
105  return 1;
106  *ey = *sy + (*ey - *sy) * (int64_t)(maxx - *sx) / (*ex - *sx);
107  *ex = maxx;
108  }
109  return 0;
110 }
111 
112 /**
113  * Draw a line from (ex, ey) -> (sx, sy).
114  * @param w width of the image
115  * @param h height of the image
116  * @param stride stride/linesize of the image
117  * @param color color of the arrow
118  */
119 static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey,
120  int w, int h, int stride, int color)
121 {
122  int x, y, fr, f;
123 
124  if (clip_line(&sx, &sy, &ex, &ey, w - 1))
125  return;
126  if (clip_line(&sy, &sx, &ey, &ex, h - 1))
127  return;
128 
129  sx = av_clip(sx, 0, w - 1);
130  sy = av_clip(sy, 0, h - 1);
131  ex = av_clip(ex, 0, w - 1);
132  ey = av_clip(ey, 0, h - 1);
133 
134  buf[sy * stride + sx] += color;
135 
136  if (FFABS(ex - sx) > FFABS(ey - sy)) {
137  if (sx > ex) {
138  FFSWAP(int, sx, ex);
139  FFSWAP(int, sy, ey);
140  }
141  buf += sx + sy * stride;
142  ex -= sx;
143  f = ((ey - sy) << 16) / ex;
144  for (x = 0; x <= ex; x++) {
145  y = (x * f) >> 16;
146  fr = (x * f) & 0xFFFF;
147  buf[ y * stride + x] += (color * (0x10000 - fr)) >> 16;
148  if(fr) buf[(y + 1) * stride + x] += (color * fr ) >> 16;
149  }
150  } else {
151  if (sy > ey) {
152  FFSWAP(int, sx, ex);
153  FFSWAP(int, sy, ey);
154  }
155  buf += sx + sy * stride;
156  ey -= sy;
157  if (ey)
158  f = ((ex - sx) << 16) / ey;
159  else
160  f = 0;
161  for(y= 0; y <= ey; y++){
162  x = (y*f) >> 16;
163  fr = (y*f) & 0xFFFF;
164  buf[y * stride + x ] += (color * (0x10000 - fr)) >> 16;
165  if(fr) buf[y * stride + x + 1] += (color * fr ) >> 16;
166  }
167  }
168 }
169 
170 /**
171  * Draw an arrow from (ex, ey) -> (sx, sy).
172  * @param w width of the image
173  * @param h height of the image
174  * @param stride stride/linesize of the image
175  * @param color color of the arrow
176  */
177 static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
178  int ey, int w, int h, int stride, int color, int tail, int direction)
179 {
180  int dx,dy;
181 
182  if (direction) {
183  FFSWAP(int, sx, ex);
184  FFSWAP(int, sy, ey);
185  }
186 
187  sx = av_clip(sx, -100, w + 100);
188  sy = av_clip(sy, -100, h + 100);
189  ex = av_clip(ex, -100, w + 100);
190  ey = av_clip(ey, -100, h + 100);
191 
192  dx = ex - sx;
193  dy = ey - sy;
194 
195  if (dx * dx + dy * dy > 3 * 3) {
196  int rx = dx + dy;
197  int ry = -dx + dy;
198  int length = sqrt((rx * rx + ry * ry) << 8);
199 
200  // FIXME subpixel accuracy
201  rx = ROUNDED_DIV(rx * 3 << 4, length);
202  ry = ROUNDED_DIV(ry * 3 << 4, length);
203 
204  if (tail) {
205  rx = -rx;
206  ry = -ry;
207  }
208 
209  draw_line(buf, sx, sy, sx + rx, sy + ry, w, h, stride, color);
210  draw_line(buf, sx, sy, sx - ry, sy + rx, w, h, stride, color);
211  }
212  draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
213 }
214 
216 {
217  AVFilterContext *ctx = inlink->dst;
218  CodecViewContext *s = ctx->priv;
219  AVFilterLink *outlink = ctx->outputs[0];
220 
221  if (s->qp) {
222  int qstride, qp_type;
223  int8_t *qp_table = av_frame_get_qp_table(frame, &qstride, &qp_type);
224 
225  if (qp_table) {
226  int x, y;
227  const int w = AV_CEIL_RSHIFT(frame->width, s->hsub);
228  const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
229  uint8_t *pu = frame->data[1];
230  uint8_t *pv = frame->data[2];
231  const int lzu = frame->linesize[1];
232  const int lzv = frame->linesize[2];
233 
234  for (y = 0; y < h; y++) {
235  for (x = 0; x < w; x++) {
236  const int qp = ff_norm_qscale(qp_table[(y >> 3) * qstride + (x >> 3)], qp_type) * 128/31;
237  pu[x] = pv[x] = qp;
238  }
239  pu += lzu;
240  pv += lzv;
241  }
242  }
243  }
244 
245  if (s->mv || s->mv_type) {
247  if (sd) {
248  int i;
249  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
250  const int is_iframe = (s->frame_type & FRAME_TYPE_I) && frame->pict_type == AV_PICTURE_TYPE_I;
251  const int is_pframe = (s->frame_type & FRAME_TYPE_P) && frame->pict_type == AV_PICTURE_TYPE_P;
252  const int is_bframe = (s->frame_type & FRAME_TYPE_B) && frame->pict_type == AV_PICTURE_TYPE_B;
253 
254  for (i = 0; i < sd->size / sizeof(*mvs); i++) {
255  const AVMotionVector *mv = &mvs[i];
256  const int direction = mv->source > 0;
257 
258  if (s->mv_type) {
259  const int is_fp = direction == 0 && (s->mv_type & MV_TYPE_FOR);
260  const int is_bp = direction == 1 && (s->mv_type & MV_TYPE_BACK);
261 
262  if ((!s->frame_type && (is_fp || is_bp)) ||
263  is_iframe && is_fp || is_iframe && is_bp ||
264  is_pframe && is_fp ||
265  is_bframe && is_fp || is_bframe && is_bp)
266  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
267  frame->width, frame->height, frame->linesize[0],
268  100, 0, direction);
269  } else if (s->mv)
270  if ((direction == 0 && (s->mv & MV_P_FOR) && frame->pict_type == AV_PICTURE_TYPE_P) ||
271  (direction == 0 && (s->mv & MV_B_FOR) && frame->pict_type == AV_PICTURE_TYPE_B) ||
272  (direction == 1 && (s->mv & MV_B_BACK) && frame->pict_type == AV_PICTURE_TYPE_B))
273  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
274  frame->width, frame->height, frame->linesize[0],
275  100, 0, direction);
276  }
277  }
278  }
279 
280  return ff_filter_frame(outlink, frame);
281 }
282 
284 {
285  AVFilterContext *ctx = inlink->dst;
286  CodecViewContext *s = ctx->priv;
288 
289  s->hsub = desc->log2_chroma_w;
290  s->vsub = desc->log2_chroma_h;
291  return 0;
292 }
293 
294 static const AVFilterPad codecview_inputs[] = {
295  {
296  .name = "default",
297  .type = AVMEDIA_TYPE_VIDEO,
298  .filter_frame = filter_frame,
299  .config_props = config_input,
300  .needs_writable = 1,
301  },
302  { NULL }
303 };
304 
305 static const AVFilterPad codecview_outputs[] = {
306  {
307  .name = "default",
308  .type = AVMEDIA_TYPE_VIDEO,
309  },
310  { NULL }
311 };
312 
314  .name = "codecview",
315  .description = NULL_IF_CONFIG_SMALL("Visualize information about some codecs."),
316  .priv_size = sizeof(CodecViewContext),
318  .inputs = codecview_inputs,
319  .outputs = codecview_outputs,
320  .priv_class = &codecview_class,
322 };
#define MV_B_BACK
Definition: vf_codecview.c:40
#define NULL
Definition: coverity.c:32
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2522
This structure describes decoded (raw) audio or video data.
Definition: frame.h:295
AVOption.
Definition: opt.h:246
#define FRAME_TYPE_I
Definition: vf_codecview.c:43
misc image utilities
int16_t src_x
Absolute source position.
Definition: motion_vector.h:38
Main libavfilter public API header.
const char * desc
Definition: nvenc.c:68
unsigned mv_type
Definition: vf_codecview.c:51
static const AVFilterPad codecview_outputs[]
Definition: vf_codecview.c:305
static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_codecview.c:215
static const AVOption codecview_options[]
Definition: vf_codecview.c:60
#define FLAGS
Definition: vf_codecview.c:57
uint8_t log2_chroma_w
Amount to shift the luma width right to find the chroma width.
Definition: pixdesc.h:92
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:283
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:734
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:125
const char * name
Pad name.
Definition: internal.h:60
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1093
uint8_t
AVOptions.
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
#define f(width, name)
Definition: cbs_vp9.c:255
int16_t dst_x
Absolute destination position.
Definition: motion_vector.h:42
Structure to hold side data for an AVFrame.
Definition: frame.h:201
int32_t source
Where the current macroblock comes from; negative value when it comes from the past, positive value when it comes from the future.
Definition: motion_vector.h:30
#define MV_P_FOR
Definition: vf_codecview.c:38
static void draw_arrow(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color, int tail, int direction)
Draw an arrow from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:177
#define ROUNDED_DIV(a, b)
Definition: common.h:56
A filter pad used for either input or output.
Definition: internal.h:54
static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color)
Draw a line from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:119
#define MV_B_FOR
Definition: vf_codecview.c:39
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
int width
Definition: frame.h:353
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:569
#define CONST(name, help, val, unit)
Definition: vf_codecview.c:58
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
#define pv
Definition: regdef.h:60
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
void * priv
private data for use by the filter
Definition: avfilter.h:353
AVFilter ff_vf_codecview
Definition: vf_codecview.c:313
GLsizei GLsizei * length
Definition: opengl_enc.c:114
#define FRAME_TYPE_B
Definition: vf_codecview.c:45
AVFILTER_DEFINE_CLASS(codecview)
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:378
uint8_t w
Definition: llviddspenc.c:38
AVFormatContext * ctx
Definition: movenc.c:48
Motion vectors exported by some codecs (on demand through the export_mvs flag set in the libavcodec A...
Definition: frame.h:96
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
#define OFFSET(x)
Definition: vf_codecview.c:56
#define FFABS(a)
Absolute value, Note, INT_MIN / INT64_MIN result in undefined behavior as they are not representable ...
Definition: common.h:72
#define s(width, name)
Definition: cbs_vp9.c:257
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
#define MV_TYPE_BACK
Definition: vf_codecview.c:42
static int query_formats(AVFilterContext *ctx)
Definition: vf_codecview.c:80
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:326
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
Definition: vf_codecview.c:91
unsigned frame_type
Definition: vf_codecview.c:50
uint8_t * data
Definition: frame.h:203
void * buf
Definition: avisynth_c.h:766
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
#define MV_TYPE_FOR
Definition: vf_codecview.c:41
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:144
#define FRAME_TYPE_P
Definition: vf_codecview.c:44
const char * name
Filter name.
Definition: avfilter.h:148
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:350
static int ff_norm_qscale(int qscale, int type)
Normalize the qscale factor FIXME the H264 qscale is a log based scale, mpeg1/2 is not...
Definition: internal.h:397
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:275
#define flags(name, subs,...)
Definition: cbs_av1.c:561
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:309
static int config_input(AVFilterLink *inlink)
Definition: vf_codecview.c:283
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
static const AVFilterPad codecview_inputs[]
Definition: vf_codecview.c:294
GLint GLenum GLboolean GLsizei stride
Definition: opengl_enc.c:104
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Bi-dir predicted.
Definition: avutil.h:276
A list of supported formats for one end of a filter link.
Definition: formats.h:64
An instance of a filter.
Definition: avfilter.h:338
int height
Definition: frame.h:353
#define FFSWAP(type, a, b)
Definition: common.h:99
#define stride
internal API functions
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
int8_t * av_frame_get_qp_table(AVFrame *f, int *stride, int *type)
Definition: frame.c:90
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
Predicted.
Definition: avutil.h:275
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58