FFmpeg
vf_codecview.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (c) 2014 Clément Bœsch <u pkh me>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Codec debug viewer filter.
25  *
26  * All the MV drawing code from Michael Niedermayer is extracted from
27  * libavcodec/mpegvideo.c.
28  *
29  * TODO: segmentation
30  */
31 
32 #include "libavutil/imgutils.h"
34 #include "libavutil/opt.h"
35 #include "avfilter.h"
36 #include "qp_table.h"
37 #include "internal.h"
38 
39 #define MV_P_FOR (1<<0)
40 #define MV_B_FOR (1<<1)
41 #define MV_B_BACK (1<<2)
42 #define MV_TYPE_FOR (1<<0)
43 #define MV_TYPE_BACK (1<<1)
44 #define FRAME_TYPE_I (1<<0)
45 #define FRAME_TYPE_P (1<<1)
46 #define FRAME_TYPE_B (1<<2)
47 
48 typedef struct CodecViewContext {
49  const AVClass *class;
50  unsigned mv;
51  unsigned frame_type;
52  unsigned mv_type;
53  int hsub, vsub;
54  int qp;
56 
57 #define OFFSET(x) offsetof(CodecViewContext, x)
58 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
59 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
60 
61 static const AVOption codecview_options[] = {
62  { "mv", "set motion vectors to visualize", OFFSET(mv), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv" },
63  CONST("pf", "forward predicted MVs of P-frames", MV_P_FOR, "mv"),
64  CONST("bf", "forward predicted MVs of B-frames", MV_B_FOR, "mv"),
65  CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
66  { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
67  { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
68  { "mvt", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
69  CONST("fp", "forward predicted MVs", MV_TYPE_FOR, "mv_type"),
70  CONST("bp", "backward predicted MVs", MV_TYPE_BACK, "mv_type"),
71  { "frame_type", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
72  { "ft", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
73  CONST("if", "I-frames", FRAME_TYPE_I, "frame_type"),
74  CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"),
75  CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"),
76  { NULL }
77 };
78 
79 AVFILTER_DEFINE_CLASS(codecview);
80 
82 {
83  // TODO: we can probably add way more pixel formats without any other
84  // changes; anything with 8-bit luma in first plane should be working
86  AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
87  if (!fmts_list)
88  return AVERROR(ENOMEM);
89  return ff_set_common_formats(ctx, fmts_list);
90 }
91 
92 static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
93 {
94  if(*sx > *ex)
95  return clip_line(ex, ey, sx, sy, maxx);
96 
97  if (*sx < 0) {
98  if (*ex < 0)
99  return 1;
100  *sy = *ey + (*sy - *ey) * (int64_t)*ex / (*ex - *sx);
101  *sx = 0;
102  }
103 
104  if (*ex > maxx) {
105  if (*sx > maxx)
106  return 1;
107  *ey = *sy + (*ey - *sy) * (int64_t)(maxx - *sx) / (*ex - *sx);
108  *ex = maxx;
109  }
110  return 0;
111 }
112 
113 /**
114  * Draw a line from (ex, ey) -> (sx, sy).
115  * @param w width of the image
116  * @param h height of the image
117  * @param stride stride/linesize of the image
118  * @param color color of the arrow
119  */
120 static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey,
121  int w, int h, int stride, int color)
122 {
123  int x, y, fr, f;
124 
125  if (clip_line(&sx, &sy, &ex, &ey, w - 1))
126  return;
127  if (clip_line(&sy, &sx, &ey, &ex, h - 1))
128  return;
129 
130  sx = av_clip(sx, 0, w - 1);
131  sy = av_clip(sy, 0, h - 1);
132  ex = av_clip(ex, 0, w - 1);
133  ey = av_clip(ey, 0, h - 1);
134 
135  buf[sy * stride + sx] += color;
136 
137  if (FFABS(ex - sx) > FFABS(ey - sy)) {
138  if (sx > ex) {
139  FFSWAP(int, sx, ex);
140  FFSWAP(int, sy, ey);
141  }
142  buf += sx + sy * stride;
143  ex -= sx;
144  f = ((ey - sy) << 16) / ex;
145  for (x = 0; x <= ex; x++) {
146  y = (x * f) >> 16;
147  fr = (x * f) & 0xFFFF;
148  buf[ y * stride + x] += (color * (0x10000 - fr)) >> 16;
149  if(fr) buf[(y + 1) * stride + x] += (color * fr ) >> 16;
150  }
151  } else {
152  if (sy > ey) {
153  FFSWAP(int, sx, ex);
154  FFSWAP(int, sy, ey);
155  }
156  buf += sx + sy * stride;
157  ey -= sy;
158  if (ey)
159  f = ((ex - sx) << 16) / ey;
160  else
161  f = 0;
162  for(y= 0; y <= ey; y++){
163  x = (y*f) >> 16;
164  fr = (y*f) & 0xFFFF;
165  buf[y * stride + x ] += (color * (0x10000 - fr)) >> 16;
166  if(fr) buf[y * stride + x + 1] += (color * fr ) >> 16;
167  }
168  }
169 }
170 
171 /**
172  * Draw an arrow from (ex, ey) -> (sx, sy).
173  * @param w width of the image
174  * @param h height of the image
175  * @param stride stride/linesize of the image
176  * @param color color of the arrow
177  */
178 static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
179  int ey, int w, int h, int stride, int color, int tail, int direction)
180 {
181  int dx,dy;
182 
183  if (direction) {
184  FFSWAP(int, sx, ex);
185  FFSWAP(int, sy, ey);
186  }
187 
188  sx = av_clip(sx, -100, w + 100);
189  sy = av_clip(sy, -100, h + 100);
190  ex = av_clip(ex, -100, w + 100);
191  ey = av_clip(ey, -100, h + 100);
192 
193  dx = ex - sx;
194  dy = ey - sy;
195 
196  if (dx * dx + dy * dy > 3 * 3) {
197  int rx = dx + dy;
198  int ry = -dx + dy;
199  int length = sqrt((rx * rx + ry * ry) << 8);
200 
201  // FIXME subpixel accuracy
202  rx = ROUNDED_DIV(rx * 3 << 4, length);
203  ry = ROUNDED_DIV(ry * 3 << 4, length);
204 
205  if (tail) {
206  rx = -rx;
207  ry = -ry;
208  }
209 
210  draw_line(buf, sx, sy, sx + rx, sy + ry, w, h, stride, color);
211  draw_line(buf, sx, sy, sx - ry, sy + rx, w, h, stride, color);
212  }
213  draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
214 }
215 
217 {
218  AVFilterContext *ctx = inlink->dst;
219  CodecViewContext *s = ctx->priv;
220  AVFilterLink *outlink = ctx->outputs[0];
221 
222  if (s->qp) {
223  int qstride, qp_type, ret;
224  int8_t *qp_table;
225 
226  ret = ff_qp_table_extract(frame, &qp_table, &qstride, NULL, &qp_type);
227  if (ret < 0) {
228  av_frame_free(&frame);
229  return ret;
230  }
231 
232  if (qp_table) {
233  int x, y;
234  const int w = AV_CEIL_RSHIFT(frame->width, s->hsub);
235  const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
236  uint8_t *pu = frame->data[1];
237  uint8_t *pv = frame->data[2];
238  const int lzu = frame->linesize[1];
239  const int lzv = frame->linesize[2];
240 
241  for (y = 0; y < h; y++) {
242  for (x = 0; x < w; x++) {
243  const int qp = ff_norm_qscale(qp_table[(y >> 3) * qstride + (x >> 3)], qp_type) * 128/31;
244  pu[x] = pv[x] = qp;
245  }
246  pu += lzu;
247  pv += lzv;
248  }
249  }
250  av_freep(&qp_table);
251  }
252 
253  if (s->mv || s->mv_type) {
255  if (sd) {
256  int i;
257  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
258  const int is_iframe = (s->frame_type & FRAME_TYPE_I) && frame->pict_type == AV_PICTURE_TYPE_I;
259  const int is_pframe = (s->frame_type & FRAME_TYPE_P) && frame->pict_type == AV_PICTURE_TYPE_P;
260  const int is_bframe = (s->frame_type & FRAME_TYPE_B) && frame->pict_type == AV_PICTURE_TYPE_B;
261 
262  for (i = 0; i < sd->size / sizeof(*mvs); i++) {
263  const AVMotionVector *mv = &mvs[i];
264  const int direction = mv->source > 0;
265 
266  if (s->mv_type) {
267  const int is_fp = direction == 0 && (s->mv_type & MV_TYPE_FOR);
268  const int is_bp = direction == 1 && (s->mv_type & MV_TYPE_BACK);
269 
270  if ((!s->frame_type && (is_fp || is_bp)) ||
271  is_iframe && is_fp || is_iframe && is_bp ||
272  is_pframe && is_fp ||
273  is_bframe && is_fp || is_bframe && is_bp)
274  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
275  frame->width, frame->height, frame->linesize[0],
276  100, 0, direction);
277  } else if (s->mv)
278  if ((direction == 0 && (s->mv & MV_P_FOR) && frame->pict_type == AV_PICTURE_TYPE_P) ||
279  (direction == 0 && (s->mv & MV_B_FOR) && frame->pict_type == AV_PICTURE_TYPE_B) ||
280  (direction == 1 && (s->mv & MV_B_BACK) && frame->pict_type == AV_PICTURE_TYPE_B))
281  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
282  frame->width, frame->height, frame->linesize[0],
283  100, 0, direction);
284  }
285  }
286  }
287 
288  return ff_filter_frame(outlink, frame);
289 }
290 
292 {
293  AVFilterContext *ctx = inlink->dst;
294  CodecViewContext *s = ctx->priv;
296 
297  s->hsub = desc->log2_chroma_w;
298  s->vsub = desc->log2_chroma_h;
299  return 0;
300 }
301 
302 static const AVFilterPad codecview_inputs[] = {
303  {
304  .name = "default",
305  .type = AVMEDIA_TYPE_VIDEO,
306  .filter_frame = filter_frame,
307  .config_props = config_input,
308  .needs_writable = 1,
309  },
310  { NULL }
311 };
312 
313 static const AVFilterPad codecview_outputs[] = {
314  {
315  .name = "default",
316  .type = AVMEDIA_TYPE_VIDEO,
317  },
318  { NULL }
319 };
320 
322  .name = "codecview",
323  .description = NULL_IF_CONFIG_SMALL("Visualize information about some codecs."),
324  .priv_size = sizeof(CodecViewContext),
326  .inputs = codecview_inputs,
327  .outputs = codecview_outputs,
328  .priv_class = &codecview_class,
330 };
#define MV_B_BACK
Definition: vf_codecview.c:41
#define NULL
Definition: coverity.c:32
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2573
This structure describes decoded (raw) audio or video data.
Definition: frame.h:314
AVOption.
Definition: opt.h:248
#define FRAME_TYPE_I
Definition: vf_codecview.c:44
const char * desc
Definition: libsvtav1.c:79
misc image utilities
int16_t src_x
Absolute source position.
Definition: motion_vector.h:38
Main libavfilter public API header.
unsigned mv_type
Definition: vf_codecview.c:52
static const AVFilterPad codecview_outputs[]
Definition: vf_codecview.c:313
static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_codecview.c:216
static const AVOption codecview_options[]
Definition: vf_codecview.c:61
#define FLAGS
Definition: vf_codecview.c:58
uint8_t log2_chroma_w
Amount to shift the luma width right to find the chroma width.
Definition: pixdesc.h:92
frame_type
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:287
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:738
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:126
const char * name
Pad name.
Definition: internal.h:60
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1091
uint8_t
AVOptions.
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
#define f(width, name)
Definition: cbs_vp9.c:255
int16_t dst_x
Absolute destination position.
Definition: motion_vector.h:42
Structure to hold side data for an AVFrame.
Definition: frame.h:220
int32_t source
Where the current macroblock comes from; negative value when it comes from the past, positive value when it comes from the future.
Definition: motion_vector.h:30
#define MV_P_FOR
Definition: vf_codecview.c:39
static void draw_arrow(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color, int tail, int direction)
Draw an arrow from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:178
#define ROUNDED_DIV(a, b)
Definition: common.h:56
A filter pad used for either input or output.
Definition: internal.h:54
static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color)
Draw a line from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:120
#define MV_B_FOR
Definition: vf_codecview.c:40
int width
Definition: frame.h:372
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:588
#define CONST(name, help, val, unit)
Definition: vf_codecview.c:59
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
int ff_qp_table_extract(AVFrame *frame, int8_t **table, int *table_w, int *table_h, int *qscale_type)
Extract a libpostproc-compatible QP table - an 8-bit QP value per 16x16 macroblock, stored in raster order - from AVVideoEncParams side data.
Definition: qp_table.c:30
#define pv
Definition: regdef.h:60
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:115
void * priv
private data for use by the filter
Definition: avfilter.h:356
AVFilter ff_vf_codecview
Definition: vf_codecview.c:321
GLsizei GLsizei * length
Definition: opengl_enc.c:114
#define FRAME_TYPE_B
Definition: vf_codecview.c:46
AVFILTER_DEFINE_CLASS(codecview)
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:397
uint8_t w
Definition: llviddspenc.c:39
AVFormatContext * ctx
Definition: movenc.c:48
Motion vectors exported by some codecs (on demand through the export_mvs flag set in the libavcodec A...
Definition: frame.h:96
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
#define OFFSET(x)
Definition: vf_codecview.c:57
#define FFABS(a)
Absolute value, Note, INT_MIN / INT64_MIN result in undefined behavior as they are not representable ...
Definition: common.h:72
#define s(width, name)
Definition: cbs_vp9.c:257
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
#define MV_TYPE_BACK
Definition: vf_codecview.c:43
static int query_formats(AVFilterContext *ctx)
Definition: vf_codecview.c:81
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:345
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
Definition: vf_codecview.c:92
unsigned frame_type
Definition: vf_codecview.c:51
uint8_t * data
Definition: frame.h:222
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
#define MV_TYPE_FOR
Definition: vf_codecview.c:42
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:145
#define FRAME_TYPE_P
Definition: vf_codecview.c:45
const char * name
Filter name.
Definition: avfilter.h:149
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:353
static int ff_norm_qscale(int qscale, int type)
Normalize the qscale factor FIXME the H264 qscale is a log based scale, mpeg1/2 is not...
Definition: internal.h:351
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:303
#define flags(name, subs,...)
Definition: cbs_av1.c:561
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:328
static int config_input(AVFilterLink *inlink)
Definition: vf_codecview.c:291
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
static const AVFilterPad codecview_inputs[]
Definition: vf_codecview.c:302
GLint GLenum GLboolean GLsizei stride
Definition: opengl_enc.c:104
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Bi-dir predicted.
Definition: avutil.h:276
A list of supported formats for one end of a filter link.
Definition: formats.h:65
An instance of a filter.
Definition: avfilter.h:341
int height
Definition: frame.h:372
#define av_freep(p)
#define FFSWAP(type, a, b)
Definition: common.h:99
#define stride
internal API functions
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
int i
Definition: input.c:407
Predicted.
Definition: avutil.h:275
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58