FFmpeg
vf_transpose_vt.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2023 Zhao Zhili <zhilizhao@tencent.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 
23 #include "libavutil/hwcontext.h"
25 #include "libavutil/opt.h"
26 #include "libavutil/pixdesc.h"
27 #include "internal.h"
28 #include "transpose.h"
29 #include "video.h"
30 
31 typedef struct TransposeVtContext {
32  AVClass *class;
33 
34  VTPixelRotationSessionRef session;
35  int dir;
38 
40 {
41  TransposeVtContext *s = avctx->priv;
42  int ret;
43 
44  ret = VTPixelRotationSessionCreate(kCFAllocatorDefault, &s->session);
45  if (ret != noErr) {
46  av_log(avctx, AV_LOG_ERROR, "Rotation session create failed, %d\n", ret);
47  return AVERROR_EXTERNAL;
48  }
49 
50  return 0;
51 }
52 
54 {
55  TransposeVtContext *s = avctx->priv;
56 
57  if (s->session) {
58  VTPixelRotationSessionInvalidate(s->session);
59  CFRelease(s->session);
60  s->session = NULL;
61  }
62 }
63 
65 {
66  int ret;
67  AVFilterContext *ctx = link->dst;
68  TransposeVtContext *s = ctx->priv;
69  AVFilterLink *outlink = ctx->outputs[0];
70  CVPixelBufferRef src;
71  CVPixelBufferRef dst;
72  AVFrame *out;
73 
74  if (s->passthrough)
75  return ff_filter_frame(outlink, in);
76 
77  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
78  if (!out) {
79  ret = AVERROR(ENOMEM);
80  goto fail;
81  }
82 
84  if (ret < 0)
85  goto fail;
86 
87  src = (CVPixelBufferRef)in->data[3];
88  dst = (CVPixelBufferRef)out->data[3];
89  ret = VTPixelRotationSessionRotateImage(s->session, src, dst);
90  if (ret != noErr) {
91  av_log(ctx, AV_LOG_ERROR, "transfer image failed, %d\n", ret);
93  goto fail;
94  }
95 
96  av_frame_free(&in);
97 
98  return ff_filter_frame(outlink, out);
99 
100 fail:
101  av_frame_free(&in);
102  av_frame_free(&out);
103  return ret;
104 }
105 
107 {
108  AVFilterContext *avctx = outlink->src;
109  AVFilterLink *inlink = outlink->src->inputs[0];
110  AVHWFramesContext *hw_frame_ctx_in;
111  AVHWFramesContext *hw_frame_ctx_out;
112  int err;
113 
114  av_buffer_unref(&outlink->hw_frames_ctx);
115 
116  hw_frame_ctx_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
117  outlink->hw_frames_ctx = av_hwframe_ctx_alloc(hw_frame_ctx_in->device_ref);
118  hw_frame_ctx_out = (AVHWFramesContext *)outlink->hw_frames_ctx->data;
119  hw_frame_ctx_out->format = AV_PIX_FMT_VIDEOTOOLBOX;
120  hw_frame_ctx_out->sw_format = hw_frame_ctx_in->sw_format;
121  hw_frame_ctx_out->width = outlink->w;
122  hw_frame_ctx_out->height = outlink->h;
123 
124  err = ff_filter_init_hw_frames(avctx, outlink, 1);
125  if (err < 0)
126  return err;
127 
128  err = av_hwframe_ctx_init(outlink->hw_frames_ctx);
129  if (err < 0) {
130  av_log(avctx, AV_LOG_ERROR,
131  "Failed to init videotoolbox frame context, %s\n",
132  av_err2str(err));
133  return err;
134  }
135 
136  return 0;
137 }
138 
140 {
141  int err;
142  AVFilterContext *avctx = outlink->src;
143  TransposeVtContext *s = avctx->priv;
144  AVFilterLink *inlink = outlink->src->inputs[0];
145  CFStringRef rotation = kVTRotation_0;
146  CFBooleanRef vflip = kCFBooleanFalse;
147  CFBooleanRef hflip = kCFBooleanFalse;
148  int swap_w_h = 0;
149 
150  av_buffer_unref(&outlink->hw_frames_ctx);
151  outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
152 
153  if ((inlink->w >= inlink->h && s->passthrough == TRANSPOSE_PT_TYPE_LANDSCAPE) ||
154  (inlink->w <= inlink->h && s->passthrough == TRANSPOSE_PT_TYPE_PORTRAIT)) {
155  av_log(avctx, AV_LOG_VERBOSE,
156  "w:%d h:%d -> w:%d h:%d (passthrough mode)\n",
157  inlink->w, inlink->h, inlink->w, inlink->h);
158  return 0;
159  }
160 
161  s->passthrough = TRANSPOSE_PT_TYPE_NONE;
162 
163  switch (s->dir) {
165  rotation = kVTRotation_CCW90;
166  vflip = kCFBooleanTrue;
167  swap_w_h = 1;
168  break;
169  case TRANSPOSE_CCLOCK:
170  rotation = kVTRotation_CCW90;
171  swap_w_h = 1;
172  break;
173  case TRANSPOSE_CLOCK:
174  rotation = kVTRotation_CW90;
175  swap_w_h = 1;
176  break;
178  rotation = kVTRotation_CW90;
179  vflip = kCFBooleanTrue;
180  swap_w_h = 1;
181  break;
182  case TRANSPOSE_REVERSAL:
183  rotation = kVTRotation_180;
184  break;
185  case TRANSPOSE_HFLIP:
186  hflip = kCFBooleanTrue;
187  break;
188  case TRANSPOSE_VFLIP:
189  vflip = kCFBooleanTrue;
190  break;
191  default:
192  av_log(avctx, AV_LOG_ERROR, "Failed to set direction to %d\n", s->dir);
193  return AVERROR(EINVAL);
194  }
195 
196  err = VTSessionSetProperty(s->session, kVTPixelRotationPropertyKey_Rotation,
197  rotation);
198  if (err != noErr) {
199  av_log(avctx, AV_LOG_ERROR, "Set rotation property failed, %d\n", err);
200  return AVERROR_EXTERNAL;
201  }
202  err = VTSessionSetProperty(s->session, kVTPixelRotationPropertyKey_FlipVerticalOrientation,
203  vflip);
204  if (err != noErr) {
205  av_log(avctx, AV_LOG_ERROR, "Set vertical flip property failed, %d\n", err);
206  return AVERROR_EXTERNAL;
207  }
208  err = VTSessionSetProperty(s->session, kVTPixelRotationPropertyKey_FlipHorizontalOrientation,
209  hflip);
210  if (err != noErr) {
211  av_log(avctx, AV_LOG_ERROR, "Set horizontal flip property failed, %d\n", err);
212  return AVERROR_EXTERNAL;
213  }
214 
215  if (!swap_w_h)
216  return 0;
217 
218  outlink->w = inlink->h;
219  outlink->h = inlink->w;
220  return transpose_vt_recreate_hw_ctx(outlink);
221 }
222 
223 #define OFFSET(x) offsetof(TransposeVtContext, x)
224 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
225 static const AVOption transpose_vt_options[] = {
226  { "dir", "set transpose direction",
227  OFFSET(dir), AV_OPT_TYPE_INT, { .i64 = TRANSPOSE_CCLOCK_FLIP }, 0, 6, FLAGS, .unit = "dir" },
228  { "cclock_flip", "rotate counter-clockwise with vertical flip",
229  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK_FLIP }, .flags=FLAGS, .unit = "dir" },
230  { "clock", "rotate clockwise",
231  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK }, .flags=FLAGS, .unit = "dir" },
232  { "cclock", "rotate counter-clockwise",
233  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK }, .flags=FLAGS, .unit = "dir" },
234  { "clock_flip", "rotate clockwise with vertical flip",
235  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK_FLIP }, .flags=FLAGS, .unit = "dir" },
236  { "reversal", "rotate by half-turn",
237  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_REVERSAL }, .flags=FLAGS, .unit = "dir" },
238  { "hflip", "flip horizontally",
239  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_HFLIP }, .flags=FLAGS, .unit = "dir" },
240  { "vflip", "flip vertically",
241  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_VFLIP }, .flags=FLAGS, .unit = "dir" },
242 
243  { "passthrough", "do not apply transposition if the input matches the specified geometry",
244  OFFSET(passthrough), AV_OPT_TYPE_INT, { .i64=TRANSPOSE_PT_TYPE_NONE }, 0, INT_MAX, FLAGS, .unit = "passthrough" },
245  { "none", "always apply transposition",
246  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_NONE }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
247  { "portrait", "preserve portrait geometry",
248  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_PORTRAIT }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
249  { "landscape", "preserve landscape geometry",
250  0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_PT_TYPE_LANDSCAPE }, INT_MIN, INT_MAX, FLAGS, .unit = "passthrough" },
251 
252  { NULL }
253 };
254 
255 AVFILTER_DEFINE_CLASS(transpose_vt);
256 
258  {
259  .name = "default",
260  .type = AVMEDIA_TYPE_VIDEO,
261  .filter_frame = &transpose_vt_filter_frame,
262  },
263 };
264 
266  {
267  .name = "default",
268  .type = AVMEDIA_TYPE_VIDEO,
269  .config_props = &transpose_vt_config_output,
270  },
271 };
272 
274  .name = "transpose_vt",
275  .description = NULL_IF_CONFIG_SMALL("Transpose Videotoolbox frames"),
276  .priv_size = sizeof(TransposeVtContext),
282  .priv_class = &transpose_vt_class,
283  .flags = AVFILTER_FLAG_HWDEVICE,
284  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
285 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:112
TransposeVtContext::session
VTPixelRotationSessionRef session
Definition: vf_transpose_vt.c:34
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: internal.h:351
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1018
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
AVHWFramesContext::format
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
Definition: hwcontext.h:197
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:130
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:322
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:344
pixdesc.h
transpose_vt_config_output
static int transpose_vt_config_output(AVFilterLink *outlink)
Definition: vf_transpose_vt.c:139
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:248
AVOption
AVOption.
Definition: opt.h:346
transpose_vt_init
static av_cold int transpose_vt_init(AVFilterContext *avctx)
Definition: vf_transpose_vt.c:39
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
TRANSPOSE_CLOCK_FLIP
@ TRANSPOSE_CLOCK_FLIP
Definition: transpose.h:34
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:217
video.h
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:365
TRANSPOSE_CCLOCK
@ TRANSPOSE_CCLOCK
Definition: transpose.h:33
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:422
fail
#define fail()
Definition: checkasm.h:179
transpose_vt_recreate_hw_ctx
static int transpose_vt_recreate_hw_ctx(AVFilterLink *outlink)
Definition: vf_transpose_vt.c:106
FLAGS
#define FLAGS
Definition: vf_transpose_vt.c:224
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:33
TRANSPOSE_HFLIP
@ TRANSPOSE_HFLIP
Definition: transpose.h:36
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
AVHWFramesContext::height
int height
Definition: hwcontext.h:217
s
#define s(width, name)
Definition: cbs_vp9.c:198
ctx
AVFormatContext * ctx
Definition: movenc.c:48
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:182
OFFSET
#define OFFSET(x)
Definition: vf_transpose_vt.c:223
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
TransposeVtContext
Definition: vf_transpose_vt.c:31
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:210
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:679
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVHWFramesContext::device_ref
AVBufferRef * device_ref
A reference to the parent AVHWDeviceContext.
Definition: hwcontext.h:126
TransposeVtContext::dir
int dir
Definition: vf_transpose_vt.c:35
AVFilterContext::inputs
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:415
transpose_vt_uninit
static av_cold void transpose_vt_uninit(AVFilterContext *avctx)
Definition: vf_transpose_vt.c:53
hwcontext_videotoolbox.h
TRANSPOSE_PT_TYPE_PORTRAIT
@ TRANSPOSE_PT_TYPE_PORTRAIT
Definition: transpose.h:27
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:365
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:106
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:121
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:138
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(transpose_vt)
TRANSPOSE_PT_TYPE_NONE
@ TRANSPOSE_PT_TYPE_NONE
Definition: transpose.h:25
transpose_vt_filter_frame
static int transpose_vt_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_transpose_vt.c:64
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
internal.h
ff_vf_transpose_vt
const AVFilter ff_vf_transpose_vt
Definition: vf_transpose_vt.c:273
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:172
AV_PIX_FMT_VIDEOTOOLBOX
@ AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
Definition: pixfmt.h:305
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
TRANSPOSE_CLOCK
@ TRANSPOSE_CLOCK
Definition: transpose.h:32
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:39
AVFilter
Filter definition.
Definition: avfilter.h:166
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:115
ret
ret
Definition: filter_design.txt:187
TRANSPOSE_CCLOCK_FLIP
@ TRANSPOSE_CCLOCK_FLIP
Definition: transpose.h:31
TransposeVtContext::passthrough
int passthrough
Definition: vf_transpose_vt.c:36
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:235
transpose.h
TRANSPOSE_REVERSAL
@ TRANSPOSE_REVERSAL
Definition: transpose.h:35
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
TRANSPOSE_PT_TYPE_LANDSCAPE
@ TRANSPOSE_PT_TYPE_LANDSCAPE
Definition: transpose.h:26
TRANSPOSE_VFLIP
@ TRANSPOSE_VFLIP
Definition: transpose.h:37
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
transpose_vt_inputs
static const AVFilterPad transpose_vt_inputs[]
Definition: vf_transpose_vt.c:257
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:183
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
hwcontext.h
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AV_OPT_TYPE_CONST
@ AV_OPT_TYPE_CONST
Definition: opt.h:244
transpose_vt_options
static const AVOption transpose_vt_options[]
Definition: vf_transpose_vt.c:225
ff_filter_init_hw_frames
int ff_filter_init_hw_frames(AVFilterContext *avctx, AVFilterLink *link, int default_pool_size)
Perform any additional setup required for hardware frames.
Definition: avfilter.c:1616
transpose_vt_outputs
static const AVFilterPad transpose_vt_outputs[]
Definition: vf_transpose_vt.c:265