FFmpeg
vf_flip_vulkan.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2021 Wu Jianhua <jianhua.wu@intel.com>
3  * Copyright (c) Lynne
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/random_seed.h"
23 #include "libavutil/opt.h"
24 #include "libavutil/vulkan_spirv.h"
25 #include "vulkan_filter.h"
26 
27 #include "filters.h"
28 #include "video.h"
29 
30 enum FlipType {
34 };
35 
36 typedef struct FlipVulkanContext {
38 
43  VkSampler sampler;
45 
47 {
48  int err = 0;
49  uint8_t *spv_data;
50  size_t spv_len;
51  void *spv_opaque = NULL;
52  FlipVulkanContext *s = ctx->priv;
53  FFVulkanContext *vkctx = &s->vkctx;
54  const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
55  FFVulkanShader *shd = &s->shd;
56  FFVkSPIRVCompiler *spv;
58 
59  spv = ff_vk_spirv_init();
60  if (!spv) {
61  av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
62  return AVERROR_EXTERNAL;
63  }
64 
65  ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
66  RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
67  RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_LINEAR));
68  RET(ff_vk_shader_init(vkctx, &s->shd, "flip",
69  VK_SHADER_STAGE_COMPUTE_BIT,
70  NULL, 0,
71  32, 32, 1,
72  0));
73 
75  {
76  .name = "input_image",
77  .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
78  .dimensions = 2,
79  .elems = planes,
80  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
81  .samplers = DUP_SAMPLER(s->sampler),
82  },
83  {
84  .name = "output_image",
85  .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
86  .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format, FF_VK_REP_FLOAT),
87  .mem_quali = "writeonly",
88  .dimensions = 2,
89  .elems = planes,
90  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
91  },
92  };
93 
94  RET(ff_vk_shader_add_descriptor_set(vkctx, &s->shd, desc, 2, 0, 0));
95 
96  GLSLC(0, void main() );
97  GLSLC(0, { );
98  GLSLC(1, ivec2 size; );
99  GLSLC(1, const ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
100  for (int i = 0; i < planes; i++) {
101  GLSLC(0, );
102  GLSLF(1, size = imageSize(output_image[%i]); ,i);
103  GLSLC(1, if (IS_WITHIN(pos, size)) { );
104  switch (type)
105  {
106  case FLIP_HORIZONTAL:
107  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.x - pos.x, pos.y)); ,i);
108  break;
109  case FLIP_VERTICAL:
110  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(pos.x, size.y - pos.y)); ,i);
111  break;
112  case FLIP_BOTH:
113  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.xy - pos.xy));, i);
114  break;
115  default:
116  GLSLF(2, vec4 res = texture(input_image[%i], pos); ,i);
117  break;
118  }
119  GLSLF(2, imageStore(output_image[%i], pos, res); ,i);
120  GLSLC(1, } );
121  }
122  GLSLC(0, } );
123 
124  RET(spv->compile_shader(vkctx, spv, shd, &spv_data, &spv_len, "main",
125  &spv_opaque));
126  RET(ff_vk_shader_link(vkctx, shd, spv_data, spv_len, "main"));
127 
128  RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->shd));
129 
130  s->initialized = 1;
131 
132 fail:
133  if (spv_opaque)
134  spv->free_shader(spv, &spv_opaque);
135  if (spv)
136  spv->uninit(&spv);
137 
138  return err;
139 }
140 
142 {
143  FlipVulkanContext *s = avctx->priv;
144 
145  FFVulkanContext *vkctx = &s->vkctx;
146  FFVulkanFunctions *vk = &vkctx->vkfn;
147 
148  ff_vk_exec_pool_free(vkctx, &s->e);
149  ff_vk_shader_free(vkctx, &s->shd);
150 
151  if (s->sampler)
152  vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
153  vkctx->hwctx->alloc);
154 
155  ff_vk_uninit(&s->vkctx);
156 
157  s->initialized = 0;
158 }
159 
161 {
162  int err;
163  AVFrame *out = NULL;
164  AVFilterContext *ctx = link->dst;
165  FlipVulkanContext *s = ctx->priv;
166  AVFilterLink *outlink = ctx->outputs[0];
167 
168  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
169  if (!out) {
170  err = AVERROR(ENOMEM);
171  goto fail;
172  }
173 
174  if (!s->initialized)
175  RET(init_filter(ctx, in, type));
176 
177  RET(ff_vk_filter_process_simple(&s->vkctx, &s->e, &s->shd, out, in,
178  s->sampler, NULL, 0));
179 
181 
182  av_frame_free(&in);
183 
184  return ff_filter_frame(outlink, out);
185 
186 fail:
187  av_frame_free(&in);
188  av_frame_free(&out);
189  return err;
190 }
191 
193 {
194  return filter_frame(link, in, FLIP_HORIZONTAL);
195 }
196 
198 {
199  return filter_frame(link, in, FLIP_VERTICAL);
200 }
201 
203 {
204  return filter_frame(link, in, FLIP_BOTH);
205 }
206 
208  {
209  .name = "default",
210  .type = AVMEDIA_TYPE_VIDEO,
211  .config_props = &ff_vk_filter_config_output,
212  }
213 };
214 
215 static const AVOption hflip_vulkan_options[] = {
216  { NULL },
217 };
218 
219 AVFILTER_DEFINE_CLASS(hflip_vulkan);
220 
222  {
223  .name = "default",
224  .type = AVMEDIA_TYPE_VIDEO,
225  .filter_frame = &hflip_vulkan_filter_frame,
226  .config_props = &ff_vk_filter_config_input,
227  }
228 };
229 
231  .name = "hflip_vulkan",
232  .description = NULL_IF_CONFIG_SMALL("Horizontally flip the input video in Vulkan"),
233  .priv_size = sizeof(FlipVulkanContext),
239  .priv_class = &hflip_vulkan_class,
240  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
241 };
242 
243 static const AVOption vflip_vulkan_options[] = {
244  { NULL },
245 };
246 
247 AVFILTER_DEFINE_CLASS(vflip_vulkan);
248 
250  {
251  .name = "default",
252  .type = AVMEDIA_TYPE_VIDEO,
253  .filter_frame = &vflip_vulkan_filter_frame,
254  .config_props = &ff_vk_filter_config_input,
255  }
256 };
257 
259  .name = "vflip_vulkan",
260  .description = NULL_IF_CONFIG_SMALL("Vertically flip the input video in Vulkan"),
261  .priv_size = sizeof(FlipVulkanContext),
267  .priv_class = &vflip_vulkan_class,
268  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
269 };
270 
271 static const AVOption flip_vulkan_options[] = {
272  { NULL },
273 };
274 
275 AVFILTER_DEFINE_CLASS(flip_vulkan);
276 
277 static const AVFilterPad flip_vulkan_inputs[] = {
278  {
279  .name = "default",
280  .type = AVMEDIA_TYPE_VIDEO,
281  .filter_frame = &flip_vulkan_filter_frame,
282  .config_props = &ff_vk_filter_config_input,
283  }
284 };
285 
287  .name = "flip_vulkan",
288  .description = NULL_IF_CONFIG_SMALL("Flip both horizontally and vertically"),
289  .priv_size = sizeof(FlipVulkanContext),
295  .priv_class = &flip_vulkan_class,
296  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
297  .flags = AVFILTER_FLAG_HWDEVICE,
298 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:116
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_vk_shader_free
void ff_vk_shader_free(FFVulkanContext *s, FFVulkanShader *shd)
Free a shader.
Definition: vulkan.c:2586
ff_vk_shader_init
int ff_vk_shader_init(FFVulkanContext *s, FFVulkanShader *shd, const char *name, VkPipelineStageFlags stage, const char *extensions[], int nb_extensions, int lg_x, int lg_y, int lg_z, uint32_t required_subgroup_size)
Initialize a shader object, with a specific set of extensions, type+bind, local group size,...
Definition: vulkan.c:1737
out
FILE * out
Definition: movenc.c:55
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1061
FlipVulkanContext::shd
FFVulkanShader shd
Definition: vf_flip_vulkan.c:42
RET
#define RET(x)
Definition: vulkan.h:67
ff_vk_filter_process_simple
int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, FFVulkanShader *shd, AVFrame *out_f, AVFrame *in_f, VkSampler sampler, void *push_src, size_t push_size)
Submit a compute shader with a zero/one input and single out for execution.
Definition: vulkan_filter.c:242
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:232
flip_vulkan_filter_frame
static int flip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:202
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: filters.h:262
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
ff_vk_filter_init
int ff_vk_filter_init(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan_filter.c:233
AVOption
AVOption.
Definition: opt.h:429
planes
static const struct @467 planes[]
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2625
FFVkSPIRVCompiler::uninit
void(* uninit)(struct FFVkSPIRVCompiler **ctx)
Definition: vulkan_spirv.h:32
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:205
video.h
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
ff_vf_vflip_vulkan
const AVFilter ff_vf_vflip_vulkan
Definition: vf_flip_vulkan.c:258
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3210
FlipType
FlipType
Definition: vf_flip_vulkan.c:30
vflip_vulkan_options
static const AVOption vflip_vulkan_options[]
Definition: vf_flip_vulkan.c:243
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:472
fail
#define fail()
Definition: checkasm.h:193
vulkan_filter.h
ff_vk_shader_register_exec
int ff_vk_shader_register_exec(FFVulkanContext *s, FFVkExecPool *pool, FFVulkanShader *shd)
Register a shader with an exec pool.
Definition: vulkan.c:2226
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
ff_vk_shader_add_descriptor_set
int ff_vk_shader_add_descriptor_set(FFVulkanContext *s, FFVulkanShader *shd, FFVulkanDescriptorSetBinding *desc, int nb, int singular, int print_to_shader_only)
Add descriptor to a shader.
Definition: vulkan.c:2101
AVFilterPad
A filter pad used for either input or output.
Definition: filters.h:38
GLSLC
#define GLSLC(N, S)
Definition: vulkan.h:44
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
av_cold
#define av_cold
Definition: attributes.h:90
main
int main
Definition: dovi_rpuenc.c:37
ff_vf_hflip_vulkan
const AVFilter ff_vf_hflip_vulkan
Definition: vf_flip_vulkan.c:230
s
#define s(width, name)
Definition: cbs_vp9.c:198
FlipVulkanContext::initialized
int initialized
Definition: vf_flip_vulkan.c:39
vflip_vulkan_inputs
static const AVFilterPad vflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:249
filters.h
FF_VK_REP_FLOAT
@ FF_VK_REP_FLOAT
Definition: vulkan.h:381
ctx
AVFormatContext * ctx
Definition: movenc.c:49
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:242
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: filters.h:263
hflip_vulkan_inputs
static const AVFilterPad hflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:221
FlipVulkanContext::sampler
VkSampler sampler
Definition: vf_flip_vulkan.c:43
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
ff_vk_shader_rep_fmt
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt, enum FFVkShaderRepFormat rep_fmt)
Definition: vulkan.c:1344
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:725
DUP_SAMPLER
#define DUP_SAMPLER(x)
Definition: vulkan.h:73
init_filter
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:46
FLIP_VERTICAL
@ FLIP_VERTICAL
Definition: vf_flip_vulkan.c:31
ff_vk_filter_config_output
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan_filter.c:209
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:160
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:306
FFVulkanContext
Definition: vulkan.h:271
FlipVulkanContext::qf
FFVkQueueFamilyCtx qf
Definition: vf_flip_vulkan.c:41
FlipVulkanContext::e
FFVkExecPool e
Definition: vf_flip_vulkan.c:40
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: filters.h:206
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(hflip_vulkan)
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
FFVulkanDescriptorSetBinding
Definition: vulkan.h:75
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:173
FLIP_BOTH
@ FLIP_BOTH
Definition: vf_flip_vulkan.c:33
flip_vulkan_inputs
static const AVFilterPad flip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:277
size
int size
Definition: twinvq_data.h:10344
FFVkQueueFamilyCtx
Definition: vulkan.h:102
FFVulkanShader
Definition: vulkan.h:187
FLIP_HORIZONTAL
@ FLIP_HORIZONTAL
Definition: vf_flip_vulkan.c:32
FFVkSPIRVCompiler::compile_shader
int(* compile_shader)(FFVulkanContext *s, struct FFVkSPIRVCompiler *ctx, FFVulkanShader *shd, uint8_t **data, size_t *size, const char *entrypoint, void **opaque)
Definition: vulkan_spirv.h:28
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
hflip_vulkan_filter_frame
static int hflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:192
FFVkSPIRVCompiler
Definition: vulkan_spirv.h:26
flip_vulkan_uninit
static av_cold void flip_vulkan_uninit(AVFilterContext *avctx)
Definition: vf_flip_vulkan.c:141
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
vflip_vulkan_filter_frame
static int vflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:197
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
FlipVulkanContext::vkctx
FFVulkanContext vkctx
Definition: vf_flip_vulkan.c:37
hflip_vulkan_options
static const AVOption hflip_vulkan_options[]
Definition: vf_flip_vulkan.c:215
ff_vk_shader_link
int ff_vk_shader_link(FFVulkanContext *s, FFVulkanShader *shd, uint8_t *spirv, size_t spirv_len, const char *entrypoint)
Link a shader into an executable.
Definition: vulkan.c:2026
ff_vf_flip_vulkan
const AVFilter ff_vf_flip_vulkan
Definition: vf_flip_vulkan.c:286
vulkan_spirv.h
AVFilterPad::name
const char * name
Pad name.
Definition: filters.h:44
flip_vulkan_outputs
static const AVFilterPad flip_vulkan_outputs[]
Definition: vf_flip_vulkan.c:207
FFVkSPIRVCompiler::free_shader
void(* free_shader)(struct FFVkSPIRVCompiler *ctx, void **opaque)
Definition: vulkan_spirv.h:31
AVFilter
Filter definition.
Definition: avfilter.h:201
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:275
FFVkExecPool
Definition: vulkan.h:249
pos
unsigned int pos
Definition: spdifenc.c:414
random_seed.h
GLSLF
#define GLSLF(N, S,...)
Definition: vulkan.h:54
AVFilterContext
An instance of a filter.
Definition: avfilter.h:457
desc
const char * desc
Definition: libsvtav1.c:79
ff_vk_filter_config_input
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan_filter.c:176
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:300
FlipVulkanContext
Definition: vf_flip_vulkan.c:36
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
ff_vk_init_sampler
int ff_vk_init_sampler(FFVulkanContext *s, VkSampler *sampler, int unnorm_coords, VkFilter filt)
Create a sampler.
Definition: vulkan.c:1274
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: filters.h:252
FFVulkanFunctions
Definition: vulkan_functions.h:264
flip_vulkan_options
static const AVOption flip_vulkan_options[]
Definition: vf_flip_vulkan.c:271