FFmpeg
vf_flip_vulkan.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2021 Wu Jianhua <jianhua.wu@intel.com>
3  * Copyright (c) Lynne
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/random_seed.h"
23 #include "libavutil/opt.h"
24 #include "vulkan_filter.h"
25 #include "vulkan_spirv.h"
26 #include "internal.h"
27 #include "video.h"
28 
29 enum FlipType {
33 };
34 
35 typedef struct FlipVulkanContext {
37 
43  VkSampler sampler;
45 
47 {
48  int err = 0;
49  uint8_t *spv_data;
50  size_t spv_len;
51  void *spv_opaque = NULL;
52  FlipVulkanContext *s = ctx->priv;
53  FFVulkanContext *vkctx = &s->vkctx;
54  const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
55  FFVkSPIRVShader *shd = &s->shd;
56  FFVkSPIRVCompiler *spv;
58 
59  spv = ff_vk_spirv_init();
60  if (!spv) {
61  av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
62  return AVERROR_EXTERNAL;
63  }
64 
65  ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
66  RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
67  RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_LINEAR));
68  RET(ff_vk_shader_init(&s->pl, &s->shd, "flip_compute",
69  VK_SHADER_STAGE_COMPUTE_BIT, 0));
70 
71  ff_vk_shader_set_compute_sizes(&s->shd, 32, 32, 1);
72 
74  {
75  .name = "input_image",
76  .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
77  .dimensions = 2,
78  .elems = planes,
79  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
80  .samplers = DUP_SAMPLER(s->sampler),
81  },
82  {
83  .name = "output_image",
84  .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
85  .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
86  .mem_quali = "writeonly",
87  .dimensions = 2,
88  .elems = planes,
89  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
90  },
91  };
92 
93  RET(ff_vk_pipeline_descriptor_set_add(vkctx, &s->pl, shd, desc, 2, 0, 0));
94 
95  GLSLC(0, void main() );
96  GLSLC(0, { );
97  GLSLC(1, ivec2 size; );
98  GLSLC(1, const ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
99  for (int i = 0; i < planes; i++) {
100  GLSLC(0, );
101  GLSLF(1, size = imageSize(output_image[%i]); ,i);
102  GLSLC(1, if (IS_WITHIN(pos, size)) { );
103  switch (type)
104  {
105  case FLIP_HORIZONTAL:
106  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.x - pos.x, pos.y)); ,i);
107  break;
108  case FLIP_VERTICAL:
109  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(pos.x, size.y - pos.y)); ,i);
110  break;
111  case FLIP_BOTH:
112  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.xy - pos.xy));, i);
113  break;
114  default:
115  GLSLF(2, vec4 res = texture(input_image[%i], pos); ,i);
116  break;
117  }
118  GLSLF(2, imageStore(output_image[%i], pos, res); ,i);
119  GLSLC(1, } );
120  }
121  GLSLC(0, } );
122 
123  RET(spv->compile_shader(spv, ctx, shd, &spv_data, &spv_len, "main",
124  &spv_opaque));
125  RET(ff_vk_shader_create(vkctx, shd, spv_data, spv_len, "main"));
126 
127  RET(ff_vk_init_compute_pipeline(vkctx, &s->pl, shd));
128  RET(ff_vk_exec_pipeline_register(vkctx, &s->e, &s->pl));
129 
130  s->initialized = 1;
131 
132 fail:
133  if (spv_opaque)
134  spv->free_shader(spv, &spv_opaque);
135  if (spv)
136  spv->uninit(&spv);
137 
138  return err;
139 }
140 
142 {
143  FlipVulkanContext *s = avctx->priv;
144 
145  FFVulkanContext *vkctx = &s->vkctx;
146  FFVulkanFunctions *vk = &vkctx->vkfn;
147 
148  ff_vk_exec_pool_free(vkctx, &s->e);
149  ff_vk_pipeline_free(vkctx, &s->pl);
150  ff_vk_shader_free(vkctx, &s->shd);
151 
152  if (s->sampler)
153  vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
154  vkctx->hwctx->alloc);
155 
156  ff_vk_uninit(&s->vkctx);
157 
158  s->initialized = 0;
159 }
160 
162 {
163  int err;
164  AVFrame *out = NULL;
165  AVFilterContext *ctx = link->dst;
166  FlipVulkanContext *s = ctx->priv;
167  AVFilterLink *outlink = ctx->outputs[0];
168 
169  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
170  if (!out) {
171  err = AVERROR(ENOMEM);
172  goto fail;
173  }
174 
175  if (!s->initialized)
176  RET(init_filter(ctx, in, type));
177 
178  RET(ff_vk_filter_process_simple(&s->vkctx, &s->e, &s->pl, out, in,
179  s->sampler, NULL, 0));
180 
182 
183  av_frame_free(&in);
184 
185  return ff_filter_frame(outlink, out);
186 
187 fail:
188  av_frame_free(&in);
189  av_frame_free(&out);
190  return err;
191 }
192 
194 {
195  return filter_frame(link, in, FLIP_HORIZONTAL);
196 }
197 
199 {
200  return filter_frame(link, in, FLIP_VERTICAL);
201 }
202 
204 {
205  return filter_frame(link, in, FLIP_BOTH);
206 }
207 
209  {
210  .name = "default",
211  .type = AVMEDIA_TYPE_VIDEO,
212  .config_props = &ff_vk_filter_config_output,
213  }
214 };
215 
216 static const AVOption hflip_vulkan_options[] = {
217  { NULL },
218 };
219 
220 AVFILTER_DEFINE_CLASS(hflip_vulkan);
221 
223  {
224  .name = "default",
225  .type = AVMEDIA_TYPE_VIDEO,
226  .filter_frame = &hflip_vulkan_filter_frame,
227  .config_props = &ff_vk_filter_config_input,
228  }
229 };
230 
232  .name = "hflip_vulkan",
233  .description = NULL_IF_CONFIG_SMALL("Horizontally flip the input video in Vulkan"),
234  .priv_size = sizeof(FlipVulkanContext),
240  .priv_class = &hflip_vulkan_class,
241  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
242 };
243 
244 static const AVOption vflip_vulkan_options[] = {
245  { NULL },
246 };
247 
248 AVFILTER_DEFINE_CLASS(vflip_vulkan);
249 
251  {
252  .name = "default",
253  .type = AVMEDIA_TYPE_VIDEO,
254  .filter_frame = &vflip_vulkan_filter_frame,
255  .config_props = &ff_vk_filter_config_input,
256  }
257 };
258 
260  .name = "vflip_vulkan",
261  .description = NULL_IF_CONFIG_SMALL("Vertically flip the input video in Vulkan"),
262  .priv_size = sizeof(FlipVulkanContext),
268  .priv_class = &vflip_vulkan_class,
269  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
270 };
271 
272 static const AVOption flip_vulkan_options[] = {
273  { NULL },
274 };
275 
276 AVFILTER_DEFINE_CLASS(flip_vulkan);
277 
278 static const AVFilterPad flip_vulkan_inputs[] = {
279  {
280  .name = "default",
281  .type = AVMEDIA_TYPE_VIDEO,
282  .filter_frame = &flip_vulkan_filter_frame,
283  .config_props = &ff_vk_filter_config_input,
284  }
285 };
286 
288  .name = "flip_vulkan",
289  .description = NULL_IF_CONFIG_SMALL("Flip both horizontally and vertically"),
290  .priv_size = sizeof(FlipVulkanContext),
296  .priv_class = &flip_vulkan_class,
297  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
298  .flags = AVFILTER_FLAG_HWDEVICE,
299 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:112
ff_vk_pipeline_free
void ff_vk_pipeline_free(FFVulkanContext *s, FFVulkanPipeline *pl)
Definition: vulkan.c:1843
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: internal.h:351
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1018
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:224
flip_vulkan_filter_frame
static int flip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:203
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:130
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:344
ff_vk_filter_init
int ff_vk_filter_init(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan_filter.c:221
ff_vk_shader_create
int ff_vk_shader_create(FFVulkanContext *s, FFVkSPIRVShader *shd, uint8_t *spirv, size_t spirv_size, const char *entrypoint)
Definition: vulkan.c:1414
AVOption
AVOption.
Definition: opt.h:346
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:1872
FFVkSPIRVCompiler::uninit
void(* uninit)(struct FFVkSPIRVCompiler **ctx)
Definition: vulkan_spirv.h:33
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
ff_vk_pipeline_descriptor_set_add
int ff_vk_pipeline_descriptor_set_add(FFVulkanContext *s, FFVulkanPipeline *pl, FFVkSPIRVShader *shd, FFVulkanDescriptorSetBinding *desc, int nb, int read_only, int print_to_shader_only)
Add descriptor to a pipeline.
Definition: vulkan.c:1464
ff_vk_shader_set_compute_sizes
void ff_vk_shader_set_compute_sizes(FFVkSPIRVShader *shd, int x, int y, int z)
Definition: vulkan.c:1372
video.h
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
ff_vf_vflip_vulkan
const AVFilter ff_vf_vflip_vulkan
Definition: vf_flip_vulkan.c:259
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3002
FlipType
FlipType
Definition: vf_flip_vulkan.c:29
vflip_vulkan_options
static const AVOption vflip_vulkan_options[]
Definition: vf_flip_vulkan.c:244
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:48
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:422
fail
#define fail()
Definition: checkasm.h:179
vulkan_filter.h
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:33
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
ff_vf_hflip_vulkan
const AVFilter ff_vf_hflip_vulkan
Definition: vf_flip_vulkan.c:231
s
#define s(width, name)
Definition: cbs_vp9.c:198
FlipVulkanContext::initialized
int initialized
Definition: vf_flip_vulkan.c:38
vflip_vulkan_inputs
static const AVFilterPad vflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:250
ctx
AVFormatContext * ctx
Definition: movenc.c:48
FFVkSPIRVCompiler::compile_shader
int(* compile_shader)(struct FFVkSPIRVCompiler *ctx, void *avctx, struct FFVkSPIRVShader *shd, uint8_t **data, size_t *size, const char *entrypoint, void **opaque)
Definition: vulkan_spirv.h:29
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:255
hflip_vulkan_inputs
static const AVFilterPad hflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:222
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:182
FlipVulkanContext::sampler
VkSampler sampler
Definition: vf_flip_vulkan.c:43
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:679
init_filter
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:46
FLIP_VERTICAL
@ FLIP_VERTICAL
Definition: vf_flip_vulkan.c:30
ff_vk_filter_config_output
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan_filter.c:198
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:161
ff_vk_init_compute_pipeline
int ff_vk_init_compute_pipeline(FFVulkanContext *s, FFVulkanPipeline *pl, FFVkSPIRVShader *shd)
Definition: vulkan.c:1784
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:295
FFVulkanContext
Definition: vulkan.h:228
FFVulkanPipeline
Definition: vulkan.h:131
FlipVulkanContext::qf
FFVkQueueFamilyCtx qf
Definition: vf_flip_vulkan.c:41
FlipVulkanContext::e
FFVkExecPool e
Definition: vf_flip_vulkan.c:40
ff_vk_shader_init
int ff_vk_shader_init(FFVulkanPipeline *pl, FFVkSPIRVShader *shd, const char *name, VkShaderStageFlags stage, uint32_t required_subgroup_size)
Shader management.
Definition: vulkan.c:1346
main
int main(int argc, char **argv)
Definition: avio_http_serve_files.c:99
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(hflip_vulkan)
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:365
FFVulkanDescriptorSetBinding
Definition: vulkan.h:83
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:106
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:138
FLIP_BOTH
@ FLIP_BOTH
Definition: vf_flip_vulkan.c:32
flip_vulkan_inputs
static const AVFilterPad flip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:278
size
int size
Definition: twinvq_data.h:10344
FFVkQueueFamilyCtx
Definition: vulkan.h:110
FLIP_HORIZONTAL
@ FLIP_HORIZONTAL
Definition: vf_flip_vulkan.c:31
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
hflip_vulkan_filter_frame
static int hflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:193
internal.h
ff_vk_filter_process_simple
int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, FFVulkanPipeline *pl, AVFrame *out_f, AVFrame *in_f, VkSampler sampler, void *push_src, size_t push_size)
Submit a compute shader with a zero/one input and single out for execution.
Definition: vulkan_filter.c:230
FFVkSPIRVCompiler
Definition: vulkan_spirv.h:27
flip_vulkan_uninit
static av_cold void flip_vulkan_uninit(AVFilterContext *avctx)
Definition: vf_flip_vulkan.c:141
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:172
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
vflip_vulkan_filter_frame
static int vflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:198
FlipVulkanContext::shd
FFVkSPIRVShader shd
Definition: vf_flip_vulkan.c:42
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:255
FlipVulkanContext::vkctx
FFVulkanContext vkctx
Definition: vf_flip_vulkan.c:36
hflip_vulkan_options
static const AVOption hflip_vulkan_options[]
Definition: vf_flip_vulkan.c:216
DUP_SAMPLER
#define DUP_SAMPLER(x)
Definition: vulkan.h:73
FlipVulkanContext::pl
FFVulkanPipeline pl
Definition: vf_flip_vulkan.c:39
ff_vk_shader_rep_fmt
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt)
Returns the format to use for images in shaders.
Definition: vulkan.c:1206
ff_vf_flip_vulkan
const AVFilter ff_vf_flip_vulkan
Definition: vf_flip_vulkan.c:287
vulkan_spirv.h
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:39
GLSLF
#define GLSLF(N, S,...)
Definition: vulkan.h:54
flip_vulkan_outputs
static const AVFilterPad flip_vulkan_outputs[]
Definition: vf_flip_vulkan.c:208
FFVkSPIRVCompiler::free_shader
void(* free_shader)(struct FFVkSPIRVCompiler *ctx, void **opaque)
Definition: vulkan_spirv.h:32
AVFilter
Filter definition.
Definition: avfilter.h:166
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:231
FFVkExecPool
Definition: vulkan.h:210
pos
unsigned int pos
Definition: spdifenc.c:413
random_seed.h
FFVkSPIRVShader
Definition: vulkan.h:75
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
desc
const char * desc
Definition: libsvtav1.c:75
GLSLC
#define GLSLC(N, S)
Definition: vulkan.h:44
ff_vk_filter_config_input
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan_filter.c:166
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:253
FlipVulkanContext
Definition: vf_flip_vulkan.c:35
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:70
planes
static const struct @386 planes[]
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:183
ff_vk_init_sampler
int ff_vk_init_sampler(FFVulkanContext *s, VkSampler *sampler, int unnorm_coords, VkFilter filt)
Create a sampler.
Definition: vulkan.c:1162
ff_vk_exec_pipeline_register
int ff_vk_exec_pipeline_register(FFVulkanContext *s, FFVkExecPool *pool, FFVulkanPipeline *pl)
Register a pipeline with an exec pool.
Definition: vulkan.c:1578
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ff_vk_shader_free
void ff_vk_shader_free(FFVulkanContext *s, FFVkSPIRVShader *shd)
Definition: vulkan.c:1405
RET
#define RET(x)
Definition: vulkan.h:67
FFVulkanFunctions
Definition: vulkan_functions.h:226
flip_vulkan_options
static const AVOption flip_vulkan_options[]
Definition: vf_flip_vulkan.c:272