FFmpeg
vulkan.h
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifndef AVFILTER_VULKAN_H
20 #define AVFILTER_VULKAN_H
21 
22 #include "avfilter.h"
23 #include "libavutil/pixdesc.h"
24 #include "libavutil/bprint.h"
25 #include "libavutil/hwcontext.h"
27 
28 /* GLSL management macros */
29 #define INDENT(N) INDENT_##N
30 #define INDENT_0
31 #define INDENT_1 INDENT_0 " "
32 #define INDENT_2 INDENT_1 INDENT_1
33 #define INDENT_3 INDENT_2 INDENT_1
34 #define INDENT_4 INDENT_3 INDENT_1
35 #define INDENT_5 INDENT_4 INDENT_1
36 #define INDENT_6 INDENT_5 INDENT_1
37 #define C(N, S) INDENT(N) #S "\n"
38 #define GLSLC(N, S) av_bprintf(&shd->src, C(N, S))
39 #define GLSLA(...) av_bprintf(&shd->src, __VA_ARGS__)
40 #define GLSLF(N, S, ...) av_bprintf(&shd->src, C(N, S), __VA_ARGS__)
41 #define GLSLD(D) GLSLC(0, ); \
42  av_bprint_append_data(&shd->src, D, strlen(D)); \
43  GLSLC(0, )
44 
45 /* Helper, pretty much every Vulkan return value needs to be checked */
46 #define RET(x) \
47  do { \
48  if ((err = (x)) < 0) \
49  goto fail; \
50  } while (0)
51 
52 /* Gets the queues count for a single queue family */
53 #define GET_QUEUE_COUNT(hwctx, graph, comp, tx) ( \
54  graph ? hwctx->nb_graphics_queues : \
55  comp ? (hwctx->nb_comp_queues ? \
56  hwctx->nb_comp_queues : hwctx->nb_graphics_queues) : \
57  tx ? (hwctx->nb_tx_queues ? hwctx->nb_tx_queues : \
58  (hwctx->nb_comp_queues ? \
59  hwctx->nb_comp_queues : hwctx->nb_graphics_queues)) : \
60  0 \
61 )
62 
63 /* Useful for attaching immutable samplers to arrays */
64 #define DUP_SAMPLER_ARRAY4(x) (VkSampler []){ x, x, x, x, }
65 
66 typedef struct SPIRVShader {
67  const char *name; /* Name for id/debugging purposes */
68  AVBPrint src;
69  int local_size[3]; /* Compute shader workgroup sizes */
70  VkPipelineShaderStageCreateInfo shader;
71 } SPIRVShader;
72 
74  const char *name;
75  VkDescriptorType type;
76  const char *mem_layout; /* Storage images (rgba8, etc.) and buffers (std430, etc.) */
77  const char *mem_quali; /* readonly, writeonly, etc. */
78  const char *buf_content; /* For buffers */
79  uint32_t dimensions; /* Needed for e.g. sampler%iD */
80  uint32_t elems; /* 0 - scalar, 1 or more - vector */
81  VkShaderStageFlags stages;
82  const VkSampler *samplers; /* Immutable samplers, length - #elems */
83  void *updater; /* Pointer to VkDescriptor*Info */
85 
86 typedef struct FFVkBuffer {
87  VkBuffer buf;
88  VkDeviceMemory mem;
89  VkMemoryPropertyFlagBits flags;
90 } FFVkBuffer;
91 
92 typedef struct VulkanPipeline {
93  VkPipelineBindPoint bind_point;
94 
95  /* Contexts */
96  VkPipelineLayout pipeline_layout;
97  VkPipeline pipeline;
98 
99  /* Shaders */
102 
103  /* Push consts */
104  VkPushConstantRange *push_consts;
106 
107  /* Descriptors */
108  VkDescriptorSetLayout *desc_layout;
109  VkDescriptorPool desc_pool;
110  VkDescriptorSet *desc_set;
111  VkDescriptorUpdateTemplate *desc_template;
115 
116  /* Temporary, used to store data in between initialization stages */
117  VkDescriptorUpdateTemplateCreateInfo *desc_template_info;
118  VkDescriptorPoolSize *pool_size_desc;
120 
121 typedef struct FFVkQueueCtx {
122  VkFence fence;
123  VkQueue queue;
124 
125  /* Buffer dependencies */
129 
130  /* Frame dependencies */
134 } FFVkQueueCtx;
135 
136 typedef struct FFVkExecContext {
137  VkCommandPool pool;
138  VkCommandBuffer *bufs;
140 
142  int *nb_deps;
144 
146 
147  VkSemaphore *sem_wait;
148  int sem_wait_alloc; /* Allocated sem_wait */
150 
151  VkPipelineStageFlagBits *sem_wait_dst;
152  int sem_wait_dst_alloc; /* Allocated sem_wait_dst */
153 
154  VkSemaphore *sem_sig;
155  int sem_sig_alloc; /* Allocated sem_sig */
158 
159 typedef struct VulkanFilterContext {
160  const AVClass *class;
161 
163  AVBufferRef *frames_ref; /* For in-place filtering */
166 
167  /* State - mirrored with the exec ctx */
171 
172  /* Properties */
175  enum AVPixelFormat output_format;
176  enum AVPixelFormat input_format;
177 
178  /* Samplers */
179  VkSampler **samplers;
181 
182  /* Exec contexts */
185 
186  /* Pipelines (each can have 1 shader of each type) */
189 
190  void *scratch; /* Scratch memory used only in functions */
191  unsigned int scratch_size;
193 
194 /* Identity mapping - r = r, b = b, g = g, a = a */
195 extern const VkComponentMapping ff_comp_identity_map;
196 
197 /**
198  * General lavfi IO functions
199  */
206 
207 /**
208  * Converts Vulkan return values to strings
209  */
210 const char *ff_vk_ret2str(VkResult res);
211 
212 /**
213  * Returns 1 if the image is any sort of supported RGB
214  */
216 
217 /**
218  * Gets the glsl format string for a pixel format
219  */
220 const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt);
221 
222 /**
223  * Create a Vulkan sampler, will be auto-freed in ff_vk_filter_uninit()
224  */
225 VkSampler *ff_vk_init_sampler(AVFilterContext *avctx, int unnorm_coords,
226  VkFilter filt);
227 
228 /**
229  * Create an imageview.
230  * Guaranteed to remain alive until the queue submission has finished executing,
231  * and will be destroyed after that.
232  */
234  VkImageView *v, VkImage img, VkFormat fmt,
235  const VkComponentMapping map);
236 
237 /**
238  * Define a push constant for a given stage into a pipeline.
239  * Must be called before the pipeline layout has been initialized.
240  */
242  int offset, int size, VkShaderStageFlagBits stage);
243 
244 /**
245  * Inits a pipeline. Everything in it will be auto-freed when calling
246  * ff_vk_filter_uninit().
247  */
249 
250 /**
251  * Inits a shader for a specific pipeline. Will be auto-freed on uninit.
252  */
254  const char *name, VkShaderStageFlags stage);
255 
256 /**
257  * Writes the workgroup size for a shader.
258  */
260  int local_size[3]);
261 
262 /**
263  * Adds a descriptor set to the shader and registers them in the pipeline.
264  */
267  int num, int only_print_to_shader);
268 
269 /**
270  * Compiles the shader, entrypoint must be set to "main".
271  */
273  const char *entrypoint);
274 
275 /**
276  * Initializes the pipeline layout after all shaders and descriptor sets have
277  * been finished.
278  */
280 
281 /**
282  * Initializes a compute pipeline. Will pick the first shader with the
283  * COMPUTE flag set.
284  */
286 
287 /**
288  * Updates a descriptor set via the updaters defined.
289  * Can be called immediately after pipeline creation, but must be called
290  * at least once before queue submission.
291  */
293  int set_id);
294 
295 /**
296  * Init an execution context for command recording and queue submission.
297  * WIll be auto-freed on uninit.
298  */
300 
301 /**
302  * Begin recording to the command buffer. Previous execution must have been
303  * completed, which ff_vk_submit_exec_queue() will ensure.
304  */
306 
307 /**
308  * Add a command to bind the completed pipeline and its descriptor sets.
309  * Must be called after ff_vk_start_exec_recording() and before submission.
310  */
312  VulkanPipeline *pl);
313 
314 /**
315  * Updates push constants.
316  * Must be called after binding a pipeline if any push constants were defined.
317  */
319  VkShaderStageFlagBits stage, int offset,
320  size_t size, void *src);
321 
322 /**
323  * Gets the command buffer to use for this submission from the exe context.
324  */
325 VkCommandBuffer ff_vk_get_exec_buf(AVFilterContext *avctx, FFVkExecContext *e);
326 
327 /**
328  * Adds a generic AVBufferRef as a queue depenency.
329  */
331  AVBufferRef **deps, int nb_deps);
332 
333 /**
334  * Discards all queue dependencies
335  */
337 
338 /**
339  * Adds a frame as a queue dependency. This also manages semaphore signalling.
340  * Must be called before submission.
341  */
343  AVFrame *frame, VkPipelineStageFlagBits in_wait_dst_flag);
344 
345 /**
346  * Submits a command buffer to the queue for execution.
347  * Will block until execution has finished in order to simplify resource
348  * management.
349  */
351 
352 /**
353  * Create a VkBuffer with the specified parameters.
354  */
355 int ff_vk_create_buf(AVFilterContext *avctx, FFVkBuffer *buf, size_t size,
356  VkBufferUsageFlags usage, VkMemoryPropertyFlagBits flags);
357 
358 /**
359  * Maps the buffer to userspace. Set invalidate to 1 if reading the contents
360  * is necessary.
361  */
362 int ff_vk_map_buffers(AVFilterContext *avctx, FFVkBuffer *buf, uint8_t *mem[],
363  int nb_buffers, int invalidate);
364 
365 /**
366  * Unmaps the buffer from userspace. Set flush to 1 to write and sync.
367  */
368 int ff_vk_unmap_buffers(AVFilterContext *avctx, FFVkBuffer *buf, int nb_buffers,
369  int flush);
370 
371 /**
372  * Frees a buffer.
373  */
374 void ff_vk_free_buf(AVFilterContext *avctx, FFVkBuffer *buf);
375 
376 #endif /* AVFILTER_VULKAN_H */
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
ptrdiff_t const GLvoid GLenum usage
Definition: opengl_enc.c:100
int sem_wait_cnt
Definition: vulkan.h:149
static enum AVPixelFormat pix_fmt
VkBuffer buf
Definition: vulkan.h:87
const VkComponentMapping ff_comp_identity_map
Definition: vulkan.c:44
This structure describes decoded (raw) audio or video data.
Definition: frame.h:300
VkPipelineShaderStageCreateInfo shader
Definition: vulkan.h:70
static void flush(AVCodecContext *avctx)
VkSemaphore * sem_wait
Definition: vulkan.h:147
int * nb_deps
Definition: vulkan.h:142
int ff_vk_map_buffers(AVFilterContext *avctx, FFVkBuffer *buf, uint8_t *mem[], int nb_buffers, int invalidate)
Maps the buffer to userspace.
Definition: vulkan.c:215
const char * name
Definition: vulkan.h:74
int push_consts_num
Definition: vulkan.h:105
Main libavfilter public API header.
VkDescriptorPoolSize * pool_size_desc
Definition: vulkan.h:118
VkDescriptorUpdateTemplateCreateInfo * desc_template_info
Definition: vulkan.h:117
const char * desc
Definition: nvenc.c:79
int ff_vk_filter_config_output_inplace(AVFilterLink *outlink)
Definition: vulkan.c:675
VkSampler * ff_vk_init_sampler(AVFilterContext *avctx, int unnorm_coords, VkFilter filt)
Create a Vulkan sampler, will be auto-freed in ff_vk_filter_uninit()
Definition: vulkan.c:769
AVFrame ** frame_deps
Definition: vulkan.h:131
VkDescriptorUpdateTemplate * desc_template
Definition: vulkan.h:111
int sem_sig_cnt
Definition: vulkan.h:156
int ff_vk_unmap_buffers(AVFilterContext *avctx, FFVkBuffer *buf, int nb_buffers, int flush)
Unmaps the buffer from userspace.
Definition: vulkan.c:264
int nb_frame_deps
Definition: vulkan.h:132
VkCommandBuffer ff_vk_get_exec_buf(AVFilterContext *avctx, FFVkExecContext *e)
Gets the command buffer to use for this submission from the exe context.
Definition: vulkan.c:458
void ff_vk_bind_pipeline_exec(AVFilterContext *avctx, FFVkExecContext *e, VulkanPipeline *pl)
Add a command to bind the completed pipeline and its descriptor sets.
Definition: vulkan.c:1316
int shaders_num
Definition: vulkan.h:101
VkPushConstantRange * push_consts
Definition: vulkan.h:104
#define img
const char * name
Definition: vulkan.h:67
VkPipelineStageFlagBits * sem_wait_dst
Definition: vulkan.h:151
void ff_vk_update_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, int set_id)
Updates a descriptor set via the updaters defined.
Definition: vulkan.c:1160
int * dep_alloc_size
Definition: vulkan.h:143
AVBufferRef *** deps
Definition: vulkan.h:141
uint8_t
int ff_vk_add_exec_dep(AVFilterContext *avctx, FFVkExecContext *e, AVFrame *frame, VkPipelineStageFlagBits in_wait_dst_flag)
Adds a frame as a queue dependency.
Definition: vulkan.c:464
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
int ff_vk_mt_is_np_rgb(enum AVPixelFormat pix_fmt)
Returns 1 if the image is any sort of supported RGB.
Definition: vulkan.c:805
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:52
AVBPrint src
Definition: vulkan.h:68
AVBufferRef * frames_ref
Definition: vulkan.h:163
int sem_wait_dst_alloc
Definition: vulkan.h:152
VulkanPipeline * ff_vk_create_pipeline(AVFilterContext *avctx)
Inits a pipeline.
Definition: vulkan.c:1276
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan.c:635
VulkanPipeline * bound_pl
Definition: vulkan.h:145
ptrdiff_t size
Definition: opengl_enc.c:100
void ff_vk_discard_exec_deps(AVFilterContext *avctx, FFVkExecContext *e)
Discards all queue dependencies.
Definition: vulkan.c:400
int ff_vk_start_exec_recording(AVFilterContext *avctx, FFVkExecContext *e)
Begin recording to the command buffer.
Definition: vulkan.c:417
int sem_sig_alloc
Definition: vulkan.h:155
const VkSampler * samplers
Definition: vulkan.h:82
void ff_vk_update_push_exec(AVFilterContext *avctx, FFVkExecContext *e, VkShaderStageFlagBits stage, int offset, size_t size, void *src)
Updates push constants.
Definition: vulkan.c:1171
API-specific header for AV_HWDEVICE_TYPE_VULKAN.
SPIRVShader * ff_vk_init_shader(AVFilterContext *avctx, VulkanPipeline *pl, const char *name, VkShaderStageFlags stage)
Inits a shader for a specific pipeline.
Definition: vulkan.c:888
unsigned int scratch_size
Definition: vulkan.h:191
VulkanPipeline ** pipelines
Definition: vulkan.h:187
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt)
Gets the glsl format string for a pixel format.
Definition: vulkan.c:817
const char * mem_layout
Definition: vulkan.h:76
int sem_wait_alloc
Definition: vulkan.h:148
AVBufferRef * device_ref
Definition: vulkan.h:162
int ff_vk_init_compute_pipeline(AVFilterContext *avctx, VulkanPipeline *pl)
Initializes a compute pipeline.
Definition: vulkan.c:1281
VkPipeline pipeline
Definition: vulkan.h:97
int ff_vk_create_imageview(AVFilterContext *avctx, FFVkExecContext *e, VkImageView *v, VkImage img, VkFormat fmt, const VkComponentMapping map)
Create an imageview.
Definition: vulkan.c:836
AVFormatContext * ctx
Definition: movenc.c:48
VkDescriptorSet * desc_set
Definition: vulkan.h:110
int buf_deps_alloc_size
Definition: vulkan.h:128
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
VkMemoryPropertyFlagBits flags
Definition: vulkan.h:89
int ff_vk_add_dep_exec_ctx(AVFilterContext *avctx, FFVkExecContext *e, AVBufferRef **deps, int nb_deps)
Adds a generic AVBufferRef as a queue depenency.
Definition: vulkan.c:561
VkDescriptorType type
Definition: vulkan.h:75
VkCommandPool pool
Definition: vulkan.h:137
int ff_vk_filter_query_formats(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan.c:592
VkFence fence
Definition: vulkan.h:122
int ff_vk_init_pipeline_layout(AVFilterContext *avctx, VulkanPipeline *pl)
Initializes the pipeline layout after all shaders and descriptor sets have been finished.
Definition: vulkan.c:1180
int ff_vk_add_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, SPIRVShader *shd, VulkanDescriptorSetBinding *desc, int num, int only_print_to_shader)
Adds a descriptor set to the shader and registers them in the pipeline.
Definition: vulkan.c:1020
int ff_vk_add_push_constant(AVFilterContext *avctx, VulkanPipeline *pl, int offset, int size, VkShaderStageFlagBits stage)
Define a push constant for a given stage into a pipeline.
Definition: vulkan.c:318
int descriptor_sets_num
Definition: vulkan.h:113
VkDeviceMemory mem
Definition: vulkan.h:88
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:165
Describe the class of an AVClass context structure.
Definition: log.h:67
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan.c:705
VkShaderStageFlags stages
Definition: vulkan.h:81
int nb_buf_deps
Definition: vulkan.h:127
const VDPAUPixFmtMap * map
VkPipelineLayout pipeline_layout
Definition: vulkan.h:96
const char * buf_content
Definition: vulkan.h:78
int desc_layout_num
Definition: vulkan.h:112
static const int8_t filt[NUMTAPS]
Definition: af_earwax.c:39
#define flags(name, subs,...)
Definition: cbs_av1.c:564
const char * mem_quali
Definition: vulkan.h:77
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
A reference to a data buffer.
Definition: buffer.h:81
VkDescriptorSetLayout * desc_layout
Definition: vulkan.h:108
void ff_vk_filter_uninit(AVFilterContext *avctx)
Definition: vulkan.c:1415
VkSemaphore * sem_sig
Definition: vulkan.h:154
int ff_vk_create_buf(AVFilterContext *avctx, FFVkBuffer *buf, size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlagBits flags)
Create a VkBuffer with the specified parameters.
Definition: vulkan.c:150
int frame_deps_alloc_size
Definition: vulkan.h:133
void ff_vk_set_compute_shader_sizes(AVFilterContext *avctx, SPIRVShader *shd, int local_size[3])
Writes the workgroup size for a shader.
Definition: vulkan.c:909
int ff_vk_filter_init(AVFilterContext *avctx)
Definition: vulkan.c:756
SPIRVShader ** shaders
Definition: vulkan.h:100
VkDescriptorPool desc_pool
Definition: vulkan.h:109
FFVkQueueCtx * queues
Definition: vulkan.h:139
int ff_vk_compile_shader(AVFilterContext *avctx, SPIRVShader *shd, const char *entrypoint)
Compiles the shader, entrypoint must be set to "main".
Definition: vulkan.c:942
VkCommandBuffer * bufs
Definition: vulkan.h:138
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:202
An instance of a filter.
Definition: avfilter.h:338
int ff_vk_create_exec_ctx(AVFilterContext *avctx, FFVkExecContext **ctx)
Init an execution context for command recording and queue submission.
Definition: vulkan.c:339
int pool_size_desc_num
Definition: vulkan.h:114
AVHWDeviceContext * device
Definition: vulkan.h:164
VkPipelineBindPoint bind_point
Definition: vulkan.h:93
int ff_vk_submit_exec_queue(AVFilterContext *avctx, FFVkExecContext *e)
Submits a command buffer to the queue for execution.
Definition: vulkan.c:522
int local_size[3]
Definition: vulkan.h:69
FFVkExecContext ** exec_ctx
Definition: vulkan.h:183
void ff_vk_free_buf(AVFilterContext *avctx, FFVkBuffer *buf)
Frees a buffer.
Definition: vulkan.c:306
VkSampler ** samplers
Definition: vulkan.h:179
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
VkQueue queue
Definition: vulkan.h:123
AVBufferRef ** buf_deps
Definition: vulkan.h:126
const char * name
Definition: opengl_enc.c:102