FFmpeg
hwcontext_vulkan.h
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifndef AVUTIL_HWCONTEXT_VULKAN_H
20 #define AVUTIL_HWCONTEXT_VULKAN_H
21 
22 #if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
23 #define VK_USE_PLATFORM_WIN32_KHR
24 #endif
25 #include <vulkan/vulkan.h>
26 
27 #include "pixfmt.h"
28 #include "frame.h"
29 
30 typedef struct AVVkFrame AVVkFrame;
31 
32 /**
33  * @file
34  * API-specific header for AV_HWDEVICE_TYPE_VULKAN.
35  *
36  * For user-allocated pools, AVHWFramesContext.pool must return AVBufferRefs
37  * with the data pointer set to an AVVkFrame.
38  */
39 
40 /**
41  * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
42  * All of these can be set before init to change what the context uses
43  */
44 typedef struct AVVulkanDeviceContext {
45  /**
46  * Custom memory allocator, else NULL
47  */
48  const VkAllocationCallbacks *alloc;
49 
50  /**
51  * Pointer to the instance-provided vkGetInstanceProcAddr loading function.
52  * If NULL, will pick either libvulkan or libvolk, depending on libavutil's
53  * compilation settings, and set this field.
54  */
55  PFN_vkGetInstanceProcAddr get_proc_addr;
56 
57  /**
58  * Vulkan instance. Must be at least version 1.3.
59  */
60  VkInstance inst;
61 
62  /**
63  * Physical device
64  */
65  VkPhysicalDevice phys_dev;
66 
67  /**
68  * Active device
69  */
70  VkDevice act_dev;
71 
72  /**
73  * This structure should be set to the set of features that present and enabled
74  * during device creation. When a device is created by FFmpeg, it will default to
75  * enabling all that are present of the shaderImageGatherExtended,
76  * fragmentStoresAndAtomics, shaderInt64 and vertexPipelineStoresAndAtomics features.
77  */
78  VkPhysicalDeviceFeatures2 device_features;
79 
80  /**
81  * Enabled instance extensions.
82  * If supplying your own device context, set this to an array of strings, with
83  * each entry containing the specified Vulkan extension string to enable.
84  * Duplicates are possible and accepted.
85  * If no extensions are enabled, set these fields to NULL, and 0 respectively.
86  */
87  const char * const *enabled_inst_extensions;
89 
90  /**
91  * Enabled device extensions. By default, VK_KHR_external_memory_fd,
92  * VK_EXT_external_memory_dma_buf, VK_EXT_image_drm_format_modifier,
93  * VK_KHR_external_semaphore_fd and VK_EXT_external_memory_host are enabled if found.
94  * If supplying your own device context, these fields takes the same format as
95  * the above fields, with the same conditions that duplicates are possible
96  * and accepted, and that NULL and 0 respectively means no extensions are enabled.
97  */
98  const char * const *enabled_dev_extensions;
100 
101  /**
102  * Queue family index for graphics operations, and the number of queues
103  * enabled for it. If unavaiable, will be set to -1. Not required.
104  * av_hwdevice_create() will attempt to find a dedicated queue for each
105  * queue family, or pick the one with the least unrelated flags set.
106  * Queue indices here may overlap if a queue has to share capabilities.
107  */
110 
111  /**
112  * Queue family index for transfer operations and the number of queues
113  * enabled. Required.
114  */
117 
118  /**
119  * Queue family index for compute operations and the number of queues
120  * enabled. Required.
121  */
124 
125  /**
126  * Queue family index for video encode ops, and the amount of queues enabled.
127  * If the device doesn't support such, queue_family_encode_index will be -1.
128  * Not required.
129  */
132 
133  /**
134  * Queue family index for video decode ops, and the amount of queues enabled.
135  * If the device doesn't support such, queue_family_decode_index will be -1.
136  * Not required.
137  */
140 
141  /**
142  * Locks a queue, preventing other threads from submitting any command
143  * buffers to this queue.
144  * If set to NULL, will be set to lavu-internal functions that utilize a
145  * mutex.
146  */
147  void (*lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
148 
149  /**
150  * Similar to lock_queue(), unlocks a queue. Must only be called after locking.
151  */
152  void (*unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
154 
155 /**
156  * Defines the behaviour of frame allocation.
157  */
158 typedef enum AVVkFrameFlags {
159  /* Unless this flag is set, autodetected flags will be OR'd based on the
160  * device and tiling during av_hwframe_ctx_init(). */
161  AV_VK_FRAME_FLAG_NONE = (1ULL << 0),
162 
164  /* DEPRECATED: does nothing. Replaced by multiplane images. */
166 #endif
167 
168  /* Disables multiplane images.
169  * This is required to export/import images from CUDA. */
172 
173 /**
174  * Allocated as AVHWFramesContext.hwctx, used to set pool-specific options
175  */
176 typedef struct AVVulkanFramesContext {
177  /**
178  * Controls the tiling of allocated frames.
179  * If left as VK_IMAGE_TILING_OPTIMAL (0), will use optimal tiling.
180  * Can be set to VK_IMAGE_TILING_LINEAR to force linear images,
181  * or VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT to force DMABUF-backed
182  * images.
183  * @note Imported frames from other APIs ignore this.
184  */
185  VkImageTiling tiling;
186 
187  /**
188  * Defines extra usage of output frames. If non-zero, all flags MUST be
189  * supported by the VkFormat. Otherwise, will use supported flags amongst:
190  * - VK_IMAGE_USAGE_SAMPLED_BIT
191  * - VK_IMAGE_USAGE_STORAGE_BIT
192  * - VK_IMAGE_USAGE_TRANSFER_SRC_BIT
193  * - VK_IMAGE_USAGE_TRANSFER_DST_BIT
194  */
195  VkImageUsageFlagBits usage;
196 
197  /**
198  * Extension data for image creation.
199  * If DRM tiling is used, a VkImageDrmFormatModifierListCreateInfoEXT structure
200  * can be added to specify the exact modifier to use.
201  *
202  * Additional structures may be added at av_hwframe_ctx_init() time,
203  * which will be freed automatically on uninit(), so users must only free
204  * any structures they've allocated themselves.
205  */
207 
208  /**
209  * Extension data for memory allocation. Must have as many entries as
210  * the number of planes of the sw_format.
211  * This will be chained to VkExportMemoryAllocateInfo, which is used
212  * to make all pool images exportable to other APIs if the necessary
213  * extensions are present in enabled_dev_extensions.
214  */
216 
217  /**
218  * A combination of AVVkFrameFlags. Unless AV_VK_FRAME_FLAG_NONE is set,
219  * autodetected flags will be OR'd based on the device and tiling during
220  * av_hwframe_ctx_init().
221  */
223 
224  /**
225  * Flags to set during image creation. If unset, defaults to
226  * VK_IMAGE_CREATE_ALIAS_BIT.
227  */
228  VkImageCreateFlags img_flags;
229 
230  /**
231  * Vulkan format for each image. MUST be compatible with the pixel format.
232  * If unset, will be automatically set.
233  * There are at most two compatible formats for a frame - a multiplane
234  * format, and a single-plane multi-image format.
235  */
237 
238  /**
239  * Number of layers each image will have.
240  */
242 
243  /**
244  * Locks a frame, preventing other threads from changing frame properties.
245  * Users SHOULD only ever lock just before command submission in order
246  * to get accurate frame properties, and unlock immediately after command
247  * submission without waiting for it to finish.
248  *
249  * If unset, will be set to lavu-internal functions that utilize a mutex.
250  */
251  void (*lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
252 
253  /**
254  * Similar to lock_frame(), unlocks a frame. Must only be called after locking.
255  */
256  void (*unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
258 
259 /*
260  * Frame structure.
261  *
262  * @note the size of this structure is not part of the ABI, to allocate
263  * you must use @av_vk_frame_alloc().
264  */
265 struct AVVkFrame {
266  /**
267  * Vulkan images to which the memory is bound to.
268  * May be one for multiplane formats, or multiple.
269  */
271 
272  /**
273  * Tiling for the frame.
274  */
275  VkImageTiling tiling;
276 
277  /**
278  * Memory backing the images. Either one, or as many as there are planes
279  * in the sw_format.
280  * In case of having multiple VkImages, but one memory, the offset field
281  * will indicate the bound offset for each image.
282  */
283  VkDeviceMemory mem[AV_NUM_DATA_POINTERS];
285 
286  /**
287  * OR'd flags for all memory allocated
288  */
289  VkMemoryPropertyFlagBits flags;
290 
291  /**
292  * Updated after every barrier. One per VkImage.
293  */
294  VkAccessFlagBits access[AV_NUM_DATA_POINTERS];
295  VkImageLayout layout[AV_NUM_DATA_POINTERS];
296 
297  /**
298  * Synchronization timeline semaphores, one for each VkImage.
299  * Must not be freed manually. Must be waited on at every submission using
300  * the value in sem_value, and must be signalled at every submission,
301  * using an incremented value.
302  */
303  VkSemaphore sem[AV_NUM_DATA_POINTERS];
304 
305  /**
306  * Up to date semaphore value at which each image becomes accessible.
307  * One per VkImage.
308  * Clients must wait on this value when submitting a command queue,
309  * and increment it when signalling.
310  */
312 
313  /**
314  * Internal data.
315  */
316  struct AVVkFrameInternal *internal;
317 
318  /**
319  * Describes the binding offset of each image to the VkDeviceMemory.
320  * One per VkImage.
321  */
323 
324  /**
325  * Queue family of the images. Must be VK_QUEUE_FAMILY_IGNORED if
326  * the image was allocated with the CONCURRENT concurrency option.
327  * One per VkImage.
328  */
330 };
331 
332 /**
333  * Allocates a single AVVkFrame and initializes everything as 0.
334  * @note Must be freed via av_free()
335  */
337 
338 /**
339  * Returns the optimal per-plane Vulkan format for a given sw_format,
340  * one for each plane.
341  * Returns NULL on unsupported formats.
342  */
344 
345 #endif /* AVUTIL_HWCONTEXT_VULKAN_H */
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:65
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:170
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to the instance-provided vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:55
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:206
AVVulkanDeviceContext::queue_family_decode_index
int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:138
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:60
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:251
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:464
AVVulkanDeviceContext::nb_decode_queues
int nb_decode_queues
Definition: hwcontext_vulkan.h:139
AVVulkanDeviceContext::queue_family_index
int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:108
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:48
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:270
AVVkFrame::offset
ptrdiff_t offset[AV_NUM_DATA_POINTERS]
Describes the binding offset of each image to the VkDeviceMemory.
Definition: hwcontext_vulkan.h:322
AVVulkanDeviceContext::queue_family_comp_index
int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:122
AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY
@ AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY
Definition: hwcontext_vulkan.h:165
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:222
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:283
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:176
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
AV_VK_FRAME_FLAG_NONE
@ AV_VK_FRAME_FLAG_NONE
Definition: hwcontext_vulkan.h:161
AVVulkanDeviceContext::nb_graphics_queues
int nb_graphics_queues
Definition: hwcontext_vulkan.h:109
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:256
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:228
ctx
AVFormatContext * ctx
Definition: movenc.c:48
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:44
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:99
AVVkFrameInternal
Definition: hwcontext_vulkan.c:138
AVVkFrame::flags
VkMemoryPropertyFlagBits flags
OR'd flags for all memory allocated.
Definition: hwcontext_vulkan.h:289
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:215
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:152
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:87
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:236
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:195
AVVulkanDeviceContext::queue_family_tx_index
int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:115
index
int index
Definition: gxfenc.c:89
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:284
AVVkFrame::access
VkAccessFlagBits access[AV_NUM_DATA_POINTERS]
Updated after every barrier.
Definition: hwcontext_vulkan.h:294
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_stub.c:30
AVVkFrame
Definition: hwcontext_vulkan.h:265
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:341
FF_API_VULKAN_CONTIGUOUS_MEMORY
#define FF_API_VULKAN_CONTIGUOUS_MEMORY
Definition: version.h:121
frame.h
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_stub.c:35
AVVkFrame::queue_family
uint32_t queue_family[AV_NUM_DATA_POINTERS]
Queue family of the images.
Definition: hwcontext_vulkan.h:329
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:147
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
AVVulkanDeviceContext::nb_encode_queues
int nb_encode_queues
Definition: hwcontext_vulkan.h:131
AVVkFrameFlags
AVVkFrameFlags
Defines the behaviour of frame allocation.
Definition: hwcontext_vulkan.h:158
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:303
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:124
AVVkFrame::tiling
VkImageTiling tiling
Tiling for the frame.
Definition: hwcontext_vulkan.h:275
pixfmt.h
AVVulkanDeviceContext::nb_comp_queues
int nb_comp_queues
Definition: hwcontext_vulkan.h:123
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:185
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:311
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:98
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:241
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:295
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:70
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:88
AVVulkanDeviceContext::nb_tx_queues
int nb_tx_queues
Definition: hwcontext_vulkan.h:116
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:78
AVVulkanDeviceContext::queue_family_encode_index
int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:130