FFmpeg
ffplay_renderer.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #define VK_NO_PROTOTYPES
20 #define VK_ENABLE_BETA_EXTENSIONS
21 
22 #include "config.h"
23 #include "ffplay_renderer.h"
24 
25 #if (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO)
26 /* Get PL_API_VER */
27 #include <libplacebo/config.h>
28 #define HAVE_VULKAN_RENDERER (PL_API_VER >= 278)
29 #else
30 #define HAVE_VULKAN_RENDERER 0
31 #endif
32 
33 #if HAVE_VULKAN_RENDERER
34 
35 #if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
36 #define VK_USE_PLATFORM_WIN32_KHR
37 #endif
38 
39 #include <libplacebo/vulkan.h>
40 #include <libplacebo/utils/frame_queue.h>
41 #include <libplacebo/utils/libav.h>
42 #include <SDL_vulkan.h>
43 
44 #include "libavutil/bprint.h"
45 #include "libavutil/mem.h"
46 
47 #endif
48 
49 struct VkRenderer {
50  const AVClass *class;
51 
52  int (*create)(VkRenderer *renderer, SDL_Window *window, AVDictionary *dict);
53 
55 
57 
58  int (*resize)(VkRenderer *renderer, int width, int height);
59 
61 };
62 
63 #if HAVE_VULKAN_RENDERER
64 
65 typedef struct RendererContext {
66  VkRenderer api;
67 
68  // Can be NULL when vulkan instance is created by avutil
69  pl_vk_inst placebo_instance;
70  pl_vulkan placebo_vulkan;
71  pl_swapchain swapchain;
72  VkSurfaceKHR vk_surface;
73  pl_renderer renderer;
74  pl_tex tex[4];
75 
76  pl_log vk_log;
77 
78  AVBufferRef *hw_device_ref;
79  AVBufferRef *hw_frame_ref;
80  enum AVPixelFormat *transfer_formats;
81  AVHWFramesConstraints *constraints;
82 
83  PFN_vkGetInstanceProcAddr get_proc_addr;
84  // This field is a copy from pl_vk_inst->instance or hw_device_ref instance.
85  VkInstance inst;
86 
87  AVFrame *vk_frame;
88 } RendererContext;
89 
90 static void vk_log_cb(void *log_priv, enum pl_log_level level,
91  const char *msg)
92 {
93  static const int level_map[] = {
100  AV_LOG_TRACE,
101  };
102 
103  if (level > 0 && level < FF_ARRAY_ELEMS(level_map))
104  av_log(log_priv, level_map[level], "%s\n", msg);
105 }
106 
107 // Should keep sync with optional_device_exts inside hwcontext_vulkan.c
108 static const char *optional_device_exts[] = {
109  /* Misc or required by other extensions */
110  VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME,
111  VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
112  VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME,
113  VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
114  VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME,
115  VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
116  VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME,
117 
118  /* Imports/exports */
119  VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME,
120  VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME,
121  VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME,
122  VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME,
123  VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME,
124 #ifdef _WIN32
125  VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME,
126  VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME,
127 #endif
128 
129  /* Video encoding/decoding */
130  VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
131  VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
132  VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
133  VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
134  "VK_MESA_video_decode_av1",
135 };
136 
137 static inline int enable_debug(const AVDictionary *opt)
138 {
139  AVDictionaryEntry *entry = av_dict_get(opt, "debug", NULL, 0);
140  int debug = entry && strtol(entry->value, NULL, 10);
141  return debug;
142 }
143 
144 static void hwctx_lock_queue(void *priv, uint32_t qf, uint32_t qidx)
145 {
146  AVHWDeviceContext *avhwctx = priv;
147  const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
148  hwctx->lock_queue(avhwctx, qf, qidx);
149 }
150 
151 static void hwctx_unlock_queue(void *priv, uint32_t qf, uint32_t qidx)
152 {
153  AVHWDeviceContext *avhwctx = priv;
154  const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
155  hwctx->unlock_queue(avhwctx, qf, qidx);
156 }
157 
158 static int add_instance_extension(const char **ext, unsigned num_ext,
159  const AVDictionary *opt,
160  AVDictionary **dict)
161 {
162  const char *inst_ext_key = "instance_extensions";
164  AVBPrint buf;
165  char *ext_list = NULL;
166  int ret;
167 
169  for (int i = 0; i < num_ext; i++) {
170  if (i)
171  av_bprintf(&buf, "+%s", ext[i]);
172  else
173  av_bprintf(&buf, "%s", ext[i]);
174  }
175 
176  entry = av_dict_get(opt, inst_ext_key, NULL, 0);
177  if (entry && entry->value && entry->value[0]) {
178  if (num_ext)
179  av_bprintf(&buf, "+");
180  av_bprintf(&buf, "%s", entry->value);
181  }
182 
183  ret = av_bprint_finalize(&buf, &ext_list);
184  if (ret < 0)
185  return ret;
186  return av_dict_set(dict, inst_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
187 }
188 
189 static int add_device_extension(const AVDictionary *opt,
190  AVDictionary **dict)
191 {
192  const char *dev_ext_key = "device_extensions";
194  AVBPrint buf;
195  char *ext_list = NULL;
196  int ret;
197 
199  av_bprintf(&buf, "%s", VK_KHR_SWAPCHAIN_EXTENSION_NAME);
200  for (int i = 0; i < pl_vulkan_num_recommended_extensions; i++)
201  av_bprintf(&buf, "+%s", pl_vulkan_recommended_extensions[i]);
202 
203  entry = av_dict_get(opt, dev_ext_key, NULL, 0);
204  if (entry && entry->value && entry->value[0])
205  av_bprintf(&buf, "+%s", entry->value);
206 
207  ret = av_bprint_finalize(&buf, &ext_list);
208  if (ret < 0)
209  return ret;
210  return av_dict_set(dict, dev_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
211 }
212 
213 static const char *select_device(const AVDictionary *opt)
214 {
215  const AVDictionaryEntry *entry;
216 
217  entry = av_dict_get(opt, "device", NULL, 0);
218  if (entry)
219  return entry->value;
220  return NULL;
221 }
222 
223 static int create_vk_by_hwcontext(VkRenderer *renderer,
224  const char **ext, unsigned num_ext,
225  const AVDictionary *opt)
226 {
227  RendererContext *ctx = (RendererContext *) renderer;
228  AVHWDeviceContext *dev;
229  AVVulkanDeviceContext *hwctx;
230  AVDictionary *dict = NULL;
231  int ret;
232 
233  ret = add_instance_extension(ext, num_ext, opt, &dict);
234  if (ret < 0)
235  return ret;
236  ret = add_device_extension(opt, &dict);
237  if (ret) {
238  av_dict_free(&dict);
239  return ret;
240  }
241 
243  select_device(opt), dict, 0);
244  av_dict_free(&dict);
245  if (ret < 0)
246  return ret;
247 
248  dev = (AVHWDeviceContext *) ctx->hw_device_ref->data;
249  hwctx = dev->hwctx;
250 
251  // There is no way to pass SDL GetInstanceProcAddr to hwdevice.
252  // Check the result and return error if they don't match.
253  if (hwctx->get_proc_addr != SDL_Vulkan_GetVkGetInstanceProcAddr()) {
255  "hwdevice and SDL use different get_proc_addr. "
256  "Try -vulkan_params create_by_placebo=1\n");
257  return AVERROR_PATCHWELCOME;
258  }
259 
260  ctx->get_proc_addr = hwctx->get_proc_addr;
261  ctx->inst = hwctx->inst;
262  ctx->placebo_vulkan = pl_vulkan_import(ctx->vk_log,
263  pl_vulkan_import_params(
264  .instance = hwctx->inst,
265  .get_proc_addr = hwctx->get_proc_addr,
266  .phys_device = hwctx->phys_dev,
267  .device = hwctx->act_dev,
268  .extensions = hwctx->enabled_dev_extensions,
269  .num_extensions = hwctx->nb_enabled_dev_extensions,
270  .features = &hwctx->device_features,
271  .lock_queue = hwctx_lock_queue,
272  .unlock_queue = hwctx_unlock_queue,
273  .queue_ctx = dev,
274  .queue_graphics = {
275  .index = hwctx->queue_family_index,
276  .count = hwctx->nb_graphics_queues,
277  },
278  .queue_compute = {
279  .index = hwctx->queue_family_comp_index,
280  .count = hwctx->nb_comp_queues,
281  },
282  .queue_transfer = {
283  .index = hwctx->queue_family_tx_index,
284  .count = hwctx->nb_tx_queues,
285  },
286  ));
287  if (!ctx->placebo_vulkan)
288  return AVERROR_EXTERNAL;
289 
290  return 0;
291 }
292 
293 static void placebo_lock_queue(struct AVHWDeviceContext *dev_ctx,
294  uint32_t queue_family, uint32_t index)
295 {
296  RendererContext *ctx = dev_ctx->user_opaque;
297  pl_vulkan vk = ctx->placebo_vulkan;
298  vk->lock_queue(vk, queue_family, index);
299 }
300 
301 static void placebo_unlock_queue(struct AVHWDeviceContext *dev_ctx,
302  uint32_t queue_family,
303  uint32_t index)
304 {
305  RendererContext *ctx = dev_ctx->user_opaque;
306  pl_vulkan vk = ctx->placebo_vulkan;
307  vk->unlock_queue(vk, queue_family, index);
308 }
309 
310 static int get_decode_queue(VkRenderer *renderer, int *index, int *count)
311 {
312  RendererContext *ctx = (RendererContext *) renderer;
313  VkQueueFamilyProperties *queue_family_prop = NULL;
314  uint32_t num_queue_family_prop = 0;
315  PFN_vkGetPhysicalDeviceQueueFamilyProperties get_queue_family_prop;
316  PFN_vkGetInstanceProcAddr get_proc_addr = ctx->get_proc_addr;
317 
318  *index = -1;
319  *count = 0;
320  get_queue_family_prop = (PFN_vkGetPhysicalDeviceQueueFamilyProperties)
321  get_proc_addr(ctx->placebo_instance->instance,
322  "vkGetPhysicalDeviceQueueFamilyProperties");
323  get_queue_family_prop(ctx->placebo_vulkan->phys_device,
324  &num_queue_family_prop, NULL);
325  if (!num_queue_family_prop)
326  return AVERROR_EXTERNAL;
327 
328  queue_family_prop = av_calloc(num_queue_family_prop,
329  sizeof(*queue_family_prop));
330  if (!queue_family_prop)
331  return AVERROR(ENOMEM);
332 
333  get_queue_family_prop(ctx->placebo_vulkan->phys_device,
334  &num_queue_family_prop,
335  queue_family_prop);
336 
337  for (int i = 0; i < num_queue_family_prop; i++) {
338  if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
339  *index = i;
340  *count = queue_family_prop[i].queueCount;
341  break;
342  }
343  }
344  av_free(queue_family_prop);
345 
346  return 0;
347 }
348 
349 static int create_vk_by_placebo(VkRenderer *renderer,
350  const char **ext, unsigned num_ext,
351  const AVDictionary *opt)
352 {
353  RendererContext *ctx = (RendererContext *) renderer;
354  AVHWDeviceContext *device_ctx;
355  AVVulkanDeviceContext *vk_dev_ctx;
356  int decode_index;
357  int decode_count;
358  int ret;
359 
360  ctx->get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr();
361 
362  ctx->placebo_instance = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
363  .get_proc_addr = ctx->get_proc_addr,
364  .debug = enable_debug(opt),
365  .extensions = ext,
366  .num_extensions = num_ext
367  ));
368  if (!ctx->placebo_instance) {
369  return AVERROR_EXTERNAL;
370  }
371  ctx->inst = ctx->placebo_instance->instance;
372 
373  ctx->placebo_vulkan = pl_vulkan_create(ctx->vk_log, pl_vulkan_params(
374  .instance = ctx->placebo_instance->instance,
375  .get_proc_addr = ctx->placebo_instance->get_proc_addr,
376  .surface = ctx->vk_surface,
377  .allow_software = false,
378  .opt_extensions = optional_device_exts,
379  .num_opt_extensions = FF_ARRAY_ELEMS(optional_device_exts),
380  .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
381  .device_name = select_device(opt),
382  ));
383  if (!ctx->placebo_vulkan)
384  return AVERROR_EXTERNAL;
386  if (!ctx->hw_device_ref) {
387  return AVERROR(ENOMEM);
388  }
389 
390  device_ctx = (AVHWDeviceContext *) ctx->hw_device_ref->data;
391  device_ctx->user_opaque = ctx;
392 
393  vk_dev_ctx = device_ctx->hwctx;
394  vk_dev_ctx->lock_queue = placebo_lock_queue;
395  vk_dev_ctx->unlock_queue = placebo_unlock_queue;
396 
397  vk_dev_ctx->get_proc_addr = ctx->placebo_instance->get_proc_addr;
398 
399  vk_dev_ctx->inst = ctx->placebo_instance->instance;
400  vk_dev_ctx->phys_dev = ctx->placebo_vulkan->phys_device;
401  vk_dev_ctx->act_dev = ctx->placebo_vulkan->device;
402 
403  vk_dev_ctx->device_features = *ctx->placebo_vulkan->features;
404 
405  vk_dev_ctx->enabled_inst_extensions = ctx->placebo_instance->extensions;
406  vk_dev_ctx->nb_enabled_inst_extensions = ctx->placebo_instance->num_extensions;
407 
408  vk_dev_ctx->enabled_dev_extensions = ctx->placebo_vulkan->extensions;
409  vk_dev_ctx->nb_enabled_dev_extensions = ctx->placebo_vulkan->num_extensions;
410 
411  vk_dev_ctx->queue_family_index = ctx->placebo_vulkan->queue_graphics.index;
412  vk_dev_ctx->nb_graphics_queues = ctx->placebo_vulkan->queue_graphics.count;
413 
414  vk_dev_ctx->queue_family_tx_index = ctx->placebo_vulkan->queue_transfer.index;
415  vk_dev_ctx->nb_tx_queues = ctx->placebo_vulkan->queue_transfer.count;
416 
417  vk_dev_ctx->queue_family_comp_index = ctx->placebo_vulkan->queue_compute.index;
418  vk_dev_ctx->nb_comp_queues = ctx->placebo_vulkan->queue_compute.count;
419 
420  ret = get_decode_queue(renderer, &decode_index, &decode_count);
421  if (ret < 0)
422  return ret;
423 
425  vk_dev_ctx->nb_decode_queues = decode_count;
426 
427  ret = av_hwdevice_ctx_init(ctx->hw_device_ref);
428  if (ret < 0)
429  return ret;
430 
431  return 0;
432 }
433 
434 static int create(VkRenderer *renderer, SDL_Window *window, AVDictionary *opt)
435 {
436  int ret = 0;
437  unsigned num_ext = 0;
438  const char **ext = NULL;
439  int w, h;
440  struct pl_log_params vk_log_params = {
441  .log_cb = vk_log_cb,
442  .log_level = PL_LOG_DEBUG,
443  .log_priv = renderer,
444  };
445  RendererContext *ctx = (RendererContext *) renderer;
447 
448  ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
449 
450  if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
451  av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n",
452  SDL_GetError());
453  return AVERROR_EXTERNAL;
454  }
455 
456  ext = av_calloc(num_ext, sizeof(*ext));
457  if (!ext) {
458  ret = AVERROR(ENOMEM);
459  goto out;
460  }
461 
462  SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext);
463 
464  entry = av_dict_get(opt, "create_by_placebo", NULL, 0);
465  if (entry && strtol(entry->value, NULL, 10))
466  ret = create_vk_by_placebo(renderer, ext, num_ext, opt);
467  else
468  ret = create_vk_by_hwcontext(renderer, ext, num_ext, opt);
469  if (ret < 0)
470  goto out;
471 
472  if (!SDL_Vulkan_CreateSurface(window, ctx->inst, &ctx->vk_surface)) {
474  goto out;
475  }
476 
477  ctx->swapchain = pl_vulkan_create_swapchain(
478  ctx->placebo_vulkan,
479  pl_vulkan_swapchain_params(
480  .surface = ctx->vk_surface,
481  .present_mode = VK_PRESENT_MODE_FIFO_KHR));
482  if (!ctx->swapchain) {
484  goto out;
485  }
486 
487  SDL_Vulkan_GetDrawableSize(window, &w, &h);
488  pl_swapchain_resize(ctx->swapchain, &w, &h);
489 
490  ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->placebo_vulkan->gpu);
491  if (!ctx->renderer) {
493  goto out;
494  }
495 
496  ctx->vk_frame = av_frame_alloc();
497  if (!ctx->vk_frame) {
498  ret = AVERROR(ENOMEM);
499  goto out;
500  }
501 
502  ret = 0;
503 
504 out:
505  av_free(ext);
506  return ret;
507 }
508 
509 static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
510 {
511  RendererContext *ctx = (RendererContext *) renderer;
512 
513  *dev = ctx->hw_device_ref;
514  return 0;
515 }
516 
517 static int create_hw_frame(VkRenderer *renderer, AVFrame *frame)
518 {
519  RendererContext *ctx = (RendererContext *) renderer;
520  AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)
521  frame->hw_frames_ctx->data;
522  AVHWFramesContext *hw_frame;
523  AVVulkanFramesContext *vk_frame_ctx;
524  int ret;
525 
526  if (ctx->hw_frame_ref) {
527  hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
528 
529  if (hw_frame->width == frame->width &&
530  hw_frame->height == frame->height &&
531  hw_frame->sw_format == src_hw_frame->sw_format)
532  return 0;
533 
534  av_buffer_unref(&ctx->hw_frame_ref);
535  }
536 
537  if (!ctx->constraints) {
539  ctx->hw_device_ref, NULL);
540  if (!ctx->constraints)
541  return AVERROR(ENOMEM);
542  }
543 
544  // Check constraints and skip create hwframe. Don't take it as error since
545  // we can fallback to memory copy from GPU to CPU.
546  if ((ctx->constraints->max_width &&
547  ctx->constraints->max_width < frame->width) ||
548  (ctx->constraints->max_height &&
549  ctx->constraints->max_height < frame->height) ||
550  (ctx->constraints->min_width &&
551  ctx->constraints->min_width > frame->width) ||
552  (ctx->constraints->min_height &&
553  ctx->constraints->min_height > frame->height))
554  return 0;
555 
556  if (ctx->constraints->valid_sw_formats) {
557  enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats;
558  while (*sw_formats != AV_PIX_FMT_NONE) {
559  if (*sw_formats == src_hw_frame->sw_format)
560  break;
561  sw_formats++;
562  }
563  if (*sw_formats == AV_PIX_FMT_NONE)
564  return 0;
565  }
566 
567  ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref);
568  if (!ctx->hw_frame_ref)
569  return AVERROR(ENOMEM);
570 
571  hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
572  hw_frame->format = AV_PIX_FMT_VULKAN;
573  hw_frame->sw_format = src_hw_frame->sw_format;
574  hw_frame->width = frame->width;
575  hw_frame->height = frame->height;
576 
577  if (frame->format == AV_PIX_FMT_CUDA) {
578  vk_frame_ctx = hw_frame->hwctx;
579  vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
580  }
581 
582  ret = av_hwframe_ctx_init(ctx->hw_frame_ref);
583  if (ret < 0) {
584  av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n",
585  av_err2str(ret));
586  return ret;
587  }
588 
589  av_hwframe_transfer_get_formats(ctx->hw_frame_ref,
591  &ctx->transfer_formats, 0);
592 
593  return 0;
594 }
595 
596 static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame)
597 {
598  if (!ctx->hw_frame_ref || !ctx->transfer_formats)
599  return 0;
600 
601  for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++)
602  if (ctx->transfer_formats[i] == frame->format)
603  return 1;
604 
605  return 0;
606 }
607 
608 static inline int move_to_output_frame(RendererContext *ctx, AVFrame *frame)
609 {
610  int ret = av_frame_copy_props(ctx->vk_frame, frame);
611  if (ret < 0)
612  return ret;
614  av_frame_move_ref(frame, ctx->vk_frame);
615  return 0;
616 }
617 
618 static int map_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
619 {
620  RendererContext *ctx = (RendererContext *) renderer;
621  int ret;
622 
623  if (use_hw_frame && !ctx->hw_frame_ref)
624  return AVERROR(ENOSYS);
625 
626  // Try map data first
627  av_frame_unref(ctx->vk_frame);
628  if (use_hw_frame) {
629  ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref);
630  ctx->vk_frame->format = AV_PIX_FMT_VULKAN;
631  }
632  ret = av_hwframe_map(ctx->vk_frame, frame, 0);
633  if (!ret)
634  return move_to_output_frame(ctx, frame);
635 
636  if (ret != AVERROR(ENOSYS))
637  av_log(NULL, AV_LOG_FATAL, "Map frame failed: %s\n", av_err2str(ret));
638  return ret;
639 }
640 
641 static int transfer_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
642 {
643  RendererContext *ctx = (RendererContext *) renderer;
644  int ret;
645 
646  if (use_hw_frame && !check_hw_transfer(ctx, frame))
647  return AVERROR(ENOSYS);
648 
649  av_frame_unref(ctx->vk_frame);
650  if (use_hw_frame)
651  av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0);
652  ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1);
653  if (!ret)
654  return move_to_output_frame(ctx, frame);
655 
656  if (ret != AVERROR(ENOSYS))
657  av_log(NULL, AV_LOG_FATAL, "Transfer frame failed: %s\n",
658  av_err2str(ret));
659  return ret;
660 }
661 
663 {
664  int ret;
665 
666  if (!frame->hw_frames_ctx)
667  return 0;
668 
669  if (frame->format == AV_PIX_FMT_VULKAN)
670  return 0;
671 
672  ret = create_hw_frame(renderer, frame);
673  if (ret < 0)
674  return ret;
675 
676  for (int use_hw = 1; use_hw >=0; use_hw--) {
677  ret = map_frame(renderer, frame, use_hw);
678  if (!ret)
679  return 0;
680  if (ret != AVERROR(ENOSYS))
681  return ret;
682 
683  ret = transfer_frame(renderer, frame, use_hw);
684  if (!ret)
685  return 0;
686  if (ret != AVERROR(ENOSYS))
687  return ret;
688  }
689 
690  return ret;
691 }
692 
693 static int display(VkRenderer *renderer, AVFrame *frame)
694 {
695  struct pl_swapchain_frame swap_frame = {0};
696  struct pl_frame pl_frame = {0};
697  struct pl_frame target = {0};
698  RendererContext *ctx = (RendererContext *) renderer;
699  int ret = 0;
700  struct pl_color_space hint = {0};
701 
703  if (ret < 0)
704  return ret;
705 
706  if (!pl_map_avframe_ex(ctx->placebo_vulkan->gpu, &pl_frame, pl_avframe_params(
707  .frame = frame,
708  .tex = ctx->tex))) {
709  av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n");
710  return AVERROR_EXTERNAL;
711  }
712 
713  pl_color_space_from_avframe(&hint, frame);
714  pl_swapchain_colorspace_hint(ctx->swapchain, &hint);
715  if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) {
716  av_log(NULL, AV_LOG_ERROR, "start frame failed\n");
718  goto out;
719  }
720 
721  pl_frame_from_swapchain(&target, &swap_frame);
722  if (!pl_render_image(ctx->renderer, &pl_frame, &target,
723  &pl_render_default_params)) {
724  av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
726  goto out;
727  }
728 
729  if (!pl_swapchain_submit_frame(ctx->swapchain)) {
730  av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n");
732  goto out;
733  }
734  pl_swapchain_swap_buffers(ctx->swapchain);
735 
736 out:
737  pl_unmap_avframe(ctx->placebo_vulkan->gpu, &pl_frame);
738  return ret;
739 }
740 
741 static int resize(VkRenderer *renderer, int width, int height)
742 {
743  RendererContext *ctx = (RendererContext *) renderer;
744 
745  if (!pl_swapchain_resize(ctx->swapchain, &width, &height))
746  return AVERROR_EXTERNAL;
747  return 0;
748 }
749 
750 static void destroy(VkRenderer *renderer)
751 {
752  RendererContext *ctx = (RendererContext *) renderer;
753  PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
754 
755  av_frame_free(&ctx->vk_frame);
756  av_freep(&ctx->transfer_formats);
757  av_hwframe_constraints_free(&ctx->constraints);
758  av_buffer_unref(&ctx->hw_frame_ref);
759 
760  if (ctx->placebo_vulkan) {
761  for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
762  pl_tex_destroy(ctx->placebo_vulkan->gpu, &ctx->tex[i]);
763  pl_renderer_destroy(&ctx->renderer);
764  pl_swapchain_destroy(&ctx->swapchain);
765  pl_vulkan_destroy(&ctx->placebo_vulkan);
766  }
767 
768  if (ctx->vk_surface) {
769  vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
770  ctx->get_proc_addr(ctx->inst, "vkDestroySurfaceKHR");
771  vkDestroySurfaceKHR(ctx->inst, ctx->vk_surface, NULL);
772  ctx->vk_surface = VK_NULL_HANDLE;
773  }
774 
775  av_buffer_unref(&ctx->hw_device_ref);
776  pl_vk_inst_destroy(&ctx->placebo_instance);
777 
778  pl_log_destroy(&ctx->vk_log);
779 }
780 
781 static const AVClass vulkan_renderer_class = {
782  .class_name = "Vulkan Renderer",
783  .item_name = av_default_item_name,
784  .version = LIBAVUTIL_VERSION_INT,
785 };
786 
788 {
789  RendererContext *ctx = av_mallocz(sizeof(*ctx));
791 
792  if (!ctx)
793  return NULL;
794 
795  renderer = &ctx->api;
796  renderer->class = &vulkan_renderer_class;
797  renderer->get_hw_dev = get_hw_dev;
798  renderer->create = create;
799  renderer->display = display;
800  renderer->resize = resize;
801  renderer->destroy = destroy;
802 
803  return renderer;
804 }
805 
806 #else
807 
809 {
810  return NULL;
811 }
812 
813 #endif
814 
816  AVDictionary *opt)
817 {
818  return renderer->create(renderer, window, opt);
819 }
820 
822 {
823  return renderer->get_hw_dev(renderer, dev);
824 }
825 
827 {
828  return renderer->display(renderer, frame);
829 }
830 
832 {
833  return renderer->resize(renderer, width, height);
834 }
835 
837 {
838  renderer->destroy(renderer);
839 }
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:85
VkRenderer::create
int(* create)(VkRenderer *renderer, SDL_Window *window, AVDictionary *dict)
Definition: ffplay_renderer.c:52
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
convert_frame
static int convert_frame(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Convert a frame from linear RGB to logspace LAB, and accumulate channel totals for each row Convert f...
Definition: vf_grayworld.c:122
entry
#define entry
Definition: aom_film_grain_template.c:66
level
uint8_t level
Definition: svq3.c:205
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:207
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
out
FILE * out
Definition: movenc.c:55
destroy
static void destroy(struct ResampleContext **c)
Definition: soxr_resample.c:64
av_bprint_init
void av_bprint_init(AVBPrint *buf, unsigned size_init, unsigned size_max)
Definition: bprint.c:69
AV_LOG_QUIET
#define AV_LOG_QUIET
Print no output.
Definition: log.h:191
ffplay_renderer.h
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:322
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:579
w
uint8_t w
Definition: llviddspenc.c:38
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:248
AVHWDeviceContext::user_opaque
void * user_opaque
Arbitrary user data, to be used e.g.
Definition: hwcontext.h:102
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:778
vk_renderer_create
int vk_renderer_create(VkRenderer *renderer, SDL_Window *window, AVDictionary *opt)
Definition: ffplay_renderer.c:815
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AVDictionary
Definition: dict.c:34
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
map_frame
static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src, struct pl_frame *out)
Definition: vf_libplacebo.c:925
vk_renderer_destroy
void vk_renderer_destroy(VkRenderer *renderer)
Definition: ffplay_renderer.c:836
av_hwdevice_get_hwframe_constraints
AVHWFramesConstraints * av_hwdevice_get_hwframe_constraints(AVBufferRef *ref, const void *hwconfig)
Get the constraints on HW frames given a device and the HW-specific configuration to be used with tha...
Definition: hwcontext.c:566
av_hwdevice_ctx_init
int av_hwdevice_ctx_init(AVBufferRef *ref)
Finalize the device context before use.
Definition: hwcontext.c:208
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:441
window
static SDL_Window * window
Definition: ffplay.c:361
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:135
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:213
AV_BPRINT_SIZE_AUTOMATIC
#define AV_BPRINT_SIZE_AUTOMATIC
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:162
AV_DICT_DONT_STRDUP_VAL
#define AV_DICT_DONT_STRDUP_VAL
Take ownership of a value that's been allocated with av_malloc() or another memory allocation functio...
Definition: dict.h:79
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:60
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:150
AV_LOG_TRACE
#define AV_LOG_TRACE
Extremely verbose debugging, useful for libav* development.
Definition: log.h:235
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
vk_get_renderer
VkRenderer * vk_get_renderer(void)
Definition: ffplay_renderer.c:808
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
av_hwdevice_ctx_alloc
AVBufferRef * av_hwdevice_ctx_alloc(enum AVHWDeviceType type)
Allocate an AVHWDeviceContext for a given hardware type.
Definition: hwcontext.c:161
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:62
av_hwframe_constraints_free
void av_hwframe_constraints_free(AVHWFramesConstraints **constraints)
Free an AVHWFrameConstraints structure.
Definition: hwcontext.c:591
decode_index
static int decode_index(SGAVideoContext *s, AVFrame *frame)
Definition: sga.c:181
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
ctx
AVFormatContext * ctx
Definition: movenc.c:49
vk_renderer_get_hw_dev
int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
Definition: ffplay_renderer.c:821
vk_renderer_display
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
Definition: ffplay_renderer.c:826
renderer
static SDL_Renderer * renderer
Definition: ffplay.c:362
if
if(ret)
Definition: filter_design.txt:179
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
VkRenderer
Definition: ffplay_renderer.c:49
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:75
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:210
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:725
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
create
static struct ResampleContext * create(struct ResampleContext *c, int out_rate, int in_rate, int filter_size, int phase_shift, int linear, double cutoff, enum AVSampleFormat format, enum SwrFilterType filter_type, double kaiser_beta, double precision, int cheby, int exact_rational)
Definition: soxr_resample.c:32
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:178
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:164
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:237
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:133
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:101
index
int index
Definition: gxfenc.c:90
height
#define height
Definition: dsp.h:85
av_bprint_finalize
int av_bprint_finalize(AVBPrint *buf, char **ret_str)
Finalize a print buffer.
Definition: bprint.c:240
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
VkRenderer::get_hw_dev
int(* get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev)
Definition: ffplay_renderer.c:54
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:124
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:223
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:173
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:220
bprint.h
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:142
av_frame_move_ref
void av_frame_move_ref(AVFrame *dst, AVFrame *src)
Move everything contained in src to dst and reset src.
Definition: frame.c:649
vk_renderer_resize
int vk_renderer_resize(VkRenderer *renderer, int width, int height)
Definition: ffplay_renderer.c:831
av_frame_unref
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
Definition: frame.c:622
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:115
ret
ret
Definition: filter_design.txt:187
AV_LOG_FATAL
#define AV_LOG_FATAL
Something went wrong and recovery is not possible.
Definition: log.h:203
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:80
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
VkRenderer::destroy
void(* destroy)(VkRenderer *renderer)
Definition: ffplay_renderer.c:60
av_hwdevice_ctx_create
int av_hwdevice_ctx_create(AVBufferRef **pdevice_ref, enum AVHWDeviceType type, const char *device, AVDictionary *opts, int flags)
Open a device of the specified type and create an AVHWDeviceContext for it.
Definition: hwcontext.c:600
av_bprintf
void av_bprintf(AVBPrint *buf, const char *fmt,...)
Definition: bprint.c:99
av_hwframe_transfer_data
int av_hwframe_transfer_data(AVFrame *dst, const AVFrame *src, int flags)
Copy data to or from a hw surface.
Definition: hwcontext.c:433
AVFormatContext::debug
int debug
Flags to enable debugging.
Definition: avformat.h:1578
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:126
av_hwframe_transfer_get_formats
int av_hwframe_transfer_get_formats(AVBufferRef *hwframe_ref, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats, int flags)
Get a list of possible source or target formats usable in av_hwframe_transfer_data().
Definition: hwcontext.c:371
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:112
mem.h
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:102
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
AVDictionaryEntry
Definition: dict.h:89
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
av_dict_set
int av_dict_set(AVDictionary **pm, const char *key, const char *value, int flags)
Set the given entry in *pm, overwriting an existing entry.
Definition: dict.c:88
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
width
#define width
Definition: dsp.h:85
VkRenderer::display
int(* display)(VkRenderer *renderer, AVFrame *frame)
Definition: ffplay_renderer.c:56
av_hwframe_get_buffer
int av_hwframe_get_buffer(AVBufferRef *hwframe_ref, AVFrame *frame, int flags)
Allocate a new frame attached to the given AVHWFramesContext.
Definition: hwcontext.c:491
VkRenderer::resize
int(* resize)(VkRenderer *renderer, int width, int height)
Definition: ffplay_renderer.c:58
AV_HWFRAME_TRANSFER_DIRECTION_TO
@ AV_HWFRAME_TRANSFER_DIRECTION_TO
Transfer the data to the queried hw frame.
Definition: hwcontext.h:412
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:144