FFmpeg
vulkan_h264.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "h264dec.h"
20 #include "h264_ps.h"
21 
22 #include "vulkan_decode.h"
23 
26  .decode_extension = FF_VK_EXT_VIDEO_DECODE_H264,
27  .decode_op = VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR,
28  .ext_props = {
29  .extensionName = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_EXTENSION_NAME,
30  .specVersion = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_SPEC_VERSION,
31  },
32 };
33 
34 typedef struct H264VulkanDecodePicture {
36 
37  /* Current picture */
38  StdVideoDecodeH264ReferenceInfo h264_ref;
39  VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref;
40 
41  /* Picture refs */
43  StdVideoDecodeH264ReferenceInfo h264_refs [H264_MAX_PICTURE_COUNT];
44  VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT];
45 
46  /* Current picture (contd.) */
47  StdVideoDecodeH264PictureInfo h264pic;
48  VkVideoDecodeH264PictureInfoKHR h264_pic_info;
50 
51 const static int h264_scaling_list8_order[] = { 0, 3, 1, 4, 2, 5 };
52 
53 static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src,
54  VkVideoReferenceSlotInfoKHR *ref_slot, /* Main structure */
55  VkVideoPictureResourceInfoKHR *ref, /* Goes in ^ */
56  VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, /* Goes in ^ */
57  StdVideoDecodeH264ReferenceInfo *h264_ref, /* Goes in ^ */
58  H264Picture *pic, int is_current,
59  int is_field, int picture_structure,
60  int dpb_slot_index)
61 {
65  FFVulkanDecodePicture *vkpic = &hp->vp;
66 
67  int err = ff_vk_decode_prepare_frame(dec, pic->f, vkpic, is_current,
68  dec->dedicated_dpb);
69  if (err < 0)
70  return err;
71 
72  *h264_ref = (StdVideoDecodeH264ReferenceInfo) {
73  .FrameNum = pic->long_ref ? pic->pic_id : pic->frame_num,
74  .PicOrderCnt = { pic->field_poc[0], pic->field_poc[1] },
75  .flags = (StdVideoDecodeH264ReferenceInfoFlags) {
76  .top_field_flag = is_field ? !!(picture_structure & PICT_TOP_FIELD) : 0,
77  .bottom_field_flag = is_field ? !!(picture_structure & PICT_BOTTOM_FIELD) : 0,
78  .used_for_long_term_reference = pic->reference && pic->long_ref,
79  /*
80  * flags.is_non_existing is used to indicate whether the picture is marked as
81  * “non-existing” as defined in section 8.2.5.2 of the ITU-T H.264 Specification;
82  * 8.2.5.2 Decoding process for gaps in frame_num
83  * corresponds to the code in h264_slice.c:h264_field_start,
84  * which sets the invalid_gap flag when decoding.
85  */
86  .is_non_existing = pic->invalid_gap,
87  },
88  };
89 
90  *vkh264_ref = (VkVideoDecodeH264DpbSlotInfoKHR) {
91  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR,
92  .pStdReferenceInfo = h264_ref,
93  };
94 
95  *ref = (VkVideoPictureResourceInfoKHR) {
96  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
97  .codedOffset = (VkOffset2D){ 0, 0 },
98  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
99  .baseArrayLayer = ctx->common.layered_dpb ? dpb_slot_index : 0,
100  .imageViewBinding = vkpic->img_view_ref,
101  };
102 
103  *ref_slot = (VkVideoReferenceSlotInfoKHR) {
104  .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
105  .pNext = vkh264_ref,
106  .slotIndex = dpb_slot_index,
107  .pPictureResource = ref,
108  };
109 
110  if (ref_src)
111  *ref_src = pic;
112 
113  return 0;
114 }
115 
116 static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
117 {
118  switch (level_idc) {
119  case 10: return STD_VIDEO_H264_LEVEL_IDC_1_0;
120  case 11: return STD_VIDEO_H264_LEVEL_IDC_1_1;
121  case 12: return STD_VIDEO_H264_LEVEL_IDC_1_2;
122  case 13: return STD_VIDEO_H264_LEVEL_IDC_1_3;
123  case 20: return STD_VIDEO_H264_LEVEL_IDC_2_0;
124  case 21: return STD_VIDEO_H264_LEVEL_IDC_2_1;
125  case 22: return STD_VIDEO_H264_LEVEL_IDC_2_2;
126  case 30: return STD_VIDEO_H264_LEVEL_IDC_3_0;
127  case 31: return STD_VIDEO_H264_LEVEL_IDC_3_1;
128  case 32: return STD_VIDEO_H264_LEVEL_IDC_3_2;
129  case 40: return STD_VIDEO_H264_LEVEL_IDC_4_0;
130  case 41: return STD_VIDEO_H264_LEVEL_IDC_4_1;
131  case 42: return STD_VIDEO_H264_LEVEL_IDC_4_2;
132  case 50: return STD_VIDEO_H264_LEVEL_IDC_5_0;
133  case 51: return STD_VIDEO_H264_LEVEL_IDC_5_1;
134  case 52: return STD_VIDEO_H264_LEVEL_IDC_5_2;
135  case 60: return STD_VIDEO_H264_LEVEL_IDC_6_0;
136  case 61: return STD_VIDEO_H264_LEVEL_IDC_6_1;
137  default:
138  case 62: return STD_VIDEO_H264_LEVEL_IDC_6_2;
139  }
140 }
141 
142 static void set_sps(const SPS *sps,
143  StdVideoH264ScalingLists *vksps_scaling,
144  StdVideoH264HrdParameters *vksps_vui_header,
145  StdVideoH264SequenceParameterSetVui *vksps_vui,
146  StdVideoH264SequenceParameterSet *vksps)
147 {
148  *vksps_scaling = (StdVideoH264ScalingLists) {
149  .scaling_list_present_mask = sps->scaling_matrix_present_mask,
150  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
151  };
152 
153  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
154  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
155  vksps_scaling->ScalingList4x4[i][j] = sps->scaling_matrix4[i][ff_zigzag_scan[j]];
156 
157  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
158  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
159  vksps_scaling->ScalingList8x8[i][j] =
160  sps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
161 
162  *vksps_vui_header = (StdVideoH264HrdParameters) {
163  .cpb_cnt_minus1 = sps->cpb_cnt - 1,
164  .bit_rate_scale = sps->bit_rate_scale,
165  .initial_cpb_removal_delay_length_minus1 = sps->initial_cpb_removal_delay_length - 1,
166  .cpb_removal_delay_length_minus1 = sps->cpb_removal_delay_length - 1,
167  .dpb_output_delay_length_minus1 = sps->dpb_output_delay_length - 1,
168  .time_offset_length = sps->time_offset_length,
169  };
170 
171  for (int i = 0; i < sps->cpb_cnt; i++) {
172  vksps_vui_header->bit_rate_value_minus1[i] = sps->bit_rate_value[i] - 1;
173  vksps_vui_header->cpb_size_value_minus1[i] = sps->cpb_size_value[i] - 1;
174  vksps_vui_header->cbr_flag[i] = (sps->cpr_flag >> i) & 0x1;
175  }
176 
177  *vksps_vui = (StdVideoH264SequenceParameterSetVui) {
178  .aspect_ratio_idc = sps->vui.aspect_ratio_idc,
179  .sar_width = sps->vui.sar.num,
180  .sar_height = sps->vui.sar.den,
181  .video_format = sps->vui.video_format,
182  .colour_primaries = sps->vui.colour_primaries,
183  .transfer_characteristics = sps->vui.transfer_characteristics,
184  .matrix_coefficients = sps->vui.matrix_coeffs,
185  .num_units_in_tick = sps->num_units_in_tick,
186  .time_scale = sps->time_scale,
187  .pHrdParameters = vksps_vui_header,
188  .max_num_reorder_frames = sps->num_reorder_frames,
189  .max_dec_frame_buffering = sps->max_dec_frame_buffering,
190  .flags = (StdVideoH264SpsVuiFlags) {
191  .aspect_ratio_info_present_flag = sps->vui.aspect_ratio_info_present_flag,
192  .overscan_info_present_flag = sps->vui.overscan_info_present_flag,
193  .overscan_appropriate_flag = sps->vui.overscan_appropriate_flag,
194  .video_signal_type_present_flag = sps->vui.video_signal_type_present_flag,
195  .video_full_range_flag = sps->vui.video_full_range_flag,
196  .color_description_present_flag = sps->vui.colour_description_present_flag,
197  .chroma_loc_info_present_flag = sps->vui.chroma_location,
198  .timing_info_present_flag = sps->timing_info_present_flag,
199  .fixed_frame_rate_flag = sps->fixed_frame_rate_flag,
200  .bitstream_restriction_flag = sps->bitstream_restriction_flag,
201  .nal_hrd_parameters_present_flag = sps->nal_hrd_parameters_present_flag,
202  .vcl_hrd_parameters_present_flag = sps->vcl_hrd_parameters_present_flag,
203  },
204  };
205 
206  *vksps = (StdVideoH264SequenceParameterSet) {
207  .profile_idc = sps->profile_idc,
208  .level_idc = convert_to_vk_level_idc(sps->level_idc),
209  .seq_parameter_set_id = sps->sps_id,
210  .chroma_format_idc = sps->chroma_format_idc,
211  .bit_depth_luma_minus8 = sps->bit_depth_luma - 8,
212  .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
213  .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4,
214  .pic_order_cnt_type = sps->poc_type,
215  .log2_max_pic_order_cnt_lsb_minus4 = sps->poc_type ? 0 : sps->log2_max_poc_lsb - 4,
216  .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
217  .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
218  .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length,
219  .max_num_ref_frames = sps->ref_frame_count,
220  .pic_width_in_mbs_minus1 = sps->mb_width - 1,
221  .pic_height_in_map_units_minus1 = (sps->mb_height/(2 - sps->frame_mbs_only_flag)) - 1,
222  .frame_crop_left_offset = sps->crop_left,
223  .frame_crop_right_offset = sps->crop_right,
224  .frame_crop_top_offset = sps->crop_top,
225  .frame_crop_bottom_offset = sps->crop_bottom,
226  .flags = (StdVideoH264SpsFlags) {
227  .constraint_set0_flag = (sps->constraint_set_flags >> 0) & 0x1,
228  .constraint_set1_flag = (sps->constraint_set_flags >> 1) & 0x1,
229  .constraint_set2_flag = (sps->constraint_set_flags >> 2) & 0x1,
230  .constraint_set3_flag = (sps->constraint_set_flags >> 3) & 0x1,
231  .constraint_set4_flag = (sps->constraint_set_flags >> 4) & 0x1,
232  .constraint_set5_flag = (sps->constraint_set_flags >> 5) & 0x1,
233  .direct_8x8_inference_flag = sps->direct_8x8_inference_flag,
234  .mb_adaptive_frame_field_flag = sps->mb_aff,
235  .frame_mbs_only_flag = sps->frame_mbs_only_flag,
236  .delta_pic_order_always_zero_flag = sps->delta_pic_order_always_zero_flag,
237  .separate_colour_plane_flag = sps->residual_color_transform_flag,
238  .gaps_in_frame_num_value_allowed_flag = sps->gaps_in_frame_num_allowed_flag,
239  .qpprime_y_zero_transform_bypass_flag = sps->transform_bypass,
240  .frame_cropping_flag = sps->crop,
241  .seq_scaling_matrix_present_flag = sps->scaling_matrix_present,
242  .vui_parameters_present_flag = sps->vui_parameters_present_flag,
243  },
244  .pOffsetForRefFrame = sps->offset_for_ref_frame,
245  .pScalingLists = vksps_scaling,
246  .pSequenceParameterSetVui = vksps_vui,
247  };
248 }
249 
250 static void set_pps(const PPS *pps, const SPS *sps,
251  StdVideoH264ScalingLists *vkpps_scaling,
252  StdVideoH264PictureParameterSet *vkpps)
253 {
254  *vkpps_scaling = (StdVideoH264ScalingLists) {
255  .scaling_list_present_mask = pps->pic_scaling_matrix_present_mask,
256  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
257  };
258 
259  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
260  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
261  vkpps_scaling->ScalingList4x4[i][j] = pps->scaling_matrix4[i][ff_zigzag_scan[j]];
262 
263  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
264  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
265  vkpps_scaling->ScalingList8x8[i][j] =
266  pps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
267 
268  *vkpps = (StdVideoH264PictureParameterSet) {
269  .seq_parameter_set_id = pps->sps_id,
270  .pic_parameter_set_id = pps->pps_id,
271  .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1,
272  .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1,
273  .weighted_bipred_idc = pps->weighted_bipred_idc,
274  .pic_init_qp_minus26 = pps->init_qp - 26,
275  .pic_init_qs_minus26 = pps->init_qs - 26,
276  .chroma_qp_index_offset = pps->chroma_qp_index_offset[0],
277  .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1],
278  .flags = (StdVideoH264PpsFlags) {
279  .transform_8x8_mode_flag = pps->transform_8x8_mode,
280  .redundant_pic_cnt_present_flag = pps->redundant_pic_cnt_present,
281  .constrained_intra_pred_flag = pps->constrained_intra_pred,
282  .deblocking_filter_control_present_flag = pps->deblocking_filter_parameters_present,
283  .weighted_pred_flag = pps->weighted_pred,
284  .bottom_field_pic_order_in_frame_present_flag = pps->pic_order_present,
285  .entropy_coding_mode_flag = pps->cabac,
286  .pic_scaling_matrix_present_flag = pps->pic_scaling_matrix_present_flag,
287  },
288  .pScalingLists = vkpps_scaling,
289  };
290 }
291 
293 {
294  int err;
297  const H264Context *h = avctx->priv_data;
298 
299  /* SPS */
300  StdVideoH264ScalingLists vksps_scaling[MAX_SPS_COUNT];
301  StdVideoH264HrdParameters vksps_vui_header[MAX_SPS_COUNT];
302  StdVideoH264SequenceParameterSetVui vksps_vui[MAX_SPS_COUNT];
303  StdVideoH264SequenceParameterSet vksps[MAX_SPS_COUNT];
304 
305  /* PPS */
306  StdVideoH264ScalingLists vkpps_scaling[MAX_PPS_COUNT];
307  StdVideoH264PictureParameterSet vkpps[MAX_PPS_COUNT];
308 
309  VkVideoDecodeH264SessionParametersAddInfoKHR h264_params_info = {
310  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_ADD_INFO_KHR,
311  .pStdSPSs = vksps,
312  .stdSPSCount = 0,
313  .pStdPPSs = vkpps,
314  .stdPPSCount = 0,
315  };
316  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
317  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
318  .pParametersAddInfo = &h264_params_info,
319  };
320  VkVideoSessionParametersCreateInfoKHR session_params_create = {
321  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
322  .pNext = &h264_params,
323  .videoSession = ctx->common.session,
324  .videoSessionParametersTemplate = VK_NULL_HANDLE,
325  };
326 
327  /* SPS list */
328  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) {
329  if (h->ps.sps_list[i]) {
330  const SPS *sps_l = h->ps.sps_list[i];
331  int idx = h264_params_info.stdSPSCount;
332  set_sps(sps_l, &vksps_scaling[idx], &vksps_vui_header[idx], &vksps_vui[idx], &vksps[idx]);
333  h264_params_info.stdSPSCount++;
334  }
335  }
336 
337  /* PPS list */
338  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.pps_list); i++) {
339  if (h->ps.pps_list[i]) {
340  const PPS *pps_l = h->ps.pps_list[i];
341  int idx = h264_params_info.stdPPSCount;
342  set_pps(pps_l, pps_l->sps, &vkpps_scaling[idx], &vkpps[idx]);
343  h264_params_info.stdPPSCount++;
344  }
345  }
346 
347  h264_params.maxStdSPSCount = h264_params_info.stdSPSCount;
348  h264_params.maxStdPPSCount = h264_params_info.stdPPSCount;
349 
350  err = ff_vk_decode_create_params(buf, avctx, ctx, &session_params_create);
351  if (err < 0)
352  return err;
353 
354  av_log(avctx, AV_LOG_DEBUG, "Created frame parameters: %i SPS %i PPS\n",
355  h264_params_info.stdSPSCount, h264_params_info.stdPPSCount);
356 
357  return 0;
358 }
359 
361  av_unused const uint8_t *buffer,
362  av_unused uint32_t size)
363 {
364  int err;
365  int dpb_slot_index = 0;
366  H264Context *h = avctx->priv_data;
367  H264Picture *pic = h->cur_pic_ptr;
370  FFVulkanDecodePicture *vp = &hp->vp;
371 
372  if (!dec->session_params) {
373  err = vk_h264_create_params(avctx, &dec->session_params);
374  if (err < 0)
375  return err;
376  }
377 
378  /* Fill in main slot */
379  dpb_slot_index = 0;
380  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
381  if (pic == &h->DPB[slot]) {
382  dpb_slot_index = slot;
383  break;
384  }
385  }
386 
387  err = vk_h264_fill_pict(avctx, NULL, &vp->ref_slot, &vp->ref,
388  &hp->vkh264_ref, &hp->h264_ref, pic, 1,
389  h->DPB[dpb_slot_index].field_picture,
390  h->DPB[dpb_slot_index].reference,
391  dpb_slot_index);
392  if (err < 0)
393  return err;
394 
395  /* Fill in short-term references */
396  for (int i = 0; i < h->short_ref_count; i++) {
397  dpb_slot_index = 0;
398  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
399  if (h->short_ref[i] == &h->DPB[slot]) {
400  dpb_slot_index = slot;
401  break;
402  }
403  }
404  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
405  &vp->refs[i], &hp->vkh264_refs[i],
406  &hp->h264_refs[i], h->short_ref[i], 0,
407  h->DPB[dpb_slot_index].field_picture,
408  h->DPB[dpb_slot_index].reference,
409  dpb_slot_index);
410  if (err < 0)
411  return err;
412  }
413 
414  /* Fill in long-term refs */
415  for (int r = 0, i = h->short_ref_count; r < H264_MAX_DPB_FRAMES &&
416  i < h->short_ref_count + h->long_ref_count; r++) {
417  if (!h->long_ref[r])
418  continue;
419 
420  dpb_slot_index = 0;
421  for (unsigned slot = 0; slot < 16; slot++) {
422  if (h->long_ref[r] == &h->DPB[slot]) {
423  dpb_slot_index = slot;
424  break;
425  }
426  }
427  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
428  &vp->refs[i], &hp->vkh264_refs[i],
429  &hp->h264_refs[i], h->long_ref[r], 0,
430  h->DPB[dpb_slot_index].field_picture,
431  h->DPB[dpb_slot_index].reference,
432  dpb_slot_index);
433  if (err < 0)
434  return err;
435  i++;
436  }
437 
438  hp->h264pic = (StdVideoDecodeH264PictureInfo) {
439  .seq_parameter_set_id = pic->pps->sps_id,
440  .pic_parameter_set_id = pic->pps->pps_id,
441  .frame_num = 0, /* Set later */
442  .idr_pic_id = 0, /* Set later */
443  .PicOrderCnt[0] = pic->field_poc[0],
444  .PicOrderCnt[1] = pic->field_poc[1],
445  .flags = (StdVideoDecodeH264PictureInfoFlags) {
446  .field_pic_flag = FIELD_PICTURE(h),
447  .is_intra = 1, /* Set later */
448  .IdrPicFlag = h->picture_idr,
449  .bottom_field_flag = h->picture_structure != PICT_FRAME &&
450  h->picture_structure & PICT_BOTTOM_FIELD,
451  .is_reference = h->nal_ref_idc != 0,
452  .complementary_field_pair = h->first_field && FIELD_PICTURE(h),
453  },
454  };
455 
456  hp->h264_pic_info = (VkVideoDecodeH264PictureInfoKHR) {
457  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PICTURE_INFO_KHR,
458  .pStdPictureInfo = &hp->h264pic,
459  };
460 
461  vp->decode_info = (VkVideoDecodeInfoKHR) {
462  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_INFO_KHR,
463  .pNext = &hp->h264_pic_info,
464  .flags = 0x0,
465  .pSetupReferenceSlot = &vp->ref_slot,
466  .referenceSlotCount = h->short_ref_count + h->long_ref_count,
467  .pReferenceSlots = vp->ref_slots,
468  .dstPictureResource = (VkVideoPictureResourceInfoKHR) {
469  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
470  .codedOffset = (VkOffset2D){ 0, 0 },
471  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
472  .baseArrayLayer = 0,
473  .imageViewBinding = vp->img_view_out,
474  },
475  };
476 
477  return 0;
478 }
479 
481  const uint8_t *data,
482  uint32_t size)
483 {
484  const H264Context *h = avctx->priv_data;
485  const H264SliceContext *sl = &h->slice_ctx[0];
486  H264VulkanDecodePicture *hp = h->cur_pic_ptr->hwaccel_picture_private;
487  FFVulkanDecodePicture *vp = &hp->vp;
488 
489  int err = ff_vk_decode_add_slice(avctx, vp, data, size, 1,
490  &hp->h264_pic_info.sliceCount,
491  &hp->h264_pic_info.pSliceOffsets);
492  if (err < 0)
493  return err;
494 
495  hp->h264pic.frame_num = sl->frame_num;
496  hp->h264pic.idr_pic_id = sl->idr_pic_id;
497 
498  /* Frame is only intra of all slices are marked as intra */
500  hp->h264pic.flags.is_intra = 0;
501 
502  return 0;
503 }
504 
506 {
507  const H264Context *h = avctx->priv_data;
508  H264Picture *pic = h->cur_pic_ptr;
511  FFVulkanDecodePicture *vp = &hp->vp;
513  AVFrame *rav[H264_MAX_PICTURE_COUNT] = { 0 };
514 
515  if (!hp->h264_pic_info.sliceCount)
516  return 0;
517 
518  if (!vp->slices_buf)
519  return AVERROR(EINVAL);
520 
521  if (!dec->session_params) {
522  int err = vk_h264_create_params(avctx, &dec->session_params);
523  if (err < 0)
524  return err;
525 
526  hp->h264pic.seq_parameter_set_id = pic->pps->sps_id;
527  hp->h264pic.pic_parameter_set_id = pic->pps->pps_id;
528  }
529 
530  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
531  H264Picture *rp = hp->ref_src[i];
533 
534  rvp[i] = &rhp->vp;
535  rav[i] = hp->ref_src[i]->f;
536  }
537 
538  av_log(avctx, AV_LOG_VERBOSE, "Decoding frame, %"SIZE_SPECIFIER" bytes, %i slices\n",
539  vp->slices_size, hp->h264_pic_info.sliceCount);
540 
541  return ff_vk_decode_frame(avctx, pic->f, vp, rav, rvp);
542 }
543 
545 {
546  AVHWDeviceContext *hwctx = _hwctx.nc;
548 
549  /* Free frame resources, this also destroys the session parameters. */
550  ff_vk_decode_free_frame(hwctx, &hp->vp);
551 }
552 
554  .p.name = "h264_vulkan",
555  .p.type = AVMEDIA_TYPE_VIDEO,
556  .p.id = AV_CODEC_ID_H264,
557  .p.pix_fmt = AV_PIX_FMT_VULKAN,
558  .start_frame = &vk_h264_start_frame,
559  .decode_slice = &vk_h264_decode_slice,
560  .end_frame = &vk_h264_end_frame,
561  .free_frame_priv = &vk_h264_free_frame_priv,
562  .frame_priv_data_size = sizeof(H264VulkanDecodePicture),
565  .decode_params = &ff_vk_params_invalidate,
568  .frame_params = &ff_vk_frame_params,
569  .priv_data_size = sizeof(FFVulkanDecodeContext),
571 };
PICT_FRAME
#define PICT_FRAME
Definition: mpegutils.h:33
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:105
r
const char * r
Definition: vf_curves.c:127
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
H264Picture::f
AVFrame * f
Definition: h264dec.h:115
H264VulkanDecodePicture::h264_ref
StdVideoDecodeH264ReferenceInfo h264_ref
Definition: vulkan_h264.c:38
set_sps
static void set_sps(const SPS *sps, StdVideoH264ScalingLists *vksps_scaling, StdVideoH264HrdParameters *vksps_vui_header, StdVideoH264SequenceParameterSetVui *vksps_vui, StdVideoH264SequenceParameterSet *vksps)
Definition: vulkan_h264.c:142
FFVulkanDecodeContext::shared_ctx
FFVulkanDecodeShared * shared_ctx
Definition: vulkan_decode.h:60
H264VulkanDecodePicture::h264_pic_info
VkVideoDecodeH264PictureInfoKHR h264_pic_info
Definition: vulkan_h264.c:48
av_unused
#define av_unused
Definition: attributes.h:131
H264Picture::pps
const PPS * pps
Definition: h264dec.h:158
FFHWAccel::p
AVHWAccel p
The public AVHWAccel.
Definition: hwaccel_internal.h:38
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
AVFrame::width
int width
Definition: frame.h:461
level_idc
int level_idc
Definition: h264_levels.c:29
set_pps
static void set_pps(const PPS *pps, const SPS *sps, StdVideoH264ScalingLists *vkpps_scaling, StdVideoH264PictureParameterSet *vkpps)
Definition: vulkan_h264.c:250
data
const char data[16]
Definition: mxf.c:149
FFVulkanDecodeDescriptor::codec_id
enum AVCodecID codec_id
Definition: vulkan_decode.h:30
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:225
PICT_BOTTOM_FIELD
#define PICT_BOTTOM_FIELD
Definition: mpegutils.h:32
H264VulkanDecodePicture::vkh264_ref
VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref
Definition: vulkan_h264.c:39
H264VulkanDecodePicture::h264pic
StdVideoDecodeH264PictureInfo h264pic
Definition: vulkan_h264.c:47
FFRefStructOpaque
RefStruct is an API for creating reference-counted objects with minimal overhead.
Definition: refstruct.h:58
vk_h264_create_params
static int vk_h264_create_params(AVCodecContext *avctx, AVBufferRef **buf)
Definition: vulkan_h264.c:292
H264Picture::invalid_gap
int invalid_gap
Definition: h264dec.h:154
FFVulkanDecodeContext
Definition: vulkan_decode.h:59
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:141
FFVulkanDecodePicture::ref
VkVideoPictureResourceInfoKHR ref
Definition: vulkan_decode.h:93
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:61
H264Picture::frame_num
int frame_num
frame_num (raw frame_num from slice header)
Definition: h264dec.h:136
H264SliceContext
Definition: h264dec.h:180
ff_h264_vulkan_hwaccel
const FFHWAccel ff_h264_vulkan_hwaccel
Definition: vulkan_h264.c:553
FFHWAccel
Definition: hwaccel_internal.h:34
HWACCEL_CAP_THREAD_SAFE
#define HWACCEL_CAP_THREAD_SAFE
Definition: hwaccel_internal.h:32
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:312
FFVulkanDecodeShared
Definition: vulkan_decode.h:46
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:60
vk_h264_free_frame_priv
static void vk_h264_free_frame_priv(FFRefStructOpaque _hwctx, void *data)
Definition: vulkan_h264.c:544
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:84
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
vk_h264_decode_slice
static int vk_h264_decode_slice(AVCodecContext *avctx, const uint8_t *data, uint32_t size)
Definition: vulkan_h264.c:480
FFVulkanDecodePicture::refs
VkVideoPictureResourceInfoKHR refs[36]
Definition: vulkan_decode.h:97
H264VulkanDecodePicture
Definition: vulkan_h264.c:34
FFVulkanDecodePicture
Definition: vulkan_decode.h:80
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1037
PPS::pps_id
unsigned int pps_id
Definition: h264_ps.h:111
H264VulkanDecodePicture::h264_refs
StdVideoDecodeH264ReferenceInfo h264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:43
FIELD_PICTURE
#define FIELD_PICTURE(h)
Definition: h264dec.h:67
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
ctx
AVFormatContext * ctx
Definition: movenc.c:49
PICT_TOP_FIELD
#define PICT_TOP_FIELD
Definition: mpegutils.h:31
H264VulkanDecodePicture::ref_src
H264Picture * ref_src[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:42
H264SliceContext::slice_type
int slice_type
Definition: h264dec.h:186
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
HWACCEL_CAP_ASYNC_SAFE
#define HWACCEL_CAP_ASYNC_SAFE
Header providing the internals of AVHWAccel.
Definition: hwaccel_internal.h:31
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:83
NULL
#define NULL
Definition: coverity.c:32
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:543
convert_to_vk_level_idc
static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
Definition: vulkan_h264.c:116
SPS
Sequence parameter set.
Definition: h264_ps.h:44
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:486
AV_PICTURE_TYPE_SI
@ AV_PICTURE_TYPE_SI
Switching Intra.
Definition: avutil.h:283
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
PPS
Picture parameter set.
Definition: h264_ps.h:110
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1066
flush
void(* flush)(AVBSFContext *ctx)
Definition: dts2pts.c:370
MAX_PPS_COUNT
#define MAX_PPS_COUNT
Definition: h264_ps.h:38
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:974
h264_ps.h
H264Picture::pic_id
int pic_id
pic_num (short -> no wrap version of pic_num, pic_num & max_pic_num; long -> long_pic_num)
Definition: h264dec.h:139
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
ff_zigzag_scan
const uint8_t ff_zigzag_scan[16+1]
Definition: mathtables.c:109
H264Picture::reference
int reference
Definition: h264dec.h:152
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:130
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
size
int size
Definition: twinvq_data.h:10344
h264_scaling_list8_order
const static int h264_scaling_list8_order[]
Definition: vulkan_h264.c:51
FFRefStructOpaque::nc
void * nc
Definition: refstruct.h:59
ff_vk_dec_h264_desc
const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc
Definition: vulkan_h264.c:24
AVHWAccel::name
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:2105
h264dec.h
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
H264Context
H264Context.
Definition: h264dec.h:340
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
vk_h264_end_frame
static int vk_h264_end_frame(AVCodecContext *avctx)
Definition: vulkan_h264.c:505
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:57
H264SliceContext::frame_num
int frame_num
Definition: h264dec.h:328
H264VulkanDecodePicture::vp
FFVulkanDecodePicture vp
Definition: vulkan_h264.c:35
vk_h264_start_frame
static int vk_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vulkan_h264.c:360
FFVulkanDecodePicture::ref_slot
VkVideoReferenceSlotInfoKHR ref_slot
Definition: vulkan_decode.h:94
FFVulkanDecodePicture::ref_slots
VkVideoReferenceSlotInfoKHR ref_slots[36]
Definition: vulkan_decode.h:98
ff_zigzag_direct
const uint8_t ff_zigzag_direct[64]
Definition: mathtables.c:98
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:276
PPS::sps
const SPS * sps
RefStruct reference.
Definition: h264_ps.h:141
vk_h264_fill_pict
static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, VkVideoReferenceSlotInfoKHR *ref_slot, VkVideoPictureResourceInfoKHR *ref, VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, StdVideoDecodeH264ReferenceInfo *h264_ref, H264Picture *pic, int is_current, int is_field, int picture_structure, int dpb_slot_index)
Definition: vulkan_h264.c:53
H264_MAX_DPB_FRAMES
@ H264_MAX_DPB_FRAMES
Definition: h264.h:76
sps
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
Definition: cbs_h264_syntax_template.c:260
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:205
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:129
PPS::sps_id
unsigned int sps_id
Definition: h264_ps.h:112
AVCodecContext
main external API structure.
Definition: avcodec.h:451
AVFrame::height
int height
Definition: frame.h:461
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:64
H264Picture::field_poc
int field_poc[2]
top/bottom POC
Definition: h264dec.h:134
buffer
the frame and frame reference mechanism is intended to as much as expensive copies of that data while still allowing the filters to produce correct results The data is stored in buffers represented by AVFrame structures Several references can point to the same frame buffer
Definition: filter_design.txt:49
H264Picture
Definition: h264dec.h:114
update_thread_context
the pkt_dts and pkt_pts fields in AVFrame will work as usual Restrictions on codec whose streams don t reset across will not work because their bitstreams cannot be decoded in parallel *The contents of buffers must not be read before as well as code calling up to before the decode process starts Call have update_thread_context() run it in the next thread. Add AV_CODEC_CAP_FRAME_THREADS to the codec capabilities. There will be very little speed gain at this point but it should work. Use ff_thread_get_buffer()(or ff_progress_frame_get_buffer() in case you have inter-frame dependencies and use the ProgressFrame API) to allocate frame buffers. Call ff_progress_frame_report() after some part of the current picture has decoded. A good place to put this is where draw_horiz_band() is called - add this if it isn 't called anywhere
FFVulkanDecodeDescriptor
Definition: vulkan_decode.h:29
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:117
pps
uint64_t pps
Definition: dovi_rpuenc.c:35
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:120
MAX_SPS_COUNT
#define MAX_SPS_COUNT
Definition: h264_ps.h:37
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:80
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:104
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
H264Picture::hwaccel_picture_private
void * hwaccel_picture_private
RefStruct reference for hardware accelerator private data.
Definition: h264dec.h:130
vulkan_decode.h
H264SliceContext::idr_pic_id
int idr_pic_id
Definition: h264dec.h:329
AVCodecContext::priv_data
void * priv_data
Definition: avcodec.h:478
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1081
h
h
Definition: vp9dsp_template.c:2070
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:101
H264_MAX_PICTURE_COUNT
#define H264_MAX_PICTURE_COUNT
Definition: h264dec.h:49
H264VulkanDecodePicture::vkh264_refs
VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:44
H264Picture::long_ref
int long_ref
1->long term reference 0->short term reference
Definition: h264dec.h:141