FFmpeg
vvc_mp4toannexb.c
Go to the documentation of this file.
1 /*
2  * H.266/VVC MP4 to Annex B byte stream format filter
3  * Copyright (c) 2022, Thomas Siedel
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <string.h>
23 
24 #include "libavutil/intreadwrite.h"
25 #include "libavutil/mem.h"
26 
27 #include "bsf.h"
28 #include "bsf_internal.h"
29 #include "bytestream.h"
30 #include "defs.h"
31 #include "vvc.h"
32 
33 #define MIN_VVCC_LENGTH 23
34 
35 typedef struct VVCBSFContext {
36  uint8_t length_size;
39 
41 {
42  GetByteContext gb;
43  int length_size, num_arrays, i, j;
44  int ret = 0;
45  int temp = 0;
46  int ptl_present;
47 
48  uint8_t *new_extradata = NULL;
49  size_t new_extradata_size = 0;
50 
51  int max_picture_width = 0;
52  int max_picture_height = 0;
53  int avg_frame_rate = 0;
54 
55  bytestream2_init(&gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
56  temp = bytestream2_get_byte(&gb);
57  length_size = ((temp & 6) >> 1) + 1;
58  ptl_present = temp & 1;
59  if (ptl_present) {
60  int num_bytes_constraint_info;
61  int general_profile_idc;
62  int general_tier_flag;
63  int general_level_idc;
64  int ptl_frame_only_constraint_flag;
65  int ptl_multi_layer_enabled_flag;
66  int ptl_num_sub_profiles;
67  int temp3, temp4, temp5;
68  int temp2 = bytestream2_get_be16(&gb);
69  int ols_idx = (temp2 >> 7) & 0x1ff;
70  int num_sublayers = (temp2 >> 4) & 0x7;
71  int constant_frame_rate = (temp2 >> 2) & 0x3;
72  int chroma_format_idc = temp2 & 0x3;
73  int bit_depth_minus8 = (bytestream2_get_byte(&gb) >> 5) & 0x7;
75  "bit_depth_minus8 %d chroma_format_idc %d\n", bit_depth_minus8,
76  chroma_format_idc);
77  av_log(ctx, AV_LOG_DEBUG, "constant_frame_rate %d, ols_idx %d\n",
78  constant_frame_rate, ols_idx);
79  // VvcPTLRecord(num_sublayers) native_ptl
80  temp3 = bytestream2_get_byte(&gb);
81  num_bytes_constraint_info = (temp3) & 0x3f;
82  temp4 = bytestream2_get_byte(&gb);
83  general_profile_idc = (temp4 >> 1) & 0x7f;
84  general_tier_flag = (temp4) & 1;
85  general_level_idc = bytestream2_get_byte(&gb);
87  "general_profile_idc %d, general_tier_flag %d, general_level_idc %d, num_sublayers %d num_bytes_constraint_info %d\n",
88  general_profile_idc, general_tier_flag, general_level_idc,
89  num_sublayers, num_bytes_constraint_info);
90 
91  temp5 = bytestream2_get_byte(&gb);
92  ptl_frame_only_constraint_flag = (temp5 >> 7) & 0x1;
93  ptl_multi_layer_enabled_flag = (temp5 >> 6) & 0x1;
94  for (i = 0; i < num_bytes_constraint_info - 1; i++) {
95  // unsigned int(8*num_bytes_constraint_info - 2) general_constraint_info;
96  bytestream2_get_byte(&gb);
97  }
98 
100  "ptl_multi_layer_enabled_flag %d, ptl_frame_only_constraint_flag %d\n",
101  ptl_multi_layer_enabled_flag, ptl_frame_only_constraint_flag);
102 
103  if (num_sublayers > 1) {
104  int temp6 = bytestream2_get_byte(&gb);
105  uint8_t ptl_sublayer_level_present_flag[8] = { 0 };
106  //uint8_t sublayer_level_idc[8] = {0};
107  for (i = num_sublayers - 2; i >= 0; i--) {
108  ptl_sublayer_level_present_flag[i] =
109  (temp6 >> (7 - (num_sublayers - 2 - i))) & 0x01;
110  }
111  // for (j=num_sublayers; j<=8 && num_sublayers > 1; j++)
112  // bit(1) ptl_reserved_zero_bit = 0;
113  for (i = num_sublayers - 2; i >= 0; i--) {
114  if (ptl_sublayer_level_present_flag[i]) {
115  //sublayer_level_idc[i] = bytestream2_get_byte(&gb);
116  }
117  }
118  }
119 
120  ptl_num_sub_profiles = bytestream2_get_byte(&gb);
121  for (j = 0; j < ptl_num_sub_profiles; j++) {
122  // unsigned int(32) general_sub_profile_idc[j];
123  bytestream2_get_be16(&gb);
124  bytestream2_get_be16(&gb);
125  }
126 
127  max_picture_width = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_width;
128  max_picture_height = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_height;
129  avg_frame_rate = bytestream2_get_be16(&gb); // unsigned int(16) avg_frame_rate; }
131  "max_picture_width %d, max_picture_height %d, avg_frame_rate %d\n",
132  max_picture_width, max_picture_height, avg_frame_rate);
133  }
134 
135  num_arrays = bytestream2_get_byte(&gb);
136 
137  for (i = 0; i < num_arrays; i++) {
138  int cnt;
139  int type = bytestream2_get_byte(&gb) & 0x1f;
140 
141  if (type == VVC_OPI_NUT || type == VVC_DCI_NUT)
142  cnt = 1;
143  else
144  cnt = bytestream2_get_be16(&gb);
145 
146  av_log(ctx, AV_LOG_DEBUG, "nalu_type %d cnt %d\n", type, cnt);
147 
148  if (!(type == VVC_OPI_NUT || type == VVC_DCI_NUT ||
152  "Invalid NAL unit type in extradata: %d\n", type);
154  goto fail;
155  }
156 
157  for (j = 0; j < cnt; j++) {
158  const int nalu_len = bytestream2_get_be16(&gb);
159 
160  if (!nalu_len ||
161  nalu_len > bytestream2_get_bytes_left(&gb) ||
162  4 + AV_INPUT_BUFFER_PADDING_SIZE + nalu_len > SIZE_MAX - new_extradata_size) {
164  goto fail;
165  }
166  ret = av_reallocp(&new_extradata, new_extradata_size + nalu_len + 4
168  if (ret < 0)
169  goto fail;
170 
171  AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
172  bytestream2_get_buffer(&gb, new_extradata + new_extradata_size + 4,
173  nalu_len);
174  new_extradata_size += 4 + nalu_len;
175  memset(new_extradata + new_extradata_size, 0,
177  }
178  }
179 
180  av_freep(&ctx->par_out->extradata);
181  ctx->par_out->extradata = new_extradata;
182  ctx->par_out->extradata_size = new_extradata_size;
183 
184  if (!new_extradata_size)
185  av_log(ctx, AV_LOG_WARNING, "No parameter sets in the extradata\n");
186 
187  return length_size;
188  fail:
189  av_freep(&new_extradata);
190  return ret;
191 }
192 
194 {
196  int ret;
197 
198  if (ctx->par_in->extradata_size < MIN_VVCC_LENGTH ||
199  AV_RB24(ctx->par_in->extradata) == 1 ||
200  AV_RB32(ctx->par_in->extradata) == 1) {
202  "The input looks like it is Annex B already\n");
203  } else {
205  if (ret < 0)
206  return ret;
207  s->length_size = ret;
208  s->extradata_parsed = 1;
209  }
210 
211  return 0;
212 }
213 
215 {
217  AVPacket *in;
218  GetByteContext gb;
219 
220  int is_irap = 0;
221  int added_extra = 0;
222  int i, ret = 0;
223 
224  ret = ff_bsf_get_packet(ctx, &in);
225  if (ret < 0)
226  return ret;
227 
228  if (!s->extradata_parsed) {
229  av_packet_move_ref(out, in);
230  av_packet_free(&in);
231  return 0;
232  }
233 
234  bytestream2_init(&gb, in->data, in->size);
235 
236  /* check if this packet contains an IRAP. The extradata will need to be added before any potential PH_NUT */
237  while (bytestream2_get_bytes_left(&gb)) {
238  uint32_t nalu_size = 0;
239  int nalu_type;
240 
241  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
243  goto fail;
244  }
245 
246  for (i = 0; i < s->length_size; i++)
247  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
248 
249  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
251  goto fail;
252  }
253 
254  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
255  is_irap = nalu_type >= VVC_IDR_W_RADL && nalu_type <= VVC_RSV_IRAP_11;
256  if (is_irap) {
257  break;
258  }
259  bytestream2_seek(&gb, nalu_size, SEEK_CUR);
260  }
261 
262  bytestream2_seek(&gb, 0, SEEK_SET);
263  while (bytestream2_get_bytes_left(&gb)) {
264  uint32_t nalu_size = 0;
265  int nalu_type;
266  int add_extradata, extra_size, prev_size;
267 
268  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
270  goto fail;
271  }
272 
273  for (i = 0; i < s->length_size; i++)
274  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
275 
276  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
278  goto fail;
279  }
280 
281  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
282 
283  /* prepend extradata to IRAP frames */
284  add_extradata = is_irap && nalu_type != VVC_AUD_NUT && !added_extra;
285  extra_size = add_extradata * ctx->par_out->extradata_size;
286  added_extra |= add_extradata;
287 
288  if (FFMIN(INT_MAX, SIZE_MAX) < 4ULL + nalu_size + extra_size) {
290  goto fail;
291  }
292 
293  prev_size = out->size;
294 
295  ret = av_grow_packet(out, 4 + nalu_size + extra_size);
296  if (ret < 0)
297  goto fail;
298 
299  if (extra_size)
300  memcpy(out->data + prev_size, ctx->par_out->extradata, extra_size);
301  AV_WB32(out->data + prev_size + extra_size, 1);
302  bytestream2_get_buffer(&gb, out->data + prev_size + 4 + extra_size,
303  nalu_size);
304  }
305 
307  if (ret < 0)
308  goto fail;
309 
310  fail:
311  if (ret < 0)
313  av_packet_free(&in);
314 
315  return ret;
316 }
317 
318 static const enum AVCodecID codec_ids[] = {
320 };
321 
323  .p.name = "vvc_mp4toannexb",
324  .p.codec_ids = codec_ids,
325  .priv_data_size = sizeof(VVCBSFContext),
328 };
av_packet_unref
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: packet.c:429
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
bsf_internal.h
out
FILE * out
Definition: movenc.c:55
GetByteContext
Definition: bytestream.h:33
VVC_DCI_NUT
@ VVC_DCI_NUT
Definition: vvc.h:42
AVBitStreamFilter::name
const char * name
Definition: bsf.h:112
av_grow_packet
int av_grow_packet(AVPacket *pkt, int grow_by)
Increase packet size, correctly zeroing padding.
Definition: packet.c:121
ff_vvc_mp4toannexb_bsf
const FFBitStreamFilter ff_vvc_mp4toannexb_bsf
Definition: vvc_mp4toannexb.c:322
bytestream2_seek
static av_always_inline int bytestream2_seek(GetByteContext *g, int offset, int whence)
Definition: bytestream.h:212
AVPacket::data
uint8_t * data
Definition: packet.h:539
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:225
filter
void(* filter)(uint8_t *src, int stride, int qscale)
Definition: h263dsp.c:29
ff_bsf_get_packet
int ff_bsf_get_packet(AVBSFContext *ctx, AVPacket **pkt)
Called by the bitstream filters to get the next packet for filtering.
Definition: bsf.c:235
VVC_AUD_NUT
@ VVC_AUD_NUT
Definition: vvc.h:49
av_packet_free
void av_packet_free(AVPacket **pkt)
Free the packet, if the packet is reference counted, it will be unreferenced first.
Definition: packet.c:74
AVBSFContext
The bitstream filter state.
Definition: bsf.h:68
bsf.h
fail
#define fail()
Definition: checkasm.h:193
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
VVC_IDR_W_RADL
@ VVC_IDR_W_RADL
Definition: vvc.h:36
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
intreadwrite.h
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
ctx
AVFormatContext * ctx
Definition: movenc.c:49
vvc_mp4toannexb_filter
static int vvc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
Definition: vvc_mp4toannexb.c:214
NULL
#define NULL
Definition: coverity.c:32
FFBitStreamFilter
Definition: bsf_internal.h:27
VVC_PREFIX_SEI_NUT
@ VVC_PREFIX_SEI_NUT
Definition: vvc.h:52
bytestream2_get_buffer
static av_always_inline unsigned int bytestream2_get_buffer(GetByteContext *g, uint8_t *dst, unsigned int size)
Definition: bytestream.h:267
vvc_extradata_to_annexb
static int vvc_extradata_to_annexb(AVBSFContext *ctx)
Definition: vvc_mp4toannexb.c:40
av_packet_move_ref
void av_packet_move_ref(AVPacket *dst, AVPacket *src)
Move every field in src to dst and reset src.
Definition: packet.c:486
bytestream2_get_bytes_left
static av_always_inline int bytestream2_get_bytes_left(GetByteContext *g)
Definition: bytestream.h:158
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
VVCBSFContext::length_size
uint8_t length_size
Definition: vvc_mp4toannexb.c:36
AV_WB32
#define AV_WB32(p, v)
Definition: intreadwrite.h:415
FFBitStreamFilter::p
AVBitStreamFilter p
The public AVBitStreamFilter.
Definition: bsf_internal.h:31
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
AVPacket::size
int size
Definition: packet.h:540
VVC_VPS_NUT
@ VVC_VPS_NUT
Definition: vvc.h:43
av_reallocp
int av_reallocp(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory through a pointer to a pointer.
Definition: mem.c:188
AV_RB32
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_RB32
Definition: bytestream.h:96
AV_CODEC_ID_VVC
@ AV_CODEC_ID_VVC
Definition: codec_id.h:252
VVC_SPS_NUT
@ VVC_SPS_NUT
Definition: vvc.h:44
av_packet_copy_props
int av_packet_copy_props(AVPacket *dst, const AVPacket *src)
Copy only "properties" fields from src to dst.
Definition: packet.c:392
AV_CODEC_ID_NONE
@ AV_CODEC_ID_NONE
Definition: codec_id.h:50
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
VVC_SUFFIX_SEI_NUT
@ VVC_SUFFIX_SEI_NUT
Definition: vvc.h:53
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
ret
ret
Definition: filter_design.txt:187
VVC_RSV_IRAP_11
@ VVC_RSV_IRAP_11
Definition: vvc.h:40
MIN_VVCC_LENGTH
#define MIN_VVCC_LENGTH
Definition: vvc_mp4toannexb.c:33
VVC_PPS_NUT
@ VVC_PPS_NUT
Definition: vvc.h:45
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
codec_ids
static enum AVCodecID codec_ids[]
Definition: vvc_mp4toannexb.c:318
defs.h
vvc_mp4toannexb_init
static int vvc_mp4toannexb_init(AVBSFContext *ctx)
Definition: vvc_mp4toannexb.c:193
temp
else temp
Definition: vf_mcdeint.c:263
VVC_OPI_NUT
@ VVC_OPI_NUT
Definition: vvc.h:41
mem.h
AVPacket
This structure stores compressed data.
Definition: packet.h:516
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
VVCBSFContext::extradata_parsed
int extradata_parsed
Definition: vvc_mp4toannexb.c:37
bytestream.h
bytestream2_init
static av_always_inline void bytestream2_init(GetByteContext *g, const uint8_t *buf, int buf_size)
Definition: bytestream.h:137
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:61
AV_RB24
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_WB32 unsigned int_TMPL AV_RB24
Definition: bytestream.h:97
VVCBSFContext
Definition: vvc_mp4toannexb.c:35
AVFormatContext::priv_data
void * priv_data
Format private data.
Definition: avformat.h:1328