FFmpeg
vvc_mp4toannexb.c
Go to the documentation of this file.
1 /*
2  * H.266/VVC MP4 to Annex B byte stream format filter
3  * Copyright (c) 2022, Thomas Siedel
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <string.h>
23 
24 #include "libavutil/intreadwrite.h"
25 #include "libavutil/mem.h"
26 
27 #include "bsf.h"
28 #include "bsf_internal.h"
29 #include "bytestream.h"
30 #include "defs.h"
31 #include "vvc.h"
32 
33 #define MIN_VVCC_LENGTH 23
34 
35 typedef struct VVCBSFContext {
36  uint8_t length_size;
39 
41 {
42  GetByteContext gb;
43  int length_size, num_arrays, i, j;
44  int ret = 0;
45  int temp = 0;
46  int ptl_present;
47 
48  uint8_t *new_extradata = NULL;
49  size_t new_extradata_size = 0;
50 
51  int max_picture_width = 0;
52  int max_picture_height = 0;
53  int avg_frame_rate = 0;
54 
55  bytestream2_init(&gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
56  temp = bytestream2_get_byte(&gb);
57  length_size = ((temp & 6) >> 1) + 1;
58  ptl_present = temp & 1;
59  if (ptl_present) {
60  int num_bytes_constraint_info;
61  int general_profile_idc;
62  int general_tier_flag;
63  int general_level_idc;
64  int ptl_frame_only_constraint_flag;
65  int ptl_multi_layer_enabled_flag;
66  int ptl_num_sub_profiles;
67  int temp3, temp4, temp5;
68  int temp2 = bytestream2_get_be16(&gb);
69  int ols_idx = (temp2 >> 7) & 0x1ff;
70  int num_sublayers = (temp2 >> 4) & 0x7;
71  int constant_frame_rate = (temp2 >> 2) & 0x3;
72  int chroma_format_idc = temp2 & 0x3;
73  int bit_depth_minus8 = (bytestream2_get_byte(&gb) >> 5) & 0x7;
75  "bit_depth_minus8 %d chroma_format_idc %d\n", bit_depth_minus8,
76  chroma_format_idc);
77  av_log(ctx, AV_LOG_DEBUG, "constant_frame_rate %d, ols_idx %d\n",
78  constant_frame_rate, ols_idx);
79  // VvcPTLRecord(num_sublayers) native_ptl
80  temp3 = bytestream2_get_byte(&gb);
81  num_bytes_constraint_info = (temp3) & 0x3f;
82  temp4 = bytestream2_get_byte(&gb);
83  general_profile_idc = (temp4 >> 1) & 0x7f;
84  general_tier_flag = (temp4) & 1;
85  general_level_idc = bytestream2_get_byte(&gb);
87  "general_profile_idc %d, general_tier_flag %d, general_level_idc %d, num_sublayers %d num_bytes_constraint_info %d\n",
88  general_profile_idc, general_tier_flag, general_level_idc,
89  num_sublayers, num_bytes_constraint_info);
90 
91  temp5 = bytestream2_get_byte(&gb);
92  ptl_frame_only_constraint_flag = (temp5 >> 7) & 0x1;
93  ptl_multi_layer_enabled_flag = (temp5 >> 6) & 0x1;
94  for (i = 0; i < num_bytes_constraint_info - 1; i++) {
95  // unsigned int(8*num_bytes_constraint_info - 2) general_constraint_info;
96  bytestream2_get_byte(&gb);
97  }
98 
100  "ptl_multi_layer_enabled_flag %d, ptl_frame_only_constraint_flag %d\n",
101  ptl_multi_layer_enabled_flag, ptl_frame_only_constraint_flag);
102 
103  if (num_sublayers > 1) {
104  int temp6 = bytestream2_get_byte(&gb);
105  uint8_t ptl_sublayer_level_present_flag[8] = { 0 };
106  //uint8_t sublayer_level_idc[8] = {0};
107  for (i = num_sublayers - 2; i >= 0; i--) {
108  ptl_sublayer_level_present_flag[i] =
109  (temp6 >> (7 - (num_sublayers - 2 - i))) & 0x01;
110  }
111  // for (j=num_sublayers; j<=8 && num_sublayers > 1; j++)
112  // bit(1) ptl_reserved_zero_bit = 0;
113  for (i = num_sublayers - 2; i >= 0; i--) {
114  if (ptl_sublayer_level_present_flag[i]) {
115  //sublayer_level_idc[i] = bytestream2_get_byte(&gb);
116  }
117  }
118  }
119 
120  ptl_num_sub_profiles = bytestream2_get_byte(&gb);
121  for (j = 0; j < ptl_num_sub_profiles; j++) {
122  // unsigned int(32) general_sub_profile_idc[j];
123  bytestream2_get_be16(&gb);
124  bytestream2_get_be16(&gb);
125  }
126 
127  max_picture_width = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_width;
128  max_picture_height = bytestream2_get_be16(&gb); // unsigned_int(16) max_picture_height;
129  avg_frame_rate = bytestream2_get_be16(&gb); // unsigned int(16) avg_frame_rate; }
131  "max_picture_width %d, max_picture_height %d, avg_frame_rate %d\n",
132  max_picture_width, max_picture_height, avg_frame_rate);
133  }
134 
135  num_arrays = bytestream2_get_byte(&gb);
136 
137  for (i = 0; i < num_arrays; i++) {
138  int cnt;
139  int type = bytestream2_get_byte(&gb) & 0x1f;
140 
141  if (type == VVC_OPI_NUT || type == VVC_DCI_NUT)
142  cnt = 1;
143  else
144  cnt = bytestream2_get_be16(&gb);
145 
146  av_log(ctx, AV_LOG_DEBUG, "nalu_type %d cnt %d\n", type, cnt);
147 
148  if (!(type == VVC_OPI_NUT || type == VVC_DCI_NUT ||
152  "Invalid NAL unit type in extradata: %d\n", type);
154  goto fail;
155  }
156 
157  for (j = 0; j < cnt; j++) {
158  const int nalu_len = bytestream2_get_be16(&gb);
159 
160  if (!nalu_len ||
161  nalu_len > bytestream2_get_bytes_left(&gb) ||
162  4 + AV_INPUT_BUFFER_PADDING_SIZE + nalu_len > SIZE_MAX - new_extradata_size) {
164  goto fail;
165  }
166  ret = av_reallocp(&new_extradata, new_extradata_size + nalu_len + 4
168  if (ret < 0)
169  goto fail;
170 
171  AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
172  bytestream2_get_buffer(&gb, new_extradata + new_extradata_size + 4,
173  nalu_len);
174  new_extradata_size += 4 + nalu_len;
175  memset(new_extradata + new_extradata_size, 0,
177  }
178  }
179 
180  av_freep(&ctx->par_out->extradata);
181  ctx->par_out->extradata = new_extradata;
182  ctx->par_out->extradata_size = new_extradata_size;
183 
184  if (!new_extradata_size)
185  av_log(ctx, AV_LOG_WARNING, "No parameter sets in the extradata\n");
186 
187  return length_size;
188  fail:
189  av_freep(&new_extradata);
190  return ret;
191 }
192 
194 {
196  int ret;
197 
198  if (ctx->par_in->extradata_size < MIN_VVCC_LENGTH ||
199  AV_RB24(ctx->par_in->extradata) == 1 ||
200  AV_RB32(ctx->par_in->extradata) == 1) {
202  "The input looks like it is Annex B already\n");
203  } else {
205  if (ret < 0)
206  return ret;
207  s->length_size = ret;
208  s->extradata_parsed = 1;
209  }
210 
211  return 0;
212 }
213 
215 {
217  AVPacket *in;
218  GetByteContext gb;
219 
220  int is_irap = 0;
221  int added_extra = 0;
222  int i, ret = 0;
223 
224  ret = ff_bsf_get_packet(ctx, &in);
225  if (ret < 0)
226  return ret;
227 
228  if (!s->extradata_parsed) {
229  av_packet_move_ref(out, in);
230  av_packet_free(&in);
231  return 0;
232  }
233 
234  bytestream2_init(&gb, in->data, in->size);
235 
236  /* check if this packet contains an IRAP. The extradata will need to be added before any potential PH_NUT */
237  while (bytestream2_get_bytes_left(&gb)) {
238  uint32_t nalu_size = 0;
239  int nalu_type;
240 
241  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
243  goto fail;
244  }
245 
246  for (i = 0; i < s->length_size; i++)
247  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
248 
249  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
251  goto fail;
252  }
253 
254  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
255  is_irap = nalu_type >= VVC_IDR_W_RADL && nalu_type <= VVC_RSV_IRAP_11;
256  if (is_irap) {
257  break;
258  }
259  bytestream2_seek(&gb, nalu_size, SEEK_CUR);
260  }
261 
262  bytestream2_seek(&gb, 0, SEEK_SET);
263  while (bytestream2_get_bytes_left(&gb)) {
264  uint32_t nalu_size = 0;
265  int nalu_type;
266  int add_extradata, extra_size, prev_size;
267 
268  if (bytestream2_get_bytes_left(&gb) < s->length_size) {
270  goto fail;
271  }
272 
273  for (i = 0; i < s->length_size; i++)
274  nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb);
275 
276  if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) {
278  goto fail;
279  }
280 
281  nalu_type = (bytestream2_peek_be16(&gb) >> 3) & 0x1f;
282 
283  /* prepend extradata to IRAP frames */
284  add_extradata = is_irap && nalu_type != VVC_AUD_NUT && !added_extra;
285  extra_size = add_extradata * ctx->par_out->extradata_size;
286  added_extra |= add_extradata;
287 
288  if (FFMIN(INT_MAX, SIZE_MAX) < 4ULL + nalu_size + extra_size) {
290  goto fail;
291  }
292 
293  prev_size = out->size;
294 
295  ret = av_grow_packet(out, 4 + nalu_size + extra_size);
296  if (ret < 0)
297  goto fail;
298 
299  if (extra_size)
300  memcpy(out->data + prev_size, ctx->par_out->extradata, extra_size);
301  AV_WB32(out->data + prev_size + extra_size, 1);
302  bytestream2_get_buffer(&gb, out->data + prev_size + 4 + extra_size,
303  nalu_size);
304  }
305 
307  if (ret < 0)
308  goto fail;
309 
310  fail:
311  if (ret < 0)
313  av_packet_free(&in);
314 
315  return ret;
316 }
317 
318 static const enum AVCodecID codec_ids[] = {
320 };
321 
323  .p.name = "vvc_mp4toannexb",
324  .p.codec_ids = codec_ids,
325  .priv_data_size = sizeof(VVCBSFContext),
328 };
av_packet_unref
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:427
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
bsf_internal.h
out
FILE * out
Definition: movenc.c:54
GetByteContext
Definition: bytestream.h:33
VVC_DCI_NUT
@ VVC_DCI_NUT
Definition: vvc.h:42
AVBitStreamFilter::name
const char * name
Definition: bsf.h:112
av_grow_packet
int av_grow_packet(AVPacket *pkt, int grow_by)
Increase packet size, correctly zeroing padding.
Definition: avpacket.c:121
ff_vvc_mp4toannexb_bsf
const FFBitStreamFilter ff_vvc_mp4toannexb_bsf
Definition: vvc_mp4toannexb.c:322
bytestream2_seek
static av_always_inline int bytestream2_seek(GetByteContext *g, int offset, int whence)
Definition: bytestream.h:212
AVPacket::data
uint8_t * data
Definition: packet.h:522
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
ff_bsf_get_packet
int ff_bsf_get_packet(AVBSFContext *ctx, AVPacket **pkt)
Called by the bitstream filters to get the next packet for filtering.
Definition: bsf.c:235
VVC_AUD_NUT
@ VVC_AUD_NUT
Definition: vvc.h:49
av_packet_free
void av_packet_free(AVPacket **pkt)
Free the packet, if the packet is reference counted, it will be unreferenced first.
Definition: avpacket.c:74
AVBSFContext
The bitstream filter state.
Definition: bsf.h:68
bsf.h
fail
#define fail()
Definition: checkasm.h:179
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
VVC_IDR_W_RADL
@ VVC_IDR_W_RADL
Definition: vvc.h:36
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
intreadwrite.h
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
vvc_mp4toannexb_filter
static int vvc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
Definition: vvc_mp4toannexb.c:214
NULL
#define NULL
Definition: coverity.c:32
FFBitStreamFilter
Definition: bsf_internal.h:27
VVC_PREFIX_SEI_NUT
@ VVC_PREFIX_SEI_NUT
Definition: vvc.h:52
bytestream2_get_buffer
static av_always_inline unsigned int bytestream2_get_buffer(GetByteContext *g, uint8_t *dst, unsigned int size)
Definition: bytestream.h:267
vvc_extradata_to_annexb
static int vvc_extradata_to_annexb(AVBSFContext *ctx)
Definition: vvc_mp4toannexb.c:40
av_packet_move_ref
void av_packet_move_ref(AVPacket *dst, AVPacket *src)
Move every field in src to dst and reset src.
Definition: avpacket.c:484
bytestream2_get_bytes_left
static av_always_inline int bytestream2_get_bytes_left(GetByteContext *g)
Definition: bytestream.h:158
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
VVCBSFContext::length_size
uint8_t length_size
Definition: vvc_mp4toannexb.c:36
AV_WB32
#define AV_WB32(p, v)
Definition: intreadwrite.h:417
FFBitStreamFilter::p
AVBitStreamFilter p
The public AVBitStreamFilter.
Definition: bsf_internal.h:31
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:365
AVPacket::size
int size
Definition: packet.h:523
VVC_VPS_NUT
@ VVC_VPS_NUT
Definition: vvc.h:43
av_reallocp
int av_reallocp(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory through a pointer to a pointer.
Definition: mem.c:186
AV_RB32
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_RB32
Definition: bytestream.h:96
AV_CODEC_ID_VVC
@ AV_CODEC_ID_VVC
Definition: codec_id.h:250
VVC_SPS_NUT
@ VVC_SPS_NUT
Definition: vvc.h:44
av_packet_copy_props
int av_packet_copy_props(AVPacket *dst, const AVPacket *src)
Copy only "properties" fields from src to dst.
Definition: avpacket.c:390
AV_CODEC_ID_NONE
@ AV_CODEC_ID_NONE
Definition: codec_id.h:50
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:255
VVC_SUFFIX_SEI_NUT
@ VVC_SUFFIX_SEI_NUT
Definition: vvc.h:53
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
ret
ret
Definition: filter_design.txt:187
VVC_RSV_IRAP_11
@ VVC_RSV_IRAP_11
Definition: vvc.h:40
MIN_VVCC_LENGTH
#define MIN_VVCC_LENGTH
Definition: vvc_mp4toannexb.c:33
VVC_PPS_NUT
@ VVC_PPS_NUT
Definition: vvc.h:45
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
codec_ids
static enum AVCodecID codec_ids[]
Definition: vvc_mp4toannexb.c:318
defs.h
vvc_mp4toannexb_init
static int vvc_mp4toannexb_init(AVBSFContext *ctx)
Definition: vvc_mp4toannexb.c:193
temp
else temp
Definition: vf_mcdeint.c:263
VVC_OPI_NUT
@ VVC_OPI_NUT
Definition: vvc.h:41
mem.h
AVPacket
This structure stores compressed data.
Definition: packet.h:499
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
VVCBSFContext::extradata_parsed
int extradata_parsed
Definition: vvc_mp4toannexb.c:37
bytestream.h
bytestream2_init
static av_always_inline void bytestream2_init(GetByteContext *g, const uint8_t *buf, int buf_size)
Definition: bytestream.h:137
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:61
AV_RB24
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_WB32 unsigned int_TMPL AV_RB24
Definition: bytestream.h:97
VVCBSFContext
Definition: vvc_mp4toannexb.c:35
AVFormatContext::priv_data
void * priv_data
Format private data.
Definition: avformat.h:1283