FFmpeg
dec.h
Go to the documentation of this file.
1 /*
2  * VVC video decoder
3  *
4  * Copyright (C) 2021 Nuo Mi
5  * Copyright (C) 2022 Xu Mu
6  *
7  * This file is part of FFmpeg.
8  *
9  * FFmpeg is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU Lesser General Public
11  * License as published by the Free Software Foundation; either
12  * version 2.1 of the License, or (at your option) any later version.
13  *
14  * FFmpeg is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17  * Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public
20  * License along with FFmpeg; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22  */
23 
24 #ifndef AVCODEC_VVC_DEC_H
25 #define AVCODEC_VVC_DEC_H
26 
27 #include "libavcodec/videodsp.h"
28 #include "libavcodec/vvc.h"
29 
30 #include "ps.h"
31 #include "dsp.h"
32 
33 #define LUMA 0
34 #define CHROMA 1
35 #define CB 1
36 #define CR 2
37 #define JCBCR 3
38 
39 #define MIN_TU_LOG2 2 ///< MinTbLog2SizeY
40 #define MIN_PU_LOG2 2
41 
42 #define L0 0
43 #define L1 1
44 
45 typedef struct VVCRefPic {
46  struct VVCFrame *ref;
47  int poc;
48  int is_lt; // is long term reference
49 
50  // for RPR
51  int is_scaled; ///< RprConstraintsActiveFlag
52  int scale[2]; ///< RefPicScale[]
53 } VVCRefPic;
54 
55 typedef struct RefPicList {
57  int nb_refs;
58 } RefPicList;
59 
60 typedef struct RefPicListTab {
63 
64 typedef struct VVCWindow {
65  int16_t left_offset;
66  int16_t right_offset;
67  int16_t top_offset;
68  int16_t bottom_offset;
69 } VVCWindow;
70 
71 typedef struct VVCFrame {
72  struct AVFrame *frame;
73 
74  const VVCSPS *sps; ///< RefStruct reference
75  const VVCPPS *pps; ///< RefStruct reference
76  struct MvField *tab_dmvr_mvf; ///< RefStruct reference
77  RefPicListTab **rpl_tab; ///< RefStruct reference
78  RefPicListTab *rpl; ///< RefStruct reference
80 
81  int ctb_count;
82 
83  int poc;
84 
85  //for RPR
86  VVCWindow scaling_win; ///< pps_scaling_win_left_offset * SubWithC, pps_scaling_win_right_offset * SubWithC,
87  ///< pps_scaling_win_top_offset * SubHeigtC, pps_scaling_win_bottom_offset * SubHiehgtC
88  int ref_width; ///< CurrPicScalWinWidthL
89  int ref_height; ///< CurrPicScalWinHeightL
90 
92 
93  struct FrameProgress *progress; ///< RefStruct reference
94 
95  /**
96  * A sequence counter, so that old frames are output first
97  * after a POC reset
98  */
99  uint16_t sequence;
100  /**
101  * A combination of VVC_FRAME_FLAG_*
102  */
103  uint8_t flags;
104 
105  void *hwaccel_picture_private; ///< hardware accelerator private data
106 } VVCFrame;
107 
108 typedef struct SliceContext {
111  struct EntryPoint *eps;
112  int nb_eps;
114  void *ref; ///< RefStruct reference, backing slice data
115 } SliceContext;
116 
117 typedef struct VVCFrameContext {
118  void *log_ctx;
119 
120  // +1 for the current frame
122 
123  struct AVFrame *frame;
125 
127 
131 
133 
136 
138 
139  uint64_t decode_order;
140 
143 
146 
147  struct {
148  int16_t *slice_idx;
149 
151  struct SAOParams *sao;
152  struct ALFParams *alf;
153 
154  int *cb_pos_x[2]; ///< CbPosX[][][]
155  int *cb_pos_y[2]; ///< CbPosY[][][]
156  uint8_t *cb_width[2]; ///< CbWidth[][][]
157  uint8_t *cb_height[2]; ///< CbHeight[][][]
158  uint8_t *cqt_depth[2]; ///< CqtDepth[][][]
160 
161  uint8_t *skip; ///< CuSkipFlag[][]
162  uint8_t *ispmf; ///< intra_sub_partitions_mode_flag
163  uint8_t *msm[2]; ///< MttSplitMode[][][] in 32 pixels
164  uint8_t *imf; ///< IntraMipFlag[][]
165  uint8_t *imtf; ///< intra_mip_transposed_flag[][]
166  uint8_t *imm; ///< intra_mip_mode[][]
167  uint8_t *ipm; ///< IntraPredModeY[][]
168  uint8_t *cpm[2]; ///< CuPredMode[][][]
169  uint8_t *msf; ///< MergeSubblockFlag[][]
170  uint8_t *iaf; ///< InterAffineFlag[][]
171  uint8_t *mmi; ///< MotionModelIdc[][]
172  struct Mv *cp_mv[2]; ///< CpMvLX[][][][MAX_CONTROL_POINTS];
173  struct MvField *mvf; ///< MvDmvrL0, MvDmvrL1
174 
175  uint8_t *tu_coded_flag[VVC_MAX_SAMPLE_ARRAYS]; ///< tu_y_coded_flag[][], tu_cb_coded_flag[][], tu_cr_coded_flag[][]
176  uint8_t *tu_joint_cbcr_residual_flag; ///< tu_joint_cbcr_residual_flag[][]
177  uint8_t *tb_width[2];
178  uint8_t *tb_height[2];
179  uint8_t *pcmf[2];
180 
181  uint8_t *bs[2][VVC_MAX_SAMPLE_ARRAYS]; ///< horizontal, vertical boundary filtering strength
182  uint8_t *max_len_p[2]; ///< horizontal, vertical maxFilterLengthPs for luma
183  uint8_t *max_len_q[2]; ///< horizontal, vertical maxFilterLengthQs for luma
184 
189 
190  int *coeffs;
191  struct CTU *ctus;
192  struct CodingUnit **cus;
193 
194  uint8_t *ibc_vir_buf[VVC_MAX_SAMPLE_ARRAYS]; ///< IbcVirBuf[]
195 
196  //used in arrays_init only
197  struct {
199  int ctu_size;
205  int width;
206  int height;
209  int ibc_buffer_width; ///< IbcBufWidth
210  } sz;
211  } tab;
213 
214 typedef struct VVCContext {
216 
219 
221 
222  int temporal_id; ///< temporal_id_plus1 - 1
223  int poc_tid0;
224 
225  int eos; ///< current packet contains an EOS/EOB NAL
226  int last_eos; ///< last packet contains an EOS/EOB NAL
227 
229  int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
230  int gdr_recovery_point_poc; ///< recoveryPointPocVal
231 
232  /**
233  * Sequence counters for decoded and output frames, so that old
234  * frames are output first after a POC reset
235  */
236  uint16_t seq_decode;
237  uint16_t seq_output;
238 
240 
242  int nb_fcs;
243 
244  uint64_t nb_frames; ///< processed frames
245  int nb_delayed; ///< delayed frames
246 
247  enum AVPixelFormat pix_fmt; ///< pix format of current frame
248 } VVCContext ;
249 
250 #endif /* AVCODEC_VVC_DEC_H */
VVCSPS
Definition: ps.h:58
VVCContext::vcl_unit_type
enum VVCNALUnitType vcl_unit_type
Definition: dec.h:228
DBParams
Definition: hevcdec.h:347
VVCFrameContext::tb_width
uint8_t * tb_width[2]
Definition: dec.h:177
VVCWindow::right_offset
int16_t right_offset
Definition: dec.h:66
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VVCContext::avctx
struct AVCodecContext * avctx
Definition: dec.h:215
VVCFrameContext::output_frame
struct AVFrame * output_frame
Definition: dec.h:124
VVCPPS
Definition: ps.h:92
VVCFrameContext::decode_order
uint64_t decode_order
Definition: dec.h:139
VVCFrameContext::frame
struct AVFrame * frame
Definition: dec.h:123
VVCFrameContext::pic_size_in_min_cb
int pic_size_in_min_cb
Definition: dec.h:200
VVCFrameContext::tu_joint_cbcr_residual_flag
uint8_t * tu_joint_cbcr_residual_flag
tu_joint_cbcr_residual_flag[][]
Definition: dec.h:176
VVCFrame::pps
const VVCPPS * pps
RefStruct reference.
Definition: dec.h:75
VVCFrameContext::ctu_count
int ctu_count
Definition: dec.h:198
VVCFrameContext::mmi
uint8_t * mmi
MotionModelIdc[][].
Definition: dec.h:171
VVCFrameContext::cpm
uint8_t * cpm[2]
CuPredMode[][][].
Definition: dec.h:168
VVCFrameContext::ispmf
uint8_t * ispmf
intra_sub_partitions_mode_flag
Definition: dec.h:162
VVCFrameContext::alf_pixel_buffer_v
uint8_t * alf_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS][2]
Definition: dec.h:188
VVCContext::last_eos
int last_eos
last packet contains an EOS/EOB NAL
Definition: dec.h:226
VVCFrameContext::ctu_size
int ctu_size
Definition: dec.h:199
VVCContext::seq_output
uint16_t seq_output
Definition: dec.h:237
VVCContext::cbc
CodedBitstreamContext * cbc
Definition: dec.h:217
VVCFrameContext::cb_width
uint8_t * cb_width[2]
CbWidth[][][].
Definition: dec.h:156
VVCFrameContext::msm
uint8_t * msm[2]
MttSplitMode[][][] in 32 pixels.
Definition: dec.h:163
CodingUnit
Definition: hevcdec.h:288
VVCFrameContext::ipm
uint8_t * ipm
IntraPredModeY[][].
Definition: dec.h:167
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
VVCFrameContext::deblock
DBParams * deblock
Definition: dec.h:150
VVCRefPic
Definition: dec.h:45
VVCFrameContext::chroma_format_idc
int chroma_format_idc
Definition: dec.h:207
CodedBitstreamContext
Context structure for coded bitstream operations.
Definition: cbs.h:219
VVCFrameContext::cus
struct CodingUnit ** cus
Definition: dec.h:192
VVCFrame::collocated_ref
struct VVCFrame * collocated_ref
Definition: dec.h:91
DPB
Decoded Picture Buffer (DPB).
Definition: vaapi_h264.c:82
RefPicList
Definition: hevcdec.h:192
VVCRefPic::ref
struct VVCFrame * ref
Definition: dec.h:46
VVCFrameContext::slices
SliceContext ** slices
Definition: dec.h:128
VVCContext::no_output_before_recovery_flag
int no_output_before_recovery_flag
NoOutputBeforeRecoveryFlag.
Definition: dec.h:229
ps.h
VVCFrameParamSets
Definition: ps.h:229
VVCWindow::left_offset
int16_t left_offset
Definition: dec.h:65
VVCFrame::rpl
RefPicListTab * rpl
RefStruct reference.
Definition: dec.h:78
VVCFrame::ref_height
int ref_height
CurrPicScalWinHeightL.
Definition: dec.h:89
VVCNALUnitType
VVCNALUnitType
Table 5 – NAL unit type codes and NAL unit type classes in T-REC-H.266-202008.
Definition: vvc.h:28
VVCFrameContext::sz
struct VVCFrameContext::@288::@289 sz
VVCFrameContext::ctus
struct CTU * ctus
Definition: dec.h:191
SliceContext::rpl
RefPicList * rpl
Definition: dec.h:113
RefPicList::nb_refs
int nb_refs
Definition: hevcdec.h:196
VVCFrameContext::sao_pixel_buffer_h
uint8_t * sao_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS]
Definition: dec.h:185
VVCContext::seq_decode
uint16_t seq_decode
Sequence counters for decoded and output frames, so that old frames are output first after a POC rese...
Definition: dec.h:236
VVCFrameContext::ctu_height
int ctu_height
Definition: dec.h:204
VVCFrameContext::pcmf
uint8_t * pcmf[2]
Definition: dec.h:179
VVCFrame::scaling_win
VVCWindow scaling_win
pps_scaling_win_left_offset * SubWithC, pps_scaling_win_right_offset * SubWithC, pps_scaling_win_top_...
Definition: dec.h:86
VVCFrameThread
Definition: thread.c:85
VVCFrameContext::cb_pos_y
int * cb_pos_y[2]
CbPosY[][][].
Definition: dec.h:155
dsp.h
VVCFrameContext::tu_coded_flag
uint8_t * tu_coded_flag[VVC_MAX_SAMPLE_ARRAYS]
tu_y_coded_flag[][], tu_cb_coded_flag[][], tu_cr_coded_flag[][]
Definition: dec.h:175
RefPicList::refs
VVCRefPic refs[VVC_MAX_REF_ENTRIES]
Definition: dec.h:56
VVCFrame::flags
uint8_t flags
A combination of VVC_FRAME_FLAG_*.
Definition: dec.h:103
CodedBitstreamFragment
Coded bitstream fragment structure, combining one or more units.
Definition: cbs.h:122
VVCFrameContext::cp_mv
struct Mv * cp_mv[2]
CpMvLX[][][][MAX_CONTROL_POINTS];.
Definition: dec.h:172
VVCFrameContext::pixel_shift
int pixel_shift
Definition: dec.h:208
VVCFrameContext::imf
uint8_t * imf
IntraMipFlag[][].
Definition: dec.h:164
FFExecutor
Definition: executor.c:56
VVCFrameContext::cb_pos_x
int * cb_pos_x[2]
CbPosX[][][].
Definition: dec.h:154
VVCFrameContext::tab_dmvr_mvf_pool
struct FFRefStructPool * tab_dmvr_mvf_pool
Definition: dec.h:141
VVCFrameContext::alf
struct ALFParams * alf
Definition: dec.h:152
SliceContext::slice_idx
int slice_idx
Definition: dec.h:109
VVCFrameContext::imtf
uint8_t * imtf
intra_mip_transposed_flag[][]
Definition: dec.h:165
VVCFrameContext::tu_pool
struct FFRefStructPool * tu_pool
Definition: dec.h:145
VVCSH
Definition: ps.h:238
VVCFrameContext::max_len_p
uint8_t * max_len_p[2]
horizontal, vertical maxFilterLengthPs for luma
Definition: dec.h:182
VVCContext::temporal_id
int temporal_id
temporal_id_plus1 - 1
Definition: dec.h:222
VVCFrameContext::ft
struct VVCFrameThread * ft
Definition: dec.h:137
VVCWindow
Definition: dec.h:64
FFRefStructPool
FFRefStructPool is an API for a thread-safe pool of objects managed via the RefStruct API.
Definition: refstruct.c:183
VVCFrameContext::slice_idx
int16_t * slice_idx
Definition: dec.h:148
VVCFrameContext::log_ctx
void * log_ctx
Definition: dec.h:118
VVCFrameContext::vdsp
VideoDSPContext vdsp
Definition: dec.h:135
VVCContext::nb_frames
uint64_t nb_frames
processed frames
Definition: dec.h:244
CTU
Definition: ctu.h:334
VVC_MAX_SAMPLE_ARRAYS
@ VVC_MAX_SAMPLE_ARRAYS
Definition: vvc.h:77
VVCFrameContext::pic_size_in_min_pu
int pic_size_in_min_pu
Definition: dec.h:201
VVCFrameContext::vvcdsp
VVCDSPContext vvcdsp
Definition: dec.h:134
SliceContext::eps
struct EntryPoint * eps
Definition: dec.h:111
VVCFrameContext::alf_pixel_buffer_h
uint8_t * alf_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS][2]
Definition: dec.h:187
VVCFrame::hwaccel_picture_private
void * hwaccel_picture_private
hardware accelerator private data
Definition: dec.h:105
VVCContext::pix_fmt
enum AVPixelFormat pix_fmt
pix format of current frame
Definition: dec.h:247
VVCFrameContext::max_len_q
uint8_t * max_len_q[2]
horizontal, vertical maxFilterLengthQs for luma
Definition: dec.h:183
RefPicListTab
Definition: hevcdec.h:199
VVCContext::current_frame
CodedBitstreamFragment current_frame
Definition: dec.h:218
VVCRefPic::is_scaled
int is_scaled
RprConstraintsActiveFlag.
Definition: dec.h:51
VVCRefPic::is_lt
int is_lt
Definition: dec.h:48
VVCContext::eos
int eos
current packet contains an EOS/EOB NAL
Definition: dec.h:225
VVCFrameContext::nb_slices_allocated
int nb_slices_allocated
Definition: dec.h:130
VVCFrame::sequence
uint16_t sequence
A sequence counter, so that old frames are output first after a POC reset.
Definition: dec.h:99
SliceContext
Definition: mss12.h:70
VVCFrameContext::ctu_width
int ctu_width
Definition: dec.h:203
VVC_MAX_DPB_SIZE
@ VVC_MAX_DPB_SIZE
Definition: vvc.h:109
VVCFrameContext::sao_pixel_buffer_v
uint8_t * sao_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS]
Definition: dec.h:186
VVCContext::gdr_recovery_point_poc
int gdr_recovery_point_poc
recoveryPointPocVal
Definition: dec.h:230
VVCFrameContext::coeffs
int * coeffs
Definition: dec.h:190
VVCWindow::bottom_offset
int16_t bottom_offset
Definition: dec.h:68
VVCRefPic::poc
int poc
Definition: dec.h:47
SliceContext::ref
void * ref
RefStruct reference, backing slice data.
Definition: dec.h:114
MvField
Definition: hevcdec.h:306
VVCFrameContext::height
int height
Definition: dec.h:206
VVCFrameContext::tab
struct VVCFrameContext::@288 tab
VVCFrame
Definition: dec.h:71
VVCFrameContext::cu_pool
struct FFRefStructPool * cu_pool
Definition: dec.h:144
VVCFrame::tab_dmvr_mvf
struct MvField * tab_dmvr_mvf
RefStruct reference.
Definition: dec.h:76
VVCFrameContext::mvf
struct MvField * mvf
MvDmvrL0, MvDmvrL1.
Definition: dec.h:173
vvc.h
VVC_MAX_REF_ENTRIES
@ VVC_MAX_REF_ENTRIES
Definition: vvc.h:115
SliceContext::nb_eps
int nb_eps
Definition: dec.h:112
VVCFrame::nb_rpl_elems
int nb_rpl_elems
Definition: dec.h:79
VVCFrame::progress
struct FrameProgress * progress
RefStruct reference.
Definition: dec.h:93
VVCFrameContext::ibc_vir_buf
uint8_t * ibc_vir_buf[VVC_MAX_SAMPLE_ARRAYS]
IbcVirBuf[].
Definition: dec.h:194
VVCFrameContext::cqt_depth
uint8_t * cqt_depth[2]
CqtDepth[][][].
Definition: dec.h:158
VVCContext::nb_delayed
int nb_delayed
delayed frames
Definition: dec.h:245
EntryPoint
Definition: ctu.h:354
SAOParams
Definition: dsp.h:34
VVCFrameContext::cb_height
uint8_t * cb_height[2]
CbHeight[][][].
Definition: dec.h:157
VVCContext::executor
struct FFExecutor * executor
Definition: dec.h:239
VVCFrameContext::tb_height
uint8_t * tb_height[2]
Definition: dec.h:178
VVCFrameContext::width
int width
Definition: dec.h:205
VVCFrame::frame
struct AVFrame * frame
Definition: dec.h:72
VVCContext::ps
VVCParamSets ps
Definition: dec.h:220
VVCFrameContext::bs
uint8_t * bs[2][VVC_MAX_SAMPLE_ARRAYS]
horizontal, vertical boundary filtering strength
Definition: dec.h:181
AVCodecContext
main external API structure.
Definition: avcodec.h:451
VVCFrame::poc
int poc
Definition: dec.h:83
VVCFrameContext::rpl_tab_pool
struct FFRefStructPool * rpl_tab_pool
Definition: dec.h:142
FrameProgress
Definition: refs.c:34
VideoDSPContext
Definition: videodsp.h:40
VVCContext::fcs
VVCFrameContext * fcs
Definition: dec.h:241
VVCFrameContext::qp
int8_t * qp[VVC_MAX_SAMPLE_ARRAYS]
Definition: dec.h:159
ALFParams
Definition: ctu.h:461
VVCFrameContext::imm
uint8_t * imm
intra_mip_mode[][]
Definition: dec.h:166
Mv
Definition: hevcdec.h:301
VVCFrameContext::ps
VVCFrameParamSets ps
Definition: dec.h:126
VVCParamSets
Definition: ps.h:218
VVCFrameContext::pic_size_in_min_tu
int pic_size_in_min_tu
Definition: dec.h:202
SliceContext::sh
VVCSH sh
Definition: dec.h:110
VVCFrameContext::ref
VVCFrame * ref
Definition: dec.h:132
videodsp.h
VVCFrameContext
Definition: dec.h:117
VVCFrameContext::sao
struct SAOParams * sao
Definition: dec.h:151
VVCWindow::top_offset
int16_t top_offset
Definition: dec.h:67
VVCFrameContext::iaf
uint8_t * iaf
InterAffineFlag[][].
Definition: dec.h:170
VVCFrame::ctb_count
int ctb_count
Definition: dec.h:81
VVCFrame::rpl_tab
RefPicListTab ** rpl_tab
RefStruct reference.
Definition: dec.h:77
VVCFrameContext::msf
uint8_t * msf
MergeSubblockFlag[][].
Definition: dec.h:169
VVCFrame::ref_width
int ref_width
CurrPicScalWinWidthL.
Definition: dec.h:88
VVCFrameContext::nb_slices
int nb_slices
Definition: dec.h:129
VVCFrame::sps
const VVCSPS * sps
RefStruct reference.
Definition: dec.h:74
VVCContext::nb_fcs
int nb_fcs
Definition: dec.h:242
VVCRefPic::scale
int scale[2]
RefPicScale[].
Definition: dec.h:52
VVCContext
Definition: dec.h:214
VVCFrameContext::skip
uint8_t * skip
CuSkipFlag[][].
Definition: dec.h:161
VVCContext::poc_tid0
int poc_tid0
Definition: dec.h:223
VVCDSPContext
Definition: dsp.h:169
RefPicListTab::refPicList
RefPicList refPicList[2]
Definition: hevcdec.h:200
VVCFrameContext::ibc_buffer_width
int ibc_buffer_width
IbcBufWidth.
Definition: dec.h:209