FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
qsvdec.c
Go to the documentation of this file.
1 /*
2  * Intel MediaSDK QSV codec-independent code
3  *
4  * copyright (c) 2013 Luca Barbato
5  * copyright (c) 2015 Anton Khirnov <anton@khirnov.net>
6  *
7  * This file is part of FFmpeg.
8  *
9  * FFmpeg is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU Lesser General Public
11  * License as published by the Free Software Foundation; either
12  * version 2.1 of the License, or (at your option) any later version.
13  *
14  * FFmpeg is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17  * Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public
20  * License along with FFmpeg; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22  */
23 
24 #include <string.h>
25 #include <sys/types.h>
26 
27 #include <mfx/mfxvideo.h>
28 
29 #include "libavutil/common.h"
30 #include "libavutil/mem.h"
31 #include "libavutil/log.h"
32 #include "libavutil/pixfmt.h"
33 #include "libavutil/time.h"
34 
35 #include "avcodec.h"
36 #include "internal.h"
37 #include "qsv.h"
38 #include "qsv_internal.h"
39 #include "qsvdec.h"
40 
42 {
43  switch (format) {
44  case AV_PIX_FMT_YUV420P:
46  return AV_PIX_FMT_NV12;
47  default:
48  return AVERROR(ENOSYS);
49  }
50 }
51 
53 {
54  mfxVideoParam param = { { 0 } };
55  mfxBitstream bs = { { { 0 } } };
56  int ret;
60 
61  q->iopattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
62  if (!q->session) {
63  if (avctx->hwaccel_context) {
64  AVQSVContext *qsv = avctx->hwaccel_context;
65 
66  q->session = qsv->session;
67  q->iopattern = qsv->iopattern;
68  q->ext_buffers = qsv->ext_buffers;
70  }
71  if (!q->session) {
73  q->load_plugins);
74  if (ret < 0)
75  return ret;
76 
77  q->session = q->internal_qs.session;
78  }
79  }
80 
81  if (avpkt->size) {
82  bs.Data = avpkt->data;
83  bs.DataLength = avpkt->size;
84  bs.MaxLength = bs.DataLength;
85  bs.TimeStamp = avpkt->pts;
86  } else
87  return AVERROR_INVALIDDATA;
88 
89  ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
90  if (ret < 0) {
91  av_log(avctx, AV_LOG_ERROR, "Unsupported codec_id %08x\n", avctx->codec_id);
92  return ret;
93  }
94 
95  param.mfx.CodecId = ret;
96 
97  ret = MFXVideoDECODE_DecodeHeader(q->session, &bs, &param);
98  if (MFX_ERR_MORE_DATA==ret) {
99  /* this code means that header not found so we return packet size to skip
100  a current packet
101  */
102  return avpkt->size;
103  } else if (ret < 0) {
104  av_log(avctx, AV_LOG_ERROR, "Decode header error %d\n", ret);
105  return ff_qsv_error(ret);
106  }
107  param.IOPattern = q->iopattern;
108  param.AsyncDepth = q->async_depth;
109  param.ExtParam = q->ext_buffers;
110  param.NumExtParam = q->nb_ext_buffers;
111  param.mfx.FrameInfo.BitDepthLuma = 8;
112  param.mfx.FrameInfo.BitDepthChroma = 8;
113 
114  ret = MFXVideoDECODE_Init(q->session, &param);
115  if (ret < 0) {
116  if (MFX_ERR_INVALID_VIDEO_PARAM==ret) {
117  av_log(avctx, AV_LOG_ERROR,
118  "Error initializing the MFX video decoder, unsupported video\n");
119  } else {
120  av_log(avctx, AV_LOG_ERROR,
121  "Error initializing the MFX video decoder %d\n", ret);
122  }
123  return ff_qsv_error(ret);
124  }
125 
126  ret = ff_get_format(avctx, pix_fmts);
127  if (ret < 0)
128  return ret;
129 
130  avctx->pix_fmt = ret;
131  avctx->profile = param.mfx.CodecProfile;
132  avctx->level = param.mfx.CodecLevel;
133  avctx->coded_width = param.mfx.FrameInfo.Width;
134  avctx->coded_height = param.mfx.FrameInfo.Height;
135  avctx->width = param.mfx.FrameInfo.CropW - param.mfx.FrameInfo.CropX;
136  avctx->height = param.mfx.FrameInfo.CropH - param.mfx.FrameInfo.CropY;
137 
138  /* maximum decoder latency should be not exceed max DPB size for h.264 and
139  HEVC which is 16 for both cases.
140  So weare pre-allocating fifo big enough for 17 elements:
141  */
142  if (!q->async_fifo) {
143  q->async_fifo = av_fifo_alloc((1 + 16) *
144  (sizeof(mfxSyncPoint) + sizeof(QSVFrame*)));
145  if (!q->async_fifo)
146  return AVERROR(ENOMEM);
147  }
148 
149  q->input_fifo = av_fifo_alloc(1024*16);
150  if (!q->input_fifo)
151  return AVERROR(ENOMEM);
152 
153  q->engine_ready = 1;
154 
155  return 0;
156 }
157 
159 {
160  int ret;
161 
162  ret = ff_get_buffer(avctx, frame->frame, AV_GET_BUFFER_FLAG_REF);
163  if (ret < 0)
164  return ret;
165 
166  if (frame->frame->format == AV_PIX_FMT_QSV) {
167  frame->surface = (mfxFrameSurface1*)frame->frame->data[3];
168  } else {
169  frame->surface_internal.Info.BitDepthLuma = 8;
170  frame->surface_internal.Info.BitDepthChroma = 8;
171  frame->surface_internal.Info.FourCC = MFX_FOURCC_NV12;
172  frame->surface_internal.Info.Width = avctx->coded_width;
173  frame->surface_internal.Info.Height = avctx->coded_height;
174  frame->surface_internal.Info.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
175 
176  frame->surface_internal.Data.PitchLow = frame->frame->linesize[0];
177  frame->surface_internal.Data.Y = frame->frame->data[0];
178  frame->surface_internal.Data.UV = frame->frame->data[1];
179 
180  frame->surface = &frame->surface_internal;
181  }
182 
183  return 0;
184 }
185 
187 {
188  QSVFrame *cur = q->work_frames;
189  while (cur) {
190  if (cur->surface && !cur->surface->Data.Locked && !cur->queued) {
191  cur->surface = NULL;
192  av_frame_unref(cur->frame);
193  }
194  cur = cur->next;
195  }
196 }
197 
198 static int get_surface(AVCodecContext *avctx, QSVContext *q, mfxFrameSurface1 **surf)
199 {
200  QSVFrame *frame, **last;
201  int ret;
202 
204 
205  frame = q->work_frames;
206  last = &q->work_frames;
207  while (frame) {
208  if (!frame->surface) {
209  ret = alloc_frame(avctx, frame);
210  if (ret < 0)
211  return ret;
212  *surf = frame->surface;
213  return 0;
214  }
215 
216  last = &frame->next;
217  frame = frame->next;
218  }
219 
220  frame = av_mallocz(sizeof(*frame));
221  if (!frame)
222  return AVERROR(ENOMEM);
223  frame->frame = av_frame_alloc();
224  if (!frame->frame) {
225  av_freep(&frame);
226  return AVERROR(ENOMEM);
227  }
228  *last = frame;
229 
230  ret = alloc_frame(avctx, frame);
231  if (ret < 0)
232  return ret;
233 
234  *surf = frame->surface;
235 
236  return 0;
237 }
238 
239 static QSVFrame *find_frame(QSVContext *q, mfxFrameSurface1 *surf)
240 {
241  QSVFrame *cur = q->work_frames;
242  while (cur) {
243  if (surf == cur->surface)
244  return cur;
245  cur = cur->next;
246  }
247  return NULL;
248 }
249 
250 /* This function uses for 'smart' releasing of consumed data
251  from the input bitstream fifo.
252  Since the input fifo mapped to mfxBitstream which does not understand
253  a wrapping of data over fifo end, we should also to relocate a possible
254  data rest to fifo begin. If rest of data is absent then we just reset fifo's
255  pointers to initial positions.
256  NOTE the case when fifo does contain unconsumed data is rare and typical
257  amount of such data is 1..4 bytes.
258 */
259 static void qsv_fifo_relocate(AVFifoBuffer *f, int bytes_to_free)
260 {
261  int data_size;
262  int data_rest = 0;
263 
264  av_fifo_drain(f, bytes_to_free);
265 
266  data_size = av_fifo_size(f);
267  if (data_size > 0) {
268  if (f->buffer!=f->rptr) {
269  if ( (f->end - f->rptr) < data_size) {
270  data_rest = data_size - (f->end - f->rptr);
271  data_size-=data_rest;
272  memmove(f->buffer+data_size, f->buffer, data_rest);
273  }
274  memmove(f->buffer, f->rptr, data_size);
275  data_size+= data_rest;
276  }
277  }
278  f->rptr = f->buffer;
279  f->wptr = f->buffer + data_size;
280  f->wndx = data_size;
281  f->rndx = 0;
282 }
283 
285  AVFrame *frame, int *got_frame,
286  AVPacket *avpkt)
287 {
288  QSVFrame *out_frame;
289  mfxFrameSurface1 *insurf;
290  mfxFrameSurface1 *outsurf;
291  mfxSyncPoint sync;
292  mfxBitstream bs = { { { 0 } } };
293  int ret;
294  int n_out_frames;
295  int buffered = 0;
296 
297  if (!q->engine_ready) {
298  ret = ff_qsv_decode_init(avctx, q, avpkt);
299  if (ret)
300  return ret;
301  }
302 
303  if (avpkt->size ) {
304  if (av_fifo_size(q->input_fifo)) {
305  /* we have got rest of previous packet into buffer */
306  if (av_fifo_space(q->input_fifo) < avpkt->size) {
307  ret = av_fifo_grow(q->input_fifo, avpkt->size);
308  if (ret < 0)
309  return ret;
310  }
311  av_fifo_generic_write(q->input_fifo, avpkt->data, avpkt->size, NULL);
312  bs.Data = q->input_fifo->rptr;
313  bs.DataLength = av_fifo_size(q->input_fifo);
314  buffered = 1;
315  } else {
316  bs.Data = avpkt->data;
317  bs.DataLength = avpkt->size;
318  }
319  bs.MaxLength = bs.DataLength;
320  bs.TimeStamp = avpkt->pts;
321  }
322 
323  while (1) {
324  ret = get_surface(avctx, q, &insurf);
325  if (ret < 0)
326  return ret;
327  do {
328  ret = MFXVideoDECODE_DecodeFrameAsync(q->session, avpkt->size ? &bs : NULL,
329  insurf, &outsurf, &sync);
330  if (ret != MFX_WRN_DEVICE_BUSY)
331  break;
332  av_usleep(500);
333  } while (1);
334 
335  if (MFX_WRN_VIDEO_PARAM_CHANGED==ret) {
336  /* TODO: handle here sequence header changing */
337  }
338 
339  if (sync) {
340  QSVFrame *out_frame = find_frame(q, outsurf);
341 
342  if (!out_frame) {
343  av_log(avctx, AV_LOG_ERROR,
344  "The returned surface does not correspond to any frame\n");
345  return AVERROR_BUG;
346  }
347 
348  out_frame->queued = 1;
349  av_fifo_generic_write(q->async_fifo, &out_frame, sizeof(out_frame), NULL);
350  av_fifo_generic_write(q->async_fifo, &sync, sizeof(sync), NULL);
351 
352  continue;
353  }
354  if (MFX_ERR_MORE_SURFACE != ret && ret < 0)
355  break;
356  }
357 
358  /* make sure we do not enter an infinite loop if the SDK
359  * did not consume any data and did not return anything */
360  if (!sync && !bs.DataOffset) {
361  av_log(avctx, AV_LOG_WARNING, "A decode call did not consume any data\n");
362  bs.DataOffset = avpkt->size;
363  }
364 
365  if (buffered) {
366  qsv_fifo_relocate(q->input_fifo, bs.DataOffset);
367  } else if (bs.DataOffset!=avpkt->size) {
368  /* some data of packet was not consumed. store it to local buffer */
369  av_fifo_generic_write(q->input_fifo, avpkt->data+bs.DataOffset,
370  avpkt->size - bs.DataOffset, NULL);
371  }
372 
373  if (MFX_ERR_MORE_DATA!=ret && ret < 0) {
374  av_log(avctx, AV_LOG_ERROR, "Error %d during QSV decoding.\n", ret);
375  return ff_qsv_error(ret);
376  }
377  n_out_frames = av_fifo_size(q->async_fifo) / (sizeof(out_frame)+sizeof(sync));
378 
379  if (n_out_frames > q->async_depth || (!avpkt->size && n_out_frames) ) {
380  AVFrame *src_frame;
381 
382  av_fifo_generic_read(q->async_fifo, &out_frame, sizeof(out_frame), NULL);
383  av_fifo_generic_read(q->async_fifo, &sync, sizeof(sync), NULL);
384  out_frame->queued = 0;
385 
386  MFXVideoCORE_SyncOperation(q->session, sync, 60000);
387 
388  src_frame = out_frame->frame;
389 
390  ret = av_frame_ref(frame, src_frame);
391  if (ret < 0)
392  return ret;
393 
394  outsurf = out_frame->surface;
395 
396  frame->pkt_pts = frame->pts = outsurf->Data.TimeStamp;
397 
398  frame->repeat_pict =
399  outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_TRIPLING ? 4 :
400  outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_DOUBLING ? 2 :
401  outsurf->Info.PicStruct & MFX_PICSTRUCT_FIELD_REPEATED ? 1 : 0;
402  frame->top_field_first =
403  outsurf->Info.PicStruct & MFX_PICSTRUCT_FIELD_TFF;
404  frame->interlaced_frame =
405  !(outsurf->Info.PicStruct & MFX_PICSTRUCT_PROGRESSIVE);
406 
407  *got_frame = 1;
408  }
409 
410  return avpkt->size;
411 }
412 
414 {
415  QSVFrame *cur = q->work_frames;
416 
417  while (cur) {
418  q->work_frames = cur->next;
419  av_frame_free(&cur->frame);
420  av_freep(&cur);
421  cur = q->work_frames;
422  }
423 
425  q->async_fifo = NULL;
426 
428  q->input_fifo = NULL;
429 
430  MFXVideoDECODE_Close(q->session);
431  q->session = NULL;
432 
434 
435  q->engine_ready = 0;
436 
437  return 0;
438 }
#define NULL
Definition: coverity.c:32
int av_fifo_grow(AVFifoBuffer *f, unsigned int size)
Enlarge an AVFifoBuffer.
Definition: fifo.c:107
int iopattern
Definition: qsvdec.h:60
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
int ff_qsv_decode(AVCodecContext *avctx, QSVContext *q, AVFrame *frame, int *got_frame, AVPacket *avpkt)
Definition: qsvdec.c:284
AVFifoBuffer * input_fifo
Definition: qsvdec.h:52
This structure describes decoded (raw) audio or video data.
Definition: frame.h:171
int coded_width
Bitstream width / height, may be different from width/height e.g.
Definition: avcodec.h:1696
uint8_t * wptr
Definition: fifo.h:33
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
memory handling functions
int repeat_pict
When decoding, this signals how much the picture must be delayed.
Definition: frame.h:362
int size
Definition: avcodec.h:1424
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:1722
mfxExtBuffer ** ext_buffers
Definition: qsvdec.h:64
mfxFrameSurface1 * surface
Definition: qsv_internal.h:54
int av_usleep(unsigned usec)
Sleep for a period of time.
Definition: time.c:76
int profile
profile
Definition: avcodec.h:3115
QSVSession internal_qs
Definition: qsvdec.h:44
uint32_t rndx
Definition: fifo.h:34
int av_fifo_generic_write(AVFifoBuffer *f, void *src, int size, int(*func)(void *, void *, int))
Feed data from a user-supplied callback to an AVFifoBuffer.
Definition: fifo.c:122
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:135
void * hwaccel_context
Hardware accelerator context.
Definition: avcodec.h:2934
int ff_qsv_decode_close(QSVContext *q)
Definition: qsvdec.c:413
int av_frame_ref(AVFrame *dst, const AVFrame *src)
Set up a new reference to the data described by the source frame.
Definition: frame.c:365
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:257
int av_fifo_space(const AVFifoBuffer *f)
Return the amount of space in bytes in the AVFifoBuffer, that is the amount of data you can write int...
Definition: fifo.c:82
static AVFrame * frame
int queued
Definition: qsv_internal.h:58
uint8_t * data
Definition: avcodec.h:1423
void av_fifo_free(AVFifoBuffer *f)
Free an AVFifoBuffer.
Definition: fifo.c:55
int interlaced_frame
The content of the picture is interlaced.
Definition: frame.h:367
#define av_log(a,...)
static QSVFrame * find_frame(QSVContext *q, mfxFrameSurface1 *surf)
Definition: qsvdec.c:239
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
struct QSVFrame * next
Definition: qsv_internal.h:60
#define AVERROR(e)
Definition: error.h:43
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:148
int av_fifo_generic_read(AVFifoBuffer *f, void *dest, int buf_size, void(*func)(void *, void *, int))
Feed data from an AVFifoBuffer to a user-supplied callback.
Definition: fifo.c:177
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:91
int iopattern
Definition: qsv.h:28
int ff_qsv_close_internal_session(QSVSession *qs)
Definition: qsv.c:252
Libavcodec external API header.
int nb_ext_buffers
Definition: qsv.h:31
AVFrame * frame
Definition: qsv_internal.h:53
static void qsv_fifo_relocate(AVFifoBuffer *f, int bytes_to_free)
Definition: qsvdec.c:259
planar YUV 4:2:0, 12bpp, full scale (JPEG), deprecated in favor of AV_PIX_FMT_YUV420P and setting col...
Definition: pixfmt.h:75
uint8_t * end
Definition: fifo.h:33
int width
picture width / height.
Definition: avcodec.h:1681
uint8_t * rptr
Definition: fifo.h:33
int ff_qsv_codec_id_to_mfx(enum AVCodecID codec_id)
Definition: qsv.c:33
int engine_ready
Definition: qsvdec.h:56
int level
level
Definition: avcodec.h:3204
mfxFrameSurface1 surface_internal
Definition: qsv_internal.h:56
int ff_qsv_init_internal_session(AVCodecContext *avctx, QSVSession *qs, const char *load_plugins)
Initialize a MSDK session.
Definition: qsv.c:171
int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
Select the (possibly hardware accelerated) pixel format.
Definition: utils.c:1210
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:232
enum AVCodecID codec_id
Definition: avcodec.h:1519
int av_fifo_size(const AVFifoBuffer *f)
Return the amount of data in bytes in the AVFifoBuffer, that is the amount of data you can read from ...
Definition: fifo.c:77
mfxExtBuffer ** ext_buffers
Definition: qsv.h:30
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:199
main external API structure.
Definition: avcodec.h:1502
int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
Get a buffer for a frame.
Definition: utils.c:1040
static int get_surface(AVCodecContext *avctx, QSVContext *q, mfxFrameSurface1 **surf)
Definition: qsvdec.c:198
#define AVERROR_BUG
Internal bug, also see AVERROR_BUG2.
Definition: error.h:50
int coded_height
Definition: avcodec.h:1696
uint8_t * buffer
Definition: fifo.h:32
HW acceleration through QSV, data[3] contains a pointer to the mfxFrameSurface1 structure.
Definition: pixfmt.h:261
int ff_qsv_decode_init(AVCodecContext *avctx, QSVContext *q, AVPacket *avpkt)
Definition: qsvdec.c:52
char * load_plugins
Definition: qsvdec.h:62
static void qsv_clear_unused_frames(QSVContext *q)
Definition: qsvdec.c:186
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:209
int64_t pkt_pts
PTS copied from the AVPacket that was decoded to produce this frame.
Definition: frame.h:262
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
Definition: frame.c:464
uint32_t wndx
Definition: fifo.h:34
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:182
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:63
common internal api header.
common internal and external API header
int ff_qsv_map_pixfmt(enum AVPixelFormat format)
Definition: qsvdec.c:41
int ff_qsv_error(int mfx_err)
Convert a libmfx error code into a ffmpeg error code.
Definition: qsv.c:54
pixel format definitions
AVFifoBuffer * av_fifo_alloc(unsigned int size)
Initialize an AVFifoBuffer.
Definition: fifo.c:43
int top_field_first
If the content is interlaced, is top field displayed first.
Definition: frame.h:372
mfxSession session
Definition: qsvdec.h:40
AVFifoBuffer * async_fifo
Definition: qsvdec.h:51
static int alloc_frame(AVCodecContext *avctx, QSVFrame *frame)
Definition: qsvdec.c:158
#define av_freep(p)
int async_depth
Definition: qsvdec.h:59
void av_fifo_drain(AVFifoBuffer *f, int size)
Discard data from the FIFO.
Definition: fifo.c:197
QSVFrame * work_frames
a linked list of frames currently being used by QSV
Definition: qsvdec.h:49
AVPixelFormat
Pixel format.
Definition: pixfmt.h:61
This structure stores compressed data.
Definition: avcodec.h:1400
#define AV_GET_BUFFER_FLAG_REF
The decoder will keep a reference to the frame and may reuse it later.
Definition: avcodec.h:1216
mfxSession session
Definition: qsv.h:27
void * av_mallocz(size_t size)
Allocate a block of size bytes with alignment suitable for all memory accesses (including vectors if ...
Definition: mem.c:252
mfxSession session
Definition: qsv_internal.h:64
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1416
int nb_ext_buffers
Definition: qsvdec.h:65