FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
mmaldec.c
Go to the documentation of this file.
1 /*
2  * MMAL Video Decoder
3  * Copyright (c) 2015 Rodger Combs
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * MMAL Video Decoder
25  */
26 
27 #include <bcm_host.h>
28 #include <interface/mmal/mmal.h>
29 #include <interface/mmal/mmal_parameters_video.h>
30 #include <interface/mmal/util/mmal_util.h>
31 #include <interface/mmal/util/mmal_util_params.h>
32 #include <interface/mmal/util/mmal_default_components.h>
33 #include <interface/mmal/vc/mmal_vc_api.h>
34 #include <stdatomic.h>
35 
36 #include "avcodec.h"
37 #include "hwaccel.h"
38 #include "internal.h"
39 #include "libavutil/avassert.h"
40 #include "libavutil/buffer.h"
41 #include "libavutil/common.h"
42 #include "libavutil/imgutils.h"
43 #include "libavutil/opt.h"
44 #include "libavutil/log.h"
45 
46 typedef struct FFBufferEntry {
48  void *data;
49  size_t length;
50  int64_t pts, dts;
51  int flags;
54 
55 // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct
56 // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames
57 // have been unreferenced.
58 typedef struct FFPoolRef {
60  MMAL_POOL_T *pool;
61 } FFPoolRef;
62 
63 typedef struct FFBufferRef {
64  MMAL_BUFFER_HEADER_T *buffer;
66 } FFBufferRef;
67 
68 typedef struct MMALDecodeContext {
72 
73  MMAL_COMPONENT_T *decoder;
74  MMAL_QUEUE_T *queue_decoded_frames;
75  MMAL_POOL_T *pool_in;
77 
78  // Waiting input packets. Because the libavcodec API requires decoding and
79  // returning packets in lockstep, it can happen that queue_decoded_frames
80  // contains almost all surfaces - then the decoder input queue can quickly
81  // fill up and won't accept new input either. Without consuming input, the
82  // libavcodec API can't return new frames, and we have a logical deadlock.
83  // This is avoided by queuing such buffers here.
85 
86  int64_t packets_sent;
88  int64_t frames_output;
90  int eos_sent;
95 
96 // Assume decoder is guaranteed to produce output after at least this many
97 // packets (where each packet contains 1 frame).
98 #define MAX_DELAYED_FRAMES 16
99 
101 {
102  if (ref &&
103  atomic_fetch_add_explicit(&ref->refcount, -1, memory_order_acq_rel) == 1) {
104  mmal_pool_destroy(ref->pool);
105  av_free(ref);
106  }
107 }
108 
109 static void ffmmal_release_frame(void *opaque, uint8_t *data)
110 {
111  FFBufferRef *ref = (void *)data;
112 
113  mmal_buffer_header_release(ref->buffer);
115 
116  av_free(ref);
117 }
118 
119 // Setup frame with a new reference to buffer. The buffer must have been
120 // allocated from the given pool.
122  MMAL_BUFFER_HEADER_T *buffer)
123 {
124  FFBufferRef *ref = av_mallocz(sizeof(*ref));
125  if (!ref)
126  return AVERROR(ENOMEM);
127 
128  ref->pool = pool;
129  ref->buffer = buffer;
130 
131  frame->buf[0] = av_buffer_create((void *)ref, sizeof(*ref),
134  if (!frame->buf[0]) {
135  av_free(ref);
136  return AVERROR(ENOMEM);
137  }
138 
139  atomic_fetch_add_explicit(&ref->pool->refcount, 1, memory_order_relaxed);
140  mmal_buffer_header_acquire(buffer);
141 
142  frame->format = AV_PIX_FMT_MMAL;
143  frame->data[3] = (uint8_t *)ref->buffer;
144  return 0;
145 }
146 
148 {
149  MMALDecodeContext *ctx = avctx->priv_data;
150  MMAL_COMPONENT_T *decoder = ctx->decoder;
151  MMAL_BUFFER_HEADER_T *buffer;
152 
153  mmal_port_disable(decoder->input[0]);
154  mmal_port_disable(decoder->output[0]);
155  mmal_port_disable(decoder->control);
156 
157  mmal_port_flush(decoder->input[0]);
158  mmal_port_flush(decoder->output[0]);
159  mmal_port_flush(decoder->control);
160 
161  while ((buffer = mmal_queue_get(ctx->queue_decoded_frames)))
162  mmal_buffer_header_release(buffer);
163 
164  while (ctx->waiting_buffers) {
165  FFBufferEntry *buffer = ctx->waiting_buffers;
166 
167  ctx->waiting_buffers = buffer->next;
168 
169  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
171 
172  av_buffer_unref(&buffer->ref);
173  av_free(buffer);
174  }
175  ctx->waiting_buffers_tail = NULL;
176 
178 
179  ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0;
180 }
181 
183 {
184  MMALDecodeContext *ctx = avctx->priv_data;
185 
186  if (ctx->decoder)
187  ffmmal_stop_decoder(avctx);
188 
189  mmal_component_destroy(ctx->decoder);
190  ctx->decoder = NULL;
191  mmal_queue_destroy(ctx->queue_decoded_frames);
192  mmal_pool_destroy(ctx->pool_in);
194 
195  mmal_vc_deinit();
196 
197  return 0;
198 }
199 
200 static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
201 {
202  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
203  MMALDecodeContext *ctx = avctx->priv_data;
204 
205  if (!buffer->cmd) {
206  FFBufferEntry *entry = buffer->user_data;
207  av_buffer_unref(&entry->ref);
208  if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
209  atomic_fetch_add(&ctx->packets_buffered, -1);
210  av_free(entry);
211  }
212  mmal_buffer_header_release(buffer);
213 }
214 
215 static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
216 {
217  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
218  MMALDecodeContext *ctx = avctx->priv_data;
219 
220  mmal_queue_put(ctx->queue_decoded_frames, buffer);
221 }
222 
223 static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
224 {
225  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
226  MMAL_STATUS_T status;
227 
228  if (buffer->cmd == MMAL_EVENT_ERROR) {
229  status = *(uint32_t *)buffer->data;
230  av_log(avctx, AV_LOG_ERROR, "MMAL error %d on control port\n", (int)status);
231  } else {
232  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on control port\n",
233  av_fourcc2str(buffer->cmd));
234  }
235 
236  mmal_buffer_header_release(buffer);
237 }
238 
239 // Feed free output buffers to the decoder.
241 {
242  MMALDecodeContext *ctx = avctx->priv_data;
243  MMAL_BUFFER_HEADER_T *buffer;
244  MMAL_STATUS_T status;
245 
246  if (!ctx->pool_out)
247  return AVERROR_UNKNOWN; // format change code failed with OOM previously
248 
249  while ((buffer = mmal_queue_get(ctx->pool_out->pool->queue))) {
250  if ((status = mmal_port_send_buffer(ctx->decoder->output[0], buffer))) {
251  mmal_buffer_header_release(buffer);
252  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending output buffer.\n", (int)status);
253  return AVERROR_UNKNOWN;
254  }
255  }
256 
257  return 0;
258 }
259 
260 static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
261 {
262  switch (fourcc) {
263  case MMAL_COLOR_SPACE_BT470_2_BG:
264  case MMAL_COLOR_SPACE_BT470_2_M:
265  case MMAL_COLOR_SPACE_ITUR_BT601: return AVCOL_SPC_BT470BG;
266  case MMAL_COLOR_SPACE_ITUR_BT709: return AVCOL_SPC_BT709;
267  case MMAL_COLOR_SPACE_FCC: return AVCOL_SPC_FCC;
268  case MMAL_COLOR_SPACE_SMPTE240M: return AVCOL_SPC_SMPTE240M;
269  default: return AVCOL_SPC_UNSPECIFIED;
270  }
271 }
272 
274 {
275  MMALDecodeContext *ctx = avctx->priv_data;
276  MMAL_STATUS_T status;
277  int ret = 0;
278  MMAL_COMPONENT_T *decoder = ctx->decoder;
279  MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
280  MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T interlace_type;
281 
283  if (!(ctx->pool_out = av_mallocz(sizeof(*ctx->pool_out)))) {
284  ret = AVERROR(ENOMEM);
285  goto fail;
286  }
287  atomic_init(&ctx->pool_out->refcount, 1);
288 
289  if (!format_out)
290  goto fail;
291 
292  if ((status = mmal_port_parameter_set_uint32(decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS, ctx->extra_buffers)))
293  goto fail;
294 
295  if ((status = mmal_port_parameter_set_boolean(decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0)))
296  goto fail;
297 
298  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
299  format_out->encoding = MMAL_ENCODING_OPAQUE;
300  } else {
301  format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420;
302  }
303 
304  if ((status = mmal_port_format_commit(decoder->output[0])))
305  goto fail;
306 
307  interlace_type.hdr.id = MMAL_PARAMETER_VIDEO_INTERLACE_TYPE;
308  interlace_type.hdr.size = sizeof(MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T);
309  status = mmal_port_parameter_get(decoder->output[0], &interlace_type.hdr);
310  if (status != MMAL_SUCCESS) {
311  av_log(avctx, AV_LOG_ERROR, "Cannot read MMAL interlace information!\n");
312  } else {
313  ctx->interlaced_frame = (interlace_type.eMode != MMAL_InterlaceProgressive);
314  ctx->top_field_first = (interlace_type.eMode == MMAL_InterlaceFieldsInterleavedUpperFirst);
315  }
316 
317  if ((ret = ff_set_dimensions(avctx, format_out->es->video.crop.x + format_out->es->video.crop.width,
318  format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0)
319  goto fail;
320 
321  if (format_out->es->video.par.num && format_out->es->video.par.den) {
322  avctx->sample_aspect_ratio.num = format_out->es->video.par.num;
323  avctx->sample_aspect_ratio.den = format_out->es->video.par.den;
324  }
325  if (format_out->es->video.frame_rate.num && format_out->es->video.frame_rate.den) {
326  avctx->framerate.num = format_out->es->video.frame_rate.num;
327  avctx->framerate.den = format_out->es->video.frame_rate.den;
328  }
329 
330  avctx->colorspace = ffmmal_csp_to_av_csp(format_out->es->video.color_space);
331 
332  decoder->output[0]->buffer_size =
333  FFMAX(decoder->output[0]->buffer_size_min, decoder->output[0]->buffer_size_recommended);
334  decoder->output[0]->buffer_num =
335  FFMAX(decoder->output[0]->buffer_num_min, decoder->output[0]->buffer_num_recommended) + ctx->extra_buffers;
336  ctx->pool_out->pool = mmal_pool_create(decoder->output[0]->buffer_num,
337  decoder->output[0]->buffer_size);
338  if (!ctx->pool_out->pool) {
339  ret = AVERROR(ENOMEM);
340  goto fail;
341  }
342 
343  return 0;
344 
345 fail:
346  return ret < 0 ? ret : AVERROR_UNKNOWN;
347 }
348 
350 {
351  MMALDecodeContext *ctx = avctx->priv_data;
352  MMAL_STATUS_T status;
353  MMAL_ES_FORMAT_T *format_in;
354  MMAL_COMPONENT_T *decoder;
355  int ret = 0;
356 
357  bcm_host_init();
358 
359  if (mmal_vc_init()) {
360  av_log(avctx, AV_LOG_ERROR, "Cannot initialize MMAL VC driver!\n");
361  return AVERROR(ENOSYS);
362  }
363 
364  if ((ret = ff_get_format(avctx, avctx->codec->pix_fmts)) < 0)
365  return ret;
366 
367  avctx->pix_fmt = ret;
368 
369  if ((status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &ctx->decoder)))
370  goto fail;
371 
372  decoder = ctx->decoder;
373 
374  format_in = decoder->input[0]->format;
375  format_in->type = MMAL_ES_TYPE_VIDEO;
376  switch (avctx->codec_id) {
378  format_in->encoding = MMAL_ENCODING_MP2V;
379  break;
380  case AV_CODEC_ID_MPEG4:
381  format_in->encoding = MMAL_ENCODING_MP4V;
382  break;
383  case AV_CODEC_ID_VC1:
384  format_in->encoding = MMAL_ENCODING_WVC1;
385  break;
386  case AV_CODEC_ID_H264:
387  default:
388  format_in->encoding = MMAL_ENCODING_H264;
389  break;
390  }
391  format_in->es->video.width = FFALIGN(avctx->width, 32);
392  format_in->es->video.height = FFALIGN(avctx->height, 16);
393  format_in->es->video.crop.width = avctx->width;
394  format_in->es->video.crop.height = avctx->height;
395  format_in->es->video.frame_rate.num = 24000;
396  format_in->es->video.frame_rate.den = 1001;
397  format_in->es->video.par.num = avctx->sample_aspect_ratio.num;
398  format_in->es->video.par.den = avctx->sample_aspect_ratio.den;
399  format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;
400 
401  av_log(avctx, AV_LOG_DEBUG, "Using MMAL %s encoding.\n",
402  av_fourcc2str(format_in->encoding));
403 
404 #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
405  if (mmal_port_parameter_set_uint32(decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS,
406  -1 - ctx->extra_decoder_buffers)) {
407  av_log(avctx, AV_LOG_WARNING, "Could not set input buffering limit.\n");
408  }
409 #endif
410 
411  if ((status = mmal_port_format_commit(decoder->input[0])))
412  goto fail;
413 
414  decoder->input[0]->buffer_num =
415  FFMAX(decoder->input[0]->buffer_num_min, 20);
416  decoder->input[0]->buffer_size =
417  FFMAX(decoder->input[0]->buffer_size_min, 512 * 1024);
418  ctx->pool_in = mmal_pool_create(decoder->input[0]->buffer_num, 0);
419  if (!ctx->pool_in) {
420  ret = AVERROR(ENOMEM);
421  goto fail;
422  }
423 
424  if ((ret = ffmal_update_format(avctx)) < 0)
425  goto fail;
426 
427  ctx->queue_decoded_frames = mmal_queue_create();
428  if (!ctx->queue_decoded_frames)
429  goto fail;
430 
431  decoder->input[0]->userdata = (void*)avctx;
432  decoder->output[0]->userdata = (void*)avctx;
433  decoder->control->userdata = (void*)avctx;
434 
435  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
436  goto fail;
437  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
438  goto fail;
439  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
440  goto fail;
441 
442  if ((status = mmal_component_enable(decoder)))
443  goto fail;
444 
445  return 0;
446 
447 fail:
448  ffmmal_close_decoder(avctx);
449  return ret < 0 ? ret : AVERROR_UNKNOWN;
450 }
451 
452 static void ffmmal_flush(AVCodecContext *avctx)
453 {
454  MMALDecodeContext *ctx = avctx->priv_data;
455  MMAL_COMPONENT_T *decoder = ctx->decoder;
456  MMAL_STATUS_T status;
457 
458  ffmmal_stop_decoder(avctx);
459 
460  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
461  goto fail;
462  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
463  goto fail;
464  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
465  goto fail;
466 
467  return;
468 
469 fail:
470  av_log(avctx, AV_LOG_ERROR, "MMAL flush error: %i\n", (int)status);
471 }
472 
473 // Split packets and add them to the waiting_buffers list. We don't queue them
474 // immediately, because it can happen that the decoder is temporarily blocked
475 // (due to us not reading/returning enough output buffers) and won't accept
476 // new input. (This wouldn't be an issue if MMAL input buffers always were
477 // complete frames - then the input buffer just would have to be big enough.)
478 // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG.
479 static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
480  int is_extradata)
481 {
482  MMALDecodeContext *ctx = avctx->priv_data;
483  AVBufferRef *buf = NULL;
484  int size = 0;
485  uint8_t *data = (uint8_t *)"";
486  uint8_t *start;
487  int ret = 0;
488 
489  if (avpkt->size) {
490  if (avpkt->buf) {
491  buf = av_buffer_ref(avpkt->buf);
492  size = avpkt->size;
493  data = avpkt->data;
494  } else {
495  buf = av_buffer_alloc(avpkt->size);
496  if (buf) {
497  memcpy(buf->data, avpkt->data, avpkt->size);
498  size = buf->size;
499  data = buf->data;
500  }
501  }
502  if (!buf) {
503  ret = AVERROR(ENOMEM);
504  goto done;
505  }
506  if (!is_extradata)
507  ctx->packets_sent++;
508  } else {
509  if (ctx->eos_sent)
510  goto done;
511  if (!ctx->packets_sent) {
512  // Short-cut the flush logic to avoid upsetting MMAL.
513  ctx->eos_sent = 1;
514  ctx->eos_received = 1;
515  goto done;
516  }
517  }
518 
519  start = data;
520 
521  do {
522  FFBufferEntry *buffer = av_mallocz(sizeof(*buffer));
523  if (!buffer) {
524  ret = AVERROR(ENOMEM);
525  goto done;
526  }
527 
528  buffer->data = data;
529  buffer->length = FFMIN(size, ctx->decoder->input[0]->buffer_size);
530 
531  if (is_extradata)
532  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_CONFIG;
533 
534  if (data == start)
535  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START;
536 
537  data += buffer->length;
538  size -= buffer->length;
539 
540  buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts;
541  buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts;
542 
543  if (!size) {
544  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
546  }
547 
548  if (!buffer->length) {
549  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS;
550  ctx->eos_sent = 1;
551  }
552 
553  if (buf) {
554  buffer->ref = av_buffer_ref(buf);
555  if (!buffer->ref) {
556  av_free(buffer);
557  ret = AVERROR(ENOMEM);
558  goto done;
559  }
560  }
561 
562  // Insert at end of the list
563  if (!ctx->waiting_buffers)
564  ctx->waiting_buffers = buffer;
565  if (ctx->waiting_buffers_tail)
568  } while (size);
569 
570 done:
571  av_buffer_unref(&buf);
572  return ret;
573 }
574 
575 // Move prepared/split packets from waiting_buffers to the MMAL decoder.
577 {
578  MMALDecodeContext *ctx = avctx->priv_data;
579 
580  while (ctx->waiting_buffers) {
581  MMAL_BUFFER_HEADER_T *mbuffer;
583  MMAL_STATUS_T status;
584 
585  mbuffer = mmal_queue_get(ctx->pool_in->queue);
586  if (!mbuffer)
587  return 0;
588 
589  buffer = ctx->waiting_buffers;
590 
591  mmal_buffer_header_reset(mbuffer);
592  mbuffer->cmd = 0;
593  mbuffer->pts = buffer->pts;
594  mbuffer->dts = buffer->dts;
595  mbuffer->flags = buffer->flags;
596  mbuffer->data = buffer->data;
597  mbuffer->length = buffer->length;
598  mbuffer->user_data = buffer;
599  mbuffer->alloc_size = ctx->decoder->input[0]->buffer_size;
600 
601  // Remove from start of the list
602  ctx->waiting_buffers = buffer->next;
603  if (ctx->waiting_buffers_tail == buffer)
604  ctx->waiting_buffers_tail = NULL;
605 
606  if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) {
607  mmal_buffer_header_release(mbuffer);
608  av_buffer_unref(&buffer->ref);
609  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
611  av_free(buffer);
612  }
613 
614  if (status) {
615  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending input\n", (int)status);
616  return AVERROR_UNKNOWN;
617  }
618  }
619 
620  return 0;
621 }
622 
624  MMAL_BUFFER_HEADER_T *buffer)
625 {
626  MMALDecodeContext *ctx = avctx->priv_data;
627  int ret = 0;
628 
629  frame->interlaced_frame = ctx->interlaced_frame;
630  frame->top_field_first = ctx->top_field_first;
631 
632  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
633  if (!ctx->pool_out)
634  return AVERROR_UNKNOWN; // format change code failed with OOM previously
635 
636  if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
637  goto done;
638 
639  if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0)
640  goto done;
641  } else {
642  int w = FFALIGN(avctx->width, 32);
643  int h = FFALIGN(avctx->height, 16);
644  uint8_t *src[4];
645  int linesize[4];
646 
647  if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
648  goto done;
649 
650  av_image_fill_arrays(src, linesize,
651  buffer->data + buffer->type->video.offset[0],
652  avctx->pix_fmt, w, h, 1);
653  av_image_copy(frame->data, frame->linesize, src, linesize,
654  avctx->pix_fmt, avctx->width, avctx->height);
655  }
656 
657  frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
658 #if FF_API_PKT_PTS
660  frame->pkt_pts = frame->pts;
662 #endif
663  frame->pkt_dts = AV_NOPTS_VALUE;
664 
665 done:
666  return ret;
667 }
668 
669 // Fetch a decoded buffer and place it into the frame parameter.
670 static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
671 {
672  MMALDecodeContext *ctx = avctx->priv_data;
673  MMAL_BUFFER_HEADER_T *buffer = NULL;
674  MMAL_STATUS_T status = 0;
675  int ret = 0;
676 
677  if (ctx->eos_received)
678  goto done;
679 
680  while (1) {
681  // To ensure decoding in lockstep with a constant delay between fed packets
682  // and output frames, we always wait until an output buffer is available.
683  // Except during start we don't know after how many input packets the decoder
684  // is going to return the first buffer, and we can't distinguish decoder
685  // being busy from decoder waiting for input. So just poll at the start and
686  // keep feeding new data to the buffer.
687  // We are pretty sure the decoder will produce output if we sent more input
688  // frames than what a H.264 decoder could logically delay. This avoids too
689  // excessive buffering.
690  // We also wait if we sent eos, but didn't receive it yet (think of decoding
691  // stream with a very low number of frames).
693  (ctx->packets_sent && ctx->eos_sent)) {
694  // MMAL will ignore broken input packets, which means the frame we
695  // expect here may never arrive. Dealing with this correctly is
696  // complicated, so here's a hack to avoid that it freezes forever
697  // in this unlikely situation.
698  buffer = mmal_queue_timedwait(ctx->queue_decoded_frames, 100);
699  if (!buffer) {
700  av_log(avctx, AV_LOG_ERROR, "Did not get output frame from MMAL.\n");
701  ret = AVERROR_UNKNOWN;
702  goto done;
703  }
704  } else {
705  buffer = mmal_queue_get(ctx->queue_decoded_frames);
706  if (!buffer)
707  goto done;
708  }
709 
710  ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
711  if (ctx->eos_received)
712  goto done;
713 
714  if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
715  MMAL_COMPONENT_T *decoder = ctx->decoder;
716  MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(buffer);
717  MMAL_BUFFER_HEADER_T *stale_buffer;
718 
719  av_log(avctx, AV_LOG_INFO, "Changing output format.\n");
720 
721  if ((status = mmal_port_disable(decoder->output[0])))
722  goto done;
723 
724  while ((stale_buffer = mmal_queue_get(ctx->queue_decoded_frames)))
725  mmal_buffer_header_release(stale_buffer);
726 
727  mmal_format_copy(decoder->output[0]->format, ev->format);
728 
729  if ((ret = ffmal_update_format(avctx)) < 0)
730  goto done;
731 
732  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
733  goto done;
734 
735  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
736  goto done;
737 
738  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
739  goto done;
740 
741  mmal_buffer_header_release(buffer);
742  continue;
743  } else if (buffer->cmd) {
744  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on output port\n",
745  av_fourcc2str(buffer->cmd));
746  goto done;
747  } else if (buffer->length == 0) {
748  // Unused output buffer that got drained after format change.
749  mmal_buffer_header_release(buffer);
750  continue;
751  }
752 
753  ctx->frames_output++;
754 
755  if ((ret = ffmal_copy_frame(avctx, frame, buffer)) < 0)
756  goto done;
757 
758  *got_frame = 1;
759  break;
760  }
761 
762 done:
763  if (buffer)
764  mmal_buffer_header_release(buffer);
765  if (status && ret >= 0)
766  ret = AVERROR_UNKNOWN;
767  return ret;
768 }
769 
770 static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
771  AVPacket *avpkt)
772 {
773  MMALDecodeContext *ctx = avctx->priv_data;
774  AVFrame *frame = data;
775  int ret = 0;
776 
777  if (avctx->extradata_size && !ctx->extradata_sent) {
778  AVPacket pkt = {0};
779  av_init_packet(&pkt);
780  pkt.data = avctx->extradata;
781  pkt.size = avctx->extradata_size;
782  ctx->extradata_sent = 1;
783  if ((ret = ffmmal_add_packet(avctx, &pkt, 1)) < 0)
784  return ret;
785  }
786 
787  if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0)
788  return ret;
789 
790  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
791  return ret;
792 
793  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
794  return ret;
795 
796  if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0)
797  return ret;
798 
799  // ffmmal_read_frame() can block for a while. Since the decoder is
800  // asynchronous, it's a good idea to fill the ports again.
801 
802  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
803  return ret;
804 
805  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
806  return ret;
807 
808  return ret;
809 }
810 
812  HW_CONFIG_INTERNAL(MMAL),
813  NULL
814 };
815 
816 static const AVOption options[]={
817  {"extra_buffers", "extra buffers", offsetof(MMALDecodeContext, extra_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
818  {"extra_decoder_buffers", "extra MMAL internal buffered frames", offsetof(MMALDecodeContext, extra_decoder_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
819  {NULL}
820 };
821 
822 #define FFMMAL_DEC_CLASS(NAME) \
823  static const AVClass ffmmal_##NAME##_dec_class = { \
824  .class_name = "mmal_" #NAME "_dec", \
825  .item_name = av_default_item_name, \
826  .option = options, \
827  .version = LIBAVUTIL_VERSION_INT, \
828  };
829 
830 #define FFMMAL_DEC(NAME, ID) \
831  FFMMAL_DEC_CLASS(NAME) \
832  AVCodec ff_##NAME##_mmal_decoder = { \
833  .name = #NAME "_mmal", \
834  .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \
835  .type = AVMEDIA_TYPE_VIDEO, \
836  .id = ID, \
837  .priv_data_size = sizeof(MMALDecodeContext), \
838  .init = ffmmal_init_decoder, \
839  .close = ffmmal_close_decoder, \
840  .decode = ffmmal_decode, \
841  .flush = ffmmal_flush, \
842  .priv_class = &ffmmal_##NAME##_dec_class, \
843  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, \
844  .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
845  .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \
846  AV_PIX_FMT_YUV420P, \
847  AV_PIX_FMT_NONE}, \
848  .hw_configs = mmal_hw_configs, \
849  .wrapper_name = "mmal", \
850  };
851 
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:475
#define NULL
Definition: coverity.c:32
int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
Select the (possibly hardware accelerated) pixel format.
Definition: decode.c:1288
const struct AVCodec * codec
Definition: avcodec.h:1511
AVRational framerate
Definition: avcodec.h:3012
uint32_t fourcc
Definition: hwcontext_qsv.c:90
MMAL_POOL_T * pool
Definition: mmaldec.c:60
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
Definition: buffer.c:125
This structure describes decoded (raw) audio or video data.
Definition: frame.h:201
#define FFMMAL_DEC(NAME, ID)
Definition: mmaldec.c:830
AVOption.
Definition: opt.h:246
ptrdiff_t const GLvoid * data
Definition: opengl_enc.c:101
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
int ff_set_dimensions(AVCodecContext *s, int width, int height)
Check that the provided frame dimensions are valid and set them on the codec context.
Definition: utils.c:196
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:393
atomic_int packets_buffered
Definition: mmaldec.c:87
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:479
FFPoolRef * pool
Definition: mmaldec.c:65
int num
Numerator.
Definition: rational.h:59
int size
Definition: avcodec.h:1415
int av_image_fill_arrays(uint8_t *dst_data[4], int dst_linesize[4], const uint8_t *src, enum AVPixelFormat pix_fmt, int width, int height, int align)
Setup the data pointers and linesizes based on the specified image parameters and the provided array...
Definition: imgutils.c:410
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:1880
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:1711
static int ffmmal_fill_output_port(AVCodecContext *avctx)
Definition: mmaldec.c:240
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:236
int ff_decode_frame_props(AVCodecContext *avctx, AVFrame *frame)
Set various frame properties from the codec context / packet data.
Definition: decode.c:1762
static AVPacket pkt
atomic_int refcount
Definition: mmaldec.c:59
intptr_t atomic_int
Definition: stdatomic.h:55
int64_t dts
Definition: mmaldec.c:50
#define src
Definition: vp8dsp.c:254
static int ffmmal_fill_input_port(AVCodecContext *avctx)
Definition: mmaldec.c:576
static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
Definition: mmaldec.c:770
functionally identical to above
Definition: pixfmt.h:481
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
uint8_t
#define av_cold
Definition: attributes.h:82
AVOptions.
MMAL_BUFFER_HEADER_T * buffer
Definition: mmaldec.c:64
AVColorSpace
YUV colorspace type.
Definition: pixfmt.h:473
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:294
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:1602
static AVFrame * frame
static void ffmmal_flush(AVCodecContext *avctx)
Definition: mmaldec.c:452
uint8_t * data
Definition: avcodec.h:1414
static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:349
static const AVOption options[]
Definition: mmaldec.c:816
static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:182
int interlaced_frame
The content of the picture is interlaced.
Definition: frame.h:348
#define AV_BUFFER_FLAG_READONLY
Always treat the buffer as read-only, even when it has only one reference.
Definition: buffer.h:113
MMAL_POOL_T * pool_in
Definition: mmaldec.c:75
ptrdiff_t size
Definition: opengl_enc.c:101
int top_field_first
Definition: mmaldec.c:93
#define FFALIGN(x, a)
Definition: macros.h:48
#define av_log(a,...)
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:215
#define atomic_load(object)
Definition: stdatomic.h:93
int extradata_sent
Definition: mmaldec.c:91
#define AVERROR(e)
Definition: error.h:43
#define HW_CONFIG_INTERNAL(format)
Definition: hwaccel.h:57
static void ffmmal_poolref_unref(FFPoolRef *ref)
Definition: mmaldec.c:100
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:197
#define av_fourcc2str(fourcc)
Definition: avutil.h:348
AVBufferRef * buf
A reference to the reference-counted buffer where the packet data is stored.
Definition: avcodec.h:1397
simple assert() macros that are a bit more flexible than ISO C assert().
static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:121
FFBufferEntry * waiting_buffers_tail
Definition: mmaldec.c:84
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:28
int extra_decoder_buffers
Definition: mmaldec.c:71
#define FFMAX(a, b)
Definition: common.h:94
#define fail()
Definition: checkasm.h:113
static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:223
void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], const uint8_t *src_data[4], const int src_linesizes[4], enum AVPixelFormat pix_fmt, int width, int height)
Copy image in src_data to dst_data.
Definition: imgutils.c:386
AVBufferRef * ref
Definition: mmaldec.c:47
size_t length
Definition: mmaldec.c:49
static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:623
AVClass * av_class
Definition: mmaldec.c:69
static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
Definition: mmaldec.c:260
enum AVPixelFormat * pix_fmts
array of supported pixel formats, or NULL if unknown, array is terminated by -1
Definition: avcodec.h:3386
FFPoolRef * pool_out
Definition: mmaldec.c:76
#define FFMIN(a, b)
Definition: common.h:96
FFBufferEntry * waiting_buffers
Definition: mmaldec.c:84
int64_t frames_output
Definition: mmaldec.c:88
static const chunk_decoder decoder[8]
Definition: dfa.c:330
int width
picture width / height.
Definition: avcodec.h:1674
AVFormatContext * ctx
Definition: movenc.c:48
#define MAX_DELAYED_FRAMES
Definition: mmaldec.c:98
preferred ID for MPEG-1/2 video decoding
Definition: avcodec.h:220
FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
Definition: pixfmt.h:478
static void ffmmal_stop_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:147
MMAL_QUEUE_T * queue_decoded_frames
Definition: mmaldec.c:74
#define atomic_fetch_add_explicit(object, operand, order)
Definition: stdatomic.h:149
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:274
#define atomic_fetch_add(object, operand)
Definition: stdatomic.h:131
int interlaced_frame
Definition: mmaldec.c:92
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
struct FFBufferEntry * next
Definition: mmaldec.c:52
Libavcodec external API header.
enum AVCodecID codec_id
Definition: avcodec.h:1512
AVBufferRef * av_buffer_alloc(int size)
Allocate an AVBuffer of the given size using av_malloc().
Definition: buffer.c:67
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:232
main external API structure.
Definition: avcodec.h:1502
uint8_t * data
The data buffer.
Definition: buffer.h:89
void * data
Definition: mmaldec.c:48
static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:200
int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
Get a buffer for a frame.
Definition: decode.c:1886
void * buf
Definition: avisynth_c.h:690
int extradata_size
Definition: avcodec.h:1603
Describe the class of an AVClass context structure.
Definition: log.h:67
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:2125
refcounted data buffer API
int size
Size of data in bytes.
Definition: buffer.h:93
static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
Definition: mmaldec.c:670
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:215
attribute_deprecated int64_t pkt_pts
PTS copied from the AVPacket that was decoded to produce this frame.
Definition: frame.h:302
int64_t packets_sent
Definition: mmaldec.c:86
int64_t pkt_dts
DTS copied from the AVPacket that triggered returning this frame.
Definition: frame.h:310
A reference to a data buffer.
Definition: buffer.h:81
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:84
common internal api header.
common internal and external API header
if(ret< 0)
Definition: vf_mcdeint.c:279
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:107
static int ffmal_update_format(AVCodecContext *avctx)
Definition: mmaldec.c:273
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:93
void av_init_packet(AVPacket *pkt)
Initialize optional fields of a packet with default values.
Definition: avpacket.c:33
int den
Denominator.
Definition: rational.h:60
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:71
static void ffmmal_release_frame(void *opaque, uint8_t *data)
Definition: mmaldec.c:109
void * priv_data
Definition: avcodec.h:1529
#define av_free(p)
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:85
int top_field_first
If the content is interlaced, is top field displayed first.
Definition: frame.h:353
static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt, int is_extradata)
Definition: mmaldec.c:479
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: avcodec.h:1413
#define atomic_init(obj, value)
Definition: stdatomic.h:33
void INT64 start
Definition: avisynth_c.h:690
HW acceleration though MMAL, data[3] contains a pointer to the MMAL_BUFFER_HEADER_T structure...
Definition: pixfmt.h:223
This structure stores compressed data.
Definition: avcodec.h:1391
MMAL_COMPONENT_T * decoder
Definition: mmaldec.c:73
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1407
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248
static const AVCodecHWConfigInternal * mmal_hw_configs[]
Definition: mmaldec.c:811
GLuint buffer
Definition: opengl_enc.c:102
int64_t pts
Definition: mmaldec.c:50