FFmpeg
mmaldec.c
Go to the documentation of this file.
1 /*
2  * MMAL Video Decoder
3  * Copyright (c) 2015 Rodger Combs
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * MMAL Video Decoder
25  */
26 
27 #include <bcm_host.h>
28 #include <interface/mmal/mmal.h>
29 #include <interface/mmal/mmal_parameters_video.h>
30 #include <interface/mmal/util/mmal_util.h>
31 #include <interface/mmal/util/mmal_util_params.h>
32 #include <interface/mmal/util/mmal_default_components.h>
33 #include <interface/mmal/vc/mmal_vc_api.h>
34 #include <stdatomic.h>
35 
36 #include "avcodec.h"
37 #include "hwaccel.h"
38 #include "internal.h"
39 #include "libavutil/avassert.h"
40 #include "libavutil/buffer.h"
41 #include "libavutil/common.h"
42 #include "libavutil/imgutils.h"
43 #include "libavutil/opt.h"
44 #include "libavutil/log.h"
45 
46 typedef struct FFBufferEntry {
48  void *data;
49  size_t length;
50  int64_t pts, dts;
51  int flags;
54 
55 // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct
56 // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames
57 // have been unreferenced.
58 typedef struct FFPoolRef {
60  MMAL_POOL_T *pool;
61 } FFPoolRef;
62 
63 typedef struct FFBufferRef {
64  MMAL_BUFFER_HEADER_T *buffer;
66 } FFBufferRef;
67 
68 typedef struct MMALDecodeContext {
72 
73  MMAL_COMPONENT_T *decoder;
74  MMAL_QUEUE_T *queue_decoded_frames;
75  MMAL_POOL_T *pool_in;
77 
78  // Waiting input packets. Because the libavcodec API requires decoding and
79  // returning packets in lockstep, it can happen that queue_decoded_frames
80  // contains almost all surfaces - then the decoder input queue can quickly
81  // fill up and won't accept new input either. Without consuming input, the
82  // libavcodec API can't return new frames, and we have a logical deadlock.
83  // This is avoided by queuing such buffers here.
84  FFBufferEntry *waiting_buffers, *waiting_buffers_tail;
85 
86  int64_t packets_sent;
88  int64_t frames_output;
90  int eos_sent;
95 
96 // Assume decoder is guaranteed to produce output after at least this many
97 // packets (where each packet contains 1 frame).
98 #define MAX_DELAYED_FRAMES 16
99 
101 {
102  if (ref &&
103  atomic_fetch_add_explicit(&ref->refcount, -1, memory_order_acq_rel) == 1) {
104  mmal_pool_destroy(ref->pool);
105  av_free(ref);
106  }
107 }
108 
109 static void ffmmal_release_frame(void *opaque, uint8_t *data)
110 {
111  FFBufferRef *ref = (void *)data;
112 
113  mmal_buffer_header_release(ref->buffer);
115 
116  av_free(ref);
117 }
118 
119 // Setup frame with a new reference to buffer. The buffer must have been
120 // allocated from the given pool.
122  MMAL_BUFFER_HEADER_T *buffer)
123 {
124  FFBufferRef *ref = av_mallocz(sizeof(*ref));
125  if (!ref)
126  return AVERROR(ENOMEM);
127 
128  ref->pool = pool;
129  ref->buffer = buffer;
130 
131  frame->buf[0] = av_buffer_create((void *)ref, sizeof(*ref),
134  if (!frame->buf[0]) {
135  av_free(ref);
136  return AVERROR(ENOMEM);
137  }
138 
139  atomic_fetch_add_explicit(&ref->pool->refcount, 1, memory_order_relaxed);
140  mmal_buffer_header_acquire(buffer);
141 
142  frame->format = AV_PIX_FMT_MMAL;
143  frame->data[3] = (uint8_t *)ref->buffer;
144  return 0;
145 }
146 
148 {
149  MMALDecodeContext *ctx = avctx->priv_data;
150  MMAL_COMPONENT_T *decoder = ctx->decoder;
151  MMAL_BUFFER_HEADER_T *buffer;
152 
153  mmal_port_disable(decoder->input[0]);
154  mmal_port_disable(decoder->output[0]);
155  mmal_port_disable(decoder->control);
156 
157  mmal_port_flush(decoder->input[0]);
158  mmal_port_flush(decoder->output[0]);
159  mmal_port_flush(decoder->control);
160 
161  while ((buffer = mmal_queue_get(ctx->queue_decoded_frames)))
162  mmal_buffer_header_release(buffer);
163 
164  while (ctx->waiting_buffers) {
165  FFBufferEntry *buffer = ctx->waiting_buffers;
166 
167  ctx->waiting_buffers = buffer->next;
168 
169  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
171 
172  av_buffer_unref(&buffer->ref);
173  av_free(buffer);
174  }
175  ctx->waiting_buffers_tail = NULL;
176 
178 
179  ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0;
180 }
181 
183 {
184  MMALDecodeContext *ctx = avctx->priv_data;
185 
186  if (ctx->decoder)
187  ffmmal_stop_decoder(avctx);
188 
189  mmal_component_destroy(ctx->decoder);
190  ctx->decoder = NULL;
191  mmal_queue_destroy(ctx->queue_decoded_frames);
192  mmal_pool_destroy(ctx->pool_in);
194 
195  mmal_vc_deinit();
196 
197  return 0;
198 }
199 
200 static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
201 {
202  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
203  MMALDecodeContext *ctx = avctx->priv_data;
204 
205  if (!buffer->cmd) {
206  FFBufferEntry *entry = buffer->user_data;
207  av_buffer_unref(&entry->ref);
208  if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
209  atomic_fetch_add(&ctx->packets_buffered, -1);
210  av_free(entry);
211  }
212  mmal_buffer_header_release(buffer);
213 }
214 
215 static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
216 {
217  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
218  MMALDecodeContext *ctx = avctx->priv_data;
219 
220  mmal_queue_put(ctx->queue_decoded_frames, buffer);
221 }
222 
223 static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
224 {
225  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
226  MMAL_STATUS_T status;
227 
228  if (buffer->cmd == MMAL_EVENT_ERROR) {
229  status = *(uint32_t *)buffer->data;
230  av_log(avctx, AV_LOG_ERROR, "MMAL error %d on control port\n", (int)status);
231  } else {
232  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on control port\n",
233  av_fourcc2str(buffer->cmd));
234  }
235 
236  mmal_buffer_header_release(buffer);
237 }
238 
239 // Feed free output buffers to the decoder.
241 {
242  MMALDecodeContext *ctx = avctx->priv_data;
243  MMAL_BUFFER_HEADER_T *buffer;
244  MMAL_STATUS_T status;
245 
246  if (!ctx->pool_out)
247  return AVERROR_UNKNOWN; // format change code failed with OOM previously
248 
249  while ((buffer = mmal_queue_get(ctx->pool_out->pool->queue))) {
250  if ((status = mmal_port_send_buffer(ctx->decoder->output[0], buffer))) {
251  mmal_buffer_header_release(buffer);
252  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending output buffer.\n", (int)status);
253  return AVERROR_UNKNOWN;
254  }
255  }
256 
257  return 0;
258 }
259 
260 static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
261 {
262  switch (fourcc) {
263  case MMAL_COLOR_SPACE_BT470_2_BG:
264  case MMAL_COLOR_SPACE_BT470_2_M:
265  case MMAL_COLOR_SPACE_ITUR_BT601: return AVCOL_SPC_BT470BG;
266  case MMAL_COLOR_SPACE_ITUR_BT709: return AVCOL_SPC_BT709;
267  case MMAL_COLOR_SPACE_FCC: return AVCOL_SPC_FCC;
268  case MMAL_COLOR_SPACE_SMPTE240M: return AVCOL_SPC_SMPTE240M;
269  default: return AVCOL_SPC_UNSPECIFIED;
270  }
271 }
272 
274 {
275  MMALDecodeContext *ctx = avctx->priv_data;
276  MMAL_STATUS_T status;
277  int ret = 0;
278  MMAL_COMPONENT_T *decoder = ctx->decoder;
279  MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
280  MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T interlace_type;
281 
283  if (!(ctx->pool_out = av_mallocz(sizeof(*ctx->pool_out)))) {
284  ret = AVERROR(ENOMEM);
285  goto fail;
286  }
287  atomic_init(&ctx->pool_out->refcount, 1);
288 
289  if (!format_out)
290  goto fail;
291 
292  if ((status = mmal_port_parameter_set_uint32(decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS, ctx->extra_buffers)))
293  goto fail;
294 
295  if ((status = mmal_port_parameter_set_boolean(decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0)))
296  goto fail;
297 
298  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
299  format_out->encoding = MMAL_ENCODING_OPAQUE;
300  } else {
301  format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420;
302  }
303 
304  if ((status = mmal_port_format_commit(decoder->output[0])))
305  goto fail;
306 
307  interlace_type.hdr.id = MMAL_PARAMETER_VIDEO_INTERLACE_TYPE;
308  interlace_type.hdr.size = sizeof(MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T);
309  status = mmal_port_parameter_get(decoder->output[0], &interlace_type.hdr);
310  if (status != MMAL_SUCCESS) {
311  av_log(avctx, AV_LOG_ERROR, "Cannot read MMAL interlace information!\n");
312  } else {
313  ctx->interlaced_frame = (interlace_type.eMode != MMAL_InterlaceProgressive);
314  ctx->top_field_first = (interlace_type.eMode == MMAL_InterlaceFieldsInterleavedUpperFirst);
315  }
316 
317  if ((ret = ff_set_dimensions(avctx, format_out->es->video.crop.x + format_out->es->video.crop.width,
318  format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0)
319  goto fail;
320 
321  if (format_out->es->video.par.num && format_out->es->video.par.den) {
322  avctx->sample_aspect_ratio.num = format_out->es->video.par.num;
323  avctx->sample_aspect_ratio.den = format_out->es->video.par.den;
324  }
325  if (format_out->es->video.frame_rate.num && format_out->es->video.frame_rate.den) {
326  avctx->framerate.num = format_out->es->video.frame_rate.num;
327  avctx->framerate.den = format_out->es->video.frame_rate.den;
328  }
329 
330  avctx->colorspace = ffmmal_csp_to_av_csp(format_out->es->video.color_space);
331 
332  decoder->output[0]->buffer_size =
333  FFMAX(decoder->output[0]->buffer_size_min, decoder->output[0]->buffer_size_recommended);
334  decoder->output[0]->buffer_num =
335  FFMAX(decoder->output[0]->buffer_num_min, decoder->output[0]->buffer_num_recommended) + ctx->extra_buffers;
336  ctx->pool_out->pool = mmal_pool_create(decoder->output[0]->buffer_num,
337  decoder->output[0]->buffer_size);
338  if (!ctx->pool_out->pool) {
339  ret = AVERROR(ENOMEM);
340  goto fail;
341  }
342 
343  return 0;
344 
345 fail:
346  return ret < 0 ? ret : AVERROR_UNKNOWN;
347 }
348 
350 {
351  MMALDecodeContext *ctx = avctx->priv_data;
352  MMAL_STATUS_T status;
353  MMAL_ES_FORMAT_T *format_in;
354  MMAL_COMPONENT_T *decoder;
355  int ret = 0;
356 
357  bcm_host_init();
358 
359  if (mmal_vc_init()) {
360  av_log(avctx, AV_LOG_ERROR, "Cannot initialize MMAL VC driver!\n");
361  return AVERROR(ENOSYS);
362  }
363 
364  if ((ret = ff_get_format(avctx, avctx->codec->pix_fmts)) < 0)
365  return ret;
366 
367  avctx->pix_fmt = ret;
368 
369  if ((status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &ctx->decoder)))
370  goto fail;
371 
372  decoder = ctx->decoder;
373 
374  format_in = decoder->input[0]->format;
375  format_in->type = MMAL_ES_TYPE_VIDEO;
376  switch (avctx->codec_id) {
378  format_in->encoding = MMAL_ENCODING_MP2V;
379  break;
380  case AV_CODEC_ID_MPEG4:
381  format_in->encoding = MMAL_ENCODING_MP4V;
382  break;
383  case AV_CODEC_ID_VC1:
384  format_in->encoding = MMAL_ENCODING_WVC1;
385  break;
386  case AV_CODEC_ID_H264:
387  default:
388  format_in->encoding = MMAL_ENCODING_H264;
389  break;
390  }
391  format_in->es->video.width = FFALIGN(avctx->width, 32);
392  format_in->es->video.height = FFALIGN(avctx->height, 16);
393  format_in->es->video.crop.width = avctx->width;
394  format_in->es->video.crop.height = avctx->height;
395  format_in->es->video.frame_rate.num = 24000;
396  format_in->es->video.frame_rate.den = 1001;
397  format_in->es->video.par.num = avctx->sample_aspect_ratio.num;
398  format_in->es->video.par.den = avctx->sample_aspect_ratio.den;
399  format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;
400 
401  av_log(avctx, AV_LOG_DEBUG, "Using MMAL %s encoding.\n",
402  av_fourcc2str(format_in->encoding));
403 
404 #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
405  if (mmal_port_parameter_set_uint32(decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS,
406  -1 - ctx->extra_decoder_buffers)) {
407  av_log(avctx, AV_LOG_WARNING, "Could not set input buffering limit.\n");
408  }
409 #endif
410 
411  if ((status = mmal_port_format_commit(decoder->input[0])))
412  goto fail;
413 
414  decoder->input[0]->buffer_num =
415  FFMAX(decoder->input[0]->buffer_num_min, 20);
416  decoder->input[0]->buffer_size =
417  FFMAX(decoder->input[0]->buffer_size_min, 512 * 1024);
418  ctx->pool_in = mmal_pool_create(decoder->input[0]->buffer_num, 0);
419  if (!ctx->pool_in) {
420  ret = AVERROR(ENOMEM);
421  goto fail;
422  }
423 
424  if ((ret = ffmal_update_format(avctx)) < 0)
425  goto fail;
426 
427  ctx->queue_decoded_frames = mmal_queue_create();
428  if (!ctx->queue_decoded_frames)
429  goto fail;
430 
431  decoder->input[0]->userdata = (void*)avctx;
432  decoder->output[0]->userdata = (void*)avctx;
433  decoder->control->userdata = (void*)avctx;
434 
435  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
436  goto fail;
437  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
438  goto fail;
439  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
440  goto fail;
441 
442  if ((status = mmal_component_enable(decoder)))
443  goto fail;
444 
445  return 0;
446 
447 fail:
448  ffmmal_close_decoder(avctx);
449  return ret < 0 ? ret : AVERROR_UNKNOWN;
450 }
451 
452 static void ffmmal_flush(AVCodecContext *avctx)
453 {
454  MMALDecodeContext *ctx = avctx->priv_data;
455  MMAL_COMPONENT_T *decoder = ctx->decoder;
456  MMAL_STATUS_T status;
457 
458  ffmmal_stop_decoder(avctx);
459 
460  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
461  goto fail;
462  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
463  goto fail;
464  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
465  goto fail;
466 
467  return;
468 
469 fail:
470  av_log(avctx, AV_LOG_ERROR, "MMAL flush error: %i\n", (int)status);
471 }
472 
473 // Split packets and add them to the waiting_buffers list. We don't queue them
474 // immediately, because it can happen that the decoder is temporarily blocked
475 // (due to us not reading/returning enough output buffers) and won't accept
476 // new input. (This wouldn't be an issue if MMAL input buffers always were
477 // complete frames - then the input buffer just would have to be big enough.)
478 // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG.
479 static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
480  int is_extradata)
481 {
482  MMALDecodeContext *ctx = avctx->priv_data;
483  AVBufferRef *buf = NULL;
484  int size = 0;
485  uint8_t *data = (uint8_t *)"";
486  uint8_t *start;
487  int ret = 0;
488 
489  if (avpkt->size) {
490  if (avpkt->buf) {
491  buf = av_buffer_ref(avpkt->buf);
492  size = avpkt->size;
493  data = avpkt->data;
494  } else {
495  buf = av_buffer_alloc(avpkt->size);
496  if (buf) {
497  memcpy(buf->data, avpkt->data, avpkt->size);
498  size = buf->size;
499  data = buf->data;
500  }
501  }
502  if (!buf) {
503  ret = AVERROR(ENOMEM);
504  goto done;
505  }
506  if (!is_extradata)
507  ctx->packets_sent++;
508  } else {
509  if (ctx->eos_sent)
510  goto done;
511  if (!ctx->packets_sent) {
512  // Short-cut the flush logic to avoid upsetting MMAL.
513  ctx->eos_sent = 1;
514  ctx->eos_received = 1;
515  goto done;
516  }
517  }
518 
519  start = data;
520 
521  do {
522  FFBufferEntry *buffer = av_mallocz(sizeof(*buffer));
523  if (!buffer) {
524  ret = AVERROR(ENOMEM);
525  goto done;
526  }
527 
528  buffer->data = data;
529  buffer->length = FFMIN(size, ctx->decoder->input[0]->buffer_size);
530 
531  if (is_extradata)
532  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_CONFIG;
533 
534  if (data == start)
535  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START;
536 
537  data += buffer->length;
538  size -= buffer->length;
539 
540  buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts;
541  buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts;
542 
543  if (!size) {
544  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
546  }
547 
548  if (!buffer->length) {
549  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS;
550  ctx->eos_sent = 1;
551  }
552 
553  if (buf) {
554  buffer->ref = av_buffer_ref(buf);
555  if (!buffer->ref) {
556  av_free(buffer);
557  ret = AVERROR(ENOMEM);
558  goto done;
559  }
560  }
561 
562  // Insert at end of the list
563  if (!ctx->waiting_buffers)
564  ctx->waiting_buffers = buffer;
565  if (ctx->waiting_buffers_tail)
568  } while (size);
569 
570 done:
571  av_buffer_unref(&buf);
572  return ret;
573 }
574 
575 // Move prepared/split packets from waiting_buffers to the MMAL decoder.
577 {
578  MMALDecodeContext *ctx = avctx->priv_data;
579 
580  while (ctx->waiting_buffers) {
581  MMAL_BUFFER_HEADER_T *mbuffer;
583  MMAL_STATUS_T status;
584 
585  mbuffer = mmal_queue_get(ctx->pool_in->queue);
586  if (!mbuffer)
587  return 0;
588 
589  buffer = ctx->waiting_buffers;
590 
591  mmal_buffer_header_reset(mbuffer);
592  mbuffer->cmd = 0;
593  mbuffer->pts = buffer->pts;
594  mbuffer->dts = buffer->dts;
595  mbuffer->flags = buffer->flags;
596  mbuffer->data = buffer->data;
597  mbuffer->length = buffer->length;
598  mbuffer->user_data = buffer;
599  mbuffer->alloc_size = ctx->decoder->input[0]->buffer_size;
600 
601  // Remove from start of the list
602  ctx->waiting_buffers = buffer->next;
603  if (ctx->waiting_buffers_tail == buffer)
604  ctx->waiting_buffers_tail = NULL;
605 
606  if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) {
607  mmal_buffer_header_release(mbuffer);
608  av_buffer_unref(&buffer->ref);
609  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
611  av_free(buffer);
612  }
613 
614  if (status) {
615  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending input\n", (int)status);
616  return AVERROR_UNKNOWN;
617  }
618  }
619 
620  return 0;
621 }
622 
624  MMAL_BUFFER_HEADER_T *buffer)
625 {
626  MMALDecodeContext *ctx = avctx->priv_data;
627  int ret = 0;
628 
629  frame->interlaced_frame = ctx->interlaced_frame;
630  frame->top_field_first = ctx->top_field_first;
631 
632  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
633  if (!ctx->pool_out)
634  return AVERROR_UNKNOWN; // format change code failed with OOM previously
635 
636  if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
637  goto done;
638 
639  if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0)
640  goto done;
641  } else {
642  int w = FFALIGN(avctx->width, 32);
643  int h = FFALIGN(avctx->height, 16);
644  uint8_t *src[4];
645  int linesize[4];
646 
647  if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
648  goto done;
649 
650  av_image_fill_arrays(src, linesize,
651  buffer->data + buffer->type->video.offset[0],
652  avctx->pix_fmt, w, h, 1);
653  av_image_copy(frame->data, frame->linesize, src, linesize,
654  avctx->pix_fmt, avctx->width, avctx->height);
655  }
656 
657  frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
658 #if FF_API_PKT_PTS
660  frame->pkt_pts = frame->pts;
662 #endif
663  frame->pkt_dts = AV_NOPTS_VALUE;
664 
665 done:
666  return ret;
667 }
668 
669 // Fetch a decoded buffer and place it into the frame parameter.
670 static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
671 {
672  MMALDecodeContext *ctx = avctx->priv_data;
673  MMAL_BUFFER_HEADER_T *buffer = NULL;
674  MMAL_STATUS_T status = 0;
675  int ret = 0;
676 
677  if (ctx->eos_received)
678  goto done;
679 
680  while (1) {
681  // To ensure decoding in lockstep with a constant delay between fed packets
682  // and output frames, we always wait until an output buffer is available.
683  // Except during start we don't know after how many input packets the decoder
684  // is going to return the first buffer, and we can't distinguish decoder
685  // being busy from decoder waiting for input. So just poll at the start and
686  // keep feeding new data to the buffer.
687  // We are pretty sure the decoder will produce output if we sent more input
688  // frames than what a H.264 decoder could logically delay. This avoids too
689  // excessive buffering.
690  // We also wait if we sent eos, but didn't receive it yet (think of decoding
691  // stream with a very low number of frames).
693  (ctx->packets_sent && ctx->eos_sent)) {
694  // MMAL will ignore broken input packets, which means the frame we
695  // expect here may never arrive. Dealing with this correctly is
696  // complicated, so here's a hack to avoid that it freezes forever
697  // in this unlikely situation.
698  buffer = mmal_queue_timedwait(ctx->queue_decoded_frames, 100);
699  if (!buffer) {
700  av_log(avctx, AV_LOG_ERROR, "Did not get output frame from MMAL.\n");
701  ret = AVERROR_UNKNOWN;
702  goto done;
703  }
704  } else {
705  buffer = mmal_queue_get(ctx->queue_decoded_frames);
706  if (!buffer)
707  goto done;
708  }
709 
710  ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
711  if (ctx->eos_received)
712  goto done;
713 
714  if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
715  MMAL_COMPONENT_T *decoder = ctx->decoder;
716  MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(buffer);
717  MMAL_BUFFER_HEADER_T *stale_buffer;
718 
719  av_log(avctx, AV_LOG_INFO, "Changing output format.\n");
720 
721  if ((status = mmal_port_disable(decoder->output[0])))
722  goto done;
723 
724  while ((stale_buffer = mmal_queue_get(ctx->queue_decoded_frames)))
725  mmal_buffer_header_release(stale_buffer);
726 
727  mmal_format_copy(decoder->output[0]->format, ev->format);
728 
729  if ((ret = ffmal_update_format(avctx)) < 0)
730  goto done;
731 
732  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
733  goto done;
734 
735  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
736  goto done;
737 
738  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
739  goto done;
740 
741  mmal_buffer_header_release(buffer);
742  continue;
743  } else if (buffer->cmd) {
744  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on output port\n",
745  av_fourcc2str(buffer->cmd));
746  goto done;
747  } else if (buffer->length == 0) {
748  // Unused output buffer that got drained after format change.
749  mmal_buffer_header_release(buffer);
750  continue;
751  }
752 
753  ctx->frames_output++;
754 
755  if ((ret = ffmal_copy_frame(avctx, frame, buffer)) < 0)
756  goto done;
757 
758  *got_frame = 1;
759  break;
760  }
761 
762 done:
763  if (buffer)
764  mmal_buffer_header_release(buffer);
765  if (status && ret >= 0)
766  ret = AVERROR_UNKNOWN;
767  return ret;
768 }
769 
770 static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
771  AVPacket *avpkt)
772 {
773  MMALDecodeContext *ctx = avctx->priv_data;
774  AVFrame *frame = data;
775  int ret = 0;
776 
777  if (avctx->extradata_size && !ctx->extradata_sent) {
778  AVPacket pkt = {0};
779  av_init_packet(&pkt);
780  pkt.data = avctx->extradata;
781  pkt.size = avctx->extradata_size;
782  ctx->extradata_sent = 1;
783  if ((ret = ffmmal_add_packet(avctx, &pkt, 1)) < 0)
784  return ret;
785  }
786 
787  if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0)
788  return ret;
789 
790  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
791  return ret;
792 
793  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
794  return ret;
795 
796  if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0)
797  return ret;
798 
799  // ffmmal_read_frame() can block for a while. Since the decoder is
800  // asynchronous, it's a good idea to fill the ports again.
801 
802  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
803  return ret;
804 
805  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
806  return ret;
807 
808  return ret;
809 }
810 
812  HW_CONFIG_INTERNAL(MMAL),
813  NULL
814 };
815 
816 static const AVOption options[]={
817  {"extra_buffers", "extra buffers", offsetof(MMALDecodeContext, extra_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
818  {"extra_decoder_buffers", "extra MMAL internal buffered frames", offsetof(MMALDecodeContext, extra_decoder_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
819  {NULL}
820 };
821 
822 #define FFMMAL_DEC_CLASS(NAME) \
823  static const AVClass ffmmal_##NAME##_dec_class = { \
824  .class_name = "mmal_" #NAME "_dec", \
825  .item_name = av_default_item_name, \
826  .option = options, \
827  .version = LIBAVUTIL_VERSION_INT, \
828  };
829 
830 #define FFMMAL_DEC(NAME, ID) \
831  FFMMAL_DEC_CLASS(NAME) \
832  AVCodec ff_##NAME##_mmal_decoder = { \
833  .name = #NAME "_mmal", \
834  .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \
835  .type = AVMEDIA_TYPE_VIDEO, \
836  .id = ID, \
837  .priv_data_size = sizeof(MMALDecodeContext), \
838  .init = ffmmal_init_decoder, \
839  .close = ffmmal_close_decoder, \
840  .decode = ffmmal_decode, \
841  .flush = ffmmal_flush, \
842  .priv_class = &ffmmal_##NAME##_dec_class, \
843  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, \
844  .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
845  .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \
846  AV_PIX_FMT_YUV420P, \
847  AV_PIX_FMT_NONE}, \
848  .hw_configs = mmal_hw_configs, \
849  .wrapper_name = "mmal", \
850  };
851 
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:498
#define NULL
Definition: coverity.c:32
int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
Select the (possibly hardware accelerated) pixel format.
Definition: decode.c:1370
const struct AVCodec * codec
Definition: avcodec.h:1574
AVRational framerate
Definition: avcodec.h:3105
MMAL_POOL_T * pool
Definition: mmaldec.c:60
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
Definition: buffer.c:125
This structure describes decoded (raw) audio or video data.
Definition: frame.h:295
#define FFMMAL_DEC(NAME, ID)
Definition: mmaldec.c:830
AVOption.
Definition: opt.h:246
ptrdiff_t const GLvoid * data
Definition: opengl_enc.c:100
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
int ff_set_dimensions(AVCodecContext *s, int width, int height)
Check that the provided frame dimensions are valid and set them on the codec context.
Definition: utils.c:104
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:486
atomic_int packets_buffered
Definition: mmaldec.c:87
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:502
FFPoolRef * pool
Definition: mmaldec.c:65
uint32_t fourcc
Definition: vaapi_decode.c:238
int num
Numerator.
Definition: rational.h:59
int size
Definition: avcodec.h:1478
int av_image_fill_arrays(uint8_t *dst_data[4], int dst_linesize[4], const uint8_t *src, enum AVPixelFormat pix_fmt, int width, int height, int align)
Setup the data pointers and linesizes based on the specified image parameters and the provided array...
Definition: imgutils.c:411
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:1944
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:1775
static int ffmmal_fill_output_port(AVCodecContext *avctx)
Definition: mmaldec.c:240
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:236
int ff_decode_frame_props(AVCodecContext *avctx, AVFrame *frame)
Set various frame properties from the codec context / packet data.
Definition: decode.c:1737
static AVPacket pkt
atomic_int refcount
Definition: mmaldec.c:59
intptr_t atomic_int
Definition: stdatomic.h:55
int64_t dts
Definition: mmaldec.c:50
#define src
Definition: vp8dsp.c:254
static int ffmmal_fill_input_port(AVCodecContext *avctx)
Definition: mmaldec.c:576
static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
Definition: mmaldec.c:770
functionally identical to above
Definition: pixfmt.h:504
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
uint8_t
#define av_cold
Definition: attributes.h:82
AVOptions.
MMAL_BUFFER_HEADER_T * buffer
Definition: mmaldec.c:64
AVColorSpace
YUV colorspace type.
Definition: pixfmt.h:496
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:388
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:1666
static void ffmmal_flush(AVCodecContext *avctx)
Definition: mmaldec.c:452
uint8_t * data
Definition: avcodec.h:1477
static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:349
static const AVOption options[]
Definition: mmaldec.c:816
static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:182
int interlaced_frame
The content of the picture is interlaced.
Definition: frame.h:442
#define AV_BUFFER_FLAG_READONLY
Always treat the buffer as read-only, even when it has only one reference.
Definition: buffer.h:113
MMAL_POOL_T * pool_in
Definition: mmaldec.c:75
ptrdiff_t size
Definition: opengl_enc.c:100
int top_field_first
Definition: mmaldec.c:93
#define FFALIGN(x, a)
Definition: macros.h:48
#define av_log(a,...)
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:215
#define atomic_load(object)
Definition: stdatomic.h:93
int extradata_sent
Definition: mmaldec.c:91
#define HW_CONFIG_INTERNAL(format)
Definition: hwaccel.h:57
static void ffmmal_poolref_unref(FFPoolRef *ref)
Definition: mmaldec.c:100
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:197
#define av_fourcc2str(fourcc)
Definition: avutil.h:348
AVBufferRef * buf
A reference to the reference-counted buffer where the packet data is stored.
Definition: avcodec.h:1460
simple assert() macros that are a bit more flexible than ISO C assert().
static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:121
FFBufferEntry * waiting_buffers_tail
Definition: mmaldec.c:84
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:28
int extra_decoder_buffers
Definition: mmaldec.c:71
#define FFMAX(a, b)
Definition: common.h:94
#define fail()
Definition: checkasm.h:120
static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:223
void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], const uint8_t *src_data[4], const int src_linesizes[4], enum AVPixelFormat pix_fmt, int width, int height)
Copy image in src_data to dst_data.
Definition: imgutils.c:387
AVBufferRef * ref
Definition: mmaldec.c:47
size_t length
Definition: mmaldec.c:49
static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:623
AVClass * av_class
Definition: mmaldec.c:69
static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
Definition: mmaldec.c:260
enum AVPixelFormat * pix_fmts
array of supported pixel formats, or NULL if unknown, array is terminated by -1
Definition: avcodec.h:3502
FFPoolRef * pool_out
Definition: mmaldec.c:76
#define FFMIN(a, b)
Definition: common.h:96
FFBufferEntry * waiting_buffers
Definition: mmaldec.c:84
int64_t frames_output
Definition: mmaldec.c:88
static const chunk_decoder decoder[8]
Definition: dfa.c:330
int width
picture width / height.
Definition: avcodec.h:1738
uint8_t w
Definition: llviddspenc.c:38
AVFormatContext * ctx
Definition: movenc.c:48
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
#define MAX_DELAYED_FRAMES
Definition: mmaldec.c:98
preferred ID for MPEG-1/2 video decoding
Definition: avcodec.h:220
FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
Definition: pixfmt.h:501
static void ffmmal_stop_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:147
MMAL_QUEUE_T * queue_decoded_frames
Definition: mmaldec.c:74
if(ret)
#define atomic_fetch_add_explicit(object, operand, order)
Definition: stdatomic.h:149
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:368
#define atomic_fetch_add(object, operand)
Definition: stdatomic.h:131
int interlaced_frame
Definition: mmaldec.c:92
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
struct FFBufferEntry * next
Definition: mmaldec.c:52
Libavcodec external API header.
enum AVCodecID codec_id
Definition: avcodec.h:1575
AVBufferRef * av_buffer_alloc(int size)
Allocate an AVBuffer of the given size using av_malloc().
Definition: buffer.c:67
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:326
main external API structure.
Definition: avcodec.h:1565
uint8_t * data
The data buffer.
Definition: buffer.h:89
void * data
Definition: mmaldec.c:48
static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:200
int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
Get a buffer for a frame.
Definition: decode.c:1963
void * buf
Definition: avisynth_c.h:766
int extradata_size
Definition: avcodec.h:1667
Describe the class of an AVClass context structure.
Definition: log.h:67
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:2193
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
refcounted data buffer API
int size
Size of data in bytes.
Definition: buffer.h:93
static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
Definition: mmaldec.c:670
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:309
attribute_deprecated int64_t pkt_pts
PTS copied from the AVPacket that was decoded to produce this frame.
Definition: frame.h:396
int64_t packets_sent
Definition: mmaldec.c:86
int64_t pkt_dts
DTS copied from the AVPacket that triggered returning this frame.
Definition: frame.h:404
A reference to a data buffer.
Definition: buffer.h:81
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:84
common internal api header.
common internal and external API header
static int ffmal_update_format(AVCodecContext *avctx)
Definition: mmaldec.c:273
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:93
void av_init_packet(AVPacket *pkt)
Initialize optional fields of a packet with default values.
Definition: avpacket.c:33
int den
Denominator.
Definition: rational.h:60
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:71
static void ffmmal_release_frame(void *opaque, uint8_t *data)
Definition: mmaldec.c:109
void * priv_data
Definition: avcodec.h:1592
#define av_free(p)
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:85
int top_field_first
If the content is interlaced, is top field displayed first.
Definition: frame.h:447
static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt, int is_extradata)
Definition: mmaldec.c:479
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: avcodec.h:1476
#define atomic_init(obj, value)
Definition: stdatomic.h:33
void INT64 start
Definition: avisynth_c.h:766
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
HW acceleration though MMAL, data[3] contains a pointer to the MMAL_BUFFER_HEADER_T structure...
Definition: pixfmt.h:227
This structure stores compressed data.
Definition: avcodec.h:1454
MMAL_COMPONENT_T * decoder
Definition: mmaldec.c:73
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1470
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248
static const AVCodecHWConfigInternal * mmal_hw_configs[]
Definition: mmaldec.c:811
GLuint buffer
Definition: opengl_enc.c:101
int64_t pts
Definition: mmaldec.c:50