Libav
mmaldec.c
Go to the documentation of this file.
1 /*
2  * MMAL Video Decoder
3  * Copyright (c) 2015 Rodger Combs
4  *
5  * This file is part of Libav.
6  *
7  * Libav is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * Libav is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with Libav; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
27 #include <bcm_host.h>
28 #include <interface/mmal/mmal.h>
29 #include <interface/mmal/mmal_parameters_video.h>
30 #include <interface/mmal/util/mmal_util.h>
31 #include <interface/mmal/util/mmal_util_params.h>
32 #include <interface/mmal/util/mmal_default_components.h>
33 #include <interface/mmal/vc/mmal_vc_api.h>
34 
35 #include "avcodec.h"
36 #include "internal.h"
37 #include "libavutil/atomic.h"
38 #include "libavutil/avassert.h"
39 #include "libavutil/buffer.h"
40 #include "libavutil/common.h"
41 #include "libavutil/imgutils.h"
42 #include "libavutil/opt.h"
43 #include "libavutil/log.h"
44 
45 typedef struct FFBufferEntry {
47  void *data;
48  size_t length;
49  int64_t pts, dts;
50  int flags;
53 
54 // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct
55 // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames
56 // have been unreferenced.
57 typedef struct FFPoolRef {
58  volatile int refcount;
59  MMAL_POOL_T *pool;
60 } FFPoolRef;
61 
62 typedef struct FFBufferRef {
63  MMAL_BUFFER_HEADER_T *buffer;
65 } FFBufferRef;
66 
67 typedef struct MMALDecodeContext {
71 
72  MMAL_COMPONENT_T *decoder;
73  MMAL_QUEUE_T *queue_decoded_frames;
74  MMAL_POOL_T *pool_in;
76 
77  // Waiting input packets. Because the libavcodec API requires decoding and
78  // returning packets in lockstep, it can happen that queue_decoded_frames
79  // contains almost all surfaces - then the decoder input queue can quickly
80  // fill up and won't accept new input either. Without consuming input, the
81  // libavcodec API can't return new frames, and we have a logical deadlock.
82  // This is avoided by queuing such buffers here.
83  FFBufferEntry *waiting_buffers, *waiting_buffers_tail;
84 
85  int64_t packets_sent;
86  volatile int packets_buffered;
87  int64_t frames_output;
89  int eos_sent;
92 
93 // Assume decoder is guaranteed to produce output after at least this many
94 // packets (where each packet contains 1 frame).
95 #define MAX_DELAYED_FRAMES 16
96 
98 {
99  if (ref && avpriv_atomic_int_add_and_fetch(&ref->refcount, -1) == 0) {
100  mmal_pool_destroy(ref->pool);
101  av_free(ref);
102  }
103 }
104 
105 static void ffmmal_release_frame(void *opaque, uint8_t *data)
106 {
107  FFBufferRef *ref = (void *)data;
108 
109  mmal_buffer_header_release(ref->buffer);
111 
112  av_free(ref);
113 }
114 
115 // Setup frame with a new reference to buffer. The buffer must have been
116 // allocated from the given pool.
117 static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool,
118  MMAL_BUFFER_HEADER_T *buffer)
119 {
120  FFBufferRef *ref = av_mallocz(sizeof(*ref));
121  if (!ref)
122  return AVERROR(ENOMEM);
123 
124  ref->pool = pool;
125  ref->buffer = buffer;
126 
127  frame->buf[0] = av_buffer_create((void *)ref, sizeof(*ref),
130  if (!frame->buf[0]) {
131  av_free(ref);
132  return AVERROR(ENOMEM);
133  }
134 
135  avpriv_atomic_int_add_and_fetch(&ref->pool->refcount, 1);
136  mmal_buffer_header_acquire(buffer);
137 
138  frame->format = AV_PIX_FMT_MMAL;
139  frame->data[3] = (uint8_t *)ref->buffer;
140  return 0;
141 }
142 
144 {
145  MMALDecodeContext *ctx = avctx->priv_data;
146  MMAL_COMPONENT_T *decoder = ctx->decoder;
147  MMAL_BUFFER_HEADER_T *buffer;
148 
149  mmal_port_disable(decoder->input[0]);
150  mmal_port_disable(decoder->output[0]);
151  mmal_port_disable(decoder->control);
152 
153  mmal_port_flush(decoder->input[0]);
154  mmal_port_flush(decoder->output[0]);
155  mmal_port_flush(decoder->control);
156 
157  while ((buffer = mmal_queue_get(ctx->queue_decoded_frames)))
158  mmal_buffer_header_release(buffer);
159 
160  while (ctx->waiting_buffers) {
161  FFBufferEntry *buffer = ctx->waiting_buffers;
162 
163  ctx->waiting_buffers = buffer->next;
164 
165  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
167 
168  av_buffer_unref(&buffer->ref);
169  av_free(buffer);
170  }
171  ctx->waiting_buffers_tail = NULL;
172 
174 
175  ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0;
176 }
177 
179 {
180  MMALDecodeContext *ctx = avctx->priv_data;
181 
182  if (ctx->decoder)
183  ffmmal_stop_decoder(avctx);
184 
185  mmal_component_destroy(ctx->decoder);
186  ctx->decoder = NULL;
187  mmal_queue_destroy(ctx->queue_decoded_frames);
188  mmal_pool_destroy(ctx->pool_in);
190 
191  mmal_vc_deinit();
192 
193  return 0;
194 }
195 
196 static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
197 {
198  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
199  MMALDecodeContext *ctx = avctx->priv_data;
200 
201  if (!buffer->cmd) {
202  FFBufferEntry *entry = buffer->user_data;
203  av_buffer_unref(&entry->ref);
204  if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
205  avpriv_atomic_int_add_and_fetch(&ctx->packets_buffered, -1);
206  av_free(entry);
207  }
208  mmal_buffer_header_release(buffer);
209 }
210 
211 static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
212 {
213  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
214  MMALDecodeContext *ctx = avctx->priv_data;
215 
216  mmal_queue_put(ctx->queue_decoded_frames, buffer);
217 }
218 
219 static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
220 {
221  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
222  MMAL_STATUS_T status;
223 
224  if (buffer->cmd == MMAL_EVENT_ERROR) {
225  status = *(uint32_t *)buffer->data;
226  av_log(avctx, AV_LOG_ERROR, "MMAL error %d on control port\n", (int)status);
227  } else {
228  char s[20];
229  av_get_codec_tag_string(s, sizeof(s), buffer->cmd);
230  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on control port\n", s);
231  }
232 
233  mmal_buffer_header_release(buffer);
234 }
235 
236 // Feed free output buffers to the decoder.
238 {
239  MMALDecodeContext *ctx = avctx->priv_data;
240  MMAL_BUFFER_HEADER_T *buffer;
241  MMAL_STATUS_T status;
242 
243  if (!ctx->pool_out)
244  return AVERROR_UNKNOWN; // format change code failed with OOM previously
245 
246  while ((buffer = mmal_queue_get(ctx->pool_out->pool->queue))) {
247  if ((status = mmal_port_send_buffer(ctx->decoder->output[0], buffer))) {
248  mmal_buffer_header_release(buffer);
249  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending output buffer.\n", (int)status);
250  return AVERROR_UNKNOWN;
251  }
252  }
253 
254  return 0;
255 }
256 
257 static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
258 {
259  switch (fourcc) {
260  case MMAL_COLOR_SPACE_BT470_2_BG:
261  case MMAL_COLOR_SPACE_BT470_2_M:
262  case MMAL_COLOR_SPACE_ITUR_BT601: return AVCOL_SPC_BT470BG;
263  case MMAL_COLOR_SPACE_ITUR_BT709: return AVCOL_SPC_BT709;
264  case MMAL_COLOR_SPACE_FCC: return AVCOL_SPC_FCC;
265  case MMAL_COLOR_SPACE_SMPTE240M: return AVCOL_SPC_SMPTE240M;
266  default: return AVCOL_SPC_UNSPECIFIED;
267  }
268 }
269 
271 {
272  MMALDecodeContext *ctx = avctx->priv_data;
273  MMAL_STATUS_T status;
274  int ret = 0;
275  MMAL_COMPONENT_T *decoder = ctx->decoder;
276  MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
277 
279  if (!(ctx->pool_out = av_mallocz(sizeof(*ctx->pool_out)))) {
280  ret = AVERROR(ENOMEM);
281  goto fail;
282  }
283  ctx->pool_out->refcount = 1;
284 
285  if (!format_out)
286  goto fail;
287 
288  if ((status = mmal_port_parameter_set_uint32(decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS, ctx->extra_buffers)))
289  goto fail;
290 
291  if ((status = mmal_port_parameter_set_boolean(decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0)))
292  goto fail;
293 
294  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
295  format_out->encoding = MMAL_ENCODING_OPAQUE;
296  } else {
297  format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420;
298  }
299 
300  if ((status = mmal_port_format_commit(decoder->output[0])))
301  goto fail;
302 
303  if ((ret = ff_set_dimensions(avctx, format_out->es->video.crop.x + format_out->es->video.crop.width,
304  format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0)
305  goto fail;
306 
307  if (format_out->es->video.par.num && format_out->es->video.par.den) {
308  avctx->sample_aspect_ratio.num = format_out->es->video.par.num;
309  avctx->sample_aspect_ratio.den = format_out->es->video.par.den;
310  }
311 
312  avctx->colorspace = ffmmal_csp_to_av_csp(format_out->es->video.color_space);
313 
314  decoder->output[0]->buffer_size =
315  FFMAX(decoder->output[0]->buffer_size_min, decoder->output[0]->buffer_size_recommended);
316  decoder->output[0]->buffer_num =
317  FFMAX(decoder->output[0]->buffer_num_min, decoder->output[0]->buffer_num_recommended) + ctx->extra_buffers;
318  ctx->pool_out->pool = mmal_pool_create(decoder->output[0]->buffer_num,
319  decoder->output[0]->buffer_size);
320  if (!ctx->pool_out->pool) {
321  ret = AVERROR(ENOMEM);
322  goto fail;
323  }
324 
325  return 0;
326 
327 fail:
328  return ret < 0 ? ret : AVERROR_UNKNOWN;
329 }
330 
332 {
333  MMALDecodeContext *ctx = avctx->priv_data;
334  MMAL_STATUS_T status;
335  MMAL_ES_FORMAT_T *format_in;
336  MMAL_COMPONENT_T *decoder;
337  char tmp[32];
338  int ret = 0;
339 
340  bcm_host_init();
341 
342  if (mmal_vc_init()) {
343  av_log(avctx, AV_LOG_ERROR, "Cannot initialize MMAL VC driver!\n");
344  return AVERROR(ENOSYS);
345  }
346 
347  if ((ret = ff_get_format(avctx, avctx->codec->pix_fmts)) < 0)
348  return ret;
349 
350  avctx->pix_fmt = ret;
351 
352  if ((status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &ctx->decoder)))
353  goto fail;
354 
355  decoder = ctx->decoder;
356 
357  format_in = decoder->input[0]->format;
358  format_in->type = MMAL_ES_TYPE_VIDEO;
359  switch (avctx->codec_id) {
361  format_in->encoding = MMAL_ENCODING_MP2V;
362  break;
363  case AV_CODEC_ID_VC1:
364  format_in->encoding = MMAL_ENCODING_WVC1;
365  break;
366  case AV_CODEC_ID_H264:
367  default:
368  format_in->encoding = MMAL_ENCODING_H264;
369  break;
370  }
371  format_in->es->video.width = FFALIGN(avctx->width, 32);
372  format_in->es->video.height = FFALIGN(avctx->height, 16);
373  format_in->es->video.crop.width = avctx->width;
374  format_in->es->video.crop.height = avctx->height;
375  format_in->es->video.frame_rate.num = 24000;
376  format_in->es->video.frame_rate.den = 1001;
377  format_in->es->video.par.num = avctx->sample_aspect_ratio.num;
378  format_in->es->video.par.den = avctx->sample_aspect_ratio.den;
379  format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;
380 
381  av_get_codec_tag_string(tmp, sizeof(tmp), format_in->encoding);
382  av_log(avctx, AV_LOG_DEBUG, "Using MMAL %s encoding.\n", tmp);
383 
384 #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
385  if (mmal_port_parameter_set_uint32(decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS,
386  -1 - ctx->extra_decoder_buffers)) {
387  av_log(avctx, AV_LOG_WARNING, "Could not set input buffering limit.\n");
388  }
389 #endif
390 
391  if ((status = mmal_port_format_commit(decoder->input[0])))
392  goto fail;
393 
394  decoder->input[0]->buffer_num =
395  FFMAX(decoder->input[0]->buffer_num_min, 20);
396  decoder->input[0]->buffer_size =
397  FFMAX(decoder->input[0]->buffer_size_min, 512 * 1024);
398  ctx->pool_in = mmal_pool_create(decoder->input[0]->buffer_num, 0);
399  if (!ctx->pool_in) {
400  ret = AVERROR(ENOMEM);
401  goto fail;
402  }
403 
404  if ((ret = ffmal_update_format(avctx)) < 0)
405  goto fail;
406 
407  ctx->queue_decoded_frames = mmal_queue_create();
408  if (!ctx->queue_decoded_frames)
409  goto fail;
410 
411  decoder->input[0]->userdata = (void*)avctx;
412  decoder->output[0]->userdata = (void*)avctx;
413  decoder->control->userdata = (void*)avctx;
414 
415  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
416  goto fail;
417  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
418  goto fail;
419  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
420  goto fail;
421 
422  if ((status = mmal_component_enable(decoder)))
423  goto fail;
424 
425  return 0;
426 
427 fail:
428  ffmmal_close_decoder(avctx);
429  return ret < 0 ? ret : AVERROR_UNKNOWN;
430 }
431 
432 static void ffmmal_flush(AVCodecContext *avctx)
433 {
434  MMALDecodeContext *ctx = avctx->priv_data;
435  MMAL_COMPONENT_T *decoder = ctx->decoder;
436  MMAL_STATUS_T status;
437 
438  ffmmal_stop_decoder(avctx);
439 
440  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
441  goto fail;
442  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
443  goto fail;
444  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
445  goto fail;
446 
447  return;
448 
449 fail:
450  av_log(avctx, AV_LOG_ERROR, "MMAL flush error: %i\n", (int)status);
451 }
452 
453 // Split packets and add them to the waiting_buffers list. We don't queue them
454 // immediately, because it can happen that the decoder is temporarily blocked
455 // (due to us not reading/returning enough output buffers) and won't accept
456 // new input. (This wouldn't be an issue if MMAL input buffers always were
457 // complete frames - then the input buffer just would have to be big enough.)
458 // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG.
459 static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
460  int is_extradata)
461 {
462  MMALDecodeContext *ctx = avctx->priv_data;
463  AVBufferRef *buf = NULL;
464  int size = 0;
465  uint8_t *data = (uint8_t *)"";
466  uint8_t *start;
467  int ret = 0;
468 
469  if (avpkt->size) {
470  if (avpkt->buf) {
471  buf = av_buffer_ref(avpkt->buf);
472  size = avpkt->size;
473  data = avpkt->data;
474  } else {
475  buf = av_buffer_alloc(avpkt->size);
476  if (buf) {
477  memcpy(buf->data, avpkt->data, avpkt->size);
478  size = buf->size;
479  data = buf->data;
480  }
481  }
482  if (!buf) {
483  ret = AVERROR(ENOMEM);
484  goto done;
485  }
486  if (!is_extradata)
487  ctx->packets_sent++;
488  } else {
489  if (ctx->eos_sent)
490  goto done;
491  if (!ctx->packets_sent) {
492  // Short-cut the flush logic to avoid upsetting MMAL.
493  ctx->eos_sent = 1;
494  ctx->eos_received = 1;
495  goto done;
496  }
497  }
498 
499  start = data;
500 
501  do {
502  FFBufferEntry *buffer = av_mallocz(sizeof(*buffer));
503  if (!buffer) {
504  ret = AVERROR(ENOMEM);
505  goto done;
506  }
507 
508  buffer->data = data;
509  buffer->length = FFMIN(size, ctx->decoder->input[0]->buffer_size);
510 
511  if (is_extradata)
512  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_CONFIG;
513 
514  if (data == start)
515  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START;
516 
517  data += buffer->length;
518  size -= buffer->length;
519 
520  buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts;
521  buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts;
522 
523  if (!size) {
524  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
526  }
527 
528  if (!buffer->length) {
529  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS;
530  ctx->eos_sent = 1;
531  }
532 
533  if (buf) {
534  buffer->ref = av_buffer_ref(buf);
535  if (!buffer->ref) {
536  av_free(buffer);
537  ret = AVERROR(ENOMEM);
538  goto done;
539  }
540  }
541 
542  // Insert at end of the list
543  if (!ctx->waiting_buffers)
544  ctx->waiting_buffers = buffer;
545  if (ctx->waiting_buffers_tail)
548  } while (size);
549 
550 done:
551  av_buffer_unref(&buf);
552  return ret;
553 }
554 
555 // Move prepared/split packets from waiting_buffers to the MMAL decoder.
557 {
558  MMALDecodeContext *ctx = avctx->priv_data;
559 
560  while (ctx->waiting_buffers) {
561  MMAL_BUFFER_HEADER_T *mbuffer;
563  MMAL_STATUS_T status;
564 
565  mbuffer = mmal_queue_get(ctx->pool_in->queue);
566  if (!mbuffer)
567  return 0;
568 
569  buffer = ctx->waiting_buffers;
570 
571  mmal_buffer_header_reset(mbuffer);
572  mbuffer->cmd = 0;
573  mbuffer->pts = buffer->pts;
574  mbuffer->dts = buffer->dts;
575  mbuffer->flags = buffer->flags;
576  mbuffer->data = buffer->data;
577  mbuffer->length = buffer->length;
578  mbuffer->user_data = buffer;
579  mbuffer->alloc_size = ctx->decoder->input[0]->buffer_size;
580 
581  // Remove from start of the list
582  ctx->waiting_buffers = buffer->next;
583  if (ctx->waiting_buffers_tail == buffer)
584  ctx->waiting_buffers_tail = NULL;
585 
586  if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) {
587  mmal_buffer_header_release(mbuffer);
588  av_buffer_unref(&buffer->ref);
589  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
591  av_free(buffer);
592  }
593 
594  if (status) {
595  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending input\n", (int)status);
596  return AVERROR_UNKNOWN;
597  }
598  }
599 
600  return 0;
601 }
602 
603 static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame,
604  MMAL_BUFFER_HEADER_T *buffer)
605 {
606  MMALDecodeContext *ctx = avctx->priv_data;
607  int ret = 0;
608 
609  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
610  if (!ctx->pool_out)
611  return AVERROR_UNKNOWN; // format change code failed with OOM previously
612 
613  if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
614  goto done;
615 
616  if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0)
617  goto done;
618  } else {
619  int w = FFALIGN(avctx->width, 32);
620  int h = FFALIGN(avctx->height, 16);
621  uint8_t *src[4];
622  int linesize[4];
623 
624  if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
625  goto done;
626 
627  av_image_fill_arrays(src, linesize,
628  buffer->data + buffer->type->video.offset[0],
629  avctx->pix_fmt, w, h, 1);
630  av_image_copy(frame->data, frame->linesize, src, linesize,
631  avctx->pix_fmt, avctx->width, avctx->height);
632  }
633 
634  frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
635 #if FF_API_PKT_PTS
637  frame->pkt_pts = frame->pts;
639 #endif
640  frame->pkt_dts = AV_NOPTS_VALUE;
641 
642 done:
643  return ret;
644 }
645 
646 // Fetch a decoded buffer and place it into the frame parameter.
647 static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
648 {
649  MMALDecodeContext *ctx = avctx->priv_data;
650  MMAL_BUFFER_HEADER_T *buffer = NULL;
651  MMAL_STATUS_T status = 0;
652  int ret = 0;
653 
654  if (ctx->eos_received)
655  goto done;
656 
657  while (1) {
658  // To ensure decoding in lockstep with a constant delay between fed packets
659  // and output frames, we always wait until an output buffer is available.
660  // Except during start we don't know after how many input packets the decoder
661  // is going to return the first buffer, and we can't distinguish decoder
662  // being busy from decoder waiting for input. So just poll at the start and
663  // keep feeding new data to the buffer.
664  // We are pretty sure the decoder will produce output if we sent more input
665  // frames than what a H.264 decoder could logically delay. This avoids too
666  // excessive buffering.
667  // We also wait if we sent eos, but didn't receive it yet (think of decoding
668  // stream with a very low number of frames).
670  (ctx->packets_sent && ctx->eos_sent)) {
671  // MMAL will ignore broken input packets, which means the frame we
672  // expect here may never arrive. Dealing with this correctly is
673  // complicated, so here's a hack to avoid that it freezes forever
674  // in this unlikely situation.
675  buffer = mmal_queue_timedwait(ctx->queue_decoded_frames, 100);
676  if (!buffer) {
677  av_log(avctx, AV_LOG_ERROR, "Did not get output frame from MMAL.\n");
678  ret = AVERROR_UNKNOWN;
679  goto done;
680  }
681  } else {
682  buffer = mmal_queue_get(ctx->queue_decoded_frames);
683  if (!buffer)
684  goto done;
685  }
686 
687  ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
688  if (ctx->eos_received)
689  goto done;
690 
691  if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
692  MMAL_COMPONENT_T *decoder = ctx->decoder;
693  MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(buffer);
694  MMAL_BUFFER_HEADER_T *stale_buffer;
695 
696  av_log(avctx, AV_LOG_INFO, "Changing output format.\n");
697 
698  if ((status = mmal_port_disable(decoder->output[0])))
699  goto done;
700 
701  while ((stale_buffer = mmal_queue_get(ctx->queue_decoded_frames)))
702  mmal_buffer_header_release(stale_buffer);
703 
704  mmal_format_copy(decoder->output[0]->format, ev->format);
705 
706  if ((ret = ffmal_update_format(avctx)) < 0)
707  goto done;
708 
709  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
710  goto done;
711 
712  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
713  goto done;
714 
715  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
716  goto done;
717 
718  mmal_buffer_header_release(buffer);
719  continue;
720  } else if (buffer->cmd) {
721  char s[20];
722  av_get_codec_tag_string(s, sizeof(s), buffer->cmd);
723  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on output port\n", s);
724  goto done;
725  } else if (buffer->length == 0) {
726  // Unused output buffer that got drained after format change.
727  mmal_buffer_header_release(buffer);
728  continue;
729  }
730 
731  ctx->frames_output++;
732 
733  if ((ret = ffmal_copy_frame(avctx, frame, buffer)) < 0)
734  goto done;
735 
736  *got_frame = 1;
737  break;
738  }
739 
740 done:
741  if (buffer)
742  mmal_buffer_header_release(buffer);
743  if (status && ret >= 0)
744  ret = AVERROR_UNKNOWN;
745  return ret;
746 }
747 
748 static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
749  AVPacket *avpkt)
750 {
751  MMALDecodeContext *ctx = avctx->priv_data;
752  AVFrame *frame = data;
753  int ret = 0;
754 
755  if (avctx->extradata_size && !ctx->extradata_sent) {
756  AVPacket pkt = {0};
757  av_init_packet(&pkt);
758  pkt.data = avctx->extradata;
759  pkt.size = avctx->extradata_size;
760  ctx->extradata_sent = 1;
761  if ((ret = ffmmal_add_packet(avctx, &pkt, 1)) < 0)
762  return ret;
763  }
764 
765  if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0)
766  return ret;
767 
768  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
769  return ret;
770 
771  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
772  return ret;
773 
774  if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0)
775  return ret;
776 
777  // ffmmal_read_frame() can block for a while. Since the decoder is
778  // asynchronous, it's a good idea to fill the ports again.
779 
780  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
781  return ret;
782 
783  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
784  return ret;
785 
786  return ret;
787 }
788 
790  .name = "h264_mmal",
791  .type = AVMEDIA_TYPE_VIDEO,
792  .id = AV_CODEC_ID_H264,
793  .pix_fmt = AV_PIX_FMT_MMAL,
794 };
795 
797  .name = "mpeg2_mmal",
798  .type = AVMEDIA_TYPE_VIDEO,
800  .pix_fmt = AV_PIX_FMT_MMAL,
801 };
802 
804  .name = "vc1_mmal",
805  .type = AVMEDIA_TYPE_VIDEO,
806  .id = AV_CODEC_ID_VC1,
807  .pix_fmt = AV_PIX_FMT_MMAL,
808 };
809 
810 static const AVOption options[]={
811  {"extra_buffers", "extra buffers", offsetof(MMALDecodeContext, extra_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
812  {"extra_decoder_buffers", "extra MMAL internal buffered frames", offsetof(MMALDecodeContext, extra_decoder_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
813  {NULL}
814 };
815 
816 #define FFMMAL_DEC_CLASS(NAME) \
817  static const AVClass ffmmal_##NAME##_dec_class = { \
818  .class_name = "mmal_" #NAME "_dec", \
819  .option = options, \
820  .version = LIBAVUTIL_VERSION_INT, \
821  };
822 
823 #define FFMMAL_DEC(NAME, ID) \
824  FFMMAL_DEC_CLASS(NAME) \
825  AVCodec ff_##NAME##_mmal_decoder = { \
826  .name = #NAME "_mmal", \
827  .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \
828  .type = AVMEDIA_TYPE_VIDEO, \
829  .id = ID, \
830  .priv_data_size = sizeof(MMALDecodeContext), \
831  .init = ffmmal_init_decoder, \
832  .close = ffmmal_close_decoder, \
833  .decode = ffmmal_decode, \
834  .flush = ffmmal_flush, \
835  .priv_class = &ffmmal_##NAME##_dec_class, \
836  .capabilities = AV_CODEC_CAP_DELAY, \
837  .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
838  .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \
839  AV_PIX_FMT_YUV420P, \
840  AV_PIX_FMT_NONE}, \
841  };
842 
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:345
const struct AVCodec * codec
Definition: avcodec.h:1418
#define avpriv_atomic_int_add_and_fetch
Definition: atomic_gcc.h:42
uint32_t fourcc
Definition: hwcontext_qsv.c:90
MMAL_POOL_T * pool
Definition: mmaldec.c:59
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
Definition: buffer.c:106
int size
This structure describes decoded (raw) audio or video data.
Definition: frame.h:140
#define FFMMAL_DEC(NAME, ID)
Definition: mmaldec.c:823
AVOption.
Definition: opt.h:234
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:130
int ff_set_dimensions(AVCodecContext *s, int width, int height)
Check that the provided frame dimensions are valid and set them on the codec context.
Definition: utils.c:134
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:308
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:349
FFPoolRef * pool
Definition: mmaldec.c:64
int num
numerator
Definition: rational.h:44
int size
Definition: avcodec.h:1347
av_log(ac->avr, AV_LOG_TRACE, "%d samples - audio_convert: %s to %s (%s)\, len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt), use_generic ? ac->func_descr_generic :ac->func_descr)
int av_image_fill_arrays(uint8_t *dst_data[4], int dst_linesize[4], const uint8_t *src, enum AVPixelFormat pix_fmt, int width, int height, int align)
Setup the data pointers and linesizes based on the specified image parameters and the provided array...
Definition: imgutils.c:303
AVHWAccel ff_h264_mmal_hwaccel
Definition: mmaldec.c:789
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:1804
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:1621
static int ffmmal_fill_output_port(AVCodecContext *avctx)
Definition: mmaldec.c:237
size_t av_get_codec_tag_string(char *buf, size_t buf_size, unsigned int codec_tag)
Put a string representing the codec tag codec_tag in buf.
Definition: utils.c:2071
int64_t dts
Definition: mmaldec.c:49
static int ffmmal_fill_input_port(AVCodecContext *avctx)
Definition: mmaldec.c:556
static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
Definition: mmaldec.c:748
functionally identical to above
Definition: pixfmt.h:351
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
static char buffer[20]
Definition: seek.c:32
uint8_t
#define av_cold
Definition: attributes.h:66
AVOptions.
MMAL_BUFFER_HEADER_T * buffer
Definition: mmaldec.c:63
AVColorSpace
YUV colorspace type.
Definition: pixfmt.h:343
int ff_decode_frame_props(AVCodecContext *avctx, AVFrame *frame)
Set various frame properties from the codec context / packet data.
Definition: utils.c:534
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:211
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:1523
static void ffmmal_flush(AVCodecContext *avctx)
Definition: mmaldec.c:432
uint8_t * data
Definition: avcodec.h:1346
static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:331
static const AVOption options[]
Definition: mmaldec.c:810
static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:178
#define AV_BUFFER_FLAG_READONLY
Always treat the buffer as read-only, even when it has only one reference.
Definition: buffer.h:113
MMAL_POOL_T * pool_in
Definition: mmaldec.c:74
volatile int refcount
Definition: mmaldec.c:58
#define FFALIGN(x, a)
Definition: macros.h:48
#define src
Definition: vp8dsp.c:254
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:124
void av_free(void *ptr)
Free a memory block which has been allocated with av_malloc(z)() or av_realloc(). ...
Definition: mem.c:190
static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:211
int extradata_sent
Definition: mmaldec.c:90
#define AVERROR(e)
Definition: error.h:43
#define avpriv_atomic_int_get
Definition: atomic_gcc.h:28
static void ffmmal_poolref_unref(FFPoolRef *ref)
Definition: mmaldec.c:97
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:145
AVBufferRef * buf
A reference to the reference-counted buffer where the packet data is stored.
Definition: avcodec.h:1329
simple assert() macros that are a bit more flexible than ISO C assert().
static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:117
FFBufferEntry * waiting_buffers_tail
Definition: mmaldec.c:83
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:28
int extra_decoder_buffers
Definition: mmaldec.c:70
#define FFMAX(a, b)
Definition: common.h:64
#define fail()
Definition: checkasm.h:80
static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:219
void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], const uint8_t *src_data[4], const int src_linesizes[4], enum AVPixelFormat pix_fmt, int width, int height)
Copy image in src_data to dst_data.
Definition: imgutils.c:268
AVBufferRef * ref
Definition: mmaldec.c:46
size_t length
Definition: mmaldec.c:48
static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:603
AVClass * av_class
Definition: mmaldec.c:68
static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
Definition: mmaldec.c:257
enum AVPixelFormat * pix_fmts
array of supported pixel formats, or NULL if unknown, array is terminated by -1
Definition: avcodec.h:3141
FFPoolRef * pool_out
Definition: mmaldec.c:75
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:3244
#define FFMIN(a, b)
Definition: common.h:66
FFBufferEntry * waiting_buffers
Definition: mmaldec.c:83
int64_t frames_output
Definition: mmaldec.c:87
static const chunk_decoder decoder[8]
Definition: dfa.c:322
int width
picture width / height.
Definition: avcodec.h:1580
AVFormatContext * ctx
Definition: movenc.c:48
#define MAX_DELAYED_FRAMES
Definition: mmaldec.c:95
preferred ID for MPEG-1/2 video decoding
Definition: avcodec.h:198
FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
Definition: pixfmt.h:348
static void ffmmal_stop_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:143
MMAL_QUEUE_T * queue_decoded_frames
Definition: mmaldec.c:73
if(ac->has_optimized_func)
int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
Select the (possibly hardware accelerated) pixel format.
Definition: utils.c:785
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:191
AVHWAccel ff_mpeg2_mmal_hwaccel
Definition: mmaldec.c:796
NULL
Definition: eval.c:55
#define AV_LOG_INFO
Standard information.
Definition: log.h:135
struct FFBufferEntry * next
Definition: mmaldec.c:51
Libavcodec external API header.
enum AVCodecID codec_id
Definition: avcodec.h:1426
AVBufferRef * av_buffer_alloc(int size)
Allocate an AVBuffer of the given size using av_malloc().
Definition: buffer.c:66
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:158
main external API structure.
Definition: avcodec.h:1409
uint8_t * data
The data buffer.
Definition: buffer.h:89
void * data
Definition: mmaldec.c:47
int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
Get a buffer for a frame.
Definition: utils.c:589
static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:196
AVHWAccel ff_vc1_mmal_hwaccel
Definition: mmaldec.c:803
int extradata_size
Definition: avcodec.h:1524
Describe the class of an AVClass context structure.
Definition: log.h:34
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:2120
refcounted data buffer API
int size
Size of data in bytes.
Definition: buffer.h:93
static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
Definition: mmaldec.c:647
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:146
attribute_deprecated int64_t pkt_pts
PTS copied from the AVPacket that was decoded to produce this frame.
Definition: frame.h:219
int64_t packets_sent
Definition: mmaldec.c:85
int64_t pkt_dts
DTS copied from the AVPacket that triggered returning this frame.
Definition: frame.h:225
A reference to a data buffer.
Definition: buffer.h:81
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:77
common internal api header.
common internal and external API header
static int ffmal_update_format(AVCodecContext *avctx)
Definition: mmaldec.c:270
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:92
void av_init_packet(AVPacket *pkt)
Initialize optional fields of a packet with default values.
Definition: avpacket.c:31
int den
denominator
Definition: rational.h:45
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:61
static void ffmmal_release_frame(void *opaque, uint8_t *data)
Definition: mmaldec.c:105
void * priv_data
Definition: avcodec.h:1451
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:78
static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt, int is_extradata)
Definition: mmaldec.c:459
static uint8_t tmp[8]
Definition: des.c:38
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: avcodec.h:1345
volatile int packets_buffered
Definition: mmaldec.c:86
HW acceleration though MMAL, data[3] contains a pointer to the MMAL_BUFFER_HEADER_T structure...
Definition: pixfmt.h:220
This structure stores compressed data.
Definition: avcodec.h:1323
void * av_mallocz(size_t size)
Allocate a block of size bytes with alignment suitable for all memory accesses (including vectors if ...
Definition: mem.c:211
MMAL_COMPONENT_T * decoder
Definition: mmaldec.c:72
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1339
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:235
int64_t pts
Definition: mmaldec.c:49