FFmpeg
avfoundation.m
Go to the documentation of this file.
1 /*
2  * AVFoundation input device
3  * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * AVFoundation input device
25  * @author Thilo Borgmann <thilo.borgmann@mail.de>
26  */
27 
28 #import <AVFoundation/AVFoundation.h>
29 #include <pthread.h>
30 
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/avstring.h"
34 #include "libavformat/internal.h"
35 #include "libavutil/internal.h"
36 #include "libavutil/parseutils.h"
37 #include "libavutil/time.h"
38 #include "libavutil/imgutils.h"
39 #include "avdevice.h"
40 
41 static const int avf_time_base = 1000000;
42 
43 static const AVRational avf_time_base_q = {
44  .num = 1,
45  .den = avf_time_base
46 };
47 
50  OSType avf_id;
51 };
52 
53 static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
54  { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },
55  { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },
56  { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },
57  { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },
58  { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },
59  { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },
60  { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },
61  { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },
62  { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },
63  { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },
64  { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },
65  { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },
66  { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },
67  { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },
68  { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
69  { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },
70  { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },
71  { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },
72  { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },
73  { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },
74  { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
75  { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },
76 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
77  { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },
78 #endif
79  { AV_PIX_FMT_NONE, 0 }
80 };
81 
82 typedef struct
83 {
84  AVClass* class;
85 
88  int64_t first_pts;
89  int64_t first_audio_pts;
93 
95  int width, height;
96 
102 
108 
111 
113 
117  int audio_be;
121 
124 
125  enum AVPixelFormat pixel_format;
126 
127  AVCaptureSession *capture_session;
128  AVCaptureVideoDataOutput *video_output;
129  AVCaptureAudioDataOutput *audio_output;
130  CMSampleBufferRef current_frame;
131  CMSampleBufferRef current_audio_frame;
132 
133  AVCaptureDevice *observed_device;
134  AVCaptureDeviceTransportControlsPlaybackMode observed_mode;
136 } AVFContext;
137 
139 {
141 }
142 
144 {
146 }
147 
148 /** FrameReciever class - delegate for AVCaptureSession
149  */
150 @interface AVFFrameReceiver : NSObject
151 {
153 }
154 
155 - (id)initWithContext:(AVFContext*)context;
156 
157 - (void) captureOutput:(AVCaptureOutput *)captureOutput
158  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
159  fromConnection:(AVCaptureConnection *)connection;
160 
161 @end
162 
163 @implementation AVFFrameReceiver
164 
165 - (id)initWithContext:(AVFContext*)context
166 {
167  if (self = [super init]) {
168  _context = context;
169 
170  // start observing if a device is set for it
171  if (_context->observed_device) {
172  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
173  NSKeyValueObservingOptions options = NSKeyValueObservingOptionNew;
174 
175  [_context->observed_device addObserver: self
176  forKeyPath: keyPath
177  options: options
178  context: _context];
179  }
180  }
181  return self;
182 }
183 
184 - (void)dealloc {
185  // stop observing if a device is set for it
186  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
187  [_context->observed_device removeObserver: self forKeyPath: keyPath];
188  [super dealloc];
189 }
190 
191 - (void)observeValueForKeyPath:(NSString *)keyPath
192  ofObject:(id)object
193  change:(NSDictionary *)change
194  context:(void *)context {
195  if (context == _context) {
196  AVCaptureDeviceTransportControlsPlaybackMode mode =
197  [change[NSKeyValueChangeNewKey] integerValue];
198 
199  if (mode != _context->observed_mode) {
200  if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {
201  _context->observed_quit = 1;
202  }
204  }
205  } else {
206  [super observeValueForKeyPath: keyPath
207  ofObject: object
208  change: change
209  context: context];
210  }
211 }
212 
213 - (void) captureOutput:(AVCaptureOutput *)captureOutput
214  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
215  fromConnection:(AVCaptureConnection *)connection
216 {
218 
219  if (_context->current_frame != nil) {
220  CFRelease(_context->current_frame);
221  }
222 
223  _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
224 
226 
228 }
229 
230 @end
231 
232 /** AudioReciever class - delegate for AVCaptureSession
233  */
234 @interface AVFAudioReceiver : NSObject
235 {
237 }
238 
239 - (id)initWithContext:(AVFContext*)context;
240 
241 - (void) captureOutput:(AVCaptureOutput *)captureOutput
242  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
243  fromConnection:(AVCaptureConnection *)connection;
244 
245 @end
246 
247 @implementation AVFAudioReceiver
248 
249 - (id)initWithContext:(AVFContext*)context
250 {
251  if (self = [super init]) {
252  _context = context;
253  }
254  return self;
255 }
256 
257 - (void) captureOutput:(AVCaptureOutput *)captureOutput
258  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
259  fromConnection:(AVCaptureConnection *)connection
260 {
262 
263  if (_context->current_audio_frame != nil) {
264  CFRelease(_context->current_audio_frame);
265  }
266 
267  _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
268 
270 
272 }
273 
274 @end
275 
277 {
278  [ctx->capture_session stopRunning];
279 
280  [ctx->capture_session release];
281  [ctx->video_output release];
282  [ctx->audio_output release];
283  [ctx->avf_delegate release];
284  [ctx->avf_audio_delegate release];
285 
286  ctx->capture_session = NULL;
287  ctx->video_output = NULL;
288  ctx->audio_output = NULL;
289  ctx->avf_delegate = NULL;
290  ctx->avf_audio_delegate = NULL;
291 
292  av_freep(&ctx->audio_buffer);
293 
295 
296  if (ctx->current_frame) {
297  CFRelease(ctx->current_frame);
298  }
299 }
300 
302 {
304  char *tmp = av_strdup(s->url);
305  char *save;
306 
307  if (tmp[0] != ':') {
308  ctx->video_filename = av_strtok(tmp, ":", &save);
309  ctx->audio_filename = av_strtok(NULL, ":", &save);
310  } else {
311  ctx->audio_filename = av_strtok(tmp, ":", &save);
312  }
313 }
314 
315 /**
316  * Configure the video device.
317  *
318  * Configure the video device using a run-time approach to access properties
319  * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7
320  * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
321  *
322  * The NSUndefinedKeyException must be handled by the caller of this function.
323  *
324  */
325 static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
326 {
328 
329  double framerate = av_q2d(ctx->framerate);
330  NSObject *range = nil;
331  NSObject *format = nil;
332  NSObject *selected_range = nil;
333  NSObject *selected_format = nil;
334 
335  // try to configure format by formats list
336  // might raise an exception if no format list is given
337  // (then fallback to default, no configuration)
338  @try {
339  for (format in [video_device valueForKey:@"formats"]) {
340  CMFormatDescriptionRef formatDescription;
341  CMVideoDimensions dimensions;
342 
343  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
344  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
345 
346  if ((ctx->width == 0 && ctx->height == 0) ||
347  (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
348 
349  selected_format = format;
350 
351  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
352  double max_framerate;
353 
354  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
355  if (fabs (framerate - max_framerate) < 0.01) {
356  selected_range = range;
357  break;
358  }
359  }
360  }
361  }
362 
363  if (!selected_format) {
364  av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",
365  ctx->width, ctx->height);
366  goto unsupported_format;
367  }
368 
369  if (!selected_range) {
370  av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
371  framerate);
372  if (ctx->video_is_muxed) {
373  av_log(s, AV_LOG_ERROR, "Falling back to default.\n");
374  } else {
375  goto unsupported_format;
376  }
377  }
378 
379  if ([video_device lockForConfiguration:NULL] == YES) {
380  if (selected_format) {
381  [video_device setValue:selected_format forKey:@"activeFormat"];
382  }
383  if (selected_range) {
384  NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
385  [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
386  [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
387  }
388  } else {
389  av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");
390  return AVERROR(EINVAL);
391  }
392  } @catch(NSException *e) {
393  av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");
394  }
395 
396  return 0;
397 
398 unsupported_format:
399 
400  av_log(s, AV_LOG_ERROR, "Supported modes:\n");
401  for (format in [video_device valueForKey:@"formats"]) {
402  CMFormatDescriptionRef formatDescription;
403  CMVideoDimensions dimensions;
404 
405  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
406  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
407 
408  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
409  double min_framerate;
410  double max_framerate;
411 
412  [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
413  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
414  av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",
415  dimensions.width, dimensions.height,
416  min_framerate, max_framerate);
417  }
418  }
419  return AVERROR(EINVAL);
420 }
421 
422 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
423 {
425  int ret;
426  NSError *error = nil;
427  AVCaptureInput* capture_input = nil;
428  struct AVFPixelFormatSpec pxl_fmt_spec;
429  NSNumber *pixel_format;
430  NSDictionary *capture_dict;
431  dispatch_queue_t queue;
432 
433  if (ctx->video_device_index < ctx->num_video_devices) {
434  capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
435  } else {
436  capture_input = (AVCaptureInput*) video_device;
437  }
438 
439  if (!capture_input) {
440  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
441  [[error localizedDescription] UTF8String]);
442  return 1;
443  }
444 
445  if ([ctx->capture_session canAddInput:capture_input]) {
446  [ctx->capture_session addInput:capture_input];
447  } else {
448  av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
449  return 1;
450  }
451 
452  // Attaching output
453  ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];
454 
455  if (!ctx->video_output) {
456  av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
457  return 1;
458  }
459 
460  // Configure device framerate and video size
461  @try {
462  if ((ret = configure_video_device(s, video_device)) < 0) {
463  return ret;
464  }
465  } @catch (NSException *exception) {
466  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
467  av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
468  return AVERROR_EXTERNAL;
469  }
470  }
471 
472  // select pixel format
473  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
474 
475  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
476  if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
477  pxl_fmt_spec = avf_pixel_formats[i];
478  break;
479  }
480  }
481 
482  // check if selected pixel format is supported by AVFoundation
483  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
484  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
485  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
486  return 1;
487  }
488 
489  // check if the pixel format is available for this device
490  if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
491  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
492  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
493 
494  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
495 
496  av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
497  for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
498  struct AVFPixelFormatSpec pxl_fmt_dummy;
499  pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
500  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
501  if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
502  pxl_fmt_dummy = avf_pixel_formats[i];
503  break;
504  }
505  }
506 
507  if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
508  av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));
509 
510  // select first supported pixel format instead of user selected (or default) pixel format
511  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
512  pxl_fmt_spec = pxl_fmt_dummy;
513  }
514  }
515  }
516 
517  // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
518  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
519  return 1;
520  } else {
521  av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
522  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
523  }
524  }
525 
526  // set videoSettings to an empty dict for receiving raw data of muxed devices
527  if (ctx->capture_raw_data) {
528  ctx->pixel_format = pxl_fmt_spec.ff_id;
529  ctx->video_output.videoSettings = @{ };
530  } else {
531  ctx->pixel_format = pxl_fmt_spec.ff_id;
532  pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
533  capture_dict = [NSDictionary dictionaryWithObject:pixel_format
534  forKey:(id)kCVPixelBufferPixelFormatTypeKey];
535 
536  [ctx->video_output setVideoSettings:capture_dict];
537  }
538  [ctx->video_output setAlwaysDiscardsLateVideoFrames:ctx->drop_late_frames];
539 
540  // check for transport control support and set observer device if supported
541  int trans_ctrl = [video_device transportControlsSupported];
542  AVCaptureDeviceTransportControlsPlaybackMode trans_mode = [video_device transportControlsPlaybackMode];
543 
544  if (trans_ctrl) {
545  ctx->observed_mode = trans_mode;
546  ctx->observed_device = video_device;
547  }
548 
549  ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
550 
551  queue = dispatch_queue_create("avf_queue", NULL);
552  [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
553  dispatch_release(queue);
554 
555  if ([ctx->capture_session canAddOutput:ctx->video_output]) {
556  [ctx->capture_session addOutput:ctx->video_output];
557  } else {
558  av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
559  return 1;
560  }
561 
562  return 0;
563 }
564 
565 static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
566 {
568  NSError *error = nil;
569  AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
570  dispatch_queue_t queue;
571 
572  if (!audio_dev_input) {
573  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
574  [[error localizedDescription] UTF8String]);
575  return 1;
576  }
577 
578  if ([ctx->capture_session canAddInput:audio_dev_input]) {
579  [ctx->capture_session addInput:audio_dev_input];
580  } else {
581  av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
582  return 1;
583  }
584 
585  // Attaching output
586  ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
587 
588  if (!ctx->audio_output) {
589  av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
590  return 1;
591  }
592 
593  ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
594 
595  queue = dispatch_queue_create("avf_audio_queue", NULL);
596  [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
597  dispatch_release(queue);
598 
599  if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
600  [ctx->capture_session addOutput:ctx->audio_output];
601  } else {
602  av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
603  return 1;
604  }
605 
606  return 0;
607 }
608 
610 {
612  CVImageBufferRef image_buffer;
613  CMBlockBufferRef block_buffer;
614  CGSize image_buffer_size;
615  AVStream* stream = avformat_new_stream(s, NULL);
616 
617  if (!stream) {
618  return 1;
619  }
620 
621  // Take stream info from the first frame.
622  while (ctx->frames_captured < 1) {
623  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
624  }
625 
626  lock_frames(ctx);
627 
628  ctx->video_stream_index = stream->index;
629 
630  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
631 
632  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
633  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
634 
635  if (image_buffer) {
636  image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
637 
638  stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
639  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
640  stream->codecpar->width = (int)image_buffer_size.width;
641  stream->codecpar->height = (int)image_buffer_size.height;
642  stream->codecpar->format = ctx->pixel_format;
643  } else {
644  stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO;
645  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
646  stream->codecpar->format = ctx->pixel_format;
647  }
648 
649  CFRelease(ctx->current_frame);
650  ctx->current_frame = nil;
651 
652  unlock_frames(ctx);
653 
654  return 0;
655 }
656 
658 {
660  CMFormatDescriptionRef format_desc;
661  AVStream* stream = avformat_new_stream(s, NULL);
662 
663  if (!stream) {
664  return 1;
665  }
666 
667  // Take stream info from the first frame.
668  while (ctx->audio_frames_captured < 1) {
669  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
670  }
671 
672  lock_frames(ctx);
673 
674  ctx->audio_stream_index = stream->index;
675 
676  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
677 
678  format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
679  const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
680 
681  if (!basic_desc) {
682  av_log(s, AV_LOG_ERROR, "audio format not available\n");
683  return 1;
684  }
685 
686  stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
687  stream->codecpar->sample_rate = basic_desc->mSampleRate;
688  stream->codecpar->channels = basic_desc->mChannelsPerFrame;
689  stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
690 
691  ctx->audio_channels = basic_desc->mChannelsPerFrame;
692  ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
693  ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
694  ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
695  ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
696  ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
697  ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
698 
699  if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
700  ctx->audio_float &&
701  ctx->audio_bits_per_sample == 32 &&
702  ctx->audio_packed) {
703  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
704  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
705  ctx->audio_signed_integer &&
706  ctx->audio_bits_per_sample == 16 &&
707  ctx->audio_packed) {
708  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
709  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
710  ctx->audio_signed_integer &&
711  ctx->audio_bits_per_sample == 24 &&
712  ctx->audio_packed) {
713  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
714  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
715  ctx->audio_signed_integer &&
716  ctx->audio_bits_per_sample == 32 &&
717  ctx->audio_packed) {
718  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
719  } else {
720  av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
721  return 1;
722  }
723 
724  if (ctx->audio_non_interleaved) {
725  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
726  ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
728  if (!ctx->audio_buffer) {
729  av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
730  return 1;
731  }
732  }
733 
734  CFRelease(ctx->current_audio_frame);
735  ctx->current_audio_frame = nil;
736 
737  unlock_frames(ctx);
738 
739  return 0;
740 }
741 
743 {
744  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
745  int capture_screen = 0;
746  uint32_t num_screens = 0;
748  AVCaptureDevice *video_device = nil;
749  AVCaptureDevice *audio_device = nil;
750  // Find capture device
751  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
752  NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
753 
754  ctx->num_video_devices = [devices count] + [devices_muxed count];
755  ctx->first_pts = av_gettime();
756  ctx->first_audio_pts = av_gettime();
757 
759 
760 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
761  CGGetActiveDisplayList(0, NULL, &num_screens);
762 #endif
763 
764  // List devices if requested
765  if (ctx->list_devices) {
766  int index = 0;
767  av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
768  for (AVCaptureDevice *device in devices) {
769  const char *name = [[device localizedName] UTF8String];
770  index = [devices indexOfObject:device];
771  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
772  }
773  for (AVCaptureDevice *device in devices_muxed) {
774  const char *name = [[device localizedName] UTF8String];
775  index = [devices count] + [devices_muxed indexOfObject:device];
776  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
777  }
778 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
779  if (num_screens > 0) {
780  CGDirectDisplayID screens[num_screens];
781  CGGetActiveDisplayList(num_screens, screens, &num_screens);
782  for (int i = 0; i < num_screens; i++) {
783  av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
784  }
785  }
786 #endif
787 
788  av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
789  devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
790  for (AVCaptureDevice *device in devices) {
791  const char *name = [[device localizedName] UTF8String];
792  int index = [devices indexOfObject:device];
793  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
794  }
795  goto fail;
796  }
797 
798  // parse input filename for video and audio device
800 
801  // check for device index given in filename
802  if (ctx->video_device_index == -1 && ctx->video_filename) {
803  sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
804  }
805  if (ctx->audio_device_index == -1 && ctx->audio_filename) {
806  sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
807  }
808 
809  if (ctx->video_device_index >= 0) {
810  if (ctx->video_device_index < ctx->num_video_devices) {
811  if (ctx->video_device_index < [devices count]) {
812  video_device = [devices objectAtIndex:ctx->video_device_index];
813  } else {
814  video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
815  ctx->video_is_muxed = 1;
816  }
817  } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
818 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
819  CGDirectDisplayID screens[num_screens];
820  CGGetActiveDisplayList(num_screens, screens, &num_screens);
821  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
822 
823  if (ctx->framerate.num > 0) {
824  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
825  }
826 
827 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
828  if (ctx->capture_cursor) {
829  capture_screen_input.capturesCursor = YES;
830  } else {
831  capture_screen_input.capturesCursor = NO;
832  }
833 #endif
834 
835  if (ctx->capture_mouse_clicks) {
836  capture_screen_input.capturesMouseClicks = YES;
837  } else {
838  capture_screen_input.capturesMouseClicks = NO;
839  }
840 
841  video_device = (AVCaptureDevice*) capture_screen_input;
842  capture_screen = 1;
843 #endif
844  } else {
845  av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
846  goto fail;
847  }
848  } else if (ctx->video_filename &&
849  strncmp(ctx->video_filename, "none", 4)) {
850  if (!strncmp(ctx->video_filename, "default", 7)) {
851  video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
852  } else {
853  // looking for video inputs
854  for (AVCaptureDevice *device in devices) {
855  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
856  video_device = device;
857  break;
858  }
859  }
860  // looking for muxed inputs
861  for (AVCaptureDevice *device in devices_muxed) {
862  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
863  video_device = device;
864  ctx->video_is_muxed = 1;
865  break;
866  }
867  }
868 
869 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
870  // looking for screen inputs
871  if (!video_device) {
872  int idx;
873  if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
874  CGDirectDisplayID screens[num_screens];
875  CGGetActiveDisplayList(num_screens, screens, &num_screens);
876  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
877  video_device = (AVCaptureDevice*) capture_screen_input;
878  ctx->video_device_index = ctx->num_video_devices + idx;
879  capture_screen = 1;
880 
881  if (ctx->framerate.num > 0) {
882  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
883  }
884 
885 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
886  if (ctx->capture_cursor) {
887  capture_screen_input.capturesCursor = YES;
888  } else {
889  capture_screen_input.capturesCursor = NO;
890  }
891 #endif
892 
893  if (ctx->capture_mouse_clicks) {
894  capture_screen_input.capturesMouseClicks = YES;
895  } else {
896  capture_screen_input.capturesMouseClicks = NO;
897  }
898  }
899  }
900 #endif
901  }
902 
903  if (!video_device) {
904  av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
905  goto fail;
906  }
907  }
908 
909  // get audio device
910  if (ctx->audio_device_index >= 0) {
911  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
912 
913  if (ctx->audio_device_index >= [devices count]) {
914  av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
915  goto fail;
916  }
917 
918  audio_device = [devices objectAtIndex:ctx->audio_device_index];
919  } else if (ctx->audio_filename &&
920  strncmp(ctx->audio_filename, "none", 4)) {
921  if (!strncmp(ctx->audio_filename, "default", 7)) {
922  audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
923  } else {
924  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
925 
926  for (AVCaptureDevice *device in devices) {
927  if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
928  audio_device = device;
929  break;
930  }
931  }
932  }
933 
934  if (!audio_device) {
935  av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
936  goto fail;
937  }
938  }
939 
940  // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
941  if (!video_device && !audio_device) {
942  av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
943  goto fail;
944  }
945 
946  if (video_device) {
947  if (ctx->video_device_index < ctx->num_video_devices) {
948  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
949  } else {
950  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
951  }
952  }
953  if (audio_device) {
954  av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
955  }
956 
957  // Initialize capture session
958  ctx->capture_session = [[AVCaptureSession alloc] init];
959 
960  if (video_device && add_video_device(s, video_device)) {
961  goto fail;
962  }
963  if (audio_device && add_audio_device(s, audio_device)) {
964  }
965 
966  [ctx->capture_session startRunning];
967 
968  /* Unlock device configuration only after the session is started so it
969  * does not reset the capture formats */
970  if (!capture_screen) {
971  [video_device unlockForConfiguration];
972  }
973 
974  if (video_device && get_video_config(s)) {
975  goto fail;
976  }
977 
978  // set audio stream
979  if (audio_device && get_audio_config(s)) {
980  goto fail;
981  }
982 
983  [pool release];
984  return 0;
985 
986 fail:
987  [pool release];
988  destroy_context(ctx);
989  return AVERROR(EIO);
990 }
991 
993  CVPixelBufferRef image_buffer,
994  AVPacket *pkt)
995 {
996  AVFContext *ctx = s->priv_data;
997  int src_linesize[4];
998  const uint8_t *src_data[4];
999  int width = CVPixelBufferGetWidth(image_buffer);
1000  int height = CVPixelBufferGetHeight(image_buffer);
1001  int status;
1002 
1003  memset(src_linesize, 0, sizeof(src_linesize));
1004  memset(src_data, 0, sizeof(src_data));
1005 
1006  status = CVPixelBufferLockBaseAddress(image_buffer, 0);
1007  if (status != kCVReturnSuccess) {
1008  av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);
1009  return AVERROR_EXTERNAL;
1010  }
1011 
1012  if (CVPixelBufferIsPlanar(image_buffer)) {
1013  size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
1014  int i;
1015  for(i = 0; i < plane_count; i++){
1016  src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
1017  src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
1018  }
1019  } else {
1020  src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
1021  src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
1022  }
1023 
1024  status = av_image_copy_to_buffer(pkt->data, pkt->size,
1025  src_data, src_linesize,
1026  ctx->pixel_format, width, height, 1);
1027 
1028 
1029 
1030  CVPixelBufferUnlockBaseAddress(image_buffer, 0);
1031 
1032  return status;
1033 }
1034 
1036 {
1038 
1039  do {
1040  CVImageBufferRef image_buffer;
1041  CMBlockBufferRef block_buffer;
1042  lock_frames(ctx);
1043 
1044  if (ctx->current_frame != nil) {
1045  int status;
1046  int length = 0;
1047 
1048  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
1049  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
1050 
1051  if (image_buffer != nil) {
1052  length = (int)CVPixelBufferGetDataSize(image_buffer);
1053  } else if (block_buffer != nil) {
1054  length = (int)CMBlockBufferGetDataLength(block_buffer);
1055  } else {
1056  return AVERROR(EINVAL);
1057  }
1058 
1059  if (av_new_packet(pkt, length) < 0) {
1060  return AVERROR(EIO);
1061  }
1062 
1063  CMItemCount count;
1064  CMSampleTimingInfo timing_info;
1065 
1066  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
1067  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1068  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1069  }
1070 
1071  pkt->stream_index = ctx->video_stream_index;
1072  pkt->flags |= AV_PKT_FLAG_KEY;
1073 
1074  if (image_buffer) {
1075  status = copy_cvpixelbuffer(s, image_buffer, pkt);
1076  } else {
1077  status = 0;
1078  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1079  if (ret != kCMBlockBufferNoErr) {
1080  status = AVERROR(EIO);
1081  }
1082  }
1083  CFRelease(ctx->current_frame);
1084  ctx->current_frame = nil;
1085 
1086  if (status < 0)
1087  return status;
1088  } else if (ctx->current_audio_frame != nil) {
1089  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
1090  int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
1091 
1092  if (!block_buffer || !block_buffer_size) {
1093  return AVERROR(EIO);
1094  }
1095 
1096  if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
1097  return AVERROR_BUFFER_TOO_SMALL;
1098  }
1099 
1100  if (av_new_packet(pkt, block_buffer_size) < 0) {
1101  return AVERROR(EIO);
1102  }
1103 
1104  CMItemCount count;
1105  CMSampleTimingInfo timing_info;
1106 
1107  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
1108  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1109  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1110  }
1111 
1112  pkt->stream_index = ctx->audio_stream_index;
1113  pkt->flags |= AV_PKT_FLAG_KEY;
1114 
1115  if (ctx->audio_non_interleaved) {
1116  int sample, c, shift, num_samples;
1117 
1118  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
1119  if (ret != kCMBlockBufferNoErr) {
1120  return AVERROR(EIO);
1121  }
1122 
1123  num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
1124 
1125  // transform decoded frame into output format
1126  #define INTERLEAVE_OUTPUT(bps) \
1127  { \
1128  int##bps##_t **src; \
1129  int##bps##_t *dest; \
1130  src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \
1131  if (!src) return AVERROR(EIO); \
1132  for (c = 0; c < ctx->audio_channels; c++) { \
1133  src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
1134  } \
1135  dest = (int##bps##_t*)pkt->data; \
1136  shift = bps - ctx->audio_bits_per_sample; \
1137  for (sample = 0; sample < num_samples; sample++) \
1138  for (c = 0; c < ctx->audio_channels; c++) \
1139  *dest++ = src[c][sample] << shift; \
1140  av_freep(&src); \
1141  }
1142 
1143  if (ctx->audio_bits_per_sample <= 16) {
1144  INTERLEAVE_OUTPUT(16)
1145  } else {
1146  INTERLEAVE_OUTPUT(32)
1147  }
1148  } else {
1149  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1150  if (ret != kCMBlockBufferNoErr) {
1151  return AVERROR(EIO);
1152  }
1153  }
1154 
1155  CFRelease(ctx->current_audio_frame);
1156  ctx->current_audio_frame = nil;
1157  } else {
1158  pkt->data = NULL;
1159  unlock_frames(ctx);
1160  if (ctx->observed_quit) {
1161  return AVERROR_EOF;
1162  } else {
1163  return AVERROR(EAGAIN);
1164  }
1165  }
1166 
1167  unlock_frames(ctx);
1168  } while (!pkt->data);
1169 
1170  return 0;
1171 }
1172 
1174 {
1176  destroy_context(ctx);
1177  return 0;
1178 }
1179 
1180 static const AVOption options[] = {
1181  { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1182  { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1183  { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1184  { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1185  { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1186  { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1187  { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1188  { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1189  { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1190  { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1191 
1192  { NULL },
1193 };
1194 
1195 static const AVClass avf_class = {
1196  .class_name = "AVFoundation indev",
1197  .item_name = av_default_item_name,
1198  .option = options,
1199  .version = LIBAVUTIL_VERSION_INT,
1201 };
1202 
1204  .name = "avfoundation",
1205  .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
1206  .priv_data_size = sizeof(AVFContext),
1209  .read_close = avf_close,
1210  .flags = AVFMT_NOFILE,
1211  .priv_class = &avf_class,
1212 };
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:81
int audio_buffer_size
Definition: avfoundation.m:123
#define NULL
Definition: coverity.c:32
static const struct AVFPixelFormatSpec avf_pixel_formats[]
Definition: avfoundation.m:53
static void lock_frames(AVFContext *ctx)
Definition: avfoundation.m:138
static int shift(int a, int b)
Definition: sonic.c:82
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
#define pthread_mutex_lock(a)
Definition: ffprobe.c:61
AVCaptureSession * capture_session
Definition: avfoundation.m:127
AVOption.
Definition: opt.h:246
int32_t * audio_buffer
Definition: avfoundation.m:122
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:453
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
int list_devices
Definition: avfoundation.m:103
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4943
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:35
int audio_frames_captured
Definition: avfoundation.m:87
AVFContext * _context
Definition: avfoundation.m:236
static const AVOption options[]
int num
Numerator.
Definition: rational.h:59
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), little-endian, X=unused/undefined ...
Definition: pixfmt.h:108
int size
Definition: avcodec.h:1534
static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Definition: avfoundation.m:422
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
CMSampleBufferRef current_audio_frame
Definition: avfoundation.m:131
static int get_audio_config(AVFormatContext *s)
Definition: avfoundation.m:657
int audio_non_interleaved
Definition: avfoundation.m:120
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition: pixfmt.h:239
static AVPacket pkt
static void error(const char *err)
#define sample
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate.The lists are not just lists
int framerate
Definition: h264_levels.c:65
AudioReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:234
AVRational framerate
Definition: avfoundation.m:94
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), little-endian
Definition: pixfmt.h:106
int capture_mouse_clicks
Definition: avfoundation.m:98
Format I/O context.
Definition: avformat.h:1353
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
int audio_signed_integer
Definition: avfoundation.m:118
uint8_t
#define av_malloc(s)
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:238
int drop_late_frames
Definition: avfoundation.m:100
AVOptions.
static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Configure the video device.
Definition: avfoundation.m:325
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4524
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), big-endian
Definition: pixfmt.h:105
#define height
uint8_t * data
Definition: avcodec.h:1533
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
#define AVERROR_EOF
End of file.
Definition: error.h:55
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
static int copy_cvpixelbuffer(AVFormatContext *s, CVPixelBufferRef image_buffer, AVPacket *pkt)
Definition: avfoundation.m:992
static void parse_device_name(AVFormatContext *s)
Definition: avfoundation.m:301
#define av_log(a,...)
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: avcodec.h:1565
AVInputFormat ff_avfoundation_demuxer
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
Main libavdevice API header.
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:86
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
char * video_filename
Definition: avfoundation.m:109
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
char * url
input or output URL.
Definition: avformat.h:1449
enum AVPixelFormat ff_id
Definition: avfoundation.m:49
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:197
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
Tag description
Definition: snow.txt:206
GLsizei GLsizei * length
Definition: opengl_enc.c:114
static int avf_read_header(AVFormatContext *s)
Definition: avfoundation.m:742
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:400
int num_video_devices
Definition: avfoundation.m:112
GLsizei count
Definition: opengl_enc.c:108
int capture_cursor
Definition: avfoundation.m:97
#define fail()
Definition: checkasm.h:122
int flags
A combination of AV_PKT_FLAG values.
Definition: avcodec.h:1539
common internal API header
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define width
typedef void(APIENTRY *FF_PFNGLACTIVETEXTUREPROC)(GLenum texture)
static const int avf_time_base
Definition: avfoundation.m:41
static int capture_screen(CCaptionSubContext *ctx)
Definition: ccaption_dec.c:435
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
int frames_captured
Definition: avfoundation.m:86
#define s(width, name)
Definition: cbs_vp9.c:257
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
int audio_bits_per_sample
Definition: avfoundation.m:115
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:65
packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as big...
Definition: pixfmt.h:148
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:527
if(ret)
int64_t av_gettime(void)
Get the current time in microseconds.
Definition: time.c:39
Stream structure.
Definition: avformat.h:876
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
Definition: avfoundation.m:565
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:251
int video_stream_index
Definition: avfoundation.m:105
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:177
static AVRational av_make_q(int num, int den)
Create an AVRational.
Definition: rational.h:71
static const AVRational avf_time_base_q
Definition: avfoundation.m:43
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:67
AVCaptureVideoDataOutput * video_output
Definition: avfoundation.m:128
pthread_mutex_t frame_lock
Definition: avfoundation.m:90
int audio_device_index
Definition: avfoundation.m:106
static int get_video_config(AVFormatContext *s)
Definition: avfoundation.m:609
static const AVClass avf_class
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
planar YUV 4:4:4 64bpp, (1 Cr & Cb sample per 1x1 Y & A samples, little-endian)
Definition: pixfmt.h:195
int index
Definition: gxfenc.c:89
AVCaptureDeviceTransportControlsPlaybackMode observed_mode
Definition: avfoundation.m:134
#define INTERLEAVE_OUTPUT(bps)
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int audio_channels
Definition: avfoundation.m:114
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:277
static void unlock_frames(AVFContext *ctx)
Definition: avfoundation.m:143
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
offset must point to AVRational
Definition: opt.h:236
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:240
int64_t first_pts
Definition: avfoundation.m:88
FrameReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:150
offset must point to two consecutive integers
Definition: opt.h:233
int audio_packed
Definition: avfoundation.m:119
misc parsing utilities
CMSampleBufferRef current_frame
Definition: avfoundation.m:130
#define flags(name, subs,...)
Definition: cbs_av1.c:564
int audio_stream_index
Definition: avfoundation.m:107
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:398
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), big-endian , X=unused/undefined
Definition: pixfmt.h:107
enum AVPixelFormat pixel_format
Definition: avfoundation.m:125
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok()...
Definition: avstring.c:184
static int FUNC() timing_info(CodedBitstreamContext *ctx, RWContext *rw, AV1RawTimingInfo *current)
int video_is_muxed
Definition: avfoundation.m:101
Y , 1bpp, 0 is black, 1 is white, in each byte pixels are ordered from the msb to the lsb...
Definition: pixfmt.h:76
int
char * audio_filename
Definition: avfoundation.m:110
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Y , 8bpp.
Definition: pixfmt.h:74
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVCaptureAudioDataOutput * audio_output
Definition: avfoundation.m:129
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:458
static void destroy_context(AVFContext *ctx)
Definition: avfoundation.m:276
int den
Denominator.
Definition: rational.h:60
static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
AVFContext * _context
Definition: avfoundation.m:152
void * priv_data
Format private data.
Definition: avformat.h:1381
AVCaptureDevice * observed_device
Definition: avfoundation.m:133
id avf_audio_delegate
Definition: avfoundation.m:92
static int avf_close(AVFormatContext *s)
int64_t first_audio_pts
Definition: avfoundation.m:89
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: avcodec.h:1532
int64_t av_get_default_channel_layout(int nb_channels)
Return default channel layout for a given number of channels.
#define av_freep(p)
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:650
int observed_quit
Definition: avfoundation.m:135
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2465
int stream_index
Definition: avcodec.h:1535
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition: pixfmt.h:237
enum AVCodecID id
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
This structure stores compressed data.
Definition: avcodec.h:1510
mode
Use these values in ebur128_init (or&#39;ed).
Definition: ebur128.h:83
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:409
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1526
int capture_raw_data
Definition: avfoundation.m:99
const char * name
Definition: opengl_enc.c:102
static uint8_t tmp[11]
Definition: aes_ctr.c:26
int video_device_index
Definition: avfoundation.m:104