FFmpeg
avfoundation.m
Go to the documentation of this file.
1 /*
2  * AVFoundation input device
3  * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * AVFoundation input device
25  * @author Thilo Borgmann <thilo.borgmann@mail.de>
26  */
27 
28 #import <AVFoundation/AVFoundation.h>
29 #include <pthread.h>
30 
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/avstring.h"
34 #include "libavformat/internal.h"
35 #include "libavutil/internal.h"
36 #include "libavutil/parseutils.h"
37 #include "libavutil/time.h"
38 #include "libavutil/imgutils.h"
39 #include "avdevice.h"
40 
41 static const int avf_time_base = 1000000;
42 
43 static const AVRational avf_time_base_q = {
44  .num = 1,
45  .den = avf_time_base
46 };
47 
50  OSType avf_id;
51 };
52 
53 static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
54  { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },
55  { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },
56  { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },
57  { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },
58  { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },
59  { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },
60  { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },
61  { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },
62  { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },
63  { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },
64  { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },
65  { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },
66  { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },
67  { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },
68  { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
69  { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },
70  { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },
71  { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },
72  { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },
73  { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },
74  { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
75  { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },
76 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
77  { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },
78 #endif
79  { AV_PIX_FMT_NONE, 0 }
80 };
81 
82 typedef struct
83 {
84  AVClass* class;
85 
88  int64_t first_pts;
89  int64_t first_audio_pts;
93 
95  int width, height;
96 
103 
109 
112 
114 
118  int audio_be;
122 
125 
126  enum AVPixelFormat pixel_format;
127 
128  AVCaptureSession *capture_session;
129  AVCaptureVideoDataOutput *video_output;
130  AVCaptureAudioDataOutput *audio_output;
131  CMSampleBufferRef current_frame;
132  CMSampleBufferRef current_audio_frame;
133 
134  AVCaptureDevice *observed_device;
135 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
136  AVCaptureDeviceTransportControlsPlaybackMode observed_mode;
137 #endif
139 } AVFContext;
140 
142 {
144 }
145 
147 {
149 }
150 
151 /** FrameReciever class - delegate for AVCaptureSession
152  */
153 @interface AVFFrameReceiver : NSObject
154 {
156 }
157 
158 - (id)initWithContext:(AVFContext*)context;
159 
160 - (void) captureOutput:(AVCaptureOutput *)captureOutput
161  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
162  fromConnection:(AVCaptureConnection *)connection;
163 
164 @end
165 
166 @implementation AVFFrameReceiver
167 
168 - (id)initWithContext:(AVFContext*)context
169 {
170  if (self = [super init]) {
171  _context = context;
172 
173  // start observing if a device is set for it
174 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
175  if (_context->observed_device) {
176  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
177  NSKeyValueObservingOptions options = NSKeyValueObservingOptionNew;
178 
179  [_context->observed_device addObserver: self
180  forKeyPath: keyPath
181  options: options
182  context: _context];
183  }
184 #endif
185  }
186  return self;
187 }
188 
189 - (void)dealloc {
190  // stop observing if a device is set for it
191 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
192  if (_context->observed_device) {
193  NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
194  [_context->observed_device removeObserver: self forKeyPath: keyPath];
195  }
196 #endif
197  [super dealloc];
198 }
199 
200 - (void)observeValueForKeyPath:(NSString *)keyPath
201  ofObject:(id)object
202  change:(NSDictionary *)change
203  context:(void *)context {
204  if (context == _context) {
205 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
206  AVCaptureDeviceTransportControlsPlaybackMode mode =
207  [change[NSKeyValueChangeNewKey] integerValue];
208 
209  if (mode != _context->observed_mode) {
210  if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {
211  _context->observed_quit = 1;
212  }
213  _context->observed_mode = mode;
214  }
215 #endif
216  } else {
217  [super observeValueForKeyPath: keyPath
218  ofObject: object
219  change: change
220  context: context];
221  }
222 }
223 
224 - (void) captureOutput:(AVCaptureOutput *)captureOutput
225  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
226  fromConnection:(AVCaptureConnection *)connection
227 {
229 
230  if (_context->current_frame != nil) {
231  CFRelease(_context->current_frame);
232  }
233 
234  _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
235 
237 
239 }
240 
241 @end
242 
243 /** AudioReciever class - delegate for AVCaptureSession
244  */
245 @interface AVFAudioReceiver : NSObject
246 {
248 }
249 
250 - (id)initWithContext:(AVFContext*)context;
251 
252 - (void) captureOutput:(AVCaptureOutput *)captureOutput
253  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
254  fromConnection:(AVCaptureConnection *)connection;
255 
256 @end
257 
258 @implementation AVFAudioReceiver
259 
260 - (id)initWithContext:(AVFContext*)context
261 {
262  if (self = [super init]) {
263  _context = context;
264  }
265  return self;
266 }
267 
268 - (void) captureOutput:(AVCaptureOutput *)captureOutput
269  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
270  fromConnection:(AVCaptureConnection *)connection
271 {
273 
274  if (_context->current_audio_frame != nil) {
275  CFRelease(_context->current_audio_frame);
276  }
277 
278  _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
279 
281 
283 }
284 
285 @end
286 
288 {
289  [ctx->capture_session stopRunning];
290 
291  [ctx->capture_session release];
292  [ctx->video_output release];
293  [ctx->audio_output release];
294  [ctx->avf_delegate release];
295  [ctx->avf_audio_delegate release];
296 
297  ctx->capture_session = NULL;
298  ctx->video_output = NULL;
299  ctx->audio_output = NULL;
300  ctx->avf_delegate = NULL;
301  ctx->avf_audio_delegate = NULL;
302 
303  av_freep(&ctx->audio_buffer);
304 
306 
307  if (ctx->current_frame) {
308  CFRelease(ctx->current_frame);
309  }
310 }
311 
313 {
315  char *tmp = av_strdup(s->url);
316  char *save;
317 
318  if (tmp[0] != ':') {
319  ctx->video_filename = av_strtok(tmp, ":", &save);
320  ctx->audio_filename = av_strtok(NULL, ":", &save);
321  } else {
322  ctx->audio_filename = av_strtok(tmp, ":", &save);
323  }
324 }
325 
326 /**
327  * Configure the video device.
328  *
329  * Configure the video device using a run-time approach to access properties
330  * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7
331  * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
332  *
333  * The NSUndefinedKeyException must be handled by the caller of this function.
334  *
335  */
336 static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
337 {
339 
340  double framerate = av_q2d(ctx->framerate);
341  NSObject *range = nil;
342  NSObject *format = nil;
343  NSObject *selected_range = nil;
344  NSObject *selected_format = nil;
345 
346  // try to configure format by formats list
347  // might raise an exception if no format list is given
348  // (then fallback to default, no configuration)
349  @try {
350  for (format in [video_device valueForKey:@"formats"]) {
351  CMFormatDescriptionRef formatDescription;
352  CMVideoDimensions dimensions;
353 
354  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
355  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
356 
357  if ((ctx->width == 0 && ctx->height == 0) ||
358  (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
359 
360  selected_format = format;
361 
362  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
363  double max_framerate;
364 
365  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
366  if (fabs (framerate - max_framerate) < 0.01) {
367  selected_range = range;
368  break;
369  }
370  }
371  }
372  }
373 
374  if (!selected_format) {
375  av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",
376  ctx->width, ctx->height);
377  goto unsupported_format;
378  }
379 
380  if (!selected_range) {
381  av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
382  framerate);
383  if (ctx->video_is_muxed) {
384  av_log(s, AV_LOG_ERROR, "Falling back to default.\n");
385  } else {
386  goto unsupported_format;
387  }
388  }
389 
390  if ([video_device lockForConfiguration:NULL] == YES) {
391  if (selected_format) {
392  [video_device setValue:selected_format forKey:@"activeFormat"];
393  }
394  if (selected_range) {
395  NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
396  [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
397  [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
398  }
399  } else {
400  av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");
401  return AVERROR(EINVAL);
402  }
403  } @catch(NSException *e) {
404  av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");
405  }
406 
407  return 0;
408 
409 unsupported_format:
410 
411  av_log(s, AV_LOG_ERROR, "Supported modes:\n");
412  for (format in [video_device valueForKey:@"formats"]) {
413  CMFormatDescriptionRef formatDescription;
414  CMVideoDimensions dimensions;
415 
416  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
417  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
418 
419  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
420  double min_framerate;
421  double max_framerate;
422 
423  [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
424  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
425  av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",
426  dimensions.width, dimensions.height,
427  min_framerate, max_framerate);
428  }
429  }
430  return AVERROR(EINVAL);
431 }
432 
433 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
434 {
436  int ret;
437  NSError *error = nil;
438  AVCaptureInput* capture_input = nil;
439  struct AVFPixelFormatSpec pxl_fmt_spec;
440  NSNumber *pixel_format;
441  NSDictionary *capture_dict;
442  dispatch_queue_t queue;
443 
444  if (ctx->video_device_index < ctx->num_video_devices) {
445  capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
446  } else {
447  capture_input = (AVCaptureInput*) video_device;
448  }
449 
450  if (!capture_input) {
451  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
452  [[error localizedDescription] UTF8String]);
453  return 1;
454  }
455 
456  if ([ctx->capture_session canAddInput:capture_input]) {
457  [ctx->capture_session addInput:capture_input];
458  } else {
459  av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
460  return 1;
461  }
462 
463  // Attaching output
464  ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];
465 
466  if (!ctx->video_output) {
467  av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
468  return 1;
469  }
470 
471  // Configure device framerate and video size
472  @try {
473  if ((ret = configure_video_device(s, video_device)) < 0) {
474  return ret;
475  }
476  } @catch (NSException *exception) {
477  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
478  av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
479  return AVERROR_EXTERNAL;
480  }
481  }
482 
483  // select pixel format
484  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
485 
486  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
487  if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
488  pxl_fmt_spec = avf_pixel_formats[i];
489  break;
490  }
491  }
492 
493  // check if selected pixel format is supported by AVFoundation
494  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
495  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
496  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
497  return 1;
498  }
499 
500  // check if the pixel format is available for this device
501  if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
502  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
503  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
504 
505  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
506 
507  av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
508  for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
509  struct AVFPixelFormatSpec pxl_fmt_dummy;
510  pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
511  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
512  if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
513  pxl_fmt_dummy = avf_pixel_formats[i];
514  break;
515  }
516  }
517 
518  if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
519  av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));
520 
521  // select first supported pixel format instead of user selected (or default) pixel format
522  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
523  pxl_fmt_spec = pxl_fmt_dummy;
524  }
525  }
526  }
527 
528  // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
529  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
530  return 1;
531  } else {
532  av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
533  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
534  }
535  }
536 
537  // set videoSettings to an empty dict for receiving raw data of muxed devices
538  if (ctx->capture_raw_data) {
539  ctx->pixel_format = pxl_fmt_spec.ff_id;
540  ctx->video_output.videoSettings = @{ };
541  } else {
542  ctx->pixel_format = pxl_fmt_spec.ff_id;
543  pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
544  capture_dict = [NSDictionary dictionaryWithObject:pixel_format
545  forKey:(id)kCVPixelBufferPixelFormatTypeKey];
546 
547  [ctx->video_output setVideoSettings:capture_dict];
548  }
549  [ctx->video_output setAlwaysDiscardsLateVideoFrames:ctx->drop_late_frames];
550 
551 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
552  // check for transport control support and set observer device if supported
553  if (!ctx->video_is_screen) {
554  int trans_ctrl = [video_device transportControlsSupported];
555  AVCaptureDeviceTransportControlsPlaybackMode trans_mode = [video_device transportControlsPlaybackMode];
556 
557  if (trans_ctrl) {
558  ctx->observed_mode = trans_mode;
559  ctx->observed_device = video_device;
560  }
561  }
562 #endif
563 
564  ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
565 
566  queue = dispatch_queue_create("avf_queue", NULL);
567  [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
568  dispatch_release(queue);
569 
570  if ([ctx->capture_session canAddOutput:ctx->video_output]) {
571  [ctx->capture_session addOutput:ctx->video_output];
572  } else {
573  av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
574  return 1;
575  }
576 
577  return 0;
578 }
579 
580 static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
581 {
583  NSError *error = nil;
584  AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
585  dispatch_queue_t queue;
586 
587  if (!audio_dev_input) {
588  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
589  [[error localizedDescription] UTF8String]);
590  return 1;
591  }
592 
593  if ([ctx->capture_session canAddInput:audio_dev_input]) {
594  [ctx->capture_session addInput:audio_dev_input];
595  } else {
596  av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
597  return 1;
598  }
599 
600  // Attaching output
601  ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
602 
603  if (!ctx->audio_output) {
604  av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
605  return 1;
606  }
607 
608  ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
609 
610  queue = dispatch_queue_create("avf_audio_queue", NULL);
611  [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
612  dispatch_release(queue);
613 
614  if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
615  [ctx->capture_session addOutput:ctx->audio_output];
616  } else {
617  av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
618  return 1;
619  }
620 
621  return 0;
622 }
623 
625 {
627  CVImageBufferRef image_buffer;
628  CMBlockBufferRef block_buffer;
629  CGSize image_buffer_size;
630  AVStream* stream = avformat_new_stream(s, NULL);
631 
632  if (!stream) {
633  return 1;
634  }
635 
636  // Take stream info from the first frame.
637  while (ctx->frames_captured < 1) {
638  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
639  }
640 
641  lock_frames(ctx);
642 
643  ctx->video_stream_index = stream->index;
644 
645  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
646 
647  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
648  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
649 
650  if (image_buffer) {
651  image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
652 
653  stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
654  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
655  stream->codecpar->width = (int)image_buffer_size.width;
656  stream->codecpar->height = (int)image_buffer_size.height;
657  stream->codecpar->format = ctx->pixel_format;
658  } else {
659  stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO;
660  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
661  stream->codecpar->format = ctx->pixel_format;
662  }
663 
664  CFRelease(ctx->current_frame);
665  ctx->current_frame = nil;
666 
667  unlock_frames(ctx);
668 
669  return 0;
670 }
671 
673 {
675  CMFormatDescriptionRef format_desc;
676  AVStream* stream = avformat_new_stream(s, NULL);
677 
678  if (!stream) {
679  return 1;
680  }
681 
682  // Take stream info from the first frame.
683  while (ctx->audio_frames_captured < 1) {
684  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
685  }
686 
687  lock_frames(ctx);
688 
689  ctx->audio_stream_index = stream->index;
690 
691  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
692 
693  format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
694  const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
695 
696  if (!basic_desc) {
697  av_log(s, AV_LOG_ERROR, "audio format not available\n");
698  return 1;
699  }
700 
701  stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
702  stream->codecpar->sample_rate = basic_desc->mSampleRate;
703  stream->codecpar->channels = basic_desc->mChannelsPerFrame;
704  stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
705 
706  ctx->audio_channels = basic_desc->mChannelsPerFrame;
707  ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
708  ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
709  ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
710  ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
711  ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
712  ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
713 
714  if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
715  ctx->audio_float &&
716  ctx->audio_bits_per_sample == 32 &&
717  ctx->audio_packed) {
718  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
719  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
720  ctx->audio_signed_integer &&
721  ctx->audio_bits_per_sample == 16 &&
722  ctx->audio_packed) {
723  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
724  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
725  ctx->audio_signed_integer &&
726  ctx->audio_bits_per_sample == 24 &&
727  ctx->audio_packed) {
728  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
729  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
730  ctx->audio_signed_integer &&
731  ctx->audio_bits_per_sample == 32 &&
732  ctx->audio_packed) {
733  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
734  } else {
735  av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
736  return 1;
737  }
738 
739  if (ctx->audio_non_interleaved) {
740  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
741  ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
743  if (!ctx->audio_buffer) {
744  av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
745  return 1;
746  }
747  }
748 
749  CFRelease(ctx->current_audio_frame);
750  ctx->current_audio_frame = nil;
751 
752  unlock_frames(ctx);
753 
754  return 0;
755 }
756 
758 {
759  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
760  uint32_t num_screens = 0;
762  AVCaptureDevice *video_device = nil;
763  AVCaptureDevice *audio_device = nil;
764  // Find capture device
765  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
766  NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
767 
768  ctx->num_video_devices = [devices count] + [devices_muxed count];
769  ctx->first_pts = av_gettime();
770  ctx->first_audio_pts = av_gettime();
771 
773 
774 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
775  CGGetActiveDisplayList(0, NULL, &num_screens);
776 #endif
777 
778  // List devices if requested
779  if (ctx->list_devices) {
780  int index = 0;
781  av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
782  for (AVCaptureDevice *device in devices) {
783  const char *name = [[device localizedName] UTF8String];
784  index = [devices indexOfObject:device];
785  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
786  }
787  for (AVCaptureDevice *device in devices_muxed) {
788  const char *name = [[device localizedName] UTF8String];
789  index = [devices count] + [devices_muxed indexOfObject:device];
790  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
791  }
792 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
793  if (num_screens > 0) {
794  CGDirectDisplayID screens[num_screens];
795  CGGetActiveDisplayList(num_screens, screens, &num_screens);
796  for (int i = 0; i < num_screens; i++) {
797  av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
798  }
799  }
800 #endif
801 
802  av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
803  devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
804  for (AVCaptureDevice *device in devices) {
805  const char *name = [[device localizedName] UTF8String];
806  int index = [devices indexOfObject:device];
807  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
808  }
809  goto fail;
810  }
811 
812  // parse input filename for video and audio device
814 
815  // check for device index given in filename
816  if (ctx->video_device_index == -1 && ctx->video_filename) {
817  sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
818  }
819  if (ctx->audio_device_index == -1 && ctx->audio_filename) {
820  sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
821  }
822 
823  if (ctx->video_device_index >= 0) {
824  if (ctx->video_device_index < ctx->num_video_devices) {
825  if (ctx->video_device_index < [devices count]) {
826  video_device = [devices objectAtIndex:ctx->video_device_index];
827  } else {
828  video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
829  ctx->video_is_muxed = 1;
830  }
831  } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
832 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
833  CGDirectDisplayID screens[num_screens];
834  CGGetActiveDisplayList(num_screens, screens, &num_screens);
835  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
836 
837  if (ctx->framerate.num > 0) {
838  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
839  }
840 
841 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
842  if (ctx->capture_cursor) {
843  capture_screen_input.capturesCursor = YES;
844  } else {
845  capture_screen_input.capturesCursor = NO;
846  }
847 #endif
848 
849  if (ctx->capture_mouse_clicks) {
850  capture_screen_input.capturesMouseClicks = YES;
851  } else {
852  capture_screen_input.capturesMouseClicks = NO;
853  }
854 
855  video_device = (AVCaptureDevice*) capture_screen_input;
856  ctx->video_is_screen = 1;
857 #endif
858  } else {
859  av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
860  goto fail;
861  }
862  } else if (ctx->video_filename &&
863  strncmp(ctx->video_filename, "none", 4)) {
864  if (!strncmp(ctx->video_filename, "default", 7)) {
865  video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
866  } else {
867  // looking for video inputs
868  for (AVCaptureDevice *device in devices) {
869  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
870  video_device = device;
871  break;
872  }
873  }
874  // looking for muxed inputs
875  for (AVCaptureDevice *device in devices_muxed) {
876  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
877  video_device = device;
878  ctx->video_is_muxed = 1;
879  break;
880  }
881  }
882 
883 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
884  // looking for screen inputs
885  if (!video_device) {
886  int idx;
887  if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
888  CGDirectDisplayID screens[num_screens];
889  CGGetActiveDisplayList(num_screens, screens, &num_screens);
890  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
891  video_device = (AVCaptureDevice*) capture_screen_input;
892  ctx->video_device_index = ctx->num_video_devices + idx;
893  ctx->video_is_screen = 1;
894 
895  if (ctx->framerate.num > 0) {
896  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
897  }
898 
899 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
900  if (ctx->capture_cursor) {
901  capture_screen_input.capturesCursor = YES;
902  } else {
903  capture_screen_input.capturesCursor = NO;
904  }
905 #endif
906 
907  if (ctx->capture_mouse_clicks) {
908  capture_screen_input.capturesMouseClicks = YES;
909  } else {
910  capture_screen_input.capturesMouseClicks = NO;
911  }
912  }
913  }
914 #endif
915  }
916 
917  if (!video_device) {
918  av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
919  goto fail;
920  }
921  }
922 
923  // get audio device
924  if (ctx->audio_device_index >= 0) {
925  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
926 
927  if (ctx->audio_device_index >= [devices count]) {
928  av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
929  goto fail;
930  }
931 
932  audio_device = [devices objectAtIndex:ctx->audio_device_index];
933  } else if (ctx->audio_filename &&
934  strncmp(ctx->audio_filename, "none", 4)) {
935  if (!strncmp(ctx->audio_filename, "default", 7)) {
936  audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
937  } else {
938  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
939 
940  for (AVCaptureDevice *device in devices) {
941  if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
942  audio_device = device;
943  break;
944  }
945  }
946  }
947 
948  if (!audio_device) {
949  av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
950  goto fail;
951  }
952  }
953 
954  // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
955  if (!video_device && !audio_device) {
956  av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
957  goto fail;
958  }
959 
960  if (video_device) {
961  if (ctx->video_device_index < ctx->num_video_devices) {
962  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
963  } else {
964  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
965  }
966  }
967  if (audio_device) {
968  av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
969  }
970 
971  // Initialize capture session
972  ctx->capture_session = [[AVCaptureSession alloc] init];
973 
974  if (video_device && add_video_device(s, video_device)) {
975  goto fail;
976  }
977  if (audio_device && add_audio_device(s, audio_device)) {
978  }
979 
980  [ctx->capture_session startRunning];
981 
982  /* Unlock device configuration only after the session is started so it
983  * does not reset the capture formats */
984  if (!ctx->video_is_screen) {
985  [video_device unlockForConfiguration];
986  }
987 
988  if (video_device && get_video_config(s)) {
989  goto fail;
990  }
991 
992  // set audio stream
993  if (audio_device && get_audio_config(s)) {
994  goto fail;
995  }
996 
997  [pool release];
998  return 0;
999 
1000 fail:
1001  [pool release];
1002  destroy_context(ctx);
1003  return AVERROR(EIO);
1004 }
1005 
1007  CVPixelBufferRef image_buffer,
1008  AVPacket *pkt)
1009 {
1010  AVFContext *ctx = s->priv_data;
1011  int src_linesize[4];
1012  const uint8_t *src_data[4];
1013  int width = CVPixelBufferGetWidth(image_buffer);
1014  int height = CVPixelBufferGetHeight(image_buffer);
1015  int status;
1016 
1017  memset(src_linesize, 0, sizeof(src_linesize));
1018  memset(src_data, 0, sizeof(src_data));
1019 
1020  status = CVPixelBufferLockBaseAddress(image_buffer, 0);
1021  if (status != kCVReturnSuccess) {
1022  av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);
1023  return AVERROR_EXTERNAL;
1024  }
1025 
1026  if (CVPixelBufferIsPlanar(image_buffer)) {
1027  size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
1028  int i;
1029  for(i = 0; i < plane_count; i++){
1030  src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
1031  src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
1032  }
1033  } else {
1034  src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
1035  src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
1036  }
1037 
1038  status = av_image_copy_to_buffer(pkt->data, pkt->size,
1039  src_data, src_linesize,
1040  ctx->pixel_format, width, height, 1);
1041 
1042 
1043 
1044  CVPixelBufferUnlockBaseAddress(image_buffer, 0);
1045 
1046  return status;
1047 }
1048 
1050 {
1052 
1053  do {
1054  CVImageBufferRef image_buffer;
1055  CMBlockBufferRef block_buffer;
1056  lock_frames(ctx);
1057 
1058  if (ctx->current_frame != nil) {
1059  int status;
1060  int length = 0;
1061 
1062  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
1063  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
1064 
1065  if (image_buffer != nil) {
1066  length = (int)CVPixelBufferGetDataSize(image_buffer);
1067  } else if (block_buffer != nil) {
1068  length = (int)CMBlockBufferGetDataLength(block_buffer);
1069  } else {
1070  return AVERROR(EINVAL);
1071  }
1072 
1073  if (av_new_packet(pkt, length) < 0) {
1074  return AVERROR(EIO);
1075  }
1076 
1077  CMItemCount count;
1078  CMSampleTimingInfo timing_info;
1079 
1080  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
1081  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1082  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1083  }
1084 
1085  pkt->stream_index = ctx->video_stream_index;
1086  pkt->flags |= AV_PKT_FLAG_KEY;
1087 
1088  if (image_buffer) {
1089  status = copy_cvpixelbuffer(s, image_buffer, pkt);
1090  } else {
1091  status = 0;
1092  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1093  if (ret != kCMBlockBufferNoErr) {
1094  status = AVERROR(EIO);
1095  }
1096  }
1097  CFRelease(ctx->current_frame);
1098  ctx->current_frame = nil;
1099 
1100  if (status < 0)
1101  return status;
1102  } else if (ctx->current_audio_frame != nil) {
1103  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
1104  int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
1105 
1106  if (!block_buffer || !block_buffer_size) {
1107  return AVERROR(EIO);
1108  }
1109 
1110  if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
1111  return AVERROR_BUFFER_TOO_SMALL;
1112  }
1113 
1114  if (av_new_packet(pkt, block_buffer_size) < 0) {
1115  return AVERROR(EIO);
1116  }
1117 
1118  CMItemCount count;
1119  CMSampleTimingInfo timing_info;
1120 
1121  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
1122  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1123  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1124  }
1125 
1126  pkt->stream_index = ctx->audio_stream_index;
1127  pkt->flags |= AV_PKT_FLAG_KEY;
1128 
1129  if (ctx->audio_non_interleaved) {
1130  int sample, c, shift, num_samples;
1131 
1132  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
1133  if (ret != kCMBlockBufferNoErr) {
1134  return AVERROR(EIO);
1135  }
1136 
1137  num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
1138 
1139  // transform decoded frame into output format
1140  #define INTERLEAVE_OUTPUT(bps) \
1141  { \
1142  int##bps##_t **src; \
1143  int##bps##_t *dest; \
1144  src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \
1145  if (!src) return AVERROR(EIO); \
1146  for (c = 0; c < ctx->audio_channels; c++) { \
1147  src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
1148  } \
1149  dest = (int##bps##_t*)pkt->data; \
1150  shift = bps - ctx->audio_bits_per_sample; \
1151  for (sample = 0; sample < num_samples; sample++) \
1152  for (c = 0; c < ctx->audio_channels; c++) \
1153  *dest++ = src[c][sample] << shift; \
1154  av_freep(&src); \
1155  }
1156 
1157  if (ctx->audio_bits_per_sample <= 16) {
1158  INTERLEAVE_OUTPUT(16)
1159  } else {
1160  INTERLEAVE_OUTPUT(32)
1161  }
1162  } else {
1163  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1164  if (ret != kCMBlockBufferNoErr) {
1165  return AVERROR(EIO);
1166  }
1167  }
1168 
1169  CFRelease(ctx->current_audio_frame);
1170  ctx->current_audio_frame = nil;
1171  } else {
1172  pkt->data = NULL;
1173  unlock_frames(ctx);
1174  if (ctx->observed_quit) {
1175  return AVERROR_EOF;
1176  } else {
1177  return AVERROR(EAGAIN);
1178  }
1179  }
1180 
1181  unlock_frames(ctx);
1182  } while (!pkt->data);
1183 
1184  return 0;
1185 }
1186 
1188 {
1190  destroy_context(ctx);
1191  return 0;
1192 }
1193 
1194 static const AVOption options[] = {
1195  { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1196  { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1197  { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1198  { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1199  { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1200  { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1201  { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1202  { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1203  { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1204  { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1205 
1206  { NULL },
1207 };
1208 
1209 static const AVClass avf_class = {
1210  .class_name = "AVFoundation indev",
1211  .item_name = av_default_item_name,
1212  .option = options,
1213  .version = LIBAVUTIL_VERSION_INT,
1215 };
1216 
1218  .name = "avfoundation",
1219  .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
1220  .priv_data_size = sizeof(AVFContext),
1223  .read_close = avf_close,
1224  .flags = AVFMT_NOFILE,
1225  .priv_class = &avf_class,
1226 };
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:81
int audio_buffer_size
Definition: avfoundation.m:124
#define NULL
Definition: coverity.c:32
static const struct AVFPixelFormatSpec avf_pixel_formats[]
Definition: avfoundation.m:53
static void lock_frames(AVFContext *ctx)
Definition: avfoundation.m:141
static int shift(int a, int b)
Definition: sonic.c:82
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
#define pthread_mutex_lock(a)
Definition: ffprobe.c:62
AVCaptureSession * capture_session
Definition: avfoundation.m:128
AVOption.
Definition: opt.h:248
int32_t * audio_buffer
Definition: avfoundation.m:123
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:505
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
int list_devices
Definition: avfoundation.m:104
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4869
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:35
int audio_frames_captured
Definition: avfoundation.m:87
AVFContext * _context
Definition: avfoundation.m:247
static const AVOption options[]
int num
Numerator.
Definition: rational.h:59
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), little-endian, X=unused/undefined ...
Definition: pixfmt.h:108
int size
Definition: packet.h:364
static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Definition: avfoundation.m:433
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
CMSampleBufferRef current_audio_frame
Definition: avfoundation.m:132
static int get_audio_config(AVFormatContext *s)
Definition: avfoundation.m:672
int audio_non_interleaved
Definition: avfoundation.m:121
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition: pixfmt.h:239
static AVPacket pkt
static void error(const char *err)
#define sample
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate.The lists are not just lists
int framerate
Definition: h264_levels.c:65
AudioReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:245
AVRational framerate
Definition: avfoundation.m:94
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), little-endian
Definition: pixfmt.h:106
int capture_mouse_clicks
Definition: avfoundation.m:98
Format I/O context.
Definition: avformat.h:1355
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
int audio_signed_integer
Definition: avfoundation.m:119
uint8_t
#define av_malloc(s)
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:238
int drop_late_frames
Definition: avfoundation.m:100
AVOptions.
static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Configure the video device.
Definition: avfoundation.m:336
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4450
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), big-endian
Definition: pixfmt.h:105
#define height
uint8_t * data
Definition: packet.h:363
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
#define AVERROR_EOF
End of file.
Definition: error.h:55
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
static int copy_cvpixelbuffer(AVFormatContext *s, CVPixelBufferRef image_buffer, AVPacket *pkt)
static void parse_device_name(AVFormatContext *s)
Definition: avfoundation.m:312
#define av_log(a,...)
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:401
AVInputFormat ff_avfoundation_demuxer
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
Main libavdevice API header.
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:88
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
char * video_filename
Definition: avfoundation.m:110
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:153
char * url
input or output URL.
Definition: avformat.h:1451
enum AVPixelFormat ff_id
Definition: avfoundation.m:49
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:215
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
Tag description
Definition: snow.txt:206
GLsizei GLsizei * length
Definition: opengl_enc.c:114
static int avf_read_header(AVFormatContext *s)
Definition: avfoundation.m:757
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:402
int num_video_devices
Definition: avfoundation.m:113
GLsizei count
Definition: opengl_enc.c:108
int capture_cursor
Definition: avfoundation.m:97
#define fail()
Definition: checkasm.h:123
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:369
common internal API header
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define width
typedef void(APIENTRY *FF_PFNGLACTIVETEXTUREPROC)(GLenum texture)
static const int avf_time_base
Definition: avfoundation.m:41
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
int frames_captured
Definition: avfoundation.m:86
#define s(width, name)
Definition: cbs_vp9.c:257
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
int audio_bits_per_sample
Definition: avfoundation.m:116
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:66
packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as big...
Definition: pixfmt.h:148
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:527
if(ret)
int64_t av_gettime(void)
Get the current time in microseconds.
Definition: time.c:39
Stream structure.
Definition: avformat.h:880
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
Definition: avfoundation.m:580
#define AV_LOG_INFO
Standard information.
Definition: log.h:205
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:253
int video_stream_index
Definition: avfoundation.m:106
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:177
static AVRational av_make_q(int num, int den)
Create an AVRational.
Definition: rational.h:71
static const AVRational avf_time_base_q
Definition: avfoundation.m:43
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:67
AVCaptureVideoDataOutput * video_output
Definition: avfoundation.m:129
pthread_mutex_t frame_lock
Definition: avfoundation.m:90
int audio_device_index
Definition: avfoundation.m:107
static int get_video_config(AVFormatContext *s)
Definition: avfoundation.m:624
static const AVClass avf_class
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
planar YUV 4:4:4 64bpp, (1 Cr & Cb sample per 1x1 Y & A samples, little-endian)
Definition: pixfmt.h:195
int index
Definition: gxfenc.c:89
#define INTERLEAVE_OUTPUT(bps)
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int audio_channels
Definition: avfoundation.m:115
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:279
static void unlock_frames(AVFContext *ctx)
Definition: avfoundation.m:146
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
offset must point to AVRational
Definition: opt.h:238
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:240
int64_t first_pts
Definition: avfoundation.m:88
FrameReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:153
offset must point to two consecutive integers
Definition: opt.h:235
int audio_packed
Definition: avfoundation.m:120
misc parsing utilities
CMSampleBufferRef current_frame
Definition: avfoundation.m:131
#define flags(name, subs,...)
Definition: cbs_av1.c:560
int audio_stream_index
Definition: avfoundation.m:108
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:400
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), big-endian , X=unused/undefined
Definition: pixfmt.h:107
int video_is_screen
Definition: avfoundation.m:102
enum AVPixelFormat pixel_format
Definition: avfoundation.m:126
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok()...
Definition: avstring.c:186
static int FUNC() timing_info(CodedBitstreamContext *ctx, RWContext *rw, AV1RawTimingInfo *current)
int video_is_muxed
Definition: avfoundation.m:101
Y , 1bpp, 0 is black, 1 is white, in each byte pixels are ordered from the msb to the lsb...
Definition: pixfmt.h:76
int
char * audio_filename
Definition: avfoundation.m:111
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Y , 8bpp.
Definition: pixfmt.h:74
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVCaptureAudioDataOutput * audio_output
Definition: avfoundation.m:130
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:458
static void destroy_context(AVFContext *ctx)
Definition: avfoundation.m:287
int den
Denominator.
Definition: rational.h:60
static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
AVFContext * _context
Definition: avfoundation.m:155
void * priv_data
Format private data.
Definition: avformat.h:1383
AVCaptureDevice * observed_device
Definition: avfoundation.m:134
id avf_audio_delegate
Definition: avfoundation.m:92
static int avf_close(AVFormatContext *s)
int64_t first_audio_pts
Definition: avfoundation.m:89
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: packet.h:362
int64_t av_get_default_channel_layout(int nb_channels)
Return default channel layout for a given number of channels.
#define av_freep(p)
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:652
int observed_quit
Definition: avfoundation.m:138
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
int stream_index
Definition: packet.h:365
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition: pixfmt.h:237
enum AVCodecID id
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
This structure stores compressed data.
Definition: packet.h:340
mode
Use these values in ebur128_init (or&#39;ed).
Definition: ebur128.h:83
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:411
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:356
int i
Definition: input.c:407
int capture_raw_data
Definition: avfoundation.m:99
const char * name
Definition: opengl_enc.c:102
static uint8_t tmp[11]
Definition: aes_ctr.c:26
int video_device_index
Definition: avfoundation.m:105