FFmpeg
avfoundation.m
Go to the documentation of this file.
1 /*
2  * AVFoundation input device
3  * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * AVFoundation input device
25  * @author Thilo Borgmann <thilo.borgmann@mail.de>
26  */
27 
28 #import <AVFoundation/AVFoundation.h>
29 #include <pthread.h>
30 
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/avstring.h"
34 #include "libavformat/internal.h"
35 #include "libavutil/internal.h"
36 #include "libavutil/parseutils.h"
37 #include "libavutil/time.h"
38 #include "libavutil/imgutils.h"
39 #include "avdevice.h"
40 
41 static const int avf_time_base = 1000000;
42 
43 static const AVRational avf_time_base_q = {
44  .num = 1,
45  .den = avf_time_base
46 };
47 
50  OSType avf_id;
51 };
52 
53 static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
54  { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },
55  { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },
56  { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },
57  { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },
58  { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },
59  { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },
60  { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },
61  { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },
62  { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },
63  { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },
64  { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },
65  { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },
66  { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },
67  { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },
68  { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
69  { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },
70  { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },
71  { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },
72  { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },
73  { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },
74  { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
75  { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },
76 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
77  { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },
78 #endif
79  { AV_PIX_FMT_NONE, 0 }
80 };
81 
82 typedef struct
83 {
84  AVClass* class;
85 
88  int64_t first_pts;
89  int64_t first_audio_pts;
94 
96  int width, height;
97 
102 
108 
111 
113 
117  int audio_be;
121 
124 
125  enum AVPixelFormat pixel_format;
126 
127  AVCaptureSession *capture_session;
128  AVCaptureVideoDataOutput *video_output;
129  AVCaptureAudioDataOutput *audio_output;
130  CMSampleBufferRef current_frame;
131  CMSampleBufferRef current_audio_frame;
132 } AVFContext;
133 
135 {
137 }
138 
140 {
142 }
143 
144 /** FrameReciever class - delegate for AVCaptureSession
145  */
146 @interface AVFFrameReceiver : NSObject
147 {
149 }
150 
151 - (id)initWithContext:(AVFContext*)context;
152 
153 - (void) captureOutput:(AVCaptureOutput *)captureOutput
154  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
155  fromConnection:(AVCaptureConnection *)connection;
156 
157 @end
158 
159 @implementation AVFFrameReceiver
160 
161 - (id)initWithContext:(AVFContext*)context
162 {
163  if (self = [super init]) {
164  _context = context;
165  }
166  return self;
167 }
168 
169 - (void) captureOutput:(AVCaptureOutput *)captureOutput
170  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
171  fromConnection:(AVCaptureConnection *)connection
172 {
174 
175  if (_context->current_frame != nil) {
176  CFRelease(_context->current_frame);
177  }
178 
179  _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
180 
182 
184 
186 }
187 
188 @end
189 
190 /** AudioReciever class - delegate for AVCaptureSession
191  */
192 @interface AVFAudioReceiver : NSObject
193 {
195 }
196 
197 - (id)initWithContext:(AVFContext*)context;
198 
199 - (void) captureOutput:(AVCaptureOutput *)captureOutput
200  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
201  fromConnection:(AVCaptureConnection *)connection;
202 
203 @end
204 
205 @implementation AVFAudioReceiver
206 
207 - (id)initWithContext:(AVFContext*)context
208 {
209  if (self = [super init]) {
210  _context = context;
211  }
212  return self;
213 }
214 
215 - (void) captureOutput:(AVCaptureOutput *)captureOutput
216  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
217  fromConnection:(AVCaptureConnection *)connection
218 {
220 
221  if (_context->current_audio_frame != nil) {
222  CFRelease(_context->current_audio_frame);
223  }
224 
225  _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
226 
228 
230 
232 }
233 
234 @end
235 
237 {
238  [ctx->capture_session stopRunning];
239 
240  [ctx->capture_session release];
241  [ctx->video_output release];
242  [ctx->audio_output release];
243  [ctx->avf_delegate release];
244  [ctx->avf_audio_delegate release];
245 
246  ctx->capture_session = NULL;
247  ctx->video_output = NULL;
248  ctx->audio_output = NULL;
249  ctx->avf_delegate = NULL;
250  ctx->avf_audio_delegate = NULL;
251 
252  av_freep(&ctx->audio_buffer);
253 
256 
257  if (ctx->current_frame) {
258  CFRelease(ctx->current_frame);
259  }
260 }
261 
263 {
265  char *tmp = av_strdup(s->url);
266  char *save;
267 
268  if (tmp[0] != ':') {
269  ctx->video_filename = av_strtok(tmp, ":", &save);
270  ctx->audio_filename = av_strtok(NULL, ":", &save);
271  } else {
272  ctx->audio_filename = av_strtok(tmp, ":", &save);
273  }
274 }
275 
276 /**
277  * Configure the video device.
278  *
279  * Configure the video device using a run-time approach to access properties
280  * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7
281  * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
282  *
283  * The NSUndefinedKeyException must be handled by the caller of this function.
284  *
285  */
286 static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
287 {
289 
290  double framerate = av_q2d(ctx->framerate);
291  NSObject *range = nil;
292  NSObject *format = nil;
293  NSObject *selected_range = nil;
294  NSObject *selected_format = nil;
295 
296  // try to configure format by formats list
297  // might raise an exception if no format list is given
298  // (then fallback to default, no configuration)
299  @try {
300  for (format in [video_device valueForKey:@"formats"]) {
301  CMFormatDescriptionRef formatDescription;
302  CMVideoDimensions dimensions;
303 
304  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
305  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
306 
307  if ((ctx->width == 0 && ctx->height == 0) ||
308  (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
309 
310  selected_format = format;
311 
312  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
313  double max_framerate;
314 
315  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
316  if (fabs (framerate - max_framerate) < 0.01) {
317  selected_range = range;
318  break;
319  }
320  }
321  }
322  }
323 
324  if (!selected_format) {
325  av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",
326  ctx->width, ctx->height);
327  goto unsupported_format;
328  }
329 
330  if (!selected_range) {
331  av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
332  framerate);
333  if (ctx->video_is_muxed) {
334  av_log(s, AV_LOG_ERROR, "Falling back to default.\n");
335  } else {
336  goto unsupported_format;
337  }
338  }
339 
340  if ([video_device lockForConfiguration:NULL] == YES) {
341  if (selected_format) {
342  [video_device setValue:selected_format forKey:@"activeFormat"];
343  }
344  if (selected_range) {
345  NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
346  [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
347  [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
348  }
349  } else {
350  av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");
351  return AVERROR(EINVAL);
352  }
353  } @catch(NSException *e) {
354  av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");
355  }
356 
357  return 0;
358 
359 unsupported_format:
360 
361  av_log(s, AV_LOG_ERROR, "Supported modes:\n");
362  for (format in [video_device valueForKey:@"formats"]) {
363  CMFormatDescriptionRef formatDescription;
364  CMVideoDimensions dimensions;
365 
366  formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
367  dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
368 
369  for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
370  double min_framerate;
371  double max_framerate;
372 
373  [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
374  [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
375  av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",
376  dimensions.width, dimensions.height,
377  min_framerate, max_framerate);
378  }
379  }
380  return AVERROR(EINVAL);
381 }
382 
383 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
384 {
386  int ret;
387  NSError *error = nil;
388  AVCaptureInput* capture_input = nil;
389  struct AVFPixelFormatSpec pxl_fmt_spec;
390  NSNumber *pixel_format;
391  NSDictionary *capture_dict;
392  dispatch_queue_t queue;
393 
394  if (ctx->video_device_index < ctx->num_video_devices) {
395  capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
396  } else {
397  capture_input = (AVCaptureInput*) video_device;
398  }
399 
400  if (!capture_input) {
401  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
402  [[error localizedDescription] UTF8String]);
403  return 1;
404  }
405 
406  if ([ctx->capture_session canAddInput:capture_input]) {
407  [ctx->capture_session addInput:capture_input];
408  } else {
409  av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
410  return 1;
411  }
412 
413  // Attaching output
414  ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];
415 
416  if (!ctx->video_output) {
417  av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
418  return 1;
419  }
420 
421  // Configure device framerate and video size
422  @try {
423  if ((ret = configure_video_device(s, video_device)) < 0) {
424  return ret;
425  }
426  } @catch (NSException *exception) {
427  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
428  av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
429  return AVERROR_EXTERNAL;
430  }
431  }
432 
433  // select pixel format
434  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
435 
436  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
437  if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
438  pxl_fmt_spec = avf_pixel_formats[i];
439  break;
440  }
441  }
442 
443  // check if selected pixel format is supported by AVFoundation
444  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
445  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
446  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
447  return 1;
448  }
449 
450  // check if the pixel format is available for this device
451  if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
452  av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
453  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
454 
455  pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
456 
457  av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
458  for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
459  struct AVFPixelFormatSpec pxl_fmt_dummy;
460  pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
461  for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
462  if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
463  pxl_fmt_dummy = avf_pixel_formats[i];
464  break;
465  }
466  }
467 
468  if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
469  av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));
470 
471  // select first supported pixel format instead of user selected (or default) pixel format
472  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
473  pxl_fmt_spec = pxl_fmt_dummy;
474  }
475  }
476  }
477 
478  // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
479  if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
480  return 1;
481  } else {
482  av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
483  av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
484  }
485  }
486 
487  // set videoSettings to an empty dict for receiving raw data of muxed devices
488  if (ctx->capture_raw_data) {
489  ctx->pixel_format = pxl_fmt_spec.ff_id;
490  ctx->video_output.videoSettings = @{ };
491  } else {
492  ctx->pixel_format = pxl_fmt_spec.ff_id;
493  pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
494  capture_dict = [NSDictionary dictionaryWithObject:pixel_format
495  forKey:(id)kCVPixelBufferPixelFormatTypeKey];
496 
497  [ctx->video_output setVideoSettings:capture_dict];
498  }
499  [ctx->video_output setAlwaysDiscardsLateVideoFrames:YES];
500 
501  ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
502 
503  queue = dispatch_queue_create("avf_queue", NULL);
504  [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
505  dispatch_release(queue);
506 
507  if ([ctx->capture_session canAddOutput:ctx->video_output]) {
508  [ctx->capture_session addOutput:ctx->video_output];
509  } else {
510  av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
511  return 1;
512  }
513 
514  return 0;
515 }
516 
517 static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
518 {
520  NSError *error = nil;
521  AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
522  dispatch_queue_t queue;
523 
524  if (!audio_dev_input) {
525  av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
526  [[error localizedDescription] UTF8String]);
527  return 1;
528  }
529 
530  if ([ctx->capture_session canAddInput:audio_dev_input]) {
531  [ctx->capture_session addInput:audio_dev_input];
532  } else {
533  av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
534  return 1;
535  }
536 
537  // Attaching output
538  ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
539 
540  if (!ctx->audio_output) {
541  av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
542  return 1;
543  }
544 
545  ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
546 
547  queue = dispatch_queue_create("avf_audio_queue", NULL);
548  [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
549  dispatch_release(queue);
550 
551  if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
552  [ctx->capture_session addOutput:ctx->audio_output];
553  } else {
554  av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
555  return 1;
556  }
557 
558  return 0;
559 }
560 
562 {
564  CVImageBufferRef image_buffer;
565  CMBlockBufferRef block_buffer;
566  CGSize image_buffer_size;
567  AVStream* stream = avformat_new_stream(s, NULL);
568 
569  if (!stream) {
570  return 1;
571  }
572 
573  // Take stream info from the first frame.
574  while (ctx->frames_captured < 1) {
575  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
576  }
577 
578  lock_frames(ctx);
579 
580  ctx->video_stream_index = stream->index;
581 
582  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
583 
584  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
585  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
586 
587  if (image_buffer) {
588  image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
589 
590  stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
591  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
592  stream->codecpar->width = (int)image_buffer_size.width;
593  stream->codecpar->height = (int)image_buffer_size.height;
594  stream->codecpar->format = ctx->pixel_format;
595  } else {
596  stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO;
597  stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
598  stream->codecpar->format = ctx->pixel_format;
599  }
600 
601  CFRelease(ctx->current_frame);
602  ctx->current_frame = nil;
603 
604  unlock_frames(ctx);
605 
606  return 0;
607 }
608 
610 {
612  CMFormatDescriptionRef format_desc;
613  AVStream* stream = avformat_new_stream(s, NULL);
614 
615  if (!stream) {
616  return 1;
617  }
618 
619  // Take stream info from the first frame.
620  while (ctx->audio_frames_captured < 1) {
621  CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
622  }
623 
624  lock_frames(ctx);
625 
626  ctx->audio_stream_index = stream->index;
627 
628  avpriv_set_pts_info(stream, 64, 1, avf_time_base);
629 
630  format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
631  const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
632 
633  if (!basic_desc) {
634  av_log(s, AV_LOG_ERROR, "audio format not available\n");
635  return 1;
636  }
637 
638  stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
639  stream->codecpar->sample_rate = basic_desc->mSampleRate;
640  stream->codecpar->channels = basic_desc->mChannelsPerFrame;
641  stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
642 
643  ctx->audio_channels = basic_desc->mChannelsPerFrame;
644  ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
645  ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
646  ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
647  ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
648  ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
649  ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
650 
651  if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
652  ctx->audio_float &&
653  ctx->audio_bits_per_sample == 32 &&
654  ctx->audio_packed) {
655  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
656  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
657  ctx->audio_signed_integer &&
658  ctx->audio_bits_per_sample == 16 &&
659  ctx->audio_packed) {
660  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
661  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
662  ctx->audio_signed_integer &&
663  ctx->audio_bits_per_sample == 24 &&
664  ctx->audio_packed) {
665  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
666  } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
667  ctx->audio_signed_integer &&
668  ctx->audio_bits_per_sample == 32 &&
669  ctx->audio_packed) {
670  stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
671  } else {
672  av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
673  return 1;
674  }
675 
676  if (ctx->audio_non_interleaved) {
677  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
678  ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
680  if (!ctx->audio_buffer) {
681  av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
682  return 1;
683  }
684  }
685 
686  CFRelease(ctx->current_audio_frame);
687  ctx->current_audio_frame = nil;
688 
689  unlock_frames(ctx);
690 
691  return 0;
692 }
693 
695 {
696  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
697  int capture_screen = 0;
698  uint32_t num_screens = 0;
700  AVCaptureDevice *video_device = nil;
701  AVCaptureDevice *audio_device = nil;
702  // Find capture device
703  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
704  NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
705 
706  ctx->num_video_devices = [devices count] + [devices_muxed count];
707  ctx->first_pts = av_gettime();
708  ctx->first_audio_pts = av_gettime();
709 
712 
713 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
714  CGGetActiveDisplayList(0, NULL, &num_screens);
715 #endif
716 
717  // List devices if requested
718  if (ctx->list_devices) {
719  int index = 0;
720  av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
721  for (AVCaptureDevice *device in devices) {
722  const char *name = [[device localizedName] UTF8String];
723  index = [devices indexOfObject:device];
724  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
725  }
726  for (AVCaptureDevice *device in devices_muxed) {
727  const char *name = [[device localizedName] UTF8String];
728  index = [devices count] + [devices_muxed indexOfObject:device];
729  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
730  }
731 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
732  if (num_screens > 0) {
733  CGDirectDisplayID screens[num_screens];
734  CGGetActiveDisplayList(num_screens, screens, &num_screens);
735  for (int i = 0; i < num_screens; i++) {
736  av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
737  }
738  }
739 #endif
740 
741  av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
742  devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
743  for (AVCaptureDevice *device in devices) {
744  const char *name = [[device localizedName] UTF8String];
745  int index = [devices indexOfObject:device];
746  av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
747  }
748  goto fail;
749  }
750 
751  // parse input filename for video and audio device
753 
754  // check for device index given in filename
755  if (ctx->video_device_index == -1 && ctx->video_filename) {
756  sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
757  }
758  if (ctx->audio_device_index == -1 && ctx->audio_filename) {
759  sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
760  }
761 
762  if (ctx->video_device_index >= 0) {
763  if (ctx->video_device_index < ctx->num_video_devices) {
764  if (ctx->video_device_index < [devices count]) {
765  video_device = [devices objectAtIndex:ctx->video_device_index];
766  } else {
767  video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
768  ctx->video_is_muxed = 1;
769  }
770  } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
771 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
772  CGDirectDisplayID screens[num_screens];
773  CGGetActiveDisplayList(num_screens, screens, &num_screens);
774  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
775 
776  if (ctx->framerate.num > 0) {
777  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
778  }
779 
780 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
781  if (ctx->capture_cursor) {
782  capture_screen_input.capturesCursor = YES;
783  } else {
784  capture_screen_input.capturesCursor = NO;
785  }
786 #endif
787 
788  if (ctx->capture_mouse_clicks) {
789  capture_screen_input.capturesMouseClicks = YES;
790  } else {
791  capture_screen_input.capturesMouseClicks = NO;
792  }
793 
794  video_device = (AVCaptureDevice*) capture_screen_input;
795  capture_screen = 1;
796 #endif
797  } else {
798  av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
799  goto fail;
800  }
801  } else if (ctx->video_filename &&
802  strncmp(ctx->video_filename, "none", 4)) {
803  if (!strncmp(ctx->video_filename, "default", 7)) {
804  video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
805  } else {
806  // looking for video inputs
807  for (AVCaptureDevice *device in devices) {
808  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
809  video_device = device;
810  break;
811  }
812  }
813  // looking for muxed inputs
814  for (AVCaptureDevice *device in devices_muxed) {
815  if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
816  video_device = device;
817  ctx->video_is_muxed = 1;
818  break;
819  }
820  }
821 
822 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
823  // looking for screen inputs
824  if (!video_device) {
825  int idx;
826  if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
827  CGDirectDisplayID screens[num_screens];
828  CGGetActiveDisplayList(num_screens, screens, &num_screens);
829  AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
830  video_device = (AVCaptureDevice*) capture_screen_input;
831  ctx->video_device_index = ctx->num_video_devices + idx;
832  capture_screen = 1;
833 
834  if (ctx->framerate.num > 0) {
835  capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
836  }
837 
838 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
839  if (ctx->capture_cursor) {
840  capture_screen_input.capturesCursor = YES;
841  } else {
842  capture_screen_input.capturesCursor = NO;
843  }
844 #endif
845 
846  if (ctx->capture_mouse_clicks) {
847  capture_screen_input.capturesMouseClicks = YES;
848  } else {
849  capture_screen_input.capturesMouseClicks = NO;
850  }
851  }
852  }
853 #endif
854  }
855 
856  if (!video_device) {
857  av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
858  goto fail;
859  }
860  }
861 
862  // get audio device
863  if (ctx->audio_device_index >= 0) {
864  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
865 
866  if (ctx->audio_device_index >= [devices count]) {
867  av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
868  goto fail;
869  }
870 
871  audio_device = [devices objectAtIndex:ctx->audio_device_index];
872  } else if (ctx->audio_filename &&
873  strncmp(ctx->audio_filename, "none", 4)) {
874  if (!strncmp(ctx->audio_filename, "default", 7)) {
875  audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
876  } else {
877  NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
878 
879  for (AVCaptureDevice *device in devices) {
880  if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
881  audio_device = device;
882  break;
883  }
884  }
885  }
886 
887  if (!audio_device) {
888  av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
889  goto fail;
890  }
891  }
892 
893  // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
894  if (!video_device && !audio_device) {
895  av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
896  goto fail;
897  }
898 
899  if (video_device) {
900  if (ctx->video_device_index < ctx->num_video_devices) {
901  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
902  } else {
903  av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
904  }
905  }
906  if (audio_device) {
907  av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
908  }
909 
910  // Initialize capture session
911  ctx->capture_session = [[AVCaptureSession alloc] init];
912 
913  if (video_device && add_video_device(s, video_device)) {
914  goto fail;
915  }
916  if (audio_device && add_audio_device(s, audio_device)) {
917  }
918 
919  [ctx->capture_session startRunning];
920 
921  /* Unlock device configuration only after the session is started so it
922  * does not reset the capture formats */
923  if (!capture_screen) {
924  [video_device unlockForConfiguration];
925  }
926 
927  if (video_device && get_video_config(s)) {
928  goto fail;
929  }
930 
931  // set audio stream
932  if (audio_device && get_audio_config(s)) {
933  goto fail;
934  }
935 
936  [pool release];
937  return 0;
938 
939 fail:
940  [pool release];
941  destroy_context(ctx);
942  return AVERROR(EIO);
943 }
944 
946  CVPixelBufferRef image_buffer,
947  AVPacket *pkt)
948 {
949  AVFContext *ctx = s->priv_data;
950  int src_linesize[4];
951  const uint8_t *src_data[4];
952  int width = CVPixelBufferGetWidth(image_buffer);
953  int height = CVPixelBufferGetHeight(image_buffer);
954  int status;
955 
956  memset(src_linesize, 0, sizeof(src_linesize));
957  memset(src_data, 0, sizeof(src_data));
958 
959  status = CVPixelBufferLockBaseAddress(image_buffer, 0);
960  if (status != kCVReturnSuccess) {
961  av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);
962  return AVERROR_EXTERNAL;
963  }
964 
965  if (CVPixelBufferIsPlanar(image_buffer)) {
966  size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
967  int i;
968  for(i = 0; i < plane_count; i++){
969  src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
970  src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
971  }
972  } else {
973  src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
974  src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
975  }
976 
977  status = av_image_copy_to_buffer(pkt->data, pkt->size,
978  src_data, src_linesize,
979  ctx->pixel_format, width, height, 1);
980 
981 
982 
983  CVPixelBufferUnlockBaseAddress(image_buffer, 0);
984 
985  return status;
986 }
987 
989 {
991 
992  do {
993  CVImageBufferRef image_buffer;
994  CMBlockBufferRef block_buffer;
995  lock_frames(ctx);
996 
997  if (ctx->current_frame != nil) {
998  int status;
999  int length = 0;
1000 
1001  image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
1002  block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
1003 
1004  if (image_buffer != nil) {
1005  length = (int)CVPixelBufferGetDataSize(image_buffer);
1006  } else if (block_buffer != nil) {
1007  length = (int)CMBlockBufferGetDataLength(block_buffer);
1008  } else {
1009  return AVERROR(EINVAL);
1010  }
1011 
1012  if (av_new_packet(pkt, length) < 0) {
1013  return AVERROR(EIO);
1014  }
1015 
1016  CMItemCount count;
1017  CMSampleTimingInfo timing_info;
1018 
1019  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
1020  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1021  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1022  }
1023 
1024  pkt->stream_index = ctx->video_stream_index;
1025  pkt->flags |= AV_PKT_FLAG_KEY;
1026 
1027  if (image_buffer) {
1028  status = copy_cvpixelbuffer(s, image_buffer, pkt);
1029  } else {
1030  status = 0;
1031  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1032  if (ret != kCMBlockBufferNoErr) {
1033  status = AVERROR(EIO);
1034  }
1035  }
1036  CFRelease(ctx->current_frame);
1037  ctx->current_frame = nil;
1038 
1039  if (status < 0)
1040  return status;
1041  } else if (ctx->current_audio_frame != nil) {
1042  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
1043  int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
1044 
1045  if (!block_buffer || !block_buffer_size) {
1046  return AVERROR(EIO);
1047  }
1048 
1049  if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
1050  return AVERROR_BUFFER_TOO_SMALL;
1051  }
1052 
1053  if (av_new_packet(pkt, block_buffer_size) < 0) {
1054  return AVERROR(EIO);
1055  }
1056 
1057  CMItemCount count;
1058  CMSampleTimingInfo timing_info;
1059 
1060  if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
1061  AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
1062  pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
1063  }
1064 
1065  pkt->stream_index = ctx->audio_stream_index;
1066  pkt->flags |= AV_PKT_FLAG_KEY;
1067 
1068  if (ctx->audio_non_interleaved) {
1069  int sample, c, shift, num_samples;
1070 
1071  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
1072  if (ret != kCMBlockBufferNoErr) {
1073  return AVERROR(EIO);
1074  }
1075 
1076  num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
1077 
1078  // transform decoded frame into output format
1079  #define INTERLEAVE_OUTPUT(bps) \
1080  { \
1081  int##bps##_t **src; \
1082  int##bps##_t *dest; \
1083  src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \
1084  if (!src) return AVERROR(EIO); \
1085  for (c = 0; c < ctx->audio_channels; c++) { \
1086  src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
1087  } \
1088  dest = (int##bps##_t*)pkt->data; \
1089  shift = bps - ctx->audio_bits_per_sample; \
1090  for (sample = 0; sample < num_samples; sample++) \
1091  for (c = 0; c < ctx->audio_channels; c++) \
1092  *dest++ = src[c][sample] << shift; \
1093  av_freep(&src); \
1094  }
1095 
1096  if (ctx->audio_bits_per_sample <= 16) {
1097  INTERLEAVE_OUTPUT(16)
1098  } else {
1099  INTERLEAVE_OUTPUT(32)
1100  }
1101  } else {
1102  OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
1103  if (ret != kCMBlockBufferNoErr) {
1104  return AVERROR(EIO);
1105  }
1106  }
1107 
1108  CFRelease(ctx->current_audio_frame);
1109  ctx->current_audio_frame = nil;
1110  } else {
1111  pkt->data = NULL;
1113  }
1114 
1115  unlock_frames(ctx);
1116  } while (!pkt->data);
1117 
1118  return 0;
1119 }
1120 
1122 {
1124  destroy_context(ctx);
1125  return 0;
1126 }
1127 
1128 static const AVOption options[] = {
1129  { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1130  { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1131  { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1132  { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1133  { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1134  { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1135  { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1136  { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1137  { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1138 
1139  { NULL },
1140 };
1141 
1142 static const AVClass avf_class = {
1143  .class_name = "AVFoundation input device",
1144  .item_name = av_default_item_name,
1145  .option = options,
1146  .version = LIBAVUTIL_VERSION_INT,
1148 };
1149 
1151  .name = "avfoundation",
1152  .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
1153  .priv_data_size = sizeof(AVFContext),
1156  .read_close = avf_close,
1157  .flags = AVFMT_NOFILE,
1158  .priv_class = &avf_class,
1159 };
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:81
int audio_buffer_size
Definition: avfoundation.m:123
#define NULL
Definition: coverity.c:32
static const struct AVFPixelFormatSpec avf_pixel_formats[]
Definition: avfoundation.m:53
static void lock_frames(AVFContext *ctx)
Definition: avfoundation.m:134
static int shift(int a, int b)
Definition: sonic.c:82
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:108
#define pthread_mutex_lock(a)
Definition: ffprobe.c:61
AVCaptureSession * capture_session
Definition: avfoundation.m:127
static av_always_inline int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
Definition: os2threads.h:166
AVOption.
Definition: opt.h:246
int32_t * audio_buffer
Definition: avfoundation.m:122
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:453
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
int list_devices
Definition: avfoundation.m:103
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4886
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:35
int audio_frames_captured
Definition: avfoundation.m:87
AVFContext * _context
Definition: avfoundation.m:194
static const AVOption options[]
int num
Numerator.
Definition: rational.h:59
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), little-endian, X=unused/undefined ...
Definition: pixfmt.h:108
int size
Definition: avcodec.h:1478
static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Definition: avfoundation.m:383
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:191
CMSampleBufferRef current_audio_frame
Definition: avfoundation.m:131
static int get_audio_config(AVFormatContext *s)
Definition: avfoundation.m:609
int audio_non_interleaved
Definition: avfoundation.m:120
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition: pixfmt.h:239
static AVPacket pkt
#define sample
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate.The lists are not just lists
int framerate
Definition: h264_levels.c:65
AudioReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:192
static av_always_inline int pthread_cond_destroy(pthread_cond_t *cond)
Definition: os2threads.h:140
AVRational framerate
Definition: avfoundation.m:95
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), little-endian
Definition: pixfmt.h:106
int capture_mouse_clicks
Definition: avfoundation.m:99
Format I/O context.
Definition: avformat.h:1358
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
int audio_signed_integer
Definition: avfoundation.m:118
uint8_t
#define av_malloc(s)
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:238
AVOptions.
static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
Configure the video device.
Definition: avfoundation.m:286
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4459
packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), big-endian
Definition: pixfmt.h:105
#define height
uint8_t * data
Definition: avcodec.h:1477
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
static av_always_inline int pthread_cond_signal(pthread_cond_t *cond)
Definition: os2threads.h:148
static int copy_cvpixelbuffer(AVFormatContext *s, CVPixelBufferRef image_buffer, AVPacket *pkt)
Definition: avfoundation.m:945
static void parse_device_name(AVFormatContext *s)
Definition: avfoundation.m:262
#define av_log(a,...)
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: avcodec.h:1509
AVInputFormat ff_avfoundation_demuxer
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
Main libavdevice API header.
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:86
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
char * video_filename
Definition: avfoundation.m:109
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
char * url
input or output URL.
Definition: avformat.h:1454
enum AVPixelFormat ff_id
Definition: avfoundation.m:49
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:197
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
Tag description
Definition: snow.txt:206
GLsizei GLsizei * length
Definition: opengl_enc.c:114
static int avf_read_header(AVFormatContext *s)
Definition: avfoundation.m:694
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:390
int num_video_devices
Definition: avfoundation.m:112
GLsizei count
Definition: opengl_enc.c:108
int capture_cursor
Definition: avfoundation.m:98
#define fail()
Definition: checkasm.h:120
int flags
A combination of AV_PKT_FLAG values.
Definition: avcodec.h:1483
common internal API header
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define width
typedef void(APIENTRY *FF_PFNGLACTIVETEXTUREPROC)(GLenum texture)
static const int avf_time_base
Definition: avfoundation.m:41
static int capture_screen(CCaptionSubContext *ctx)
Definition: ccaption_dec.c:435
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:100
int frames_captured
Definition: avfoundation.m:86
#define s(width, name)
Definition: cbs_vp9.c:257
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
int audio_bits_per_sample
Definition: avfoundation.m:115
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:65
packed RGB 16:16:16, 48bpp, 16B, 16G, 16R, the 2-byte value for each R/G/B component is stored as big...
Definition: pixfmt.h:148
static void error(const char *err)
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:530
if(ret)
int64_t av_gettime(void)
Get the current time in microseconds.
Definition: time.c:39
Stream structure.
Definition: avformat.h:881
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
Definition: avfoundation.m:517
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:251
int video_stream_index
Definition: avfoundation.m:105
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:177
static AVRational av_make_q(int num, int den)
Create an AVRational.
Definition: rational.h:71
static const AVRational avf_time_base_q
Definition: avfoundation.m:43
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:67
AVCaptureVideoDataOutput * video_output
Definition: avfoundation.m:128
pthread_mutex_t frame_lock
Definition: avfoundation.m:90
int audio_device_index
Definition: avfoundation.m:106
static int get_video_config(AVFormatContext *s)
Definition: avfoundation.m:561
static const AVClass avf_class
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
planar YUV 4:4:4 64bpp, (1 Cr & Cb sample per 1x1 Y & A samples, little-endian)
Definition: pixfmt.h:195
int index
Definition: gxfenc.c:89
#define INTERLEAVE_OUTPUT(bps)
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int audio_channels
Definition: avfoundation.m:114
#define AV_OPT_FLAG_DECODING_PARAM
a generic parameter which can be set by the user for demuxing or decoding
Definition: opt.h:277
static void unlock_frames(AVFContext *ctx)
Definition: avfoundation.m:139
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
offset must point to AVRational
Definition: opt.h:236
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:240
int64_t first_pts
Definition: avfoundation.m:88
FrameReciever class - delegate for AVCaptureSession.
Definition: avfoundation.m:146
offset must point to two consecutive integers
Definition: opt.h:233
int audio_packed
Definition: avfoundation.m:119
misc parsing utilities
CMSampleBufferRef current_frame
Definition: avfoundation.m:130
#define flags(name, subs,...)
Definition: cbs_av1.c:561
int audio_stream_index
Definition: avfoundation.m:107
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:388
packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), big-endian , X=unused/undefined
Definition: pixfmt.h:107
enum AVPixelFormat pixel_format
Definition: avfoundation.m:125
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok()...
Definition: avstring.c:184
static int FUNC() timing_info(CodedBitstreamContext *ctx, RWContext *rw, AV1RawTimingInfo *current)
int video_is_muxed
Definition: avfoundation.m:101
Y , 1bpp, 0 is black, 1 is white, in each byte pixels are ordered from the msb to the lsb...
Definition: pixfmt.h:76
int
char * audio_filename
Definition: avfoundation.m:110
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Y , 8bpp.
Definition: pixfmt.h:74
_fmutex pthread_mutex_t
Definition: os2threads.h:49
AVCaptureAudioDataOutput * audio_output
Definition: avfoundation.m:129
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:463
static void destroy_context(AVFContext *ctx)
Definition: avfoundation.m:236
static av_always_inline int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
Definition: os2threads.h:129
int den
Denominator.
Definition: rational.h:60
static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
Definition: avfoundation.m:988
AVFContext * _context
Definition: avfoundation.m:148
void * priv_data
Format private data.
Definition: avformat.h:1386
id avf_audio_delegate
Definition: avfoundation.m:93
static int avf_close(AVFormatContext *s)
int64_t first_audio_pts
Definition: avfoundation.m:89
pthread_cond_t frame_wait_cond
Definition: avfoundation.m:91
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: avcodec.h:1476
int64_t av_get_default_channel_layout(int nb_channels)
Return default channel layout for a given number of channels.
#define av_freep(p)
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:654
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2438
int stream_index
Definition: avcodec.h:1479
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition: pixfmt.h:237
enum AVCodecID id
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
This structure stores compressed data.
Definition: avcodec.h:1454
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:399
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1470
int capture_raw_data
Definition: avfoundation.m:100
const char * name
Definition: opengl_enc.c:102
static uint8_t tmp[11]
Definition: aes_ctr.c:26
int video_device_index
Definition: avfoundation.m:104