FFmpeg
vf_coreimage.m
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Thilo Borgmann
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Video processing based on Apple's CoreImage API
24  */
25 
26 #import <CoreImage/CoreImage.h>
27 #import <AppKit/AppKit.h>
28 
29 #include "avfilter.h"
30 #include "filters.h"
31 #include "formats.h"
32 #include "video.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/mem.h"
35 #include "libavutil/opt.h"
36 #include "libavutil/pixdesc.h"
37 
38 typedef struct CoreImageContext {
39  const AVClass *class;
40 
41  int is_video_source; ///< filter is used as video source
42 
43  int w, h; ///< video size
44  AVRational sar; ///< sample aspect ratio
45  AVRational frame_rate; ///< video frame rate
46  AVRational time_base; ///< stream time base
47  int64_t duration; ///< duration expressed in microseconds
48  int64_t pts; ///< increasing presentation time stamp
49  AVFrame *picref; ///< cached reference containing the painted picture
50 
51  CFTypeRef glctx; ///< OpenGL context
52  CGContextRef cgctx; ///< Bitmap context for image copy
53  CFTypeRef input_image; ///< Input image container for passing into Core Image API
54  CGColorSpaceRef color_space; ///< Common color space for input image and cgcontext
55  int bits_per_component; ///< Shared bpc for input-output operation
56 
57  char *filter_string; ///< The complete user provided filter definition
58  CFTypeRef *filters; ///< CIFilter object for all requested filters
59  int num_filters; ///< Amount of filters in *filters
60 
61  char *output_rect; ///< Rectangle to be filled with filter intput
62  int list_filters; ///< Option used to list all available filters including generators
63  int list_generators; ///< Option used to list all available generators
65 
67 {
69  CoreImageContext *ctx = link->src->priv;
70 
71  link->w = ctx->w;
72  link->h = ctx->h;
74  l->frame_rate = ctx->frame_rate;
75  link->time_base = ctx->time_base;
76 
78  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
79 
80  return 0;
81 }
82 
83 /** Determine image properties from input link of filter chain.
84  */
86 {
87  CoreImageContext *ctx = link->dst->priv;
89  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
90 
91  return 0;
92 }
93 
94 /** Print a list of all available filters including options and respective value ranges and defaults.
95  */
97 {
98  // querying filters and attributes
99  NSArray *filter_categories = nil;
100 
101  if (ctx->list_generators && !ctx->list_filters) {
102  filter_categories = [NSArray arrayWithObjects:kCICategoryGenerator, nil];
103  }
104 
105  for (NSString *filter_name in [CIFilter filterNamesInCategories:filter_categories]) {
106  CIFilter *filter = [CIFilter filterWithName:filter_name];
107  NSDictionary<NSString *, id> *filter_attribs = [filter attributes];
108 
109  av_log(ctx, AV_LOG_INFO, "Filter: %s\n", [filter_name UTF8String]);
110 
111  for (NSString *input in [filter inputKeys]) {
112  NSDictionary *input_attribs = [filter_attribs valueForKey:input];
113  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
114  if ([input_class isEqualToString:@"NSNumber"]) {
115  NSNumber *value_default = [input_attribs valueForKey:kCIAttributeDefault];
116  NSNumber *value_min = [input_attribs valueForKey:kCIAttributeSliderMin];
117  NSNumber *value_max = [input_attribs valueForKey:kCIAttributeSliderMax];
118 
119  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\t[%s %s][%s]\n",
120  [input UTF8String],
121  [input_class UTF8String],
122  [[value_min stringValue] UTF8String],
123  [[value_max stringValue] UTF8String],
124  [[value_default stringValue] UTF8String]);
125  } else {
126  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\n",
127  [input UTF8String],
128  [input_class UTF8String]);
129  }
130  }
131  }
132 }
133 
135 {
136  int i;
137 
138  // (re-)initialize input image
139  const CGSize frame_size = {
140  frame->width,
141  frame->height
142  };
143 
144  NSData *data = [NSData dataWithBytesNoCopy:frame->data[0]
145  length:frame->height*frame->linesize[0]
146  freeWhenDone:NO];
147 
148  CIImage *ret = [(__bridge CIImage*)ctx->input_image initWithBitmapData:data
149  bytesPerRow:frame->linesize[0]
150  size:frame_size
151  format:kCIFormatARGB8
152  colorSpace:ctx->color_space]; //kCGColorSpaceGenericRGB
153  if (!ret) {
154  av_log(ctx, AV_LOG_ERROR, "Input image could not be initialized.\n");
155  return AVERROR_EXTERNAL;
156  }
157 
158  CIFilter *filter = NULL;
159  CIImage *filter_input = (__bridge CIImage*)ctx->input_image;
160  CIImage *filter_output = NULL;
161 
162  // successively apply all filters
163  for (i = 0; i < ctx->num_filters; i++) {
164  if (i) {
165  // set filter input to previous filter output
166  filter_input = [(__bridge CIImage*)ctx->filters[i-1] valueForKey:kCIOutputImageKey];
167  CGRect out_rect = [filter_input extent];
168  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
169  // do not keep padded image regions after filtering
170  out_rect.origin.x = 0.0f;
171  out_rect.origin.y = 0.0f;
172  out_rect.size.width = frame->width;
173  out_rect.size.height = frame->height;
174  }
175  filter_input = [filter_input imageByCroppingToRect:out_rect];
176  }
177 
178  filter = (__bridge CIFilter*)ctx->filters[i];
179 
180  // do not set input image for the first filter if used as video source
181  if (!ctx->is_video_source || i) {
182  @try {
183  [filter setValue:filter_input forKey:kCIInputImageKey];
184  } @catch (NSException *exception) {
185  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
186  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
187  return AVERROR_EXTERNAL;
188  } else {
189  av_log(ctx, AV_LOG_WARNING, "Selected filter does not accept an input image.\n");
190  }
191  }
192  }
193  }
194 
195  // get output of last filter
196  filter_output = [filter valueForKey:kCIOutputImageKey];
197 
198  if (!filter_output) {
199  av_log(ctx, AV_LOG_ERROR, "Filter output not available.\n");
200  return AVERROR_EXTERNAL;
201  }
202 
203  // do not keep padded image regions after filtering
204  CGRect out_rect = [filter_output extent];
205  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
206  av_log(ctx, AV_LOG_DEBUG, "Cropping output image.\n");
207  out_rect.origin.x = 0.0f;
208  out_rect.origin.y = 0.0f;
209  out_rect.size.width = frame->width;
210  out_rect.size.height = frame->height;
211  }
212 
213  CGImageRef out = [(__bridge CIContext*)ctx->glctx createCGImage:filter_output
214  fromRect:out_rect];
215 
216  if (!out) {
217  av_log(ctx, AV_LOG_ERROR, "Cannot create valid output image.\n");
218  }
219 
220  // create bitmap context on the fly for rendering into current frame->data[]
221  if (ctx->cgctx) {
222  CGContextRelease(ctx->cgctx);
223  ctx->cgctx = NULL;
224  }
225  size_t out_width = CGImageGetWidth(out);
226  size_t out_height = CGImageGetHeight(out);
227 
228  if (out_width > frame->width || out_height > frame->height) { // this might result in segfault
229  av_log(ctx, AV_LOG_WARNING, "Output image has unexpected size: %lux%lu (expected: %ix%i). This may crash...\n",
230  out_width, out_height, frame->width, frame->height);
231  }
232  ctx->cgctx = CGBitmapContextCreate(frame->data[0],
233  frame->width,
234  frame->height,
235  ctx->bits_per_component,
236  frame->linesize[0],
237  ctx->color_space,
238  (uint32_t)kCGImageAlphaPremultipliedFirst); // ARGB
239  if (!ctx->cgctx) {
240  av_log(ctx, AV_LOG_ERROR, "CGBitmap context cannot be created.\n");
241  return AVERROR_EXTERNAL;
242  }
243 
244  // copy ("draw") the output image into the frame data
245  CGRect rect = {{0,0},{frame->width, frame->height}};
246  if (ctx->output_rect) {
247  @try {
248  NSString *tmp_string = [NSString stringWithUTF8String:ctx->output_rect];
249  NSRect tmp = NSRectFromString(tmp_string);
250  rect = NSRectToCGRect(tmp);
251  } @catch (NSException *exception) {
252  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
253  return AVERROR_EXTERNAL;
254  }
255  if (rect.size.width == 0.0f) {
256  av_log(ctx, AV_LOG_WARNING, "Width of output rect is zero.\n");
257  }
258  if (rect.size.height == 0.0f) {
259  av_log(ctx, AV_LOG_WARNING, "Height of output rect is zero.\n");
260  }
261  }
262 
263  CGContextDrawImage(ctx->cgctx, rect, out);
264 
265  return ff_filter_frame(link, frame);
266 }
267 
268 /** Apply all valid filters successively to the input image.
269  * The final output image is copied from the GPU by "drawing" using a bitmap context.
270  */
272 {
273  return apply_filter(link->dst->priv, link->dst->outputs[0], frame);
274 }
275 
277 {
278  CoreImageContext *ctx = link->src->priv;
279  AVFrame *frame;
280 
281  if (ctx->duration >= 0 &&
282  av_rescale_q(ctx->pts, ctx->time_base, AV_TIME_BASE_Q) >= ctx->duration) {
283  return AVERROR_EOF;
284  }
285 
286  if (!ctx->picref) {
287  ctx->picref = ff_get_video_buffer(link, ctx->w, ctx->h);
288  if (!ctx->picref) {
289  return AVERROR(ENOMEM);
290  }
291  }
292 
293  frame = av_frame_clone(ctx->picref);
294  if (!frame) {
295  return AVERROR(ENOMEM);
296  }
297 
298  frame->pts = ctx->pts;
299  frame->duration = 1;
300  frame->flags |= AV_FRAME_FLAG_KEY;
301  frame->flags &= ~AV_FRAME_FLAG_INTERLACED;
302 
304 #if FF_API_FRAME_KEY
305  frame->key_frame = 1;
306 #endif
307 #if FF_API_INTERLACED_FRAME
308  frame->interlaced_frame = 0;
309 #endif
311 
312  frame->pict_type = AV_PICTURE_TYPE_I;
313  frame->sample_aspect_ratio = ctx->sar;
314 
315  ctx->pts++;
316 
317  return apply_filter(ctx, link, frame);
318 }
319 
320 /** Set an option of the given filter to the provided key-value pair.
321  */
322 static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
323 {
324  NSString *input_key = [NSString stringWithUTF8String:key];
325  NSString *input_val = [NSString stringWithUTF8String:value];
326 
327  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
328  NSDictionary *input_attribs = [filter_attribs valueForKey:input_key];
329 
330  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
331  NSString *input_type = [input_attribs valueForKey:kCIAttributeType];
332 
333  if (!input_attribs) {
334  av_log(ctx, AV_LOG_WARNING, "Skipping unknown option: \"%s\".\n",
335  [input_key UTF8String]); // [[filter name] UTF8String]) not currently defined...
336  return;
337  }
338 
339  av_log(ctx, AV_LOG_DEBUG, "key: %s, val: %s, #attribs: %lu, class: %s, type: %s\n",
340  [input_key UTF8String],
341  [input_val UTF8String],
342  input_attribs ? (unsigned long)[input_attribs count] : -1,
343  [input_class UTF8String],
344  [input_type UTF8String]);
345 
346  if ([input_class isEqualToString:@"NSNumber"]) {
347  float input = input_val.floatValue;
348  NSNumber *max_value = [input_attribs valueForKey:kCIAttributeSliderMax];
349  NSNumber *min_value = [input_attribs valueForKey:kCIAttributeSliderMin];
350  NSNumber *used_value = nil;
351 
352 #define CLAMP_WARNING do { \
353 av_log(ctx, AV_LOG_WARNING, "Value of \"%f\" for option \"%s\" is out of range [%f %f], clamping to \"%f\".\n", \
354  input, \
355  [input_key UTF8String], \
356  min_value.floatValue, \
357  max_value.floatValue, \
358  used_value.floatValue); \
359 } while(0)
360  if (input > max_value.floatValue) {
361  used_value = max_value;
363  } else if (input < min_value.floatValue) {
364  used_value = min_value;
366  } else {
367  used_value = [NSNumber numberWithFloat:input];
368  }
369 
370  [filter setValue:used_value forKey:input_key];
371  } else if ([input_class isEqualToString:@"CIVector"]) {
372  CIVector *input = [CIVector vectorWithString:input_val];
373 
374  if (!input) {
375  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIVctor description: \"%s\".\n",
376  [input_val UTF8String]);
377  return;
378  }
379 
380  [filter setValue:input forKey:input_key];
381  } else if ([input_class isEqualToString:@"CIColor"]) {
382  CIColor *input = [CIColor colorWithString:input_val];
383 
384  if (!input) {
385  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIColor description: \"%s\".\n",
386  [input_val UTF8String]);
387  return;
388  }
389 
390  [filter setValue:input forKey:input_key];
391  } else if ([input_class isEqualToString:@"NSString"]) { // set display name as string with latin1 encoding
392  [filter setValue:input_val forKey:input_key];
393  } else if ([input_class isEqualToString:@"NSData"]) { // set display name as string with latin1 encoding
394  NSData *input = [NSData dataWithBytes:(const void*)[input_val cStringUsingEncoding:NSISOLatin1StringEncoding]
395  length:[input_val lengthOfBytesUsingEncoding:NSISOLatin1StringEncoding]];
396 
397  if (!input) {
398  av_log(ctx, AV_LOG_WARNING, "Skipping invalid NSData description: \"%s\".\n",
399  [input_val UTF8String]);
400  return;
401  }
402 
403  [filter setValue:input forKey:input_key];
404  } else {
405  av_log(ctx, AV_LOG_WARNING, "Skipping unsupported option class: \"%s\".\n",
406  [input_class UTF8String]);
407  avpriv_report_missing_feature(ctx, "Handling of some option classes");
408  return;
409  }
410 }
411 
412 /** Create a filter object by a given name and set all options to defaults.
413  * Overwrite any option given by the user to the provided value in filter_options.
414  */
415 static CIFilter* create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
416 {
417  // create filter object
418  CIFilter *filter = [CIFilter filterWithName:[NSString stringWithUTF8String:filter_name]];
419 
420  // set default options
421  [filter setDefaults];
422 
423  // set user options
424  if (filter_options) {
425  const AVDictionaryEntry *o = NULL;
426  while ((o = av_dict_iterate(filter_options, o))) {
427  set_option(ctx, filter, o->key, o->value);
428  }
429  }
430 
431  return filter;
432 }
433 
434 static av_cold int init(AVFilterContext *fctx)
435 {
436  CoreImageContext *ctx = fctx->priv;
437  AVDictionary *filter_dict = NULL;
438  const AVDictionaryEntry *f = NULL;
439  const AVDictionaryEntry *o = NULL;
440  int ret;
441  int i;
442 
443  if (ctx->list_filters || ctx->list_generators) {
444  list_filters(ctx);
445  return AVERROR_EXIT;
446  }
447 
448  if (ctx->filter_string) {
449  // parse filter string (filter=name@opt=val@opt2=val2#name2@opt3=val3) for filters separated by #
450  av_log(ctx, AV_LOG_DEBUG, "Filter_string: %s\n", ctx->filter_string);
451  ret = av_dict_parse_string(&filter_dict, ctx->filter_string, "@", "#", AV_DICT_MULTIKEY); // parse filter_name:all_filter_options
452  if (ret) {
453  av_dict_free(&filter_dict);
454  av_log(ctx, AV_LOG_ERROR, "Parsing of filters failed.\n");
455  return AVERROR(EIO);
456  }
457  ctx->num_filters = av_dict_count(filter_dict);
458  av_log(ctx, AV_LOG_DEBUG, "Filter count: %i\n", ctx->num_filters);
459 
460  // allocate CIFilter array
461  ctx->filters = av_calloc(ctx->num_filters, sizeof(CIFilter*));
462  if (!ctx->filters) {
463  av_log(ctx, AV_LOG_ERROR, "Could not allocate filter array.\n");
464  return AVERROR(ENOMEM);
465  }
466 
467  // parse filters for option key-value pairs (opt=val@opt2=val2) separated by @
468  i = 0;
469  while ((f = av_dict_iterate(filter_dict, f))) {
470  AVDictionary *filter_options = NULL;
471 
472  if (strncmp(f->value, "default", 7)) { // not default
473  ret = av_dict_parse_string(&filter_options, f->value, "=", "@", 0); // parse option_name:option_value
474  if (ret) {
475  av_dict_free(&filter_options);
476  av_log(ctx, AV_LOG_ERROR, "Parsing of filter options for \"%s\" failed.\n", f->key);
477  return AVERROR(EIO);
478  }
479  }
480 
481  if (av_log_get_level() >= AV_LOG_DEBUG) {
482  av_log(ctx, AV_LOG_DEBUG, "Creating filter %i: \"%s\":\n", i, f->key);
483  if (!filter_options) {
484  av_log(ctx, AV_LOG_DEBUG, "\tusing default options\n");
485  } else {
486  while ((o = av_dict_iterate(filter_options, o))) {
487  av_log(ctx, AV_LOG_DEBUG, "\t%s: %s\n", o->key, o->value);
488  }
489  }
490  }
491 
492  ctx->filters[i] = CFBridgingRetain(create_filter(ctx, f->key, filter_options));
493  if (!ctx->filters[i]) {
494  av_log(ctx, AV_LOG_ERROR, "Could not create filter \"%s\".\n", f->key);
495  return AVERROR(EINVAL);
496  }
497 
498  i++;
499  }
500  } else {
501  av_log(ctx, AV_LOG_ERROR, "No filters specified.\n");
502  return AVERROR(EINVAL);
503  }
504 
505  // create GPU context on OSX
506  const NSOpenGLPixelFormatAttribute attr[] = {
507  NSOpenGLPFAAccelerated,
508  NSOpenGLPFANoRecovery,
509  NSOpenGLPFAColorSize, 32,
510  0
511  };
512 
513  NSOpenGLPixelFormat *pixel_format = [[NSOpenGLPixelFormat alloc] initWithAttributes:(void *)&attr];
514  ctx->color_space = CGColorSpaceCreateWithName(kCGColorSpaceGenericRGB);
515  ctx->glctx = CFBridgingRetain([CIContext contextWithCGLContext:CGLGetCurrentContext()
516  pixelFormat:[pixel_format CGLPixelFormatObj]
517  colorSpace:ctx->color_space
518  options:nil]);
519 
520  if (!ctx->glctx) {
521  av_log(ctx, AV_LOG_ERROR, "CIContext not created.\n");
522  return AVERROR_EXTERNAL;
523  }
524 
525  // Creating an empty input image as input container for the context
526  ctx->input_image = CFBridgingRetain([CIImage emptyImage]);
527 
528  return 0;
529 }
530 
532 {
533  CoreImageContext *ctx = fctx->priv;
534 
535  ctx->is_video_source = 1;
536  ctx->time_base = av_inv_q(ctx->frame_rate);
537  ctx->pts = 0;
538 
539  return init(fctx);
540 }
541 
542 static av_cold void uninit(AVFilterContext *fctx)
543 {
544 #define SafeCFRelease(ptr) do { \
545  if (ptr) { \
546  CFRelease(ptr); \
547  ptr = NULL; \
548  } \
549 } while (0)
550 
551  CoreImageContext *ctx = fctx->priv;
552 
553  SafeCFRelease(ctx->glctx);
554  SafeCFRelease(ctx->cgctx);
555  SafeCFRelease(ctx->color_space);
556  SafeCFRelease(ctx->input_image);
557 
558  if (ctx->filters) {
559  for (int i = 0; i < ctx->num_filters; i++) {
560  SafeCFRelease(ctx->filters[i]);
561  }
562  av_freep(&ctx->filters);
563  }
564 
565  av_frame_free(&ctx->picref);
566 }
567 
569  {
570  .name = "default",
571  .type = AVMEDIA_TYPE_VIDEO,
572  .filter_frame = filter_frame,
573  .config_props = config_input,
574  },
575 };
576 
578  {
579  .name = "default",
580  .type = AVMEDIA_TYPE_VIDEO,
581  },
582 };
583 
585  {
586  .name = "default",
587  .type = AVMEDIA_TYPE_VIDEO,
588  .request_frame = request_frame,
589  .config_props = config_output,
590  },
591 };
592 
593 #define OFFSET(x) offsetof(CoreImageContext, x)
594 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
595 
596 #define GENERATOR_OPTIONS \
597  {"size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
598  {"s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
599  {"rate", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
600  {"r", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
601  {"duration", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
602  {"d", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
603  {"sar", "set video sample aspect ratio", OFFSET(sar), AV_OPT_TYPE_RATIONAL, {.dbl = 1}, 0, INT_MAX, FLAGS},
604 
605 #define FILTER_OPTIONS \
606  {"list_filters", "list available filters", OFFSET(list_filters), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
607  {"list_generators", "list available generators", OFFSET(list_generators), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
608  {"filter", "names and options of filters to apply", OFFSET(filter_string), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS}, \
609  {"output_rect", "output rectangle within output image", OFFSET(output_rect), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS},
610 
611 
612 // definitions for coreimage video filter
613 static const AVOption coreimage_options[] = {
615  { NULL }
616 };
617 
618 AVFILTER_DEFINE_CLASS(coreimage);
619 
621  .name = "coreimage",
622  .description = NULL_IF_CONFIG_SMALL("Video filtering using CoreImage API."),
623  .init = init,
624  .uninit = uninit,
625  .priv_size = sizeof(CoreImageContext),
626  .priv_class = &coreimage_class,
630 };
631 
632 // definitions for coreimagesrc video source
633 static const AVOption coreimagesrc_options[] = {
636  { NULL }
637 };
638 
639 AVFILTER_DEFINE_CLASS(coreimagesrc);
640 
642  .name = "coreimagesrc",
643  .description = NULL_IF_CONFIG_SMALL("Video source using image generators of CoreImage API."),
644  .init = init_src,
645  .uninit = uninit,
646  .priv_size = sizeof(CoreImageContext),
647  .priv_class = &coreimagesrc_class,
648  .inputs = NULL,
651 };
ff_vsrc_coreimagesrc
const AVFilter ff_vsrc_coreimagesrc
Definition: vf_coreimage.m:641
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:116
CoreImageContext::glctx
CFTypeRef glctx
OpenGL context.
Definition: vf_coreimage.m:51
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
CoreImageContext::w
int w
Definition: vf_coreimage.m:43
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
CoreImageContext::duration
int64_t duration
duration expressed in microseconds
Definition: vf_coreimage.m:47
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:55
CoreImageContext::cgctx
CGContextRef cgctx
Bitmap context for image copy.
Definition: vf_coreimage.m:52
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1061
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3170
AVERROR_EOF
#define AVERROR_EOF
End of file.
Definition: error.h:57
av_dict_count
int av_dict_count(const AVDictionary *m)
Get number of entries in dictionary.
Definition: dict.c:39
rect
Definition: f_ebur128.c:76
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(coreimage)
AV_TIME_BASE_Q
#define AV_TIME_BASE_Q
Internal time base represented as fractional value.
Definition: avutil.h:264
int64_t
long long int64_t
Definition: coverity.c:34
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: filters.h:262
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
pixdesc.h
AVOption
AVOption.
Definition: opt.h:429
create_filter
static CIFilter * create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
Create a filter object by a given name and set all options to defaults.
Definition: vf_coreimage.m:415
data
const char data[16]
Definition: mxf.c:149
CoreImageContext::color_space
CGColorSpaceRef color_space
Common color space for input image and cgcontext.
Definition: vf_coreimage.m:54
vf_coreimage_outputs
static const AVFilterPad vf_coreimage_outputs[]
Definition: vf_coreimage.m:577
filter
void(* filter)(uint8_t *src, int stride, int qscale)
Definition: h263dsp.c:29
av_get_bits_per_pixel
int av_get_bits_per_pixel(const AVPixFmtDescriptor *pixdesc)
Return the number of bits per pixel used by the pixel format described by pixdesc.
Definition: pixdesc.c:3122
AVDictionary
Definition: dict.c:34
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:205
coreimage_options
static const AVOption coreimage_options[]
Definition: vf_coreimage.m:613
video.h
CoreImageContext
Definition: vf_coreimage.m:38
formats.h
ff_vf_coreimage
const AVFilter ff_vf_coreimage
Definition: vf_coreimage.m:620
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *frame)
Apply all valid filters successively to the input image.
Definition: vf_coreimage.m:271
CLAMP_WARNING
#define CLAMP_WARNING
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:472
vsrc_coreimagesrc_outputs
static const AVFilterPad vsrc_coreimagesrc_outputs[]
Definition: vf_coreimage.m:584
request_frame
static int request_frame(AVFilterLink *link)
Definition: vf_coreimage.m:276
init
static av_cold int init(AVFilterContext *fctx)
Definition: vf_coreimage.m:434
AVFilterPad
A filter pad used for either input or output.
Definition: filters.h:38
config_output
static int config_output(AVFilterLink *link)
Definition: vf_coreimage.m:66
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
av_cold
#define av_cold
Definition: attributes.h:90
AV_FRAME_FLAG_KEY
#define AV_FRAME_FLAG_KEY
A flag to mark frames that are keyframes.
Definition: frame.h:640
CoreImageContext::picref
AVFrame * picref
cached reference containing the painted picture
Definition: vf_coreimage.m:49
AVDictionaryEntry::key
char * key
Definition: dict.h:90
frame_size
int frame_size
Definition: mxfenc.c:2429
FILTER_OPTIONS
#define FILTER_OPTIONS
Definition: vf_coreimage.m:605
CoreImageContext::h
int h
video size
Definition: vf_coreimage.m:43
filters.h
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
ctx
AVFormatContext * ctx
Definition: movenc.c:49
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:609
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
CoreImageContext::time_base
AVRational time_base
stream time base
Definition: vf_coreimage.m:46
key
const char * key
Definition: hwcontext_opencl.c:189
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: filters.h:263
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
if
if(ret)
Definition: filter_design.txt:179
config_input
static int config_input(AVFilterLink *link)
Determine image properties from input link of filter chain.
Definition: vf_coreimage.m:85
av_log_get_level
int av_log_get_level(void)
Get the current log level.
Definition: log.c:442
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:75
NULL
#define NULL
Definition: coverity.c:32
AV_DICT_MULTIKEY
#define AV_DICT_MULTIKEY
Allow to store several equal keys in the dictionary.
Definition: dict.h:84
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
CoreImageContext::filters
CFTypeRef * filters
CIFilter object for all requested filters.
Definition: vf_coreimage.m:58
CoreImageContext::bits_per_component
int bits_per_component
Shared bpc for input-output operation.
Definition: vf_coreimage.m:55
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
options
Definition: swscale.c:42
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
ff_filter_link
static FilterLink * ff_filter_link(AVFilterLink *link)
Definition: filters.h:197
CoreImageContext::pts
int64_t pts
increasing presentation time stamp
Definition: vf_coreimage.m:48
f
f
Definition: af_crystalizer.c:122
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
CoreImageContext::sar
AVRational sar
sample aspect ratio
Definition: vf_coreimage.m:44
CoreImageContext::input_image
CFTypeRef input_image
Input image container for passing into Core Image API.
Definition: vf_coreimage.m:53
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
AVFrame::time_base
AVRational time_base
Time base for the timestamps in this frame.
Definition: frame.h:516
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:476
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:223
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:220
AV_PIX_FMT_ARGB
@ AV_PIX_FMT_ARGB
packed ARGB 8:8:8:8, 32bpp, ARGBARGB...
Definition: pixfmt.h:99
list_filters
static void list_filters(CoreImageContext *ctx)
Print a list of all available filters including options and respective value ranges and defaults.
Definition: vf_coreimage.m:96
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
coreimagesrc_options
static const AVOption coreimagesrc_options[]
Definition: vf_coreimage.m:633
internal.h
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
av_inv_q
static av_always_inline AVRational av_inv_q(AVRational q)
Invert a rational.
Definition: rational.h:159
CoreImageContext::filter_string
char * filter_string
The complete user provided filter definition.
Definition: vf_coreimage.m:57
AVFilterPad::name
const char * name
Pad name.
Definition: filters.h:44
set_option
static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
Set an option of the given filter to the provided key-value pair.
Definition: vf_coreimage.m:322
AV_FRAME_FLAG_INTERLACED
#define AV_FRAME_FLAG_INTERLACED
A flag to mark frames whose content is interlaced.
Definition: frame.h:648
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
CoreImageContext::frame_rate
AVRational frame_rate
video frame rate
Definition: vf_coreimage.m:45
AVFilter
Filter definition.
Definition: avfilter.h:201
ret
ret
Definition: filter_design.txt:187
CoreImageContext::output_rect
char * output_rect
Rectangle to be filled with filter intput.
Definition: vf_coreimage.m:61
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:496
CoreImageContext::num_filters
int num_filters
Amount of filters in *filters.
Definition: vf_coreimage.m:59
CoreImageContext::list_generators
int list_generators
Option used to list all available generators.
Definition: vf_coreimage.m:63
init_src
static av_cold int init_src(AVFilterContext *fctx)
Definition: vf_coreimage.m:531
avfilter.h
av_dict_parse_string
int av_dict_parse_string(AVDictionary **pm, const char *str, const char *key_val_sep, const char *pairs_sep, int flags)
Parse the key/value pairs list and add the parsed entries to a dictionary.
Definition: dict.c:200
CoreImageContext::is_video_source
int is_video_source
filter is used as video source
Definition: vf_coreimage.m:41
apply_filter
static int apply_filter(CoreImageContext *ctx, AVFilterLink *link, AVFrame *frame)
Definition: vf_coreimage.m:134
AVFormatContext::duration
int64_t duration
Duration of the stream, in AV_TIME_BASE fractional seconds.
Definition: avformat.h:1435
AVFilterContext
An instance of a filter.
Definition: avfilter.h:457
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
desc
const char * desc
Definition: libsvtav1.c:79
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
uninit
static av_cold void uninit(AVFilterContext *fctx)
Definition: vf_coreimage.m:542
mem.h
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVDictionaryEntry
Definition: dict.h:89
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
vf_coreimage_inputs
static const AVFilterPad vf_coreimage_inputs[]
Definition: vf_coreimage.m:568
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
GENERATOR_OPTIONS
#define GENERATOR_OPTIONS
Definition: vf_coreimage.m:596
AVERROR_EXIT
#define AVERROR_EXIT
Immediate exit was requested; the called function should not be restarted.
Definition: error.h:58
SafeCFRelease
#define SafeCFRelease(ptr)
AVDictionaryEntry::value
char * value
Definition: dict.h:91
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: filters.h:252
av_dict_iterate
const AVDictionaryEntry * av_dict_iterate(const AVDictionary *m, const AVDictionaryEntry *prev)
Iterate over a dictionary.
Definition: dict.c:44
CoreImageContext::list_filters
int list_filters
Option used to list all available filters including generators.
Definition: vf_coreimage.m:62