FFmpeg
vf_coreimage.m
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Thilo Borgmann
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Video processing based on Apple's CoreImage API
24  */
25 
26 #import <CoreImage/CoreImage.h>
27 #import <AppKit/AppKit.h>
28 
29 #include "avfilter.h"
30 #include "formats.h"
31 #include "internal.h"
32 #include "video.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/opt.h"
35 #include "libavutil/pixdesc.h"
36 
37 typedef struct CoreImageContext {
38  const AVClass *class;
39 
40  int is_video_source; ///< filter is used as video source
41 
42  int w, h; ///< video size
43  AVRational sar; ///< sample aspect ratio
44  AVRational frame_rate; ///< video frame rate
45  AVRational time_base; ///< stream time base
46  int64_t duration; ///< duration expressed in microseconds
47  int64_t pts; ///< increasing presentation time stamp
48  AVFrame *picref; ///< cached reference containing the painted picture
49 
50  CFTypeRef glctx; ///< OpenGL context
51  CGContextRef cgctx; ///< Bitmap context for image copy
52  CFTypeRef input_image; ///< Input image container for passing into Core Image API
53  CGColorSpaceRef color_space; ///< Common color space for input image and cgcontext
54  int bits_per_component; ///< Shared bpc for input-output operation
55 
56  char *filter_string; ///< The complete user provided filter definition
57  CFTypeRef *filters; ///< CIFilter object for all requested filters
58  int num_filters; ///< Amount of filters in *filters
59 
60  char *output_rect; ///< Rectangle to be filled with filter intput
61  int list_filters; ///< Option used to list all available filters including generators
62  int list_generators; ///< Option used to list all available generators
64 
66 {
67  CoreImageContext *ctx = link->src->priv;
68 
69  link->w = ctx->w;
70  link->h = ctx->h;
72  link->frame_rate = ctx->frame_rate;
73  link->time_base = ctx->time_base;
74 
76  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
77 
78  return 0;
79 }
80 
81 /** Determine image properties from input link of filter chain.
82  */
84 {
85  CoreImageContext *ctx = link->dst->priv;
87  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
88 
89  return 0;
90 }
91 
92 /** Print a list of all available filters including options and respective value ranges and defaults.
93  */
95 {
96  // querying filters and attributes
97  NSArray *filter_categories = nil;
98 
99  if (ctx->list_generators && !ctx->list_filters) {
100  filter_categories = [NSArray arrayWithObjects:kCICategoryGenerator, nil];
101  }
102 
103  NSArray *filter_names = [CIFilter filterNamesInCategories:filter_categories];
104  NSEnumerator *filters = [filter_names objectEnumerator];
105 
106  NSString *filter_name;
107  while (filter_name = [filters nextObject]) {
108  av_log(ctx, AV_LOG_INFO, "Filter: %s\n", [filter_name UTF8String]);
109  NSString *input;
110 
111  CIFilter *filter = [CIFilter filterWithName:filter_name];
112  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
113  NSArray *filter_inputs = [filter inputKeys]; // <nsstring>
114 
115  for (input in filter_inputs) {
116  NSDictionary *input_attribs = [filter_attribs valueForKey:input];
117  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
118  if ([input_class isEqualToString:@"NSNumber"]) {
119  NSNumber *value_default = [input_attribs valueForKey:kCIAttributeDefault];
120  NSNumber *value_min = [input_attribs valueForKey:kCIAttributeSliderMin];
121  NSNumber *value_max = [input_attribs valueForKey:kCIAttributeSliderMax];
122 
123  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\t[%s %s][%s]\n",
124  [input UTF8String],
125  [input_class UTF8String],
126  [[value_min stringValue] UTF8String],
127  [[value_max stringValue] UTF8String],
128  [[value_default stringValue] UTF8String]);
129  } else {
130  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\n",
131  [input UTF8String],
132  [input_class UTF8String]);
133  }
134  }
135  }
136 }
137 
139 {
140  int i;
141 
142  // (re-)initialize input image
143  const CGSize frame_size = {
144  frame->width,
145  frame->height
146  };
147 
148  NSData *data = [NSData dataWithBytesNoCopy:frame->data[0]
149  length:frame->height*frame->linesize[0]
150  freeWhenDone:NO];
151 
152  CIImage *ret = [(__bridge CIImage*)ctx->input_image initWithBitmapData:data
153  bytesPerRow:frame->linesize[0]
154  size:frame_size
155  format:kCIFormatARGB8
156  colorSpace:ctx->color_space]; //kCGColorSpaceGenericRGB
157  if (!ret) {
158  av_log(ctx, AV_LOG_ERROR, "Input image could not be initialized.\n");
159  return AVERROR_EXTERNAL;
160  }
161 
162  CIFilter *filter = NULL;
163  CIImage *filter_input = (__bridge CIImage*)ctx->input_image;
164  CIImage *filter_output = NULL;
165 
166  // successively apply all filters
167  for (i = 0; i < ctx->num_filters; i++) {
168  if (i) {
169  // set filter input to previous filter output
170  filter_input = [(__bridge CIImage*)ctx->filters[i-1] valueForKey:kCIOutputImageKey];
171  CGRect out_rect = [filter_input extent];
172  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
173  // do not keep padded image regions after filtering
174  out_rect.origin.x = 0.0f;
175  out_rect.origin.y = 0.0f;
176  out_rect.size.width = frame->width;
177  out_rect.size.height = frame->height;
178  }
179  filter_input = [filter_input imageByCroppingToRect:out_rect];
180  }
181 
182  filter = (__bridge CIFilter*)ctx->filters[i];
183 
184  // do not set input image for the first filter if used as video source
185  if (!ctx->is_video_source || i) {
186  @try {
187  [filter setValue:filter_input forKey:kCIInputImageKey];
188  } @catch (NSException *exception) {
189  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
190  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
191  return AVERROR_EXTERNAL;
192  } else {
193  av_log(ctx, AV_LOG_WARNING, "Selected filter does not accept an input image.\n");
194  }
195  }
196  }
197  }
198 
199  // get output of last filter
200  filter_output = [filter valueForKey:kCIOutputImageKey];
201 
202  if (!filter_output) {
203  av_log(ctx, AV_LOG_ERROR, "Filter output not available.\n");
204  return AVERROR_EXTERNAL;
205  }
206 
207  // do not keep padded image regions after filtering
208  CGRect out_rect = [filter_output extent];
209  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
210  av_log(ctx, AV_LOG_DEBUG, "Cropping output image.\n");
211  out_rect.origin.x = 0.0f;
212  out_rect.origin.y = 0.0f;
213  out_rect.size.width = frame->width;
214  out_rect.size.height = frame->height;
215  }
216 
217  CGImageRef out = [(__bridge CIContext*)ctx->glctx createCGImage:filter_output
218  fromRect:out_rect];
219 
220  if (!out) {
221  av_log(ctx, AV_LOG_ERROR, "Cannot create valid output image.\n");
222  }
223 
224  // create bitmap context on the fly for rendering into current frame->data[]
225  if (ctx->cgctx) {
226  CGContextRelease(ctx->cgctx);
227  ctx->cgctx = NULL;
228  }
229  size_t out_width = CGImageGetWidth(out);
230  size_t out_height = CGImageGetHeight(out);
231 
232  if (out_width > frame->width || out_height > frame->height) { // this might result in segfault
233  av_log(ctx, AV_LOG_WARNING, "Output image has unexpected size: %lux%lu (expected: %ix%i). This may crash...\n",
234  out_width, out_height, frame->width, frame->height);
235  }
236  ctx->cgctx = CGBitmapContextCreate(frame->data[0],
237  frame->width,
238  frame->height,
239  ctx->bits_per_component,
240  frame->linesize[0],
241  ctx->color_space,
242  (uint32_t)kCGImageAlphaPremultipliedFirst); // ARGB
243  if (!ctx->cgctx) {
244  av_log(ctx, AV_LOG_ERROR, "CGBitmap context cannot be created.\n");
245  return AVERROR_EXTERNAL;
246  }
247 
248  // copy ("draw") the output image into the frame data
249  CGRect rect = {{0,0},{frame->width, frame->height}};
250  if (ctx->output_rect) {
251  @try {
252  NSString *tmp_string = [NSString stringWithUTF8String:ctx->output_rect];
253  NSRect tmp = NSRectFromString(tmp_string);
254  rect = NSRectToCGRect(tmp);
255  } @catch (NSException *exception) {
256  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
257  return AVERROR_EXTERNAL;
258  }
259  if (rect.size.width == 0.0f) {
260  av_log(ctx, AV_LOG_WARNING, "Width of output rect is zero.\n");
261  }
262  if (rect.size.height == 0.0f) {
263  av_log(ctx, AV_LOG_WARNING, "Height of output rect is zero.\n");
264  }
265  }
266 
267  CGContextDrawImage(ctx->cgctx, rect, out);
268 
269  return ff_filter_frame(link, frame);
270 }
271 
272 /** Apply all valid filters successively to the input image.
273  * The final output image is copied from the GPU by "drawing" using a bitmap context.
274  */
276 {
277  return apply_filter(link->dst->priv, link->dst->outputs[0], frame);
278 }
279 
281 {
282  CoreImageContext *ctx = link->src->priv;
283  AVFrame *frame;
284 
285  if (ctx->duration >= 0 &&
286  av_rescale_q(ctx->pts, ctx->time_base, AV_TIME_BASE_Q) >= ctx->duration) {
287  return AVERROR_EOF;
288  }
289 
290  if (!ctx->picref) {
291  ctx->picref = ff_get_video_buffer(link, ctx->w, ctx->h);
292  if (!ctx->picref) {
293  return AVERROR(ENOMEM);
294  }
295  }
296 
297  frame = av_frame_clone(ctx->picref);
298  if (!frame) {
299  return AVERROR(ENOMEM);
300  }
301 
302  frame->pts = ctx->pts;
303  frame->duration = 1;
304 #if FF_API_FRAME_KEY
305  frame->key_frame = 1;
306 #endif
308 #if FF_API_INTERLACED_FRAME
309  frame->interlaced_frame = 0;
310 #endif
313  frame->sample_aspect_ratio = ctx->sar;
314 
315  ctx->pts++;
316 
317  return apply_filter(ctx, link, frame);
318 }
319 
320 /** Set an option of the given filter to the provided key-value pair.
321  */
322 static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
323 {
324  NSString *input_key = [NSString stringWithUTF8String:key];
325  NSString *input_val = [NSString stringWithUTF8String:value];
326 
327  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
328  NSDictionary *input_attribs = [filter_attribs valueForKey:input_key];
329 
330  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
331  NSString *input_type = [input_attribs valueForKey:kCIAttributeType];
332 
333  if (!input_attribs) {
334  av_log(ctx, AV_LOG_WARNING, "Skipping unknown option: \"%s\".\n",
335  [input_key UTF8String]); // [[filter name] UTF8String]) not currently defined...
336  return;
337  }
338 
339  av_log(ctx, AV_LOG_DEBUG, "key: %s, val: %s, #attribs: %lu, class: %s, type: %s\n",
340  [input_key UTF8String],
341  [input_val UTF8String],
342  input_attribs ? (unsigned long)[input_attribs count] : -1,
343  [input_class UTF8String],
344  [input_type UTF8String]);
345 
346  if ([input_class isEqualToString:@"NSNumber"]) {
347  float input = input_val.floatValue;
348  NSNumber *max_value = [input_attribs valueForKey:kCIAttributeSliderMax];
349  NSNumber *min_value = [input_attribs valueForKey:kCIAttributeSliderMin];
350  NSNumber *used_value = nil;
351 
352 #define CLAMP_WARNING do { \
353 av_log(ctx, AV_LOG_WARNING, "Value of \"%f\" for option \"%s\" is out of range [%f %f], clamping to \"%f\".\n", \
354  input, \
355  [input_key UTF8String], \
356  min_value.floatValue, \
357  max_value.floatValue, \
358  used_value.floatValue); \
359 } while(0)
360  if (input > max_value.floatValue) {
361  used_value = max_value;
363  } else if (input < min_value.floatValue) {
364  used_value = min_value;
366  } else {
367  used_value = [NSNumber numberWithFloat:input];
368  }
369 
370  [filter setValue:used_value forKey:input_key];
371  } else if ([input_class isEqualToString:@"CIVector"]) {
372  CIVector *input = [CIVector vectorWithString:input_val];
373 
374  if (!input) {
375  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIVctor description: \"%s\".\n",
376  [input_val UTF8String]);
377  return;
378  }
379 
380  [filter setValue:input forKey:input_key];
381  } else if ([input_class isEqualToString:@"CIColor"]) {
382  CIColor *input = [CIColor colorWithString:input_val];
383 
384  if (!input) {
385  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIColor description: \"%s\".\n",
386  [input_val UTF8String]);
387  return;
388  }
389 
390  [filter setValue:input forKey:input_key];
391  } else if ([input_class isEqualToString:@"NSString"]) { // set display name as string with latin1 encoding
392  [filter setValue:input_val forKey:input_key];
393  } else if ([input_class isEqualToString:@"NSData"]) { // set display name as string with latin1 encoding
394  NSData *input = [NSData dataWithBytes:(const void*)[input_val cStringUsingEncoding:NSISOLatin1StringEncoding]
395  length:[input_val lengthOfBytesUsingEncoding:NSISOLatin1StringEncoding]];
396 
397  if (!input) {
398  av_log(ctx, AV_LOG_WARNING, "Skipping invalid NSData description: \"%s\".\n",
399  [input_val UTF8String]);
400  return;
401  }
402 
403  [filter setValue:input forKey:input_key];
404  } else {
405  av_log(ctx, AV_LOG_WARNING, "Skipping unsupported option class: \"%s\".\n",
406  [input_class UTF8String]);
407  avpriv_report_missing_feature(ctx, "Handling of some option classes");
408  return;
409  }
410 }
411 
412 /** Create a filter object by a given name and set all options to defaults.
413  * Overwrite any option given by the user to the provided value in filter_options.
414  */
415 static CIFilter* create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
416 {
417  // create filter object
418  CIFilter *filter = [CIFilter filterWithName:[NSString stringWithUTF8String:filter_name]];
419 
420  // set default options
421  [filter setDefaults];
422 
423  // set user options
424  if (filter_options) {
425  const AVDictionaryEntry *o = NULL;
426  while ((o = av_dict_iterate(filter_options, o))) {
427  set_option(ctx, filter, o->key, o->value);
428  }
429  }
430 
431  return filter;
432 }
433 
434 static av_cold int init(AVFilterContext *fctx)
435 {
436  CoreImageContext *ctx = fctx->priv;
437  AVDictionary *filter_dict = NULL;
438  const AVDictionaryEntry *f = NULL;
439  const AVDictionaryEntry *o = NULL;
440  int ret;
441  int i;
442 
443  if (ctx->list_filters || ctx->list_generators) {
444  list_filters(ctx);
445  return AVERROR_EXIT;
446  }
447 
448  if (ctx->filter_string) {
449  // parse filter string (filter=name@opt=val@opt2=val2#name2@opt3=val3) for filters separated by #
450  av_log(ctx, AV_LOG_DEBUG, "Filter_string: %s\n", ctx->filter_string);
451  ret = av_dict_parse_string(&filter_dict, ctx->filter_string, "@", "#", AV_DICT_MULTIKEY); // parse filter_name:all_filter_options
452  if (ret) {
453  av_dict_free(&filter_dict);
454  av_log(ctx, AV_LOG_ERROR, "Parsing of filters failed.\n");
455  return AVERROR(EIO);
456  }
457  ctx->num_filters = av_dict_count(filter_dict);
458  av_log(ctx, AV_LOG_DEBUG, "Filter count: %i\n", ctx->num_filters);
459 
460  // allocate CIFilter array
461  ctx->filters = av_calloc(ctx->num_filters, sizeof(CIFilter*));
462  if (!ctx->filters) {
463  av_log(ctx, AV_LOG_ERROR, "Could not allocate filter array.\n");
464  return AVERROR(ENOMEM);
465  }
466 
467  // parse filters for option key-value pairs (opt=val@opt2=val2) separated by @
468  i = 0;
469  while ((f = av_dict_iterate(filter_dict, f))) {
470  AVDictionary *filter_options = NULL;
471 
472  if (strncmp(f->value, "default", 7)) { // not default
473  ret = av_dict_parse_string(&filter_options, f->value, "=", "@", 0); // parse option_name:option_value
474  if (ret) {
475  av_dict_free(&filter_options);
476  av_log(ctx, AV_LOG_ERROR, "Parsing of filter options for \"%s\" failed.\n", f->key);
477  return AVERROR(EIO);
478  }
479  }
480 
481  if (av_log_get_level() >= AV_LOG_DEBUG) {
482  av_log(ctx, AV_LOG_DEBUG, "Creating filter %i: \"%s\":\n", i, f->key);
483  if (!filter_options) {
484  av_log(ctx, AV_LOG_DEBUG, "\tusing default options\n");
485  } else {
486  while ((o = av_dict_iterate(filter_options, o))) {
487  av_log(ctx, AV_LOG_DEBUG, "\t%s: %s\n", o->key, o->value);
488  }
489  }
490  }
491 
492  ctx->filters[i] = CFBridgingRetain(create_filter(ctx, f->key, filter_options));
493  if (!ctx->filters[i]) {
494  av_log(ctx, AV_LOG_ERROR, "Could not create filter \"%s\".\n", f->key);
495  return AVERROR(EINVAL);
496  }
497 
498  i++;
499  }
500  } else {
501  av_log(ctx, AV_LOG_ERROR, "No filters specified.\n");
502  return AVERROR(EINVAL);
503  }
504 
505  // create GPU context on OSX
506  const NSOpenGLPixelFormatAttribute attr[] = {
507  NSOpenGLPFAAccelerated,
508  NSOpenGLPFANoRecovery,
509  NSOpenGLPFAColorSize, 32,
510  0
511  };
512 
513  NSOpenGLPixelFormat *pixel_format = [[NSOpenGLPixelFormat alloc] initWithAttributes:(void *)&attr];
514  ctx->color_space = CGColorSpaceCreateWithName(kCGColorSpaceGenericRGB);
515  ctx->glctx = CFBridgingRetain([CIContext contextWithCGLContext:CGLGetCurrentContext()
516  pixelFormat:[pixel_format CGLPixelFormatObj]
517  colorSpace:ctx->color_space
518  options:nil]);
519 
520  if (!ctx->glctx) {
521  av_log(ctx, AV_LOG_ERROR, "CIContext not created.\n");
522  return AVERROR_EXTERNAL;
523  }
524 
525  // Creating an empty input image as input container for the context
526  ctx->input_image = CFBridgingRetain([CIImage emptyImage]);
527 
528  return 0;
529 }
530 
532 {
533  CoreImageContext *ctx = fctx->priv;
534 
535  ctx->is_video_source = 1;
536  ctx->time_base = av_inv_q(ctx->frame_rate);
537  ctx->pts = 0;
538 
539  return init(fctx);
540 }
541 
542 static av_cold void uninit(AVFilterContext *fctx)
543 {
544 #define SafeCFRelease(ptr) do { \
545  if (ptr) { \
546  CFRelease(ptr); \
547  ptr = NULL; \
548  } \
549 } while (0)
550 
551  CoreImageContext *ctx = fctx->priv;
552 
553  SafeCFRelease(ctx->glctx);
554  SafeCFRelease(ctx->cgctx);
555  SafeCFRelease(ctx->color_space);
556  SafeCFRelease(ctx->input_image);
557 
558  if (ctx->filters) {
559  for (int i = 0; i < ctx->num_filters; i++) {
560  SafeCFRelease(ctx->filters[i]);
561  }
562  av_freep(&ctx->filters);
563  }
564 
565  av_frame_free(&ctx->picref);
566 }
567 
569  {
570  .name = "default",
571  .type = AVMEDIA_TYPE_VIDEO,
572  .filter_frame = filter_frame,
573  .config_props = config_input,
574  },
575 };
576 
578  {
579  .name = "default",
580  .type = AVMEDIA_TYPE_VIDEO,
581  },
582 };
583 
585  {
586  .name = "default",
587  .type = AVMEDIA_TYPE_VIDEO,
588  .request_frame = request_frame,
589  .config_props = config_output,
590  },
591 };
592 
593 #define OFFSET(x) offsetof(CoreImageContext, x)
594 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
595 
596 #define GENERATOR_OPTIONS \
597  {"size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
598  {"s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
599  {"rate", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
600  {"r", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
601  {"duration", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
602  {"d", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
603  {"sar", "set video sample aspect ratio", OFFSET(sar), AV_OPT_TYPE_RATIONAL, {.dbl = 1}, 0, INT_MAX, FLAGS},
604 
605 #define FILTER_OPTIONS \
606  {"list_filters", "list available filters", OFFSET(list_filters), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
607  {"list_generators", "list available generators", OFFSET(list_generators), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
608  {"filter", "names and options of filters to apply", OFFSET(filter_string), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS}, \
609  {"output_rect", "output rectangle within output image", OFFSET(output_rect), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS},
610 
611 
612 // definitions for coreimage video filter
613 static const AVOption coreimage_options[] = {
615  { NULL }
616 };
617 
618 AVFILTER_DEFINE_CLASS(coreimage);
619 
621  .name = "coreimage",
622  .description = NULL_IF_CONFIG_SMALL("Video filtering using CoreImage API."),
623  .init = init,
624  .uninit = uninit,
625  .priv_size = sizeof(CoreImageContext),
626  .priv_class = &coreimage_class,
630 };
631 
632 // definitions for coreimagesrc video source
633 static const AVOption coreimagesrc_options[] = {
636  { NULL }
637 };
638 
639 AVFILTER_DEFINE_CLASS(coreimagesrc);
640 
642  .name = "coreimagesrc",
643  .description = NULL_IF_CONFIG_SMALL("Video source using image generators of CoreImage API."),
644  .init = init_src,
645  .uninit = uninit,
646  .priv_size = sizeof(CoreImageContext),
647  .priv_class = &coreimagesrc_class,
648  .inputs = NULL,
651 };
ff_vsrc_coreimagesrc
const AVFilter ff_vsrc_coreimagesrc
Definition: vf_coreimage.m:641
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:112
CoreImageContext::glctx
CFTypeRef glctx
OpenGL context.
Definition: vf_coreimage.m:50
CoreImageContext::w
int w
Definition: vf_coreimage.m:42
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
CoreImageContext::duration
int64_t duration
duration expressed in microseconds
Definition: vf_coreimage.m:46
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
CoreImageContext::cgctx
CGContextRef cgctx
Bitmap context for image copy.
Definition: vf_coreimage.m:51
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1007
AVFrame::duration
int64_t duration
Duration of the frame, in the same units as pts.
Definition: frame.h:807
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2968
AVERROR_EOF
#define AVERROR_EOF
End of file.
Definition: error.h:57
av_dict_count
int av_dict_count(const AVDictionary *m)
Get number of entries in dictionary.
Definition: dict.c:39
rect
Definition: f_ebur128.c:78
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(coreimage)
AV_TIME_BASE_Q
#define AV_TIME_BASE_Q
Internal time base represented as fractional value.
Definition: avutil.h:264
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
pixdesc.h
AVFrame::pts
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:452
AVFrame::width
int width
Definition: frame.h:412
AVOption
AVOption.
Definition: opt.h:251
create_filter
static CIFilter * create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
Create a filter object by a given name and set all options to defaults.
Definition: vf_coreimage.m:415
data
const char data[16]
Definition: mxf.c:148
CoreImageContext::color_space
CGColorSpaceRef color_space
Common color space for input image and cgcontext.
Definition: vf_coreimage.m:53
vf_coreimage_outputs
static const AVFilterPad vf_coreimage_outputs[]
Definition: vf_coreimage.m:577
av_get_bits_per_pixel
int av_get_bits_per_pixel(const AVPixFmtDescriptor *pixdesc)
Return the number of bits per pixel used by the pixel format described by pixdesc.
Definition: pixdesc.c:2920
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
AVDictionary
Definition: dict.c:34
AVFrame::flags
int flags
Frame flags, a combination of AV_FRAME_FLAGS.
Definition: frame.h:649
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
coreimage_options
static const AVOption coreimage_options[]
Definition: vf_coreimage.m:613
video.h
CoreImageContext
Definition: vf_coreimage.m:37
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:361
formats.h
ff_vf_coreimage
const AVFilter ff_vf_coreimage
Definition: vf_coreimage.m:620
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *frame)
Apply all valid filters successively to the input image.
Definition: vf_coreimage.m:275
CLAMP_WARNING
#define CLAMP_WARNING
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:424
vsrc_coreimagesrc_outputs
static const AVFilterPad vsrc_coreimagesrc_outputs[]
Definition: vf_coreimage.m:584
request_frame
static int request_frame(AVFilterLink *link)
Definition: vf_coreimage.m:280
init
static av_cold int init(AVFilterContext *fctx)
Definition: vf_coreimage.m:434
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:33
config_output
static int config_output(AVFilterLink *link)
Definition: vf_coreimage.m:65
AVFrame::interlaced_frame
attribute_deprecated int interlaced_frame
The content of the picture is interlaced.
Definition: frame.h:530
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
AV_FRAME_FLAG_KEY
#define AV_FRAME_FLAG_KEY
A flag to mark frames that are keyframes.
Definition: frame.h:628
CoreImageContext::picref
AVFrame * picref
cached reference containing the painted picture
Definition: vf_coreimage.m:48
AVDictionaryEntry::key
char * key
Definition: dict.h:90
frame_size
int frame_size
Definition: mxfenc.c:2422
FILTER_OPTIONS
#define FILTER_OPTIONS
Definition: vf_coreimage.m:605
filters
#define filters(fmt, type, inverse, clp, inverset, clip, one, clip_fn, packed)
Definition: af_crystalizer.c:54
CoreImageContext::h
int h
video size
Definition: vf_coreimage.m:42
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:609
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
CoreImageContext::time_base
AVRational time_base
stream time base
Definition: vf_coreimage.m:45
key
const char * key
Definition: hwcontext_opencl.c:174
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:182
AVFrame::key_frame
attribute_deprecated int key_frame
1 -> keyframe, 0-> not
Definition: frame.h:436
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
frame
static AVFrame * frame
Definition: demux_decode.c:54
if
if(ret)
Definition: filter_design.txt:179
config_input
static int config_input(AVFilterLink *link)
Determine image properties from input link of filter chain.
Definition: vf_coreimage.m:83
av_log_get_level
int av_log_get_level(void)
Get the current log level.
Definition: log.c:442
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AV_DICT_MULTIKEY
#define AV_DICT_MULTIKEY
Allow to store several equal keys in the dictionary.
Definition: dict.h:84
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
CoreImageContext::filters
CFTypeRef * filters
CIFilter object for all requested filters.
Definition: vf_coreimage.m:57
CoreImageContext::bits_per_component
int bits_per_component
Shared bpc for input-output operation.
Definition: vf_coreimage.m:54
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
CoreImageContext::pts
int64_t pts
increasing presentation time stamp
Definition: vf_coreimage.m:47
options
const OptionDef options[]
f
f
Definition: af_crystalizer.c:121
AVFrame::pict_type
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:442
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:106
CoreImageContext::sar
AVRational sar
sample aspect ratio
Definition: vf_coreimage.m:43
CoreImageContext::input_image
CFTypeRef input_image
Input image container for passing into Core Image API.
Definition: vf_coreimage.m:52
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
AVFrame::time_base
AVRational time_base
Time base for the timestamps in this frame.
Definition: frame.h:467
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:427
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:225
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
internal.h
AV_PIX_FMT_ARGB
@ AV_PIX_FMT_ARGB
packed ARGB 8:8:8:8, 32bpp, ARGBARGB...
Definition: pixfmt.h:99
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:172
list_filters
static void list_filters(CoreImageContext *ctx)
Print a list of all available filters including options and respective value ranges and defaults.
Definition: vf_coreimage.m:94
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:255
coreimagesrc_options
static const AVOption coreimagesrc_options[]
Definition: vf_coreimage.m:633
internal.h
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
av_inv_q
static av_always_inline AVRational av_inv_q(AVRational q)
Invert a rational.
Definition: rational.h:159
CoreImageContext::filter_string
char * filter_string
The complete user provided filter definition.
Definition: vf_coreimage.m:56
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:39
set_option
static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
Set an option of the given filter to the provided key-value pair.
Definition: vf_coreimage.m:322
AV_FRAME_FLAG_INTERLACED
#define AV_FRAME_FLAG_INTERLACED
A flag to mark frames whose content is interlaced.
Definition: frame.h:636
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:262
CoreImageContext::frame_rate
AVRational frame_rate
video frame rate
Definition: vf_coreimage.m:44
AVFilter
Filter definition.
Definition: avfilter.h:166
ret
ret
Definition: filter_design.txt:187
CoreImageContext::output_rect
char * output_rect
Rectangle to be filled with filter intput.
Definition: vf_coreimage.m:60
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:447
AVFrame::height
int height
Definition: frame.h:412
CoreImageContext::num_filters
int num_filters
Amount of filters in *filters.
Definition: vf_coreimage.m:58
CoreImageContext::list_generators
int list_generators
Option used to list all available generators.
Definition: vf_coreimage.m:62
init_src
static av_cold int init_src(AVFilterContext *fctx)
Definition: vf_coreimage.m:531
avfilter.h
av_dict_parse_string
int av_dict_parse_string(AVDictionary **pm, const char *str, const char *key_val_sep, const char *pairs_sep, int flags)
Parse the key/value pairs list and add the parsed entries to a dictionary.
Definition: dict.c:202
CoreImageContext::is_video_source
int is_video_source
filter is used as video source
Definition: vf_coreimage.m:40
apply_filter
static int apply_filter(CoreImageContext *ctx, AVFilterLink *link, AVFrame *frame)
Definition: vf_coreimage.m:138
AVFormatContext::duration
int64_t duration
Duration of the stream, in AV_TIME_BASE fractional seconds.
Definition: avformat.h:1465
AVFilterContext
An instance of a filter.
Definition: avfilter.h:409
desc
const char * desc
Definition: libsvtav1.c:83
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
uninit
static av_cold void uninit(AVFilterContext *fctx)
Definition: vf_coreimage.m:542
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVDictionaryEntry
Definition: dict.h:89
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:183
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
vf_coreimage_inputs
static const AVFilterPad vf_coreimage_inputs[]
Definition: vf_coreimage.m:568
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:385
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
GENERATOR_OPTIONS
#define GENERATOR_OPTIONS
Definition: vf_coreimage.m:596
AVERROR_EXIT
#define AVERROR_EXIT
Immediate exit was requested; the called function should not be restarted.
Definition: error.h:58
SafeCFRelease
#define SafeCFRelease(ptr)
AVDictionaryEntry::value
char * value
Definition: dict.h:91
av_dict_iterate
const AVDictionaryEntry * av_dict_iterate(const AVDictionary *m, const AVDictionaryEntry *prev)
Iterate over a dictionary.
Definition: dict.c:44
CoreImageContext::list_filters
int list_filters
Option used to list all available filters including generators.
Definition: vf_coreimage.m:61