FFmpeg
vf_coreimage.m
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Thilo Borgmann
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Video processing based on Apple's CoreImage API
24  */
25 
26 #import <CoreImage/CoreImage.h>
27 #import <AppKit/AppKit.h>
28 
29 #include "avfilter.h"
30 #include "formats.h"
31 #include "internal.h"
32 #include "video.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/opt.h"
35 #include "libavutil/pixdesc.h"
36 
37 typedef struct CoreImageContext {
38  const AVClass *class;
39 
40  int is_video_source; ///< filter is used as video source
41 
42  int w, h; ///< video size
43  AVRational sar; ///< sample aspect ratio
44  AVRational frame_rate; ///< video frame rate
45  AVRational time_base; ///< stream time base
46  int64_t duration; ///< duration expressed in microseconds
47  int64_t pts; ///< increasing presentation time stamp
48  AVFrame *picref; ///< cached reference containing the painted picture
49 
50  CFTypeRef glctx; ///< OpenGL context
51  CGContextRef cgctx; ///< Bitmap context for image copy
52  CFTypeRef input_image; ///< Input image container for passing into Core Image API
53  CGColorSpaceRef color_space; ///< Common color space for input image and cgcontext
54  int bits_per_component; ///< Shared bpc for input-output operation
55 
56  char *filter_string; ///< The complete user provided filter definition
57  CFTypeRef *filters; ///< CIFilter object for all requested filters
58  int num_filters; ///< Amount of filters in *filters
59 
60  char *output_rect; ///< Rectangle to be filled with filter intput
61  int list_filters; ///< Option used to list all available filters including generators
62  int list_generators; ///< Option used to list all available generators
64 
66 {
67  CoreImageContext *ctx = link->src->priv;
68 
69  link->w = ctx->w;
70  link->h = ctx->h;
72  link->frame_rate = ctx->frame_rate;
73  link->time_base = ctx->time_base;
74 
76  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
77 
78  return 0;
79 }
80 
81 /** Determine image properties from input link of filter chain.
82  */
84 {
85  CoreImageContext *ctx = link->dst->priv;
87  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
88 
89  return 0;
90 }
91 
92 /** Print a list of all available filters including options and respective value ranges and defaults.
93  */
95 {
96  // querying filters and attributes
97  NSArray *filter_categories = nil;
98 
99  if (ctx->list_generators && !ctx->list_filters) {
100  filter_categories = [NSArray arrayWithObjects:kCICategoryGenerator, nil];
101  }
102 
103  NSArray *filter_names = [CIFilter filterNamesInCategories:filter_categories];
104  NSEnumerator *filters = [filter_names objectEnumerator];
105 
106  NSString *filter_name;
107  while (filter_name = [filters nextObject]) {
108  av_log(ctx, AV_LOG_INFO, "Filter: %s\n", [filter_name UTF8String]);
109  NSString *input;
110 
111  CIFilter *filter = [CIFilter filterWithName:filter_name];
112  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
113  NSArray *filter_inputs = [filter inputKeys]; // <nsstring>
114 
115  for (input in filter_inputs) {
116  NSDictionary *input_attribs = [filter_attribs valueForKey:input];
117  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
118  if ([input_class isEqualToString:@"NSNumber"]) {
119  NSNumber *value_default = [input_attribs valueForKey:kCIAttributeDefault];
120  NSNumber *value_min = [input_attribs valueForKey:kCIAttributeSliderMin];
121  NSNumber *value_max = [input_attribs valueForKey:kCIAttributeSliderMax];
122 
123  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\t[%s %s][%s]\n",
124  [input UTF8String],
125  [input_class UTF8String],
126  [[value_min stringValue] UTF8String],
127  [[value_max stringValue] UTF8String],
128  [[value_default stringValue] UTF8String]);
129  } else {
130  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\n",
131  [input UTF8String],
132  [input_class UTF8String]);
133  }
134  }
135  }
136 }
137 
139 {
140  int i;
141 
142  // (re-)initialize input image
143  const CGSize frame_size = {
144  frame->width,
145  frame->height
146  };
147 
148  NSData *data = [NSData dataWithBytesNoCopy:frame->data[0]
149  length:frame->height*frame->linesize[0]
150  freeWhenDone:NO];
151 
152  CIImage *ret = [(__bridge CIImage*)ctx->input_image initWithBitmapData:data
153  bytesPerRow:frame->linesize[0]
154  size:frame_size
155  format:kCIFormatARGB8
156  colorSpace:ctx->color_space]; //kCGColorSpaceGenericRGB
157  if (!ret) {
158  av_log(ctx, AV_LOG_ERROR, "Input image could not be initialized.\n");
159  return AVERROR_EXTERNAL;
160  }
161 
162  CIFilter *filter = NULL;
163  CIImage *filter_input = (__bridge CIImage*)ctx->input_image;
164  CIImage *filter_output = NULL;
165 
166  // successively apply all filters
167  for (i = 0; i < ctx->num_filters; i++) {
168  if (i) {
169  // set filter input to previous filter output
170  filter_input = [(__bridge CIImage*)ctx->filters[i-1] valueForKey:kCIOutputImageKey];
171  CGRect out_rect = [filter_input extent];
172  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
173  // do not keep padded image regions after filtering
174  out_rect.origin.x = 0.0f;
175  out_rect.origin.y = 0.0f;
176  out_rect.size.width = frame->width;
177  out_rect.size.height = frame->height;
178  }
179  filter_input = [filter_input imageByCroppingToRect:out_rect];
180  }
181 
182  filter = (__bridge CIFilter*)ctx->filters[i];
183 
184  // do not set input image for the first filter if used as video source
185  if (!ctx->is_video_source || i) {
186  @try {
187  [filter setValue:filter_input forKey:kCIInputImageKey];
188  } @catch (NSException *exception) {
189  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
190  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
191  return AVERROR_EXTERNAL;
192  } else {
193  av_log(ctx, AV_LOG_WARNING, "Selected filter does not accept an input image.\n");
194  }
195  }
196  }
197  }
198 
199  // get output of last filter
200  filter_output = [filter valueForKey:kCIOutputImageKey];
201 
202  if (!filter_output) {
203  av_log(ctx, AV_LOG_ERROR, "Filter output not available.\n");
204  return AVERROR_EXTERNAL;
205  }
206 
207  // do not keep padded image regions after filtering
208  CGRect out_rect = [filter_output extent];
209  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
210  av_log(ctx, AV_LOG_DEBUG, "Cropping output image.\n");
211  out_rect.origin.x = 0.0f;
212  out_rect.origin.y = 0.0f;
213  out_rect.size.width = frame->width;
214  out_rect.size.height = frame->height;
215  }
216 
217  CGImageRef out = [(__bridge CIContext*)ctx->glctx createCGImage:filter_output
218  fromRect:out_rect];
219 
220  if (!out) {
221  av_log(ctx, AV_LOG_ERROR, "Cannot create valid output image.\n");
222  }
223 
224  // create bitmap context on the fly for rendering into current frame->data[]
225  if (ctx->cgctx) {
226  CGContextRelease(ctx->cgctx);
227  ctx->cgctx = NULL;
228  }
229  size_t out_width = CGImageGetWidth(out);
230  size_t out_height = CGImageGetHeight(out);
231 
232  if (out_width > frame->width || out_height > frame->height) { // this might result in segfault
233  av_log(ctx, AV_LOG_WARNING, "Output image has unexpected size: %lux%lu (expected: %ix%i). This may crash...\n",
234  out_width, out_height, frame->width, frame->height);
235  }
236  ctx->cgctx = CGBitmapContextCreate(frame->data[0],
237  frame->width,
238  frame->height,
239  ctx->bits_per_component,
240  frame->linesize[0],
241  ctx->color_space,
242  (uint32_t)kCGImageAlphaPremultipliedFirst); // ARGB
243  if (!ctx->cgctx) {
244  av_log(ctx, AV_LOG_ERROR, "CGBitmap context cannot be created.\n");
245  return AVERROR_EXTERNAL;
246  }
247 
248  // copy ("draw") the output image into the frame data
249  CGRect rect = {{0,0},{frame->width, frame->height}};
250  if (ctx->output_rect) {
251  @try {
252  NSString *tmp_string = [NSString stringWithUTF8String:ctx->output_rect];
253  NSRect tmp = NSRectFromString(tmp_string);
254  rect = NSRectToCGRect(tmp);
255  } @catch (NSException *exception) {
256  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
257  return AVERROR_EXTERNAL;
258  }
259  if (rect.size.width == 0.0f) {
260  av_log(ctx, AV_LOG_WARNING, "Width of output rect is zero.\n");
261  }
262  if (rect.size.height == 0.0f) {
263  av_log(ctx, AV_LOG_WARNING, "Height of output rect is zero.\n");
264  }
265  }
266 
267  CGContextDrawImage(ctx->cgctx, rect, out);
268 
269  return ff_filter_frame(link, frame);
270 }
271 
272 /** Apply all valid filters successively to the input image.
273  * The final output image is copied from the GPU by "drawing" using a bitmap context.
274  */
276 {
277  return apply_filter(link->dst->priv, link->dst->outputs[0], frame);
278 }
279 
281 {
282  CoreImageContext *ctx = link->src->priv;
283  AVFrame *frame;
284 
285  if (ctx->duration >= 0 &&
286  av_rescale_q(ctx->pts, ctx->time_base, AV_TIME_BASE_Q) >= ctx->duration) {
287  return AVERROR_EOF;
288  }
289 
290  if (!ctx->picref) {
291  ctx->picref = ff_get_video_buffer(link, ctx->w, ctx->h);
292  if (!ctx->picref) {
293  return AVERROR(ENOMEM);
294  }
295  }
296 
297  frame = av_frame_clone(ctx->picref);
298  if (!frame) {
299  return AVERROR(ENOMEM);
300  }
301 
302  frame->pts = ctx->pts;
303  frame->key_frame = 1;
304  frame->interlaced_frame = 0;
305  frame->pict_type = AV_PICTURE_TYPE_I;
306  frame->sample_aspect_ratio = ctx->sar;
307 
308  ctx->pts++;
309 
310  return apply_filter(ctx, link, frame);
311 }
312 
313 /** Set an option of the given filter to the provided key-value pair.
314  */
315 static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
316 {
317  NSString *input_key = [NSString stringWithUTF8String:key];
318  NSString *input_val = [NSString stringWithUTF8String:value];
319 
320  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
321  NSDictionary *input_attribs = [filter_attribs valueForKey:input_key];
322 
323  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
324  NSString *input_type = [input_attribs valueForKey:kCIAttributeType];
325 
326  if (!input_attribs) {
327  av_log(ctx, AV_LOG_WARNING, "Skipping unknown option: \"%s\".\n",
328  [input_key UTF8String]); // [[filter name] UTF8String]) not currently defined...
329  return;
330  }
331 
332  av_log(ctx, AV_LOG_DEBUG, "key: %s, val: %s, #attribs: %lu, class: %s, type: %s\n",
333  [input_key UTF8String],
334  [input_val UTF8String],
335  input_attribs ? (unsigned long)[input_attribs count] : -1,
336  [input_class UTF8String],
337  [input_type UTF8String]);
338 
339  if ([input_class isEqualToString:@"NSNumber"]) {
340  float input = input_val.floatValue;
341  NSNumber *max_value = [input_attribs valueForKey:kCIAttributeSliderMax];
342  NSNumber *min_value = [input_attribs valueForKey:kCIAttributeSliderMin];
343  NSNumber *used_value = nil;
344 
345 #define CLAMP_WARNING do { \
346 av_log(ctx, AV_LOG_WARNING, "Value of \"%f\" for option \"%s\" is out of range [%f %f], clamping to \"%f\".\n", \
347  input, \
348  [input_key UTF8String], \
349  min_value.floatValue, \
350  max_value.floatValue, \
351  used_value.floatValue); \
352 } while(0)
353  if (input > max_value.floatValue) {
354  used_value = max_value;
356  } else if (input < min_value.floatValue) {
357  used_value = min_value;
359  } else {
360  used_value = [NSNumber numberWithFloat:input];
361  }
362 
363  [filter setValue:used_value forKey:input_key];
364  } else if ([input_class isEqualToString:@"CIVector"]) {
365  CIVector *input = [CIVector vectorWithString:input_val];
366 
367  if (!input) {
368  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIVctor description: \"%s\".\n",
369  [input_val UTF8String]);
370  return;
371  }
372 
373  [filter setValue:input forKey:input_key];
374  } else if ([input_class isEqualToString:@"CIColor"]) {
375  CIColor *input = [CIColor colorWithString:input_val];
376 
377  if (!input) {
378  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIColor description: \"%s\".\n",
379  [input_val UTF8String]);
380  return;
381  }
382 
383  [filter setValue:input forKey:input_key];
384  } else if ([input_class isEqualToString:@"NSString"]) { // set display name as string with latin1 encoding
385  [filter setValue:input_val forKey:input_key];
386  } else if ([input_class isEqualToString:@"NSData"]) { // set display name as string with latin1 encoding
387  NSData *input = [NSData dataWithBytes:(const void*)[input_val cStringUsingEncoding:NSISOLatin1StringEncoding]
388  length:[input_val lengthOfBytesUsingEncoding:NSISOLatin1StringEncoding]];
389 
390  if (!input) {
391  av_log(ctx, AV_LOG_WARNING, "Skipping invalid NSData description: \"%s\".\n",
392  [input_val UTF8String]);
393  return;
394  }
395 
396  [filter setValue:input forKey:input_key];
397  } else {
398  av_log(ctx, AV_LOG_WARNING, "Skipping unsupported option class: \"%s\".\n",
399  [input_class UTF8String]);
400  avpriv_report_missing_feature(ctx, "Handling of some option classes");
401  return;
402  }
403 }
404 
405 /** Create a filter object by a given name and set all options to defaults.
406  * Overwrite any option given by the user to the provided value in filter_options.
407  */
408 static CIFilter* create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
409 {
410  // create filter object
411  CIFilter *filter = [CIFilter filterWithName:[NSString stringWithUTF8String:filter_name]];
412 
413  // set default options
414  [filter setDefaults];
415 
416  // set user options
417  if (filter_options) {
418  AVDictionaryEntry *o = NULL;
419  while ((o = av_dict_get(filter_options, "", o, AV_DICT_IGNORE_SUFFIX))) {
420  set_option(ctx, filter, o->key, o->value);
421  }
422  }
423 
424  return filter;
425 }
426 
427 static av_cold int init(AVFilterContext *fctx)
428 {
429  CoreImageContext *ctx = fctx->priv;
430  AVDictionary *filter_dict = NULL;
432  AVDictionaryEntry *o = NULL;
433  int ret;
434  int i;
435 
436  if (ctx->list_filters || ctx->list_generators) {
437  list_filters(ctx);
438  return AVERROR_EXIT;
439  }
440 
441  if (ctx->filter_string) {
442  // parse filter string (filter=name@opt=val@opt2=val2#name2@opt3=val3) for filters separated by #
443  av_log(ctx, AV_LOG_DEBUG, "Filter_string: %s\n", ctx->filter_string);
444  ret = av_dict_parse_string(&filter_dict, ctx->filter_string, "@", "#", AV_DICT_MULTIKEY); // parse filter_name:all_filter_options
445  if (ret) {
446  av_dict_free(&filter_dict);
447  av_log(ctx, AV_LOG_ERROR, "Parsing of filters failed.\n");
448  return AVERROR(EIO);
449  }
450  ctx->num_filters = av_dict_count(filter_dict);
451  av_log(ctx, AV_LOG_DEBUG, "Filter count: %i\n", ctx->num_filters);
452 
453  // allocate CIFilter array
454  ctx->filters = av_calloc(ctx->num_filters, sizeof(CIFilter*));
455  if (!ctx->filters) {
456  av_log(ctx, AV_LOG_ERROR, "Could not allocate filter array.\n");
457  return AVERROR(ENOMEM);
458  }
459 
460  // parse filters for option key-value pairs (opt=val@opt2=val2) separated by @
461  i = 0;
462  while ((f = av_dict_get(filter_dict, "", f, AV_DICT_IGNORE_SUFFIX))) {
463  AVDictionary *filter_options = NULL;
464 
465  if (strncmp(f->value, "default", 7)) { // not default
466  ret = av_dict_parse_string(&filter_options, f->value, "=", "@", 0); // parse option_name:option_value
467  if (ret) {
468  av_dict_free(&filter_options);
469  av_log(ctx, AV_LOG_ERROR, "Parsing of filter options for \"%s\" failed.\n", f->key);
470  return AVERROR(EIO);
471  }
472  }
473 
474  if (av_log_get_level() >= AV_LOG_DEBUG) {
475  av_log(ctx, AV_LOG_DEBUG, "Creating filter %i: \"%s\":\n", i, f->key);
476  if (!filter_options) {
477  av_log(ctx, AV_LOG_DEBUG, "\tusing default options\n");
478  } else {
479  while ((o = av_dict_get(filter_options, "", o, AV_DICT_IGNORE_SUFFIX))) {
480  av_log(ctx, AV_LOG_DEBUG, "\t%s: %s\n", o->key, o->value);
481  }
482  }
483  }
484 
485  ctx->filters[i] = CFBridgingRetain(create_filter(ctx, f->key, filter_options));
486  if (!ctx->filters[i]) {
487  av_log(ctx, AV_LOG_ERROR, "Could not create filter \"%s\".\n", f->key);
488  return AVERROR(EINVAL);
489  }
490 
491  i++;
492  }
493  } else {
494  av_log(ctx, AV_LOG_ERROR, "No filters specified.\n");
495  return AVERROR(EINVAL);
496  }
497 
498  // create GPU context on OSX
499  const NSOpenGLPixelFormatAttribute attr[] = {
500  NSOpenGLPFAAccelerated,
501  NSOpenGLPFANoRecovery,
502  NSOpenGLPFAColorSize, 32,
503  0
504  };
505 
506  NSOpenGLPixelFormat *pixel_format = [[NSOpenGLPixelFormat alloc] initWithAttributes:(void *)&attr];
507  ctx->color_space = CGColorSpaceCreateWithName(kCGColorSpaceGenericRGB);
508  ctx->glctx = CFBridgingRetain([CIContext contextWithCGLContext:CGLGetCurrentContext()
509  pixelFormat:[pixel_format CGLPixelFormatObj]
510  colorSpace:ctx->color_space
511  options:nil]);
512 
513  if (!ctx->glctx) {
514  av_log(ctx, AV_LOG_ERROR, "CIContext not created.\n");
515  return AVERROR_EXTERNAL;
516  }
517 
518  // Creating an empty input image as input container for the context
519  ctx->input_image = CFBridgingRetain([CIImage emptyImage]);
520 
521  return 0;
522 }
523 
525 {
526  CoreImageContext *ctx = fctx->priv;
527 
528  ctx->is_video_source = 1;
529  ctx->time_base = av_inv_q(ctx->frame_rate);
530  ctx->pts = 0;
531 
532  return init(fctx);
533 }
534 
535 static av_cold void uninit(AVFilterContext *fctx)
536 {
537 #define SafeCFRelease(ptr) do { \
538  if (ptr) { \
539  CFRelease(ptr); \
540  ptr = NULL; \
541  } \
542 } while (0)
543 
544  CoreImageContext *ctx = fctx->priv;
545 
546  SafeCFRelease(ctx->glctx);
547  SafeCFRelease(ctx->cgctx);
548  SafeCFRelease(ctx->color_space);
549  SafeCFRelease(ctx->input_image);
550 
551  if (ctx->filters) {
552  for (int i = 0; i < ctx->num_filters; i++) {
553  SafeCFRelease(ctx->filters[i]);
554  }
555  av_freep(&ctx->filters);
556  }
557 
558  av_frame_free(&ctx->picref);
559 }
560 
562  {
563  .name = "default",
564  .type = AVMEDIA_TYPE_VIDEO,
565  .filter_frame = filter_frame,
566  .config_props = config_input,
567  },
568 };
569 
571  {
572  .name = "default",
573  .type = AVMEDIA_TYPE_VIDEO,
574  },
575 };
576 
578  {
579  .name = "default",
580  .type = AVMEDIA_TYPE_VIDEO,
581  .request_frame = request_frame,
582  .config_props = config_output,
583  },
584 };
585 
586 #define OFFSET(x) offsetof(CoreImageContext, x)
587 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
588 
589 #define GENERATOR_OPTIONS \
590  {"size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
591  {"s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
592  {"rate", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
593  {"r", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
594  {"duration", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
595  {"d", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
596  {"sar", "set video sample aspect ratio", OFFSET(sar), AV_OPT_TYPE_RATIONAL, {.dbl = 1}, 0, INT_MAX, FLAGS},
597 
598 #define FILTER_OPTIONS \
599  {"list_filters", "list available filters", OFFSET(list_filters), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
600  {"list_generators", "list available generators", OFFSET(list_generators), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
601  {"filter", "names and options of filters to apply", OFFSET(filter_string), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS}, \
602  {"output_rect", "output rectangle within output image", OFFSET(output_rect), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS},
603 
604 
605 // definitions for coreimage video filter
606 static const AVOption coreimage_options[] = {
608  { NULL }
609 };
610 
611 AVFILTER_DEFINE_CLASS(coreimage);
612 
614  .name = "coreimage",
615  .description = NULL_IF_CONFIG_SMALL("Video filtering using CoreImage API."),
616  .init = init,
617  .uninit = uninit,
618  .priv_size = sizeof(CoreImageContext),
619  .priv_class = &coreimage_class,
623 };
624 
625 // definitions for coreimagesrc video source
626 static const AVOption coreimagesrc_options[] = {
629  { NULL }
630 };
631 
632 AVFILTER_DEFINE_CLASS(coreimagesrc);
633 
635  .name = "coreimagesrc",
636  .description = NULL_IF_CONFIG_SMALL("Video source using image generators of CoreImage API."),
637  .init = init_src,
638  .uninit = uninit,
639  .priv_size = sizeof(CoreImageContext),
640  .priv_class = &coreimagesrc_class,
641  .inputs = NULL,
644 };
ff_vsrc_coreimagesrc
const AVFilter ff_vsrc_coreimagesrc
Definition: vf_coreimage.m:634
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:98
CoreImageContext::glctx
CFTypeRef glctx
OpenGL context.
Definition: vf_coreimage.m:50
CoreImageContext::w
int w
Definition: vf_coreimage.m:42
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
CoreImageContext::duration
int64_t duration
duration expressed in microseconds
Definition: vf_coreimage.m:46
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
CoreImageContext::cgctx
CGContextRef cgctx
Bitmap context for image copy.
Definition: vf_coreimage.m:51
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1018
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2660
AVERROR_EOF
#define AVERROR_EOF
End of file.
Definition: error.h:57
av_dict_count
int av_dict_count(const AVDictionary *m)
Get number of entries in dictionary.
Definition: dict.c:35
rect
Definition: f_ebur128.c:77
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(coreimage)
AV_TIME_BASE_Q
#define AV_TIME_BASE_Q
Internal time base represented as fractional value.
Definition: avutil.h:260
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:109
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:317
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:26
pixdesc.h
AVOption
AVOption.
Definition: opt.h:247
create_filter
static CIFilter * create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
Create a filter object by a given name and set all options to defaults.
Definition: vf_coreimage.m:408
data
const char data[16]
Definition: mxf.c:143
AV_DICT_IGNORE_SUFFIX
#define AV_DICT_IGNORE_SUFFIX
Return first entry in a dictionary whose first part corresponds to the search key,...
Definition: dict.h:68
CoreImageContext::color_space
CGColorSpaceRef color_space
Common color space for input image and cgcontext.
Definition: vf_coreimage.m:53
vf_coreimage_outputs
static const AVFilterPad vf_coreimage_outputs[]
Definition: vf_coreimage.m:570
av_get_bits_per_pixel
int av_get_bits_per_pixel(const AVPixFmtDescriptor *pixdesc)
Return the number of bits per pixel used by the pixel format described by pixdesc.
Definition: pixdesc.c:2612
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
AVDictionary
Definition: dict.c:30
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:169
coreimage_options
static const AVOption coreimage_options[]
Definition: vf_coreimage.m:606
video.h
CoreImageContext
Definition: vf_coreimage.m:37
formats.h
ff_vf_coreimage
const AVFilter ff_vf_coreimage
Definition: vf_coreimage.m:613
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *frame)
Apply all valid filters successively to the input image.
Definition: vf_coreimage.m:275
CLAMP_WARNING
#define CLAMP_WARNING
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:417
vsrc_coreimagesrc_outputs
static const AVFilterPad vsrc_coreimagesrc_outputs[]
Definition: vf_coreimage.m:577
request_frame
static int request_frame(AVFilterLink *link)
Definition: vf_coreimage.m:280
init
static av_cold int init(AVFilterContext *fctx)
Definition: vf_coreimage.m:427
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:50
filters
#define filters(fmt, inverse, clip, i, c)
Definition: af_crystalizer.c:221
config_output
static int config_output(AVFilterLink *link)
Definition: vf_coreimage.m:65
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:40
CoreImageContext::picref
AVFrame * picref
cached reference containing the painted picture
Definition: vf_coreimage.m:48
AVDictionaryEntry::key
char * key
Definition: dict.h:80
frame_size
int frame_size
Definition: mxfenc.c:2199
FILTER_OPTIONS
#define FILTER_OPTIONS
Definition: vf_coreimage.m:598
CoreImageContext::h
int h
video size
Definition: vf_coreimage.m:42
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:422
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:141
CoreImageContext::time_base
AVRational time_base
stream time base
Definition: vf_coreimage.m:45
key
const char * key
Definition: hwcontext_opencl.c:168
f
#define f(width, name)
Definition: cbs_vp9.c:255
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:191
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
if
if(ret)
Definition: filter_design.txt:179
config_input
static int config_input(AVFilterLink *link)
Determine image properties from input link of filter chain.
Definition: vf_coreimage.m:83
av_log_get_level
int av_log_get_level(void)
Get the current log level.
Definition: log.c:435
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AV_DICT_MULTIKEY
#define AV_DICT_MULTIKEY
Allow to store several equal keys in the dictionary.
Definition: dict.h:77
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
CoreImageContext::filters
CFTypeRef * filters
CIFilter object for all requested filters.
Definition: vf_coreimage.m:57
CoreImageContext::bits_per_component
int bits_per_component
Shared bpc for input-output operation.
Definition: vf_coreimage.m:54
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
for
for(j=16;j >0;--j)
Definition: h264pred_template.c:469
CoreImageContext::pts
int64_t pts
increasing presentation time stamp
Definition: vf_coreimage.m:47
options
const OptionDef options[]
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
CoreImageContext::sar
AVRational sar
sample aspect ratio
Definition: vf_coreimage.m:43
CoreImageContext::input_image
CFTypeRef input_image
Input image container for passing into Core Image API.
Definition: vf_coreimage.m:52
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
AVFrame::time_base
AVRational time_base
Time base for the timestamps in this frame.
Definition: frame.h:439
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:404
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:203
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
internal.h
AV_PIX_FMT_ARGB
@ AV_PIX_FMT_ARGB
packed ARGB 8:8:8:8, 32bpp, ARGBARGB...
Definition: pixfmt.h:92
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:181
list_filters
static void list_filters(CoreImageContext *ctx)
Print a list of all available filters including options and respective value ranges and defaults.
Definition: vf_coreimage.m:94
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:271
coreimagesrc_options
static const AVOption coreimagesrc_options[]
Definition: vf_coreimage.m:626
internal.h
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
av_inv_q
static av_always_inline AVRational av_inv_q(AVRational q)
Invert a rational.
Definition: rational.h:159
CoreImageContext::filter_string
char * filter_string
The complete user provided filter definition.
Definition: vf_coreimage.m:56
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:56
set_option
static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
Set an option of the given filter to the provided key-value pair.
Definition: vf_coreimage.m:315
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:271
CoreImageContext::frame_rate
AVRational frame_rate
video frame rate
Definition: vf_coreimage.m:44
AVFilter
Filter definition.
Definition: avfilter.h:165
ret
ret
Definition: filter_design.txt:187
CoreImageContext::output_rect
char * output_rect
Rectangle to be filled with filter intput.
Definition: vf_coreimage.m:60
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:419
CoreImageContext::num_filters
int num_filters
Amount of filters in *filters.
Definition: vf_coreimage.m:58
CoreImageContext::list_generators
int list_generators
Option used to list all available generators.
Definition: vf_coreimage.m:62
init_src
static av_cold int init_src(AVFilterContext *fctx)
Definition: vf_coreimage.m:524
avfilter.h
av_dict_parse_string
int av_dict_parse_string(AVDictionary **pm, const char *str, const char *key_val_sep, const char *pairs_sep, int flags)
Parse the key/value pairs list and add the parsed entries to a dictionary.
Definition: dict.c:180
CoreImageContext::is_video_source
int is_video_source
filter is used as video source
Definition: vf_coreimage.m:40
apply_filter
static int apply_filter(CoreImageContext *ctx, AVFilterLink *link, AVFrame *frame)
Definition: vf_coreimage.m:138
AVFormatContext::duration
int64_t duration
Duration of the stream, in AV_TIME_BASE fractional seconds.
Definition: avformat.h:1302
AVFilterContext
An instance of a filter.
Definition: avfilter.h:402
desc
const char * desc
Definition: libsvtav1.c:79
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
uninit
static av_cold void uninit(AVFilterContext *fctx)
Definition: vf_coreimage.m:535
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVDictionaryEntry
Definition: dict.h:79
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:192
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
vf_coreimage_inputs
static const AVFilterPad vf_coreimage_inputs[]
Definition: vf_coreimage.m:561
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
GENERATOR_OPTIONS
#define GENERATOR_OPTIONS
Definition: vf_coreimage.m:589
AVERROR_EXIT
#define AVERROR_EXIT
Immediate exit was requested; the called function should not be restarted.
Definition: error.h:58
SafeCFRelease
#define SafeCFRelease(ptr)
AVDictionaryEntry::value
char * value
Definition: dict.h:81
CoreImageContext::list_filters
int list_filters
Option used to list all available filters including generators.
Definition: vf_coreimage.m:61