FFmpeg
af_headphone.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2017 Paul B Mahol
3  * Copyright (C) 2013-2015 Andreas Fuchs, Wolfgang Hrauda
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <math.h>
22 
23 #include "libavutil/avstring.h"
25 #include "libavutil/float_dsp.h"
26 #include "libavutil/intmath.h"
27 #include "libavutil/opt.h"
28 #include "libavcodec/avfft.h"
29 
30 #include "avfilter.h"
31 #include "filters.h"
32 #include "internal.h"
33 #include "audio.h"
34 
35 #define TIME_DOMAIN 0
36 #define FREQUENCY_DOMAIN 1
37 
38 #define HRIR_STEREO 0
39 #define HRIR_MULTI 1
40 
41 typedef struct HeadphoneContext {
42  const AVClass *class;
43 
44  char *map;
45  int type;
46 
48 
50  int eof_hrirs;
51 
52  int ir_len;
53  int air_len;
54 
56 
57  int nb_irs;
58 
59  float gain;
61 
62  float *ringbuffer[2];
63  int write[2];
64 
66  int n_fft;
67  int size;
68  int hrir_fmt;
69 
70  float *data_ir[2];
71  float *temp_src[2];
74 
75  FFTContext *fft[2], *ifft[2];
77 
78  float (*scalarproduct_float)(const float *v1, const float *v2, int len);
79  struct hrir_inputs {
80  int ir_len;
81  int eof;
82  } hrir_in[64];
83  uint64_t mapping[64];
85 
86 static int parse_channel_name(const char *arg, uint64_t *rchannel)
87 {
88  uint64_t layout = av_get_channel_layout(arg);
89 
90  if (av_get_channel_layout_nb_channels(layout) != 1)
91  return AVERROR(EINVAL);
92  *rchannel = layout;
93  return 0;
94 }
95 
97 {
98  HeadphoneContext *s = ctx->priv;
99  char *arg, *tokenizer, *p;
100  uint64_t used_channels = 0;
101 
102  p = s->map;
103  while ((arg = av_strtok(p, "|", &tokenizer))) {
104  uint64_t out_channel;
105 
106  p = NULL;
107  if (parse_channel_name(arg, &out_channel)) {
108  av_log(ctx, AV_LOG_WARNING, "Failed to parse \'%s\' as channel name.\n", arg);
109  continue;
110  }
111  if (used_channels & out_channel) {
112  av_log(ctx, AV_LOG_WARNING, "Ignoring duplicate channel '%s'.\n", arg);
113  continue;
114  }
115  used_channels |= out_channel;
116  s->mapping[s->nb_irs] = out_channel;
117  s->nb_irs++;
118  }
119 
120  if (s->hrir_fmt == HRIR_MULTI)
121  s->nb_hrir_inputs = 1;
122  else
123  s->nb_hrir_inputs = s->nb_irs;
124 }
125 
126 typedef struct ThreadData {
127  AVFrame *in, *out;
128  int *write;
129  float **ir;
131  float **ringbuffer;
132  float **temp_src;
135 } ThreadData;
136 
137 static int headphone_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
138 {
139  HeadphoneContext *s = ctx->priv;
140  ThreadData *td = arg;
141  AVFrame *in = td->in, *out = td->out;
142  int offset = jobnr;
143  int *write = &td->write[jobnr];
144  const float *const ir = td->ir[jobnr];
145  int *n_clippings = &td->n_clippings[jobnr];
146  float *ringbuffer = td->ringbuffer[jobnr];
147  float *temp_src = td->temp_src[jobnr];
148  const int ir_len = s->ir_len;
149  const int air_len = s->air_len;
150  const float *src = (const float *)in->data[0];
151  float *dst = (float *)out->data[0];
152  const int in_channels = in->channels;
153  const int buffer_length = s->buffer_length;
154  const uint32_t modulo = (uint32_t)buffer_length - 1;
155  float *buffer[64];
156  int wr = *write;
157  int read;
158  int i, l;
159 
160  dst += offset;
161  for (l = 0; l < in_channels; l++) {
162  buffer[l] = ringbuffer + l * buffer_length;
163  }
164 
165  for (i = 0; i < in->nb_samples; i++) {
166  const float *cur_ir = ir;
167 
168  *dst = 0;
169  for (l = 0; l < in_channels; l++) {
170  *(buffer[l] + wr) = src[l];
171  }
172 
173  for (l = 0; l < in_channels; cur_ir += air_len, l++) {
174  const float *const bptr = buffer[l];
175 
176  if (l == s->lfe_channel) {
177  *dst += *(buffer[s->lfe_channel] + wr) * s->gain_lfe;
178  continue;
179  }
180 
181  read = (wr - (ir_len - 1)) & modulo;
182 
183  if (read + ir_len < buffer_length) {
184  memcpy(temp_src, bptr + read, ir_len * sizeof(*temp_src));
185  } else {
186  int len = FFMIN(air_len - (read % ir_len), buffer_length - read);
187 
188  memcpy(temp_src, bptr + read, len * sizeof(*temp_src));
189  memcpy(temp_src + len, bptr, (air_len - len) * sizeof(*temp_src));
190  }
191 
192  dst[0] += s->scalarproduct_float(cur_ir, temp_src, FFALIGN(ir_len, 32));
193  }
194 
195  if (fabsf(dst[0]) > 1)
196  n_clippings[0]++;
197 
198  dst += 2;
199  src += in_channels;
200  wr = (wr + 1) & modulo;
201  }
202 
203  *write = wr;
204 
205  return 0;
206 }
207 
208 static int headphone_fast_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
209 {
210  HeadphoneContext *s = ctx->priv;
211  ThreadData *td = arg;
212  AVFrame *in = td->in, *out = td->out;
213  int offset = jobnr;
214  int *write = &td->write[jobnr];
215  FFTComplex *hrtf = s->data_hrtf[jobnr];
216  int *n_clippings = &td->n_clippings[jobnr];
217  float *ringbuffer = td->ringbuffer[jobnr];
218  const int ir_len = s->ir_len;
219  const float *src = (const float *)in->data[0];
220  float *dst = (float *)out->data[0];
221  const int in_channels = in->channels;
222  const int buffer_length = s->buffer_length;
223  const uint32_t modulo = (uint32_t)buffer_length - 1;
224  FFTComplex *fft_in = s->temp_fft[jobnr];
225  FFTComplex *fft_acc = s->temp_afft[jobnr];
226  FFTContext *ifft = s->ifft[jobnr];
227  FFTContext *fft = s->fft[jobnr];
228  const int n_fft = s->n_fft;
229  const float fft_scale = 1.0f / s->n_fft;
230  FFTComplex *hrtf_offset;
231  int wr = *write;
232  int n_read;
233  int i, j;
234 
235  dst += offset;
236 
237  n_read = FFMIN(ir_len, in->nb_samples);
238  for (j = 0; j < n_read; j++) {
239  dst[2 * j] = ringbuffer[wr];
240  ringbuffer[wr] = 0.0;
241  wr = (wr + 1) & modulo;
242  }
243 
244  for (j = n_read; j < in->nb_samples; j++) {
245  dst[2 * j] = 0;
246  }
247 
248  memset(fft_acc, 0, sizeof(FFTComplex) * n_fft);
249 
250  for (i = 0; i < in_channels; i++) {
251  if (i == s->lfe_channel) {
252  for (j = 0; j < in->nb_samples; j++) {
253  dst[2 * j] += src[i + j * in_channels] * s->gain_lfe;
254  }
255  continue;
256  }
257 
258  offset = i * n_fft;
259  hrtf_offset = hrtf + offset;
260 
261  memset(fft_in, 0, sizeof(FFTComplex) * n_fft);
262 
263  for (j = 0; j < in->nb_samples; j++) {
264  fft_in[j].re = src[j * in_channels + i];
265  }
266 
267  av_fft_permute(fft, fft_in);
268  av_fft_calc(fft, fft_in);
269  for (j = 0; j < n_fft; j++) {
270  const FFTComplex *hcomplex = hrtf_offset + j;
271  const float re = fft_in[j].re;
272  const float im = fft_in[j].im;
273 
274  fft_acc[j].re += re * hcomplex->re - im * hcomplex->im;
275  fft_acc[j].im += re * hcomplex->im + im * hcomplex->re;
276  }
277  }
278 
279  av_fft_permute(ifft, fft_acc);
280  av_fft_calc(ifft, fft_acc);
281 
282  for (j = 0; j < in->nb_samples; j++) {
283  dst[2 * j] += fft_acc[j].re * fft_scale;
284  if (fabsf(dst[2 * j]) > 1)
285  n_clippings[0]++;
286  }
287 
288  for (j = 0; j < ir_len - 1; j++) {
289  int write_pos = (wr + j) & modulo;
290 
291  *(ringbuffer + write_pos) += fft_acc[in->nb_samples + j].re * fft_scale;
292  }
293 
294  *write = wr;
295 
296  return 0;
297 }
298 
299 static int check_ir(AVFilterLink *inlink, int input_number)
300 {
301  AVFilterContext *ctx = inlink->dst;
302  HeadphoneContext *s = ctx->priv;
303  int ir_len, max_ir_len;
304 
305  ir_len = ff_inlink_queued_samples(inlink);
306  max_ir_len = 65536;
307  if (ir_len > max_ir_len) {
308  av_log(ctx, AV_LOG_ERROR, "Too big length of IRs: %d > %d.\n", ir_len, max_ir_len);
309  return AVERROR(EINVAL);
310  }
311  s->hrir_in[input_number].ir_len = ir_len;
312  s->ir_len = FFMAX(ir_len, s->ir_len);
313 
314  return 0;
315 }
316 
318 {
319  AVFilterContext *ctx = outlink->src;
320  int n_clippings[2] = { 0 };
321  ThreadData td;
322  AVFrame *out;
323 
324  out = ff_get_audio_buffer(outlink, in->nb_samples);
325  if (!out) {
326  av_frame_free(&in);
327  return AVERROR(ENOMEM);
328  }
329  out->pts = in->pts;
330 
331  td.in = in; td.out = out; td.write = s->write;
332  td.ir = s->data_ir; td.n_clippings = n_clippings;
333  td.ringbuffer = s->ringbuffer; td.temp_src = s->temp_src;
334  td.temp_fft = s->temp_fft;
335  td.temp_afft = s->temp_afft;
336 
337  if (s->type == TIME_DOMAIN) {
338  ctx->internal->execute(ctx, headphone_convolute, &td, NULL, 2);
339  } else {
340  ctx->internal->execute(ctx, headphone_fast_convolute, &td, NULL, 2);
341  }
342  emms_c();
343 
344  if (n_clippings[0] + n_clippings[1] > 0) {
345  av_log(ctx, AV_LOG_WARNING, "%d of %d samples clipped. Please reduce gain.\n",
346  n_clippings[0] + n_clippings[1], out->nb_samples * 2);
347  }
348 
349  av_frame_free(&in);
350  return ff_filter_frame(outlink, out);
351 }
352 
354 {
355  struct HeadphoneContext *s = ctx->priv;
356  const int ir_len = s->ir_len;
357  int nb_input_channels = ctx->inputs[0]->channels;
358  float gain_lin = expf((s->gain - 3 * nb_input_channels) / 20 * M_LN10);
359  AVFrame *frame;
360  int ret = 0;
361  int n_fft;
362  int i, j, k;
363 
364  s->air_len = 1 << (32 - ff_clz(ir_len));
365  if (s->type == TIME_DOMAIN) {
366  s->air_len = FFALIGN(s->air_len, 32);
367  }
368  s->buffer_length = 1 << (32 - ff_clz(s->air_len));
369  s->n_fft = n_fft = 1 << (32 - ff_clz(ir_len + s->size));
370 
371  if (s->type == FREQUENCY_DOMAIN) {
372  s->fft[0] = av_fft_init(av_log2(s->n_fft), 0);
373  s->fft[1] = av_fft_init(av_log2(s->n_fft), 0);
374  s->ifft[0] = av_fft_init(av_log2(s->n_fft), 1);
375  s->ifft[1] = av_fft_init(av_log2(s->n_fft), 1);
376 
377  if (!s->fft[0] || !s->fft[1] || !s->ifft[0] || !s->ifft[1]) {
378  av_log(ctx, AV_LOG_ERROR, "Unable to create FFT contexts of size %d.\n", s->n_fft);
379  ret = AVERROR(ENOMEM);
380  goto fail;
381  }
382  }
383 
384  if (s->type == TIME_DOMAIN) {
385  s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels);
386  s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels);
387  } else {
388  s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float));
389  s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float));
390  s->temp_fft[0] = av_calloc(s->n_fft, sizeof(FFTComplex));
391  s->temp_fft[1] = av_calloc(s->n_fft, sizeof(FFTComplex));
392  s->temp_afft[0] = av_calloc(s->n_fft, sizeof(FFTComplex));
393  s->temp_afft[1] = av_calloc(s->n_fft, sizeof(FFTComplex));
394  if (!s->temp_fft[0] || !s->temp_fft[1] ||
395  !s->temp_afft[0] || !s->temp_afft[1]) {
396  ret = AVERROR(ENOMEM);
397  goto fail;
398  }
399  }
400 
401  if (!s->ringbuffer[0] || !s->ringbuffer[1]) {
402  ret = AVERROR(ENOMEM);
403  goto fail;
404  }
405 
406  if (s->type == TIME_DOMAIN) {
407  s->temp_src[0] = av_calloc(s->air_len, sizeof(float));
408  s->temp_src[1] = av_calloc(s->air_len, sizeof(float));
409 
410  s->data_ir[0] = av_calloc(nb_input_channels * s->air_len, sizeof(*s->data_ir[0]));
411  s->data_ir[1] = av_calloc(nb_input_channels * s->air_len, sizeof(*s->data_ir[1]));
412  if (!s->data_ir[0] || !s->data_ir[1] || !s->temp_src[0] || !s->temp_src[1]) {
413  ret = AVERROR(ENOMEM);
414  goto fail;
415  }
416  } else {
417  s->data_hrtf[0] = av_calloc(n_fft, sizeof(*s->data_hrtf[0]) * nb_input_channels);
418  s->data_hrtf[1] = av_calloc(n_fft, sizeof(*s->data_hrtf[1]) * nb_input_channels);
419  if (!s->data_hrtf[0] || !s->data_hrtf[1]) {
420  ret = AVERROR(ENOMEM);
421  goto fail;
422  }
423  }
424 
425  for (i = 0; i < s->nb_hrir_inputs; av_frame_free(&frame), i++) {
426  int len = s->hrir_in[i].ir_len;
427  float *ptr;
428 
429  ret = ff_inlink_consume_samples(ctx->inputs[i + 1], len, len, &frame);
430  if (ret < 0)
431  goto fail;
432  ptr = (float *)frame->extended_data[0];
433 
434  if (s->hrir_fmt == HRIR_STEREO) {
436  s->mapping[i]);
437  if (idx < 0)
438  continue;
439  if (s->type == TIME_DOMAIN) {
440  float *data_ir_l = s->data_ir[0] + idx * s->air_len;
441  float *data_ir_r = s->data_ir[1] + idx * s->air_len;
442 
443  for (j = 0; j < len; j++) {
444  data_ir_l[j] = ptr[len * 2 - j * 2 - 2] * gain_lin;
445  data_ir_r[j] = ptr[len * 2 - j * 2 - 1] * gain_lin;
446  }
447  } else {
448  FFTComplex *fft_in_l = s->data_hrtf[0] + idx * n_fft;
449  FFTComplex *fft_in_r = s->data_hrtf[1] + idx * n_fft;
450 
451  for (j = 0; j < len; j++) {
452  fft_in_l[j].re = ptr[j * 2 ] * gain_lin;
453  fft_in_r[j].re = ptr[j * 2 + 1] * gain_lin;
454  }
455 
456  av_fft_permute(s->fft[0], fft_in_l);
457  av_fft_calc(s->fft[0], fft_in_l);
458  av_fft_permute(s->fft[0], fft_in_r);
459  av_fft_calc(s->fft[0], fft_in_r);
460  }
461  } else {
462  int I, N = ctx->inputs[1]->channels;
463 
464  for (k = 0; k < N / 2; k++) {
466  s->mapping[k]);
467  if (idx < 0)
468  continue;
469 
470  I = k * 2;
471  if (s->type == TIME_DOMAIN) {
472  float *data_ir_l = s->data_ir[0] + idx * s->air_len;
473  float *data_ir_r = s->data_ir[1] + idx * s->air_len;
474 
475  for (j = 0; j < len; j++) {
476  data_ir_l[j] = ptr[len * N - j * N - N + I ] * gain_lin;
477  data_ir_r[j] = ptr[len * N - j * N - N + I + 1] * gain_lin;
478  }
479  } else {
480  FFTComplex *fft_in_l = s->data_hrtf[0] + idx * n_fft;
481  FFTComplex *fft_in_r = s->data_hrtf[1] + idx * n_fft;
482 
483  for (j = 0; j < len; j++) {
484  fft_in_l[j].re = ptr[j * N + I ] * gain_lin;
485  fft_in_r[j].re = ptr[j * N + I + 1] * gain_lin;
486  }
487 
488  av_fft_permute(s->fft[0], fft_in_l);
489  av_fft_calc(s->fft[0], fft_in_l);
490  av_fft_permute(s->fft[0], fft_in_r);
491  av_fft_calc(s->fft[0], fft_in_r);
492  }
493  }
494  }
495  }
496 
497  s->have_hrirs = 1;
498 
499 fail:
500  return ret;
501 }
502 
504 {
505  HeadphoneContext *s = ctx->priv;
506  AVFilterLink *inlink = ctx->inputs[0];
507  AVFilterLink *outlink = ctx->outputs[0];
508  AVFrame *in = NULL;
509  int i, ret;
510 
512  if (!s->eof_hrirs) {
513  int eof = 1;
514  for (i = 0; i < s->nb_hrir_inputs; i++) {
515  AVFilterLink *input = ctx->inputs[i + 1];
516 
517  if (s->hrir_in[i].eof)
518  continue;
519 
520  if ((ret = check_ir(input, i)) < 0)
521  return ret;
522 
523  if (ff_outlink_get_status(input) == AVERROR_EOF) {
524  if (!ff_inlink_queued_samples(input)) {
525  av_log(ctx, AV_LOG_ERROR, "No samples provided for "
526  "HRIR stream %d.\n", i);
527  return AVERROR_INVALIDDATA;
528  }
529  s->hrir_in[i].eof = 1;
530  } else {
531  if (ff_outlink_frame_wanted(ctx->outputs[0]))
533  eof = 0;
534  }
535  }
536  if (!eof)
537  return 0;
538  s->eof_hrirs = 1;
539 
540  ret = convert_coeffs(ctx, inlink);
541  if (ret < 0)
542  return ret;
543  } else if (!s->have_hrirs)
544  return AVERROR_EOF;
545 
546  if ((ret = ff_inlink_consume_samples(ctx->inputs[0], s->size, s->size, &in)) > 0) {
547  ret = headphone_frame(s, in, outlink);
548  if (ret < 0)
549  return ret;
550  }
551 
552  if (ret < 0)
553  return ret;
554 
555  FF_FILTER_FORWARD_STATUS(ctx->inputs[0], ctx->outputs[0]);
556  if (ff_outlink_frame_wanted(ctx->outputs[0]))
558 
559  return 0;
560 }
561 
563 {
564  struct HeadphoneContext *s = ctx->priv;
567  AVFilterChannelLayouts *stereo_layout = NULL;
568  AVFilterChannelLayouts *hrir_layouts = NULL;
569  int ret, i;
570 
571  ret = ff_add_format(&formats, AV_SAMPLE_FMT_FLT);
572  if (ret)
573  return ret;
574  ret = ff_set_common_formats(ctx, formats);
575  if (ret)
576  return ret;
577 
578  layouts = ff_all_channel_layouts();
579  if (!layouts)
580  return AVERROR(ENOMEM);
581 
582  ret = ff_channel_layouts_ref(layouts, &ctx->inputs[0]->outcfg.channel_layouts);
583  if (ret)
584  return ret;
585 
586  ret = ff_add_channel_layout(&stereo_layout, AV_CH_LAYOUT_STEREO);
587  if (ret)
588  return ret;
589  ret = ff_channel_layouts_ref(stereo_layout, &ctx->outputs[0]->incfg.channel_layouts);
590  if (ret)
591  return ret;
592 
593  if (s->hrir_fmt == HRIR_MULTI) {
594  hrir_layouts = ff_all_channel_counts();
595  if (!hrir_layouts)
596  return AVERROR(ENOMEM);
597  ret = ff_channel_layouts_ref(hrir_layouts, &ctx->inputs[1]->outcfg.channel_layouts);
598  if (ret)
599  return ret;
600  } else {
601  for (i = 1; i <= s->nb_hrir_inputs; i++) {
602  ret = ff_channel_layouts_ref(stereo_layout, &ctx->inputs[i]->outcfg.channel_layouts);
603  if (ret)
604  return ret;
605  }
606  }
607 
608  formats = ff_all_samplerates();
609  if (!formats)
610  return AVERROR(ENOMEM);
611  return ff_set_common_samplerates(ctx, formats);
612 }
613 
615 {
616  AVFilterContext *ctx = inlink->dst;
617  HeadphoneContext *s = ctx->priv;
618 
619  if (s->nb_irs < inlink->channels) {
620  av_log(ctx, AV_LOG_ERROR, "Number of HRIRs must be >= %d.\n", inlink->channels);
621  return AVERROR(EINVAL);
622  }
623 
626  return 0;
627 }
628 
630 {
631  HeadphoneContext *s = ctx->priv;
632  int i, ret;
633 
634  AVFilterPad pad = {
635  .name = "in0",
636  .type = AVMEDIA_TYPE_AUDIO,
637  .config_props = config_input,
638  };
639  if ((ret = ff_insert_inpad(ctx, 0, &pad)) < 0)
640  return ret;
641 
642  if (!s->map) {
643  av_log(ctx, AV_LOG_ERROR, "Valid mapping must be set.\n");
644  return AVERROR(EINVAL);
645  }
646 
647  parse_map(ctx);
648 
649  for (i = 0; i < s->nb_hrir_inputs; i++) {
650  char *name = av_asprintf("hrir%d", i);
651  AVFilterPad pad = {
652  .name = name,
653  .type = AVMEDIA_TYPE_AUDIO,
654  };
655  if (!name)
656  return AVERROR(ENOMEM);
657  if ((ret = ff_insert_inpad(ctx, i + 1, &pad)) < 0) {
658  av_freep(&pad.name);
659  return ret;
660  }
661  }
662 
663  if (s->type == TIME_DOMAIN) {
665  if (!fdsp)
666  return AVERROR(ENOMEM);
668  av_free(fdsp);
669  }
670 
671  return 0;
672 }
673 
674 static int config_output(AVFilterLink *outlink)
675 {
676  AVFilterContext *ctx = outlink->src;
677  HeadphoneContext *s = ctx->priv;
678  AVFilterLink *inlink = ctx->inputs[0];
679 
680  if (s->hrir_fmt == HRIR_MULTI) {
681  AVFilterLink *hrir_link = ctx->inputs[1];
682 
683  if (hrir_link->channels < inlink->channels * 2) {
684  av_log(ctx, AV_LOG_ERROR, "Number of channels in HRIR stream must be >= %d.\n", inlink->channels * 2);
685  return AVERROR(EINVAL);
686  }
687  }
688 
689  s->gain_lfe = expf((s->gain - 3 * inlink->channels + s->lfe_gain) / 20 * M_LN10);
690 
691  return 0;
692 }
693 
695 {
696  HeadphoneContext *s = ctx->priv;
697 
698  av_fft_end(s->ifft[0]);
699  av_fft_end(s->ifft[1]);
700  av_fft_end(s->fft[0]);
701  av_fft_end(s->fft[1]);
702  av_freep(&s->data_ir[0]);
703  av_freep(&s->data_ir[1]);
704  av_freep(&s->ringbuffer[0]);
705  av_freep(&s->ringbuffer[1]);
706  av_freep(&s->temp_src[0]);
707  av_freep(&s->temp_src[1]);
708  av_freep(&s->temp_fft[0]);
709  av_freep(&s->temp_fft[1]);
710  av_freep(&s->temp_afft[0]);
711  av_freep(&s->temp_afft[1]);
712  av_freep(&s->data_hrtf[0]);
713  av_freep(&s->data_hrtf[1]);
714 
715  for (unsigned i = 1; i < ctx->nb_inputs; i++)
716  av_freep(&ctx->input_pads[i].name);
717 }
718 
719 #define OFFSET(x) offsetof(HeadphoneContext, x)
720 #define FLAGS AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
721 
722 static const AVOption headphone_options[] = {
723  { "map", "set channels convolution mappings", OFFSET(map), AV_OPT_TYPE_STRING, {.str=NULL}, .flags = FLAGS },
724  { "gain", "set gain in dB", OFFSET(gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS },
725  { "lfe", "set lfe gain in dB", OFFSET(lfe_gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS },
726  { "type", "set processing", OFFSET(type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, .flags = FLAGS, "type" },
727  { "time", "time domain", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, .flags = FLAGS, "type" },
728  { "freq", "frequency domain", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, .flags = FLAGS, "type" },
729  { "size", "set frame size", OFFSET(size), AV_OPT_TYPE_INT, {.i64=1024},1024,96000, .flags = FLAGS },
730  { "hrir", "set hrir format", OFFSET(hrir_fmt), AV_OPT_TYPE_INT, {.i64=HRIR_STEREO}, 0, 1, .flags = FLAGS, "hrir" },
731  { "stereo", "hrir files have exactly 2 channels", 0, AV_OPT_TYPE_CONST, {.i64=HRIR_STEREO}, 0, 0, .flags = FLAGS, "hrir" },
732  { "multich", "single multichannel hrir file", 0, AV_OPT_TYPE_CONST, {.i64=HRIR_MULTI}, 0, 0, .flags = FLAGS, "hrir" },
733  { NULL }
734 };
735 
736 AVFILTER_DEFINE_CLASS(headphone);
737 
738 static const AVFilterPad outputs[] = {
739  {
740  .name = "default",
741  .type = AVMEDIA_TYPE_AUDIO,
742  .config_props = config_output,
743  },
744  { NULL }
745 };
746 
748  .name = "headphone",
749  .description = NULL_IF_CONFIG_SMALL("Apply headphone binaural spatialization with HRTFs in additional streams."),
750  .priv_size = sizeof(HeadphoneContext),
751  .priv_class = &headphone_class,
752  .init = init,
753  .uninit = uninit,
755  .activate = activate,
756  .inputs = NULL,
757  .outputs = outputs,
759 };
float(* scalarproduct_float)(const float *v1, const float *v2, int len)
Calculate the scalar product of two vectors of floats.
Definition: float_dsp.h:175
#define NULL
Definition: coverity.c:32
static int config_output(AVFilterLink *outlink)
Definition: af_headphone.c:674
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
AVFrame * out
Definition: af_adeclick.c:494
This structure describes decoded (raw) audio or video data.
Definition: frame.h:308
AVOption.
Definition: opt.h:248
av_cold void av_fft_end(FFTContext *s)
Definition: avfft.c:48
static int convert_coeffs(AVFilterContext *ctx, AVFilterLink *inlink)
Definition: af_headphone.c:353
float re
Definition: fft.c:82
float ** temp_src
Definition: af_headphone.c:132
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, filter)
Forward the status on an output link to all input links.
Definition: filters.h:212
Main libavfilter public API header.
#define AVFILTER_FLAG_DYNAMIC_INPUTS
The number of the filter inputs is not determined just by AVFilter.inputs.
Definition: avfilter.h:106
FF_FILTER_FORWARD_STATUS(inlink, outlink)
int av_log2(unsigned v)
Definition: intmath.c:26
FFTContext * fft[2]
Definition: af_headphone.c:75
static int parse_channel_name(const char *arg, uint64_t *rchannel)
Definition: af_headphone.c:86
GLint GLenum type
Definition: opengl_enc.c:104
FFTSample re
Definition: avfft.h:38
void av_fft_permute(FFTContext *s, FFTComplex *z)
Do the permutation needed BEFORE calling ff_fft_calc().
Definition: avfft.c:38
FFTComplex ** temp_afft
Definition: af_headphone.c:134
#define AV_CH_LAYOUT_STEREO
#define TIME_DOMAIN
Definition: af_headphone.c:35
void ff_inlink_request_frame(AVFilterLink *link)
Mark that a frame is wanted on the link.
Definition: avfilter.c:1615
#define N
Definition: af_mcompand.c:54
void * av_calloc(size_t nmemb, size_t size)
Non-inlined equivalent of av_mallocz_array().
Definition: mem.c:245
int av_get_channel_layout_nb_channels(uint64_t channel_layout)
Return the number of channels in the channel layout.
static int activate(AVFilterContext *ctx)
Definition: af_headphone.c:503
const char * name
Pad name.
Definition: internal.h:60
uint64_t av_get_channel_layout(const char *name)
Return a channel layout id that matches name, or 0 if no match is found.
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:347
int ff_channel_layouts_ref(AVFilterChannelLayouts *f, AVFilterChannelLayouts **ref)
Add *ref as a new reference to f.
Definition: formats.c:462
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1091
float ** ringbuffer
Definition: af_headphone.c:131
#define av_cold
Definition: attributes.h:88
AVOptions.
float(* scalarproduct_float)(const float *v1, const float *v2, int len)
Definition: af_headphone.c:78
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
#define OFFSET(x)
Definition: af_headphone.c:719
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:401
#define HRIR_MULTI
Definition: af_headphone.c:39
#define AV_CH_LOW_FREQUENCY
#define AVERROR_EOF
End of file.
Definition: error.h:55
FFTContext * ifft[2]
Definition: af_headphone.c:75
ptrdiff_t size
Definition: opengl_enc.c:100
the definition of that something depends on the semantic of the filter The callback must examine the status of the filter s links and proceed accordingly The status of output links is stored in the status_in and status_out fields and tested by the ff_outlink_frame_wanted() function.If this function returns true
#define FFALIGN(x, a)
Definition: macros.h:48
#define av_log(a,...)
static int config_input(AVFilterLink *inlink)
Definition: af_headphone.c:614
A filter pad used for either input or output.
Definition: internal.h:54
#define expf(x)
Definition: libm.h:283
#define src
Definition: vp8dsp.c:254
AVFilterPad * input_pads
array of input pads
Definition: avfilter.h:346
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:588
#define td
Definition: regdef.h:70
#define HRIR_STEREO
Definition: af_headphone.c:38
av_cold AVFloatDSPContext * avpriv_float_dsp_alloc(int bit_exact)
Allocate a float DSP context.
Definition: float_dsp.c:135
int ff_add_channel_layout(AVFilterChannelLayouts **l, uint64_t channel_layout)
Definition: formats.c:339
AVFrame * ff_get_audio_buffer(AVFilterLink *link, int nb_samples)
Request an audio samples buffer with a specific set of permissions.
Definition: audio.c:86
#define FLAGS
Definition: af_headphone.c:720
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:153
void * priv
private data for use by the filter
Definition: avfilter.h:354
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:117
static void parse_map(AVFilterContext *ctx)
Definition: af_headphone.c:96
const char * arg
Definition: jacosubdec.c:66
static const AVFilterPad outputs[]
Definition: af_headphone.c:738
int ff_add_format(AVFilterFormats **avff, int64_t fmt)
Add fmt to the list of media formats contained in *avff.
Definition: formats.c:333
FFTContext * av_fft_init(int nbits, int inverse)
Set up a complex FFT.
Definition: avfft.c:28
#define FFMAX(a, b)
Definition: common.h:94
static av_cold void uninit(AVFilterContext *ctx)
Definition: af_headphone.c:694
#define fail()
Definition: checkasm.h:123
AVFILTER_DEFINE_CLASS(headphone)
char * av_asprintf(const char *fmt,...)
Definition: avstring.c:113
static int headphone_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: af_headphone.c:137
FFTComplex * data_hrtf[2]
Definition: af_headphone.c:76
Definition: fft.h:88
int channels
number of audio channels, only used for audio.
Definition: frame.h:614
audio channel layout utility functions
unsigned nb_inputs
number of input pads
Definition: avfilter.h:348
#define FFMIN(a, b)
Definition: common.h:96
AVFilterChannelLayouts * channel_layouts
Lists of supported channel layouts, only for audio.
Definition: avfilter.h:453
#define ff_clz
Definition: intmath.h:142
int ff_inlink_queued_samples(AVFilterLink *link)
Definition: avfilter.c:1469
float * data_ir[2]
Definition: af_headphone.c:70
AVFormatContext * ctx
Definition: movenc.c:48
AVFilter ff_af_headphone
Definition: af_headphone.c:747
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
#define s(width, name)
Definition: cbs_vp9.c:257
int * n_clippings
Definition: af_headphone.c:130
static int query_formats(AVFilterContext *ctx)
Definition: af_headphone.c:562
FFTComplex * temp_fft[2]
Definition: af_headphone.c:72
AVFilterChannelLayouts * ff_all_channel_layouts(void)
Construct an empty AVFilterChannelLayouts/AVFilterFormats struct – representing any channel layout (...
Definition: formats.c:428
static const AVOption headphone_options[]
Definition: af_headphone.c:722
A list of supported channel layouts.
Definition: formats.h:86
uint64_t mapping[64]
Definition: af_headphone.c:83
if(ret)
FFTComplex ** temp_fft
Definition: af_headphone.c:133
static int headphone_frame(HeadphoneContext *s, AVFrame *in, AVFilterLink *outlink)
Definition: af_headphone.c:317
Used for passing data between threads.
Definition: dsddec.c:67
int ff_inlink_consume_samples(AVFilterLink *link, unsigned min, unsigned max, AVFrame **rframe)
Take samples from the link&#39;s FIFO and update the link&#39;s stats.
Definition: avfilter.c:1508
FFT functions.
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:145
int ff_outlink_get_status(AVFilterLink *link)
Get the status on an output link.
Definition: avfilter.c:1638
float im
Definition: fft.c:82
const char * name
Filter name.
Definition: avfilter.h:149
FFTComplex * temp_afft[2]
Definition: af_headphone.c:73
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
float * temp_src[2]
Definition: af_headphone.c:71
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:351
enum MovChannelLayoutTag * layouts
Definition: mov_chan.c:434
int av_get_channel_layout_channel_index(uint64_t channel_layout, uint64_t channel)
Get the index of a channel in channel_layout.
AVFilterFormats * ff_all_samplerates(void)
Definition: formats.c:422
static av_cold int init(AVFilterContext *ctx)
Definition: af_headphone.c:629
float ** ir
Definition: af_headphone.c:129
#define flags(name, subs,...)
Definition: cbs_av1.c:560
AVFilterInternal * internal
An opaque struct for libavfilter internal use.
Definition: avfilter.h:379
float * ringbuffer[2]
Definition: af_headphone.c:62
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:322
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok()...
Definition: avstring.c:186
#define M_LN10
Definition: mathematics.h:43
FFTSample im
Definition: avfft.h:38
struct HeadphoneContext::hrir_inputs hrir_in[64]
avfilter_execute_func * execute
Definition: internal.h:136
#define av_free(p)
int len
static int headphone_fast_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: af_headphone.c:208
A list of supported formats for one end of a filter link.
Definition: formats.h:65
An instance of a filter.
Definition: avfilter.h:339
FILE * out
Definition: movenc.c:54
#define av_freep(p)
AVFrame * in
Definition: af_afftdn.c:1083
formats
Definition: signature.h:48
static int check_ir(AVFilterLink *inlink, int input_number)
Definition: af_headphone.c:299
internal API functions
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
AVFilterChannelLayouts * ff_all_channel_counts(void)
Construct an AVFilterChannelLayouts coding for any channel layout, with known or unknown disposition...
Definition: formats.c:437
uint8_t ** extended_data
pointers to the data planes/channels.
Definition: frame.h:355
void av_fft_calc(FFTContext *s, FFTComplex *z)
Do a complex FFT with the parameters defined in av_fft_init().
Definition: avfft.c:43
#define FREQUENCY_DOMAIN
Definition: af_headphone.c:36
int nb_samples
number of audio samples (per channel) described by this frame
Definition: frame.h:374
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
int ff_set_common_samplerates(AVFilterContext *ctx, AVFilterFormats *samplerates)
Definition: formats.c:576
int i
Definition: input.c:407
GLuint buffer
Definition: opengl_enc.c:101
const char * name
Definition: opengl_enc.c:102
static int ff_insert_inpad(AVFilterContext *f, unsigned index, AVFilterPad *p)
Insert a new input pad for the filter.
Definition: internal.h:240