FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
framesync.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013 Nicolas George
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public License
8  * as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define FF_INTERNAL_FIELDS 1
22 #include "framequeue.h"
23 
24 #include "libavutil/avassert.h"
25 #include "avfilter.h"
26 #include "bufferqueue.h"
27 #include "framesync.h"
28 #include "internal.h"
29 
30 #define OFFSET(member) offsetof(FFFrameSync, member)
31 
32 static const char *framesync_name(void *ptr)
33 {
34  return "framesync";
35 }
36 
37 static const AVClass framesync_class = {
39  .class_name = "framesync",
40  .item_name = framesync_name,
41  .category = AV_CLASS_CATEGORY_FILTER,
42  .option = NULL,
43  .parent_log_context_offset = OFFSET(parent),
44 };
45 
46 enum {
50 };
51 
52 int ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in)
53 {
54  fs->class = &framesync_class;
55  fs->parent = parent;
56  fs->nb_in = nb_in;
57 
58  fs->in = av_calloc(nb_in, sizeof(*fs->in));
59  if (!fs->in)
60  return AVERROR(ENOMEM);
61  return 0;
62 }
63 
65 {
66  unsigned i, level = 0;
67 
68  for (i = 0; i < fs->nb_in; i++)
69  if (fs->in[i].state != STATE_EOF)
70  level = FFMAX(level, fs->in[i].sync);
71  av_assert0(level <= fs->sync_level);
72  if (level < fs->sync_level)
73  av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level);
74  if (level)
75  fs->sync_level = level;
76  else
77  fs->eof = 1;
78 }
79 
81 {
82  unsigned i;
83  int64_t gcd, lcm;
84 
85  if (!fs->time_base.num) {
86  for (i = 0; i < fs->nb_in; i++) {
87  if (fs->in[i].sync) {
88  if (fs->time_base.num) {
89  gcd = av_gcd(fs->time_base.den, fs->in[i].time_base.den);
90  lcm = (fs->time_base.den / gcd) * fs->in[i].time_base.den;
91  if (lcm < AV_TIME_BASE / 2) {
92  fs->time_base.den = lcm;
93  fs->time_base.num = av_gcd(fs->time_base.num,
94  fs->in[i].time_base.num);
95  } else {
96  fs->time_base.num = 1;
98  break;
99  }
100  } else {
101  fs->time_base = fs->in[i].time_base;
102  }
103  }
104  }
105  if (!fs->time_base.num) {
106  av_log(fs, AV_LOG_ERROR, "Impossible to set time base\n");
107  return AVERROR(EINVAL);
108  }
109  av_log(fs, AV_LOG_VERBOSE, "Selected %d/%d time base\n",
110  fs->time_base.num, fs->time_base.den);
111  }
112 
113  for (i = 0; i < fs->nb_in; i++)
114  fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE;
115  fs->sync_level = UINT_MAX;
117 
118  return 0;
119 }
120 
122 {
123  int latest;
124  unsigned i;
125  int64_t pts;
126 
127  if (fs->eof)
128  return;
129  while (!fs->frame_ready) {
130  latest = -1;
131  for (i = 0; i < fs->nb_in; i++) {
132  if (!fs->in[i].have_next) {
133  if (latest < 0 || fs->in[i].pts < fs->in[latest].pts)
134  latest = i;
135  }
136  }
137  if (latest >= 0) {
138  fs->in_request = latest;
139  break;
140  }
141 
142  pts = fs->in[0].pts_next;
143  for (i = 1; i < fs->nb_in; i++)
144  if (fs->in[i].pts_next < pts)
145  pts = fs->in[i].pts_next;
146  if (pts == INT64_MAX) {
147  fs->eof = 1;
148  break;
149  }
150  for (i = 0; i < fs->nb_in; i++) {
151  if (fs->in[i].pts_next == pts ||
152  (fs->in[i].before == EXT_INFINITY &&
153  fs->in[i].state == STATE_BOF)) {
154  av_frame_free(&fs->in[i].frame);
155  fs->in[i].frame = fs->in[i].frame_next;
156  fs->in[i].pts = fs->in[i].pts_next;
157  fs->in[i].frame_next = NULL;
158  fs->in[i].pts_next = AV_NOPTS_VALUE;
159  fs->in[i].have_next = 0;
160  fs->in[i].state = fs->in[i].frame ? STATE_RUN : STATE_EOF;
161  if (fs->in[i].sync == fs->sync_level && fs->in[i].frame)
162  fs->frame_ready = 1;
163  if (fs->in[i].state == STATE_EOF &&
164  fs->in[i].after == EXT_STOP)
165  fs->eof = 1;
166  }
167  }
168  if (fs->eof)
169  fs->frame_ready = 0;
170  if (fs->frame_ready)
171  for (i = 0; i < fs->nb_in; i++)
172  if ((fs->in[i].state == STATE_BOF &&
173  fs->in[i].before == EXT_STOP))
174  fs->frame_ready = 0;
175  fs->pts = pts;
176  }
177 }
178 
179 static int64_t framesync_pts_extrapolate(FFFrameSync *fs, unsigned in,
180  int64_t pts)
181 {
182  /* Possible enhancement: use the link's frame rate */
183  return pts + 1;
184 }
185 
186 static void framesync_inject_frame(FFFrameSync *fs, unsigned in, AVFrame *frame)
187 {
188  int64_t pts;
189 
190  av_assert0(!fs->in[in].have_next);
191  if (frame) {
192  pts = av_rescale_q(frame->pts, fs->in[in].time_base, fs->time_base);
193  frame->pts = pts;
194  } else {
195  pts = fs->in[in].state != STATE_RUN || fs->in[in].after == EXT_INFINITY
196  ? INT64_MAX : framesync_pts_extrapolate(fs, in, fs->in[in].pts);
197  fs->in[in].sync = 0;
199  }
200  fs->in[in].frame_next = frame;
201  fs->in[in].pts_next = pts;
202  fs->in[in].have_next = 1;
203 }
204 
206 {
207  av_assert1(in < fs->nb_in);
208  if (!fs->in[in].have_next)
209  framesync_inject_frame(fs, in, frame);
210  else
211  ff_bufqueue_add(fs, &fs->in[in].queue, frame);
212  return 0;
213 }
214 
216 {
217  unsigned i;
218 
219  av_assert0(!fs->frame_ready);
220  for (i = 0; i < fs->nb_in; i++)
221  if (!fs->in[i].have_next && fs->in[i].queue.available)
223  fs->frame_ready = 0;
224  framesync_advance(fs);
225 }
226 
228 {
229  fs->frame_ready = 0;
230 }
231 
232 int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe,
233  unsigned get)
234 {
235  AVFrame *frame;
236  unsigned need_copy = 0, i;
237  int64_t pts_next;
238  int ret;
239 
240  if (!fs->in[in].frame) {
241  *rframe = NULL;
242  return 0;
243  }
244  frame = fs->in[in].frame;
245  if (get) {
246  /* Find out if we need to copy the frame: is there another sync
247  stream, and do we know if its current frame will outlast this one? */
248  pts_next = fs->in[in].have_next ? fs->in[in].pts_next : INT64_MAX;
249  for (i = 0; i < fs->nb_in && !need_copy; i++)
250  if (i != in && fs->in[i].sync &&
251  (!fs->in[i].have_next || fs->in[i].pts_next < pts_next))
252  need_copy = 1;
253  if (need_copy) {
254  if (!(frame = av_frame_clone(frame)))
255  return AVERROR(ENOMEM);
256  if ((ret = av_frame_make_writable(frame)) < 0) {
257  av_frame_free(&frame);
258  return ret;
259  }
260  } else {
261  fs->in[in].frame = NULL;
262  }
263  fs->frame_ready = 0;
264  }
265  *rframe = frame;
266  return 0;
267 }
268 
270 {
271  unsigned i;
272 
273  for (i = 0; i < fs->nb_in; i++) {
274  av_frame_free(&fs->in[i].frame);
275  av_frame_free(&fs->in[i].frame_next);
277  }
278 
279  av_freep(&fs->in);
280 }
281 
283 {
284  int ret, count = 0;
285 
286  av_assert0(fs->on_event);
287  while (1) {
288  ff_framesync_next(fs);
289  if (fs->eof || !fs->frame_ready)
290  break;
291  if ((ret = fs->on_event(fs)) < 0)
292  return ret;
293  ff_framesync_drop(fs);
294  count++;
295  if (!all)
296  break;
297  }
298  if (!count && fs->eof)
299  return AVERROR_EOF;
300  return count;
301 }
302 
304  AVFrame *in)
305 {
306  int ret;
307 
308  if ((ret = ff_framesync_process_frame(fs, 1)) < 0)
309  return ret;
310  if ((ret = ff_framesync_add_frame(fs, FF_INLINK_IDX(inlink), in)) < 0)
311  return ret;
312  if ((ret = ff_framesync_process_frame(fs, 0)) < 0)
313  return ret;
314  return 0;
315 }
316 
318 {
319  AVFilterContext *ctx = outlink->src;
320  int input, ret, i;
321 
322  if ((ret = ff_framesync_process_frame(fs, 0)) < 0)
323  return ret;
324  if (ret > 0)
325  return 0;
326  if (fs->eof)
327  return AVERROR_EOF;
328  input = fs->in_request;
329  /* Detect status change early */
330  for (i = 0; i < fs->nb_in; i++)
331  if (!ff_framequeue_queued_frames(&ctx->inputs[i]->fifo) &&
332  ctx->inputs[i]->status_in && !ctx->inputs[i]->status_out)
333  input = i;
334  ret = ff_request_frame(ctx->inputs[input]);
335  if (ret == AVERROR_EOF) {
336  if ((ret = ff_framesync_add_frame(fs, input, NULL)) < 0)
337  return ret;
338  if ((ret = ff_framesync_process_frame(fs, 0)) < 0)
339  return ret;
340  ret = 0;
341  }
342  return ret;
343 }
static AVFrame * ff_bufqueue_get(struct FFBufQueue *queue)
Get the first buffer from the queue and remove it.
Definition: bufferqueue.h:98
#define NULL
Definition: coverity.c:32
This structure describes decoded (raw) audio or video data.
Definition: frame.h:187
static void framesync_sync_level_update(FFFrameSync *fs)
Definition: framesync.c:64
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
Main libavfilter public API header.
int(* on_event)(struct FFFrameSync *fs)
Callback called when a frame event is ready.
Definition: framesync.h:175
int num
Numerator.
Definition: rational.h:59
static void framesync_advance(FFFrameSync *fs)
Definition: framesync.c:121
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:80
static const char * framesync_name(void *ptr)
Definition: framesync.c:32
int64_t pts
Timestamp of the current event.
Definition: framesync.h:170
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:93
void * av_calloc(size_t nmemb, size_t size)
Non-inlined equivalent of av_mallocz_array().
Definition: mem.c:230
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:331
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
uint8_t have_next
Boolean flagging the next frame, for internal use.
Definition: framesync.h:128
void * parent
Definition: framesync.h:155
void ff_framesync_next(FFFrameSync *fs)
Prepare the next frame event.
Definition: framesync.c:215
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:271
unsigned sync_level
Synchronization level: only inputs with the same sync level are sync sources.
Definition: framesync.h:191
FFFrameSyncIn * in
Pointer to array of inputs.
Definition: framesync.h:206
static AVFrame * frame
#define AVERROR_EOF
End of file.
Definition: error.h:55
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:192
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:98
int ff_framesync_process_frame(FFFrameSync *fs, unsigned all)
Process one or several frame using the on_event callback.
Definition: framesync.c:282
#define av_log(a,...)
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink, AVFrame *in)
Accept a frame on a filter input.
Definition: framesync.c:303
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:269
AVFrame * frame_next
Next frame, for internal use.
Definition: framesync.h:113
Frame sync structure.
Definition: framesync.h:153
#define AVERROR(e)
Definition: error.h:43
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:163
#define OFFSET(member)
Definition: framesync.c:30
simple assert() macros that are a bit more flexible than ISO C assert().
int64_t av_gcd(int64_t a, int64_t b)
Compute the greatest common divisor of two integer operands.
Definition: mathematics.c:37
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:103
GLsizei count
Definition: opengl_enc.c:109
#define FFMAX(a, b)
Definition: common.h:94
uint8_t eof
Flag indicating that output has reached EOF.
Definition: framesync.h:201
#define av_assert1(cond)
assert() equivalent, that does not lie in speed critical code.
Definition: avassert.h:53
#define AV_TIME_BASE
Internal time base represented as integer.
Definition: avutil.h:254
unsigned in_request
Index of the input that requires a request.
Definition: framesync.h:185
int ff_framesync_request_frame(FFFrameSync *fs, AVFilterLink *outlink)
Request a frame on the filter output.
Definition: framesync.c:317
int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame)
Add a frame to an input.
Definition: framesync.c:205
AVFormatContext * ctx
Definition: movenc.c:48
AVRational time_base
Time base for the output events.
Definition: framesync.h:165
struct FFBufQueue queue
Queue of incoming AVFrame, and NULL to mark EOF.
Definition: framesync.h:88
static void ff_bufqueue_discard_all(struct FFBufQueue *queue)
Unref and remove all buffers from the queue.
Definition: bufferqueue.h:111
static int64_t framesync_pts_extrapolate(FFFrameSync *fs, unsigned in, int64_t pts)
Definition: framesync.c:179
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:485
static const AVClass framesync_class
Definition: framesync.c:37
Extend the frame to infinity.
Definition: framesync.h:77
unsigned short available
number of available buffers
Definition: bufferqueue.h:52
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(constuint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(constuint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(constint16_t *) pi >>8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(constint16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(constint16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(constint32_t *) pi >>24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(constint32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(constint32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(constfloat *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(constfloat *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(constfloat *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(constdouble *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(constdouble *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(constdouble *) pi *(1U<< 31))))#defineSET_CONV_FUNC_GROUP(ofmt, ifmt) staticvoidset_generic_function(AudioConvert *ac){}voidff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enumAVSampleFormatout_fmt, enumAVSampleFormatin_fmt, intchannels, intsample_rate, intapply_map){AudioConvert *ac;intin_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) returnNULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt)>2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);returnNULL;}returnac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}elseif(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;elseac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);returnac;}intff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){intuse_generic=1;intlen=in->nb_samples;intp;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%dsamples-audio_convert:%sto%s(dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));returnff_convert_dither(ac-> in
int ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:52
uint8_t state
State: before first, in stream or after EOF, for internal use.
Definition: framesync.h:133
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events...
Definition: framesync.h:146
Describe the class of an AVClass context structure.
Definition: log.h:67
unsigned nb_in
Number of input streams.
Definition: framesync.h:160
AVFrame * frame
Current frame, may be NULL before the first one or after EOF.
Definition: framesync.h:108
static int64_t pts
Global timestamp for the audio frames.
int av_frame_make_writable(AVFrame *frame)
Ensure that the frame data is writable, avoiding data copy if possible.
Definition: frame.c:553
uint8_t frame_ready
Flag indicating that a frame event is ready.
Definition: framesync.h:196
uint8_t level
Definition: svq3.c:207
int version
LIBAVUTIL_VERSION with which this structure was created.
Definition: log.h:93
#define FF_INLINK_IDX(link)
Find the index of a link.
Definition: internal.h:356
static size_t ff_framequeue_queued_frames(const FFFrameQueue *fq)
Get the number of queued frames.
Definition: framequeue.h:146
if(ret< 0)
Definition: vf_mcdeint.c:282
int64_t pts
PTS of the current frame.
Definition: framesync.h:118
int den
Denominator.
Definition: rational.h:60
Completely stop all streams with this one.
Definition: framesync.h:67
void ff_framesync_drop(FFFrameSync *fs)
Drop the current frame event.
Definition: framesync.c:227
An instance of a filter.
Definition: avfilter.h:323
#define av_freep(p)
static void ff_bufqueue_add(void *log, struct FFBufQueue *queue, AVFrame *buf)
Add a buffer to the queue.
Definition: bufferqueue.h:71
static void framesync_inject_frame(FFFrameSync *fs, unsigned in, AVFrame *frame)
Definition: framesync.c:186
int ff_request_frame(AVFilterLink *link)
Request an input frame from the filter at the other end of the link.
Definition: avfilter.c:405
internal API functions
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:232
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248
const AVClass * class
Definition: framesync.h:154
int64_t pts_next
PTS of the next frame, for internal use.
Definition: framesync.h:123