FFmpeg
cache.c
Go to the documentation of this file.
1 /*
2  * Input cache protocol.
3  * Copyright (c) 2011,2014 Michael Niedermayer
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  *
21  * Based on file.c by Fabrice Bellard
22  */
23 
24 /**
25  * @TODO
26  * support keeping files
27  * support filling with a background thread
28  */
29 
30 #include "libavutil/avassert.h"
31 #include "libavutil/avstring.h"
32 #include "libavutil/internal.h"
33 #include "libavutil/opt.h"
34 #include "libavutil/tree.h"
35 #include "avformat.h"
36 #include <fcntl.h>
37 #if HAVE_IO_H
38 #include <io.h>
39 #endif
40 #if HAVE_UNISTD_H
41 #include <unistd.h>
42 #endif
43 #include <sys/stat.h>
44 #include <stdlib.h>
45 #include "os_support.h"
46 #include "url.h"
47 
48 typedef struct CacheEntry {
49  int64_t logical_pos;
50  int64_t physical_pos;
51  int size;
52 } CacheEntry;
53 
54 typedef struct Context {
55  AVClass *class;
56  int fd;
57  char *filename;
58  struct AVTreeNode *root;
59  int64_t logical_pos;
60  int64_t cache_pos;
61  int64_t inner_pos;
62  int64_t end;
64  URLContext *inner;
65  int64_t cache_hit, cache_miss;
67 } Context;
68 
69 static int cmp(const void *key, const void *node)
70 {
71  return FFDIFFSIGN(*(const int64_t *)key, ((const CacheEntry *) node)->logical_pos);
72 }
73 
74 static int cache_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
75 {
76  int ret;
77  char *buffername;
78  Context *c= h->priv_data;
79 
80  av_strstart(arg, "cache:", &arg);
81 
82  c->fd = avpriv_tempfile("ffcache", &buffername, 0, h);
83  if (c->fd < 0){
84  av_log(h, AV_LOG_ERROR, "Failed to create tempfile\n");
85  return c->fd;
86  }
87 
88  ret = unlink(buffername);
89 
90  if (ret >= 0)
91  av_freep(&buffername);
92  else
93  c->filename = buffername;
94 
95  return ffurl_open_whitelist(&c->inner, arg, flags, &h->interrupt_callback,
96  options, h->protocol_whitelist, h->protocol_blacklist, h);
97 }
98 
99 static int add_entry(URLContext *h, const unsigned char *buf, int size)
100 {
101  Context *c= h->priv_data;
102  int64_t pos = -1;
103  int ret;
104  CacheEntry *entry = NULL, *next[2] = {NULL, NULL};
105  CacheEntry *entry_ret;
106  struct AVTreeNode *node = NULL;
107 
108  //FIXME avoid lseek
109  pos = lseek(c->fd, 0, SEEK_END);
110  if (pos < 0) {
111  ret = AVERROR(errno);
112  av_log(h, AV_LOG_ERROR, "seek in cache failed\n");
113  goto fail;
114  }
115  c->cache_pos = pos;
116 
117  ret = write(c->fd, buf, size);
118  if (ret < 0) {
119  ret = AVERROR(errno);
120  av_log(h, AV_LOG_ERROR, "write in cache failed\n");
121  goto fail;
122  }
123  c->cache_pos += ret;
124 
125  entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
126 
127  if (!entry)
128  entry = next[0];
129 
130  if (!entry ||
131  entry->logical_pos + entry->size != c->logical_pos ||
132  entry->physical_pos + entry->size != pos
133  ) {
134  entry = av_malloc(sizeof(*entry));
135  node = av_tree_node_alloc();
136  if (!entry || !node) {
137  ret = AVERROR(ENOMEM);
138  goto fail;
139  }
140  entry->logical_pos = c->logical_pos;
141  entry->physical_pos = pos;
142  entry->size = ret;
143 
144  entry_ret = av_tree_insert(&c->root, entry, cmp, &node);
145  if (entry_ret && entry_ret != entry) {
146  ret = -1;
147  av_log(h, AV_LOG_ERROR, "av_tree_insert failed\n");
148  goto fail;
149  }
150  } else
151  entry->size += ret;
152 
153  return 0;
154 fail:
155  //we could truncate the file to pos here if pos >=0 but ftruncate isn't available in VS so
156  //for simplicty we just leave the file a bit larger
157  av_free(entry);
158  av_free(node);
159  return ret;
160 }
161 
162 static int cache_read(URLContext *h, unsigned char *buf, int size)
163 {
164  Context *c= h->priv_data;
165  CacheEntry *entry, *next[2] = {NULL, NULL};
166  int64_t r;
167 
168  entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
169 
170  if (!entry)
171  entry = next[0];
172 
173  if (entry) {
174  int64_t in_block_pos = c->logical_pos - entry->logical_pos;
175  av_assert0(entry->logical_pos <= c->logical_pos);
176  if (in_block_pos < entry->size) {
177  int64_t physical_target = entry->physical_pos + in_block_pos;
178 
179  if (c->cache_pos != physical_target) {
180  r = lseek(c->fd, physical_target, SEEK_SET);
181  } else
182  r = c->cache_pos;
183 
184  if (r >= 0) {
185  c->cache_pos = r;
186  r = read(c->fd, buf, FFMIN(size, entry->size - in_block_pos));
187  }
188 
189  if (r > 0) {
190  c->cache_pos += r;
191  c->logical_pos += r;
192  c->cache_hit ++;
193  return r;
194  }
195  }
196  }
197 
198  // Cache miss or some kind of fault with the cache
199 
200  if (c->logical_pos != c->inner_pos) {
201  r = ffurl_seek(c->inner, c->logical_pos, SEEK_SET);
202  if (r<0) {
203  av_log(h, AV_LOG_ERROR, "Failed to perform internal seek\n");
204  return r;
205  }
206  c->inner_pos = r;
207  }
208 
209  r = ffurl_read(c->inner, buf, size);
210  if (r == AVERROR_EOF && size>0) {
211  c->is_true_eof = 1;
212  av_assert0(c->end >= c->logical_pos);
213  }
214  if (r<=0)
215  return r;
216  c->inner_pos += r;
217 
218  c->cache_miss ++;
219 
220  add_entry(h, buf, r);
221  c->logical_pos += r;
222  c->end = FFMAX(c->end, c->logical_pos);
223 
224  return r;
225 }
226 
227 static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
228 {
229  Context *c= h->priv_data;
230  int64_t ret;
231 
232  if (whence == AVSEEK_SIZE) {
233  pos= ffurl_seek(c->inner, pos, whence);
234  if(pos <= 0){
235  pos= ffurl_seek(c->inner, -1, SEEK_END);
236  if (ffurl_seek(c->inner, c->inner_pos, SEEK_SET) < 0)
237  av_log(h, AV_LOG_ERROR, "Inner protocol failed to seekback end : %"PRId64"\n", pos);
238  }
239  if (pos > 0)
240  c->is_true_eof = 1;
241  c->end = FFMAX(c->end, pos);
242  return pos;
243  }
244 
245  if (whence == SEEK_CUR) {
246  whence = SEEK_SET;
247  pos += c->logical_pos;
248  } else if (whence == SEEK_END && c->is_true_eof) {
249 resolve_eof:
250  whence = SEEK_SET;
251  pos += c->end;
252  }
253 
254  if (whence == SEEK_SET && pos >= 0 && pos < c->end) {
255  //Seems within filesize, assume it will not fail.
256  c->logical_pos = pos;
257  return pos;
258  }
259 
260  //cache miss
261  ret= ffurl_seek(c->inner, pos, whence);
262  if ((whence == SEEK_SET && pos >= c->logical_pos ||
263  whence == SEEK_END && pos <= 0) && ret < 0) {
264  if ( (whence == SEEK_SET && c->read_ahead_limit >= pos - c->logical_pos)
265  || c->read_ahead_limit < 0) {
266  uint8_t tmp[32768];
267  while (c->logical_pos < pos || whence == SEEK_END) {
268  int size = sizeof(tmp);
269  if (whence == SEEK_SET)
270  size = FFMIN(sizeof(tmp), pos - c->logical_pos);
271  ret = cache_read(h, tmp, size);
272  if (ret == AVERROR_EOF && whence == SEEK_END) {
274  goto resolve_eof;
275  }
276  if (ret < 0) {
277  return ret;
278  }
279  }
280  return c->logical_pos;
281  }
282  }
283 
284  if (ret >= 0) {
285  c->logical_pos = ret;
286  c->end = FFMAX(c->end, ret);
287  }
288 
289  return ret;
290 }
291 
292 static int enu_free(void *opaque, void *elem)
293 {
294  av_free(elem);
295  return 0;
296 }
297 
299 {
300  Context *c= h->priv_data;
301  int ret;
302 
303  av_log(h, AV_LOG_INFO, "Statistics, cache hits:%"PRId64" cache misses:%"PRId64"\n",
304  c->cache_hit, c->cache_miss);
305 
306  close(c->fd);
307  if (c->filename) {
308  ret = unlink(c->filename);
309  if (ret < 0)
310  av_log(h, AV_LOG_ERROR, "Could not delete %s.\n", c->filename);
311  av_freep(&c->filename);
312  }
313  ffurl_close(c->inner);
315  av_tree_destroy(c->root);
316 
317  return 0;
318 }
319 
320 #define OFFSET(x) offsetof(Context, x)
321 #define D AV_OPT_FLAG_DECODING_PARAM
322 
323 static const AVOption options[] = {
324  { "read_ahead_limit", "Amount in bytes that may be read ahead when seeking isn't supported, -1 for unlimited", OFFSET(read_ahead_limit), AV_OPT_TYPE_INT, { .i64 = 65536 }, -1, INT_MAX, D },
325  {NULL},
326 };
327 
328 static const AVClass cache_context_class = {
329  .class_name = "cache",
330  .item_name = av_default_item_name,
331  .option = options,
332  .version = LIBAVUTIL_VERSION_INT,
333 };
334 
336  .name = "cache",
337  .url_open2 = cache_open,
338  .url_read = cache_read,
339  .url_seek = cache_seek,
340  .url_close = cache_close,
341  .priv_data_size = sizeof(Context),
342  .priv_data_class = &cache_context_class,
343 };
Definition: async.c:56
#define NULL
Definition: coverity.c:32
int64_t inner_pos
Definition: cache.c:61
int ffurl_open_whitelist(URLContext **puc, const char *filename, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options, const char *whitelist, const char *blacklist, URLContext *parent)
Create an URLContext for accessing to the resource indicated by url, and open it. ...
Definition: avio.c:306
char * filename
Definition: cache.c:57
AVOption.
Definition: opt.h:246
support keeping files support filling with a background thread
Definition: cache.c:48
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
struct AVTreeNode * root
Definition: cache.c:58
AVIOInterruptCB interrupt_callback
Definition: url.h:47
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:191
static const AVClass cache_context_class
Definition: cache.c:328
int avpriv_tempfile(const char *prefix, char **filename, int log_offset, void *log_ctx)
Wrapper to work around the lack of mkstemp() on mingw.
Definition: file_open.c:110
void * av_tree_find(const AVTreeNode *t, void *key, int(*cmp)(const void *key, const void *b), void *next[2])
Definition: tree.c:39
const char * key
static const AVOption options[]
Definition: cache.c:323
struct AVTreeNode * av_tree_node_alloc(void)
Allocate an AVTreeNode.
Definition: tree.c:34
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
uint8_t
#define av_malloc(s)
const URLProtocol ff_cache_protocol
Definition: cache.c:335
static int enu_free(void *opaque, void *elem)
Definition: cache.c:292
AVOptions.
A tree container.
miscellaneous OS support macros and functions.
static av_cold int end(AVCodecContext *avctx)
Definition: avrndec.c:90
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
int64_t cache_hit
Definition: cache.c:65
int size
Definition: cache.c:51
int64_t end
Definition: cache.c:62
int read_ahead_limit
Definition: cache.c:66
#define AVERROR_EOF
End of file.
Definition: error.h:55
ptrdiff_t size
Definition: opengl_enc.c:100
int64_t cache_miss
Definition: cache.c:65
#define av_log(a,...)
void av_tree_destroy(AVTreeNode *t)
Definition: tree.c:146
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
int64_t logical_pos
Definition: async.c:70
int fd
Definition: cache.c:56
const char * protocol_whitelist
Definition: url.h:49
const char * r
Definition: vf_curves.c:114
const char * arg
Definition: jacosubdec.c:66
simple assert() macros that are a bit more flexible than ISO C assert().
URLContext * inner
Definition: async.c:58
#define FFMAX(a, b)
Definition: common.h:94
#define fail()
Definition: checkasm.h:122
static int cache_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
Definition: cache.c:74
#define FFDIFFSIGN(x, y)
Comparator.
Definition: common.h:92
common internal API header
static int cache_close(URLContext *h)
Definition: cache.c:298
#define FFMIN(a, b)
Definition: common.h:96
int64_t logical_pos
Definition: cache.c:49
#define D
Definition: cache.c:321
#define OFFSET(x)
Definition: cache.c:320
static int cmp(const void *key, const void *node)
Definition: cache.c:69
static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
Definition: cache.c:227
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
const char * protocol_blacklist
Definition: url.h:50
void * buf
Definition: avisynth_c.h:766
Definition: url.h:38
static int cache_read(URLContext *h, unsigned char *buf, int size)
Definition: cache.c:162
Describe the class of an AVClass context structure.
Definition: log.h:67
void * priv_data
Definition: url.h:41
const char * name
Definition: url.h:55
#define flags(name, subs,...)
Definition: cbs_av1.c:561
int ffurl_close(URLContext *h)
Definition: avio.c:465
int av_strstart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str.
Definition: avstring.c:34
Main libavformat public API header.
int64_t cache_pos
Definition: cache.c:60
int64_t ffurl_seek(URLContext *h, int64_t pos, int whence)
Change the position that will be used by the next read/write operation on the resource accessed by h...
Definition: avio.c:432
#define AVSEEK_SIZE
ORing this as the "whence" parameter to a seek function causes it to return the filesize without seek...
Definition: avio.h:531
#define av_free(p)
int64_t physical_pos
Definition: cache.c:50
static int add_entry(URLContext *h, const unsigned char *buf, int size)
Definition: cache.c:99
void * av_tree_insert(AVTreeNode **tp, void *key, int(*cmp)(const void *key, const void *b), AVTreeNode **next)
Insert or remove an element.
Definition: tree.c:59
#define av_freep(p)
unbuffered private I/O API
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
void av_tree_enumerate(AVTreeNode *t, void *opaque, int(*cmp)(void *opaque, void *elem), int(*enu)(void *opaque, void *elem))
Apply enu(opaque, &elem) to all the elements in the tree in a given range.
Definition: tree.c:155
int ffurl_read(URLContext *h, unsigned char *buf, int size)
Read up to size bytes from the resource accessed by h, and store the read bytes in buf...
Definition: avio.c:405
void * elem
Definition: tree.c:28
int is_true_eof
Definition: cache.c:63
static uint8_t tmp[11]
Definition: aes_ctr.c:26