FFmpeg
Loading...
Searching...
No Matches
shared.c
Go to the documentation of this file.
1/*
2 * Shared file cache protocol.
3 * Copyright (c) 2026 Niklas Haas
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 * Based on cache.c by Michael Niedermayer
22 */
23
25#include "libavutil/avassert.h"
26#include "libavutil/avstring.h"
27#include "libavutil/crc.h"
28#include "libavutil/error.h"
29#include "libavutil/file.h"
30#include "libavutil/hash.h"
31#include "libavutil/file_open.h"
32#include "libavutil/mem.h"
33#include "libavutil/opt.h"
34#include "libavutil/time.h"
35
36#include "internal.h"
37#include "os_support.h"
38#include "url.h"
39
40#include <assert.h>
41#include <errno.h>
42#include <fcntl.h>
43#include <inttypes.h>
44#include <stdatomic.h>
45#include <string.h>
46#include <sys/stat.h>
47#if HAVE_UNISTD_H
48#include <unistd.h>
49#endif
50#ifndef _WIN32
51#include <sys/file.h>
52#endif
53
54/**
55 * This hash should be resistant against collision attacks, so that an
56 * attacker could not generate e.g. two different URIs that map to the same
57 * cache file. This requires at least 64 bits of collision resistance in
58 * practice (i.e. 128 bits = 16 bytes of hash size). However, we can be
59 * conservative by computing e.g. a 256 bit hash and storing it inside the
60 * file header for verification.
61 *
62 * Note that due to the way we use atomics, we should avoid zero bytes in
63 * the resulting hash; hence we tweak the input slightly to avoid this.
64 * The resulting loss in hash strength is negligible, since 32 bytes is
65 * already much more than needed.
66 */
67#define HASH_METHOD "SHA512/256"
68#define HASH_SIZE 32
69#define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$')
70#define HEADER_VERSION 3
71
72/**
73 * Hard watershed of consecutive failed blocks before we give up on the cache
74 * file altogether and assume it's entirely lost to us.
75 **/
76#define MAX_CORRUPT_BLOCKS 10
77
78static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
79{
80 struct AVHashContext *ctx = NULL;
82 if (ret < 0)
83 return ret;
84
85 const int16_t version = HEADER_VERSION;
88 av_hash_update(ctx, (const uint8_t *) &version, sizeof(version));
89 av_hash_update(ctx, (const uint8_t *) uri, strlen(uri));
92
93 for (int i = 0; i < HASH_SIZE; i++)
94 hash[i] = hash[i] ? hash[i] : ~hash[i]; /* prevent zero bytes */
95 return 0;
96}
97
99 /* Reserved block state values */
100 BLOCK_NONE = 0, ///< block is not cached
101 BLOCK_PENDING, ///< a thread is currently trying to write this block
102 BLOCK_FAILED, ///< the underlying I/O source failed to read this block
103
104 /**
105 * All other block states represent valid cached blocks, with the value
106 * being the CRC of the block data.
107 */
108};
109
110static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
111{
112 uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_IEEE), 0, block, block_size);
113 switch (crc) {
114 case BLOCK_NONE:
115 case BLOCK_FAILED:
116 case BLOCK_PENDING:
117 return ~crc; /* avoid reserved block states */
118 default:
119 return crc;
120 }
121}
122
123typedef struct Block {
124 atomic_uint state; /* enum BlockState */
125} Block;
126
127typedef struct Spacemap {
131 atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */
132 atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */
133 atomic_ullong blocks_cached; /* (lower bound on) the number of blocks cached */
134 char reserved[72];
135
137} Spacemap;
138
139static_assert(offsetof(Spacemap, blocks) == 128, "Spacemap header layout mismatch");
140
141/* Set to value iff the current value is unset (zero) */
142#define DEF_SET_ONCE(ctype, atype) \
143 static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \
144 { \
145 ctype prev = 0; \
146 av_assert1(value != 0); \
147 if (atomic_compare_exchange_strong_explicit( \
148 ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
149 return 1; \
150 else if (prev == value) \
151 return 0; \
152 else \
153 return AVERROR(EINVAL); \
154 }
155
156DEF_SET_ONCE(unsigned char, uchar)
157DEF_SET_ONCE(unsigned int, uint)
158DEF_SET_ONCE(unsigned short, ushort)
159DEF_SET_ONCE(unsigned long long, ullong)
160
161typedef struct SharedContext {
162 AVClass *class;
165
166 /* options */
170 int block_shift; ///< requested shift; updated on init if it disagrees
179
180 /* misc state */
181 int64_t pos; ///< current logical position
182 uint8_t *tmp_buf;
184 int write_err; ///< write error occurred
186 int64_t filesize; ///< once known
187 int64_t blocks_max; ///< maximum number of blocks to cache
188
189 /* cache file */
191 int fd;
192
193 /* space map */
195 char *map_path;
196 off_t map_size;
197 int mapfd;
198
199 /* statistics */
203
205{
206 SharedContext *s = h->priv_data;
207
208 ffurl_close(s->inner);
209 av_file_unmap_shared(s->spacemap, s->map_size);
210 if (s->fd != -1)
211 close(s->fd);
212 if (s->mapfd != -1)
213 close(s->mapfd);
214 av_freep(&s->cache_path);
215 av_freep(&s->map_path);
216 av_freep(&s->tmp_buf);
217
218 av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n",
219 s->nb_hit, s->nb_miss);
220 return 0;
221}
222
223static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]);
225
227{
228 SharedContext *s = h->priv_data;
229 if (!s->filesize) {
230 uint64_t size = atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed);
231 if (size > INT64_MAX)
232 return AVERROR(EINVAL);
233 else if (size)
234 s->filesize = size;
235 }
236
237 return s->filesize;
238}
239
240static int set_filesize(URLContext *h, int64_t new_size)
241{
242 SharedContext *s = h->priv_data;
243 int ret;
244
245 if (!new_size)
246 return 0;
247
248 ret = set_once_ullong(&s->spacemap->filesize, new_size);
249 if (ret < 0) {
250 av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: "
251 "%"PRId64", got: %"PRIu64"!\n", new_size,
252 (uint64_t) atomic_load(&s->spacemap->filesize));
253 return ret;
254 }
255
256 return ret;
257}
258
260{
261 switch (err) {
262 case AVERROR_EXIT:
263 case AVERROR_EOF:
264 case AVERROR_BUG:
265 case AVERROR(EAGAIN):
266 case AVERROR(ENOSYS):
267 case AVERROR(EINVAL):
268 return 0;
269 default:
270 return err < 0;
271 }
272}
273
274static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
275{
276 SharedContext *s = h->priv_data;
277 int ret;
278
279 if (!s->cache_dir || !s->cache_dir[0]) {
280 av_log(h, AV_LOG_ERROR, "Missing path for shared cache! Specify a "
281 "directory using the -cache_dir option.\n");
282 return AVERROR(EINVAL);
283 }
284
285 const char *spacemap_dir = s->cache_dir;
286 if (s->metadata_dir && s->metadata_dir[0])
287 spacemap_dir = s->metadata_dir;
288
289 s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */
290
291 /* Open underlying protocol */
292 av_strstart(arg, "shared:", &arg);
293 ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback,
294 options, h->protocol_whitelist, h->protocol_blacklist, h);
295 if (is_ignorable_error(ret) && s->ignore_errors) {
296 av_log(h, AV_LOG_WARNING, "Underlying URL failed to open: %s. "
297 "Continuing with cache file only.\n", av_err2str(ret));
298 } else if (ret < 0)
299 goto fail;
300
301 const char *cache_key = s->cache_key;
302 if (cache_key) {
303 if (strchr(cache_key, '/') || strchr(cache_key, '\\')) {
304 av_log(h, AV_LOG_ERROR, "Invalid cache key '%s': must not contain "
305 "path separators\n", cache_key);
306 ret = AVERROR(EINVAL);
307 goto fail;
308 } else if (!cache_key[0])
309 cache_key = NULL;
310 }
311
312 uint8_t hash[HASH_SIZE];
313 ret = hash_uri(hash, cache_key ? cache_key : arg);
314 if (ret < 0)
315 goto fail;
316
317 /* 128 bits is enough for collision resistance; we already store the full
318 * hash inside the header for verification */
319 const char *filename = cache_key;
320 char buf[2 * 16 + 1];
321 if (!filename) {
322 ff_data_to_hex(buf, hash, sizeof(buf)/2, 0);
323 filename = buf;
324 }
325
326 s->cache_path = av_asprintf("%s/%s.cache", s->cache_dir, filename);
327 s->map_path = av_asprintf("%s/%s.spacemap", spacemap_dir, filename);
328 if (!s->cache_path || !s->map_path) {
329 ret = AVERROR(ENOMEM);
330 goto fail;
331 }
332
333 av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n",
334 s->cache_path, s->inner ? s->inner->filename : arg);
335
336 const int mode = O_RDWR | O_BINARY | (s->inner ? O_CREAT : 0);
337 s->fd = avpriv_open(s->cache_path, mode, 0660);
338 s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, mode, 0660) : -1;
339 if (s->fd < 0 || s->mapfd < 0) {
340 ret = AVERROR(errno);
341 av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n",
342 s->fd < 0 ? s->cache_path : s->map_path, av_err2str(ret));
343 goto fail;
344 }
345
347 if (ret < 0)
348 goto fail;
349
350 /* s->block_shift is fully settled after spacemap_init() */
351 s->block_size = 1 << s->block_shift;
352 s->blocks_max = s->cache_size_max >> s->block_shift;
353
355 if (filesize < 0) {
356 ret = (int) filesize;
357 goto fail;
358 } else if (!filesize) {
359 /* Filesize is not yet known, try to get it from the underlying URL;
360 * go through our own seek function to handle errors and updates */
362 if (filesize < 0 && filesize != AVERROR(ENOSYS)) {
363 ret = (int) filesize;
364 goto fail;
365 }
366 }
367
368 if (filesize > 0) {
369 int64_t last_pos = filesize - 1;
370 int64_t last_block = last_pos >> s->block_shift;
371 ret = spacemap_grow(h, last_block);
372 if (ret < 0)
373 goto fail;
374 }
375
376 /* Temporary buffer needed for pread/pwrite() fallback */
377 s->tmp_buf = av_malloc(s->block_size);
378 if (!s->tmp_buf) {
379 ret = AVERROR(ENOMEM);
380 goto fail;
381 }
382
383 h->max_packet_size = s->block_size;
384 h->min_packet_size = s->block_size;
385 s->pos = s->offset;
386 ret = 0;
387
388fail:
389 if (ret < 0)
391 return ret;
392}
393
394static int spacemap_remap(URLContext *h, size_t map_size)
395{
396 SharedContext *s = h->priv_data;
397 int ret, did_grow = 0, locked = 0;
398 if (map_size <= s->map_size)
399 return 0;
400
401 /* Opportunistically get current filesize before attempting to lock */
402 struct stat st;
403 ret = fstat(s->mapfd, &st);
404 if (ret < 0) {
405 ret = AVERROR(errno);
406 goto fail;
407 }
408
409 if (st.st_size >= map_size)
410 goto skip_resize;
411
412 /* Lock the spacemap to ensure nobody else is currently resizing it */
413 ret = flock(s->mapfd, LOCK_EX);
414 if (ret < 0) {
415 ret = AVERROR(errno);
416 goto fail;
417 }
418 locked = 1;
419
420 /* Refresh filesize after acquiring the lock */
421 ret = fstat(s->mapfd, &st);
422 if (ret < 0) {
423 ret = AVERROR(errno);
424 goto fail;
425 }
426
427 if (st.st_size >= map_size)
428 goto skip_resize;
429
430 /* The new mapping extends the file */
431 st.st_size = map_size;
432 did_grow = 1;
433
434skip_resize:
435 av_file_unmap_shared(s->spacemap, s->map_size);
436 s->spacemap = NULL;
437 s->map_size = st.st_size;
438
439 void *map;
440 ret = av_file_map_shared(s->mapfd, s->map_size, &map);
441 if (ret < 0) {
442 s->map_size = 0;
443 goto fail;
444 }
445 s->spacemap = map;
446
447 if (locked) {
448 flock(s->mapfd, LOCK_UN);
449 locked = 0;
450 }
451
452 return did_grow;
453
454fail:
455 if (locked)
456 flock(s->mapfd, LOCK_UN);
457 av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret));
458 return ret;
459}
460
462{
463 SharedContext *s = h->priv_data;
464 int64_t num_blocks = block + 1;
465 size_t map_bytes = sizeof(Spacemap) + num_blocks * sizeof(Block);
466
467 /* When streaming files without known size, round up the number of blocks
468 * to the nearest multiple of the block size to reduce the rate of resizes */
470 if (filesize < 0)
471 return (int) filesize;
472 else if (!filesize) {
473 av_assert0(s->block_size > 0);
474 map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size);
475 }
476
477 if (map_bytes < num_blocks)
478 return AVERROR(EINVAL); /* overflow */
479
480 const off_t old_size = s->map_size;
481 int ret = spacemap_remap(h, map_bytes);
482 if (ret < 0)
483 return ret;
484
485 /* Report new size after successful grow */
486 if (s->map_size > old_size) {
487 num_blocks = (s->map_size - sizeof(Spacemap)) / sizeof(Block);
489 "%s %zu bytes, capacity: %"PRId64" blocks = %"PRId64" MB\n",
490 ret ? "Resized spacemap to" : "Mapped spacemap with",
491 (size_t) s->map_size, num_blocks,
492 (num_blocks * (int64_t) s->block_size) >> 20);
493 }
494 return 0;
495}
496
497static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
498{
499 SharedContext *s = h->priv_data;
500 int ret;
501
502 ret = spacemap_remap(h, sizeof(Spacemap));
503 if (ret < 0)
504 return ret;
505
506 if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 ||
507 (ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0)
508 {
509 av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n");
510 av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d\n",
512 av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d\n",
513 atomic_load(&s->spacemap->header_magic),
514 atomic_load(&s->spacemap->version));
515 return ret;
516 }
517
518 ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift);
519 if (ret < 0) {
520 const int shift = atomic_load(&s->spacemap->block_shift);
521 av_log(h, AV_LOG_WARNING, "Shared cache uses block shift %d, "
522 "but requested block shift is %d.\n", shift, s->block_shift);
523 if (shift < 9 || shift > 30) {
524 av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift);
525 return AVERROR(EINVAL);
526 }
527 s->block_shift = shift;
528 }
529
530 for (int i = 0; i < HASH_SIZE; i++) {
531 ret = set_once_uchar(&s->spacemap->hash[i], hash[i]);
532 if (ret < 0) {
533 av_log(h, AV_LOG_ERROR, "Shared cache spacemap hash mismatch!\n");
534 char hash_hex[2 * HASH_SIZE + 1];
535 ff_data_to_hex(hash_hex, hash, HASH_SIZE, 0);
536 av_log(h, AV_LOG_ERROR, " Expected hash: %s\n", hash_hex);
537 uint8_t hash2[HASH_SIZE];
538 for (int j = 0; j < HASH_SIZE; ++j)
539 hash2[j] = atomic_load_explicit(&s->spacemap->hash[j], memory_order_relaxed);
540 ff_data_to_hex(hash_hex, hash2, HASH_SIZE, 0);
541 av_log(h, AV_LOG_ERROR, " Got hash: %s\n", hash_hex);
542 return ret;
543 }
544 }
545
546 if (ret) /* set_once() return 1 if this is the first time setting the value */
547 av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n");
548
549 return ret;
550}
551
552static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
553{
554 while (size) {
555 ssize_t ret = pread(s->fd, buf, size, offset);
556 if (ret <= 0)
557 return ret ? AVERROR(errno) : AVERROR_EOF;
558 buf += ret;
559 offset += ret;
560 size -= ret;
561 }
562
563 return 0;
564}
565
566static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
567{
568 while (size) {
569 ssize_t ret = pwrite(s->fd, buf, size, offset);
570 if (ret <= 0)
571 return ret ? AVERROR(errno) : AVERROR(EIO);
572 buf += ret;
573 offset += ret;
574 size -= ret;
575 }
576
577 return 0;
578}
579
581{
582 if (!filesize)
583 return size;
584 else if (pos > filesize)
585 return 0;
586 else
587 return FFMIN(filesize - pos, size);
588}
589
590static int shared_read(URLContext *h, unsigned char *buf, int size)
591{
592 SharedContext *s = h->priv_data;
593 uint8_t *tmp;
594 int ret;
595 if (!s->spacemap)
596 return AVERROR(EIO);
597
598 if (size <= 0)
599 return 0;
600
602 if (filesize < 0)
603 return (int) filesize;
604
605 size = clamp_size(h, size, s->pos, filesize);
606 if (size <= 0)
607 return AVERROR_EOF;
608
609 const int64_t block_id = s->pos >> s->block_shift;
610 const int64_t offset = s->pos & (s->block_size - 1);
611 const int64_t block_pos = block_id * s->block_size;
612 int block_size = clamp_size(h, s->block_size, block_pos, filesize);
613 ret = spacemap_grow(h, block_id);
614 if (ret < 0)
615 return ret;
616
617 Block *const block = &s->spacemap->blocks[block_id];
619 int64_t pending_since = 0;
620 int verify_read = 0, acquired = 0, allocated = 0;
621
622retry:
623 switch (state) {
624 default:
625 if (s->num_corrupt >= MAX_CORRUPT_BLOCKS)
626 goto read_block; /* assume broken cache file */
627
628 /* filesize may have become known in the meantime */
630 if (filesize < 0)
631 return (int) filesize;
632
633 /* We always need to read the entire block to verify integrity */
634 block_size = clamp_size(h, block_size, block_pos, filesize);
635 tmp = s->tmp_buf;
636 ret = read_cache(s, tmp, block_size, block_pos);
637 if (ret < 0) {
638 av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret));
639 if (ret == AVERROR_EOF) { /* e.g. cache appears truncated? */
640 if (s->retry_corrupt) {
641 s->num_corrupt++;
642 goto read_block;
643 }
644 ret = AVERROR(EIO); /* don't propagate EOF to caller */
645 }
646 return ret;
647 }
648
649 uint32_t crc = get_block_crc(tmp, block_size);
650 if (crc != state) {
651 av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at "
652 "offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n",
653 block_id, block_pos, state, crc);
654 if (s->retry_corrupt) {
655 s->num_corrupt++;
656 goto read_block;
657 }
658 return AVERROR(EIO);
659 } else
660 s->num_corrupt = 0; /* reset corrupt block count on success */
661
662 tmp += (ptrdiff_t) offset;
663 size = FFMIN(size, block_size - offset);
664 if (size <= 0)
665 return AVERROR_EOF;
666 if (s->verify) {
667 verify_read = 1;
668 break; /* fall through to the cache miss logic */
669 }
670
671 memcpy(buf, tmp, size);
672 s->nb_hit++;
673 s->pos += size;
674 return size;
675
676 case BLOCK_FAILED:
677 if (s->retry_errors)
678 goto read_block;
679 return AVERROR(EIO);
680
682 if (s->num_corrupt == MAX_CORRUPT_BLOCKS) {
683 av_log(h, AV_LOG_ERROR, "Too many consecutive corrupt blocks; "
684 "assuming cache file is completely broken.\n");
685 s->num_corrupt++; /* silence this log on subsequent reads */
686 }
688
689 case BLOCK_NONE:
690 if (s->read_only || s->write_err || !s->inner)
691 break; /* don't mark block as pending */
692 else if (s->cache_size_max) {
693 int64_t cached = atomic_load_explicit(&s->spacemap->blocks_cached,
695 if (cached >= s->blocks_max) {
696 av_log(h, AV_LOG_WARNING, "Cache size limit reached (%"PRId64" "
697 "blocks = %"PRId64" bytes), switching to read-only mode.\n",
698 s->blocks_max, s->blocks_max << s->block_shift);
699 s->read_only = 1;
700 break;
701 }
702 }
703
708 {
709 /* Acquired pending state, proceed to fetch the block */
710 acquired = 1;
711 allocated = (state == BLOCK_NONE || state == BLOCK_FAILED);
713 break;
714 }
715 /* CAS failed, another thread changed the state; reload it */
716 goto retry;
717
718 case BLOCK_PENDING:
719 /* Another thread is busy fetching this block, wait for it to finish */
720 if (!s->timeout) {
721 break; /* no timeout requested, immediately race to fetch block */
722 } else if (pending_since) {
724 if (new - pending_since >= s->timeout)
725 break; /* timeout expired, try to fetch the block ourselves */
726 } else {
727 pending_since = av_gettime_relative();
728 }
729
730 if (h->flags & AVIO_FLAG_NONBLOCK)
731 return AVERROR(EAGAIN);
732
733 /* Make sure we try a few times before giving up */
734 av_usleep(FFMIN(s->timeout >> 4, 10000));
735 if (ff_check_interrupt(&h->interrupt_callback))
736 return AVERROR_EXIT;
737
739 goto retry;
740 }
741
742 /* Release pending state on failure to avoid stalling other threads */
743#define RELEASE_PENDING(block, state) \
744 do { \
745 if (acquired) { \
746 av_assert1(state == BLOCK_PENDING); \
747 atomic_compare_exchange_strong_explicit( \
748 &block->state, &state, BLOCK_NONE, memory_order_relaxed, \
749 memory_order_relaxed); \
750 } \
751 } while (0)
752
753 /* Cache miss, fetch this block from underlying protocol */
754 s->nb_miss++;
755
756 if (!s->inner) {
757 av_log(h, AV_LOG_ERROR, "Cache miss for block 0x%"PRIx64" at offset "
758 "0x%"PRIx64", but underlying protocol is not available!\n",
759 block_id, block_pos);
760 av_assert0(!acquired);
761 return AVERROR(EIO);
762 }
763
764 const int read_only = s->read_only || s->write_err || verify_read;
765 int64_t inner_pos = read_only ? s->pos : block_pos;
766 if (s->inner_pos != inner_pos) {
767 inner_pos = ffurl_seek(s->inner, inner_pos, SEEK_SET);
768 if (inner_pos < 0) {
769 av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n",
770 av_err2str(inner_pos));
772 return inner_pos;
773 }
774
775 av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", inner_pos);
776 s->inner_pos = inner_pos;
777 }
778
779 if (read_only) {
780 /* Directly defer to the underlying protocol */
781 ret = ffurl_read(s->inner, buf, size);
782 if (ret < 0) {
783 av_assert1(!acquired);
784 return ret;
785 } else {
786 s->inner_pos = inner_pos + ret;
787 }
788
789 /* Verify the read data against the cached data if requested */
790 if (verify_read && memcmp(buf, tmp, ret)) {
791 av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes "
792 "in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n",
793 ret, block_id, block_pos, offset);
794 return AVERROR(EIO);
795 }
796
797 s->pos = s->inner_pos;
798 return ret;
799 }
800
801 int write_back = 1;
802 if (size >= block_size && !offset) {
803 /* Read directly into output buffer if aligned and large enough */
804 tmp = buf;
805 } else {
806 /* Read into temporary buffer and copy later */
807 tmp = s->tmp_buf;
808 }
809
810 /* Try and fetch the entire block */
811 av_assert0(inner_pos == block_pos);
812 int bytes_read = 0;
813 while (bytes_read < block_size) {
814 ret = ffurl_read(s->inner, &tmp[bytes_read], block_size - bytes_read);
815 if (!ret || ret == AVERROR_EOF)
816 break;
817 else if (ret < 0) {
818 av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n",
819 block_id, av_err2str(ret));
820 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) {
822 return ret; /* transient error, allow retries */
823 }
824
825 /* Try to mark block as failed; ignore errors - any mismatch
826 * here will mean that either another thread already marked it
827 * as failed, or successfully cached it in the meantime */
832 return ret;
833 }
834
835 bytes_read += ret;
836 s->inner_pos += ret;
837 }
838
839 if (bytes_read < block_size) {
840 /* Learned location of true EOF, update filesize */
841 ret = set_filesize(h, inner_pos + bytes_read);
842 if (ret < 0) {
844 return ret;
845 }
846 }
847
848 if (bytes_read > 0) {
849 ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0;
850 if (ret < 0) {
851 if (ret != AVERROR(EINTR)) {
852 av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n",
853 av_err2str(ret));
854 s->write_err = 1;
855 }
857 } else {
858 uint32_t crc = get_block_crc(tmp, bytes_read);
859 av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at "
860 "offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id,
861 block_pos, crc);
863 if (allocated)
864 atomic_fetch_add_explicit(&s->spacemap->blocks_cached, 1, memory_order_release);
865 }
866 } else {
868 return AVERROR_EOF;
869 }
870
871 size = FFMIN(bytes_read - offset, size);
872 if (size <= 0)
873 return AVERROR_EOF;
874 if (tmp != buf)
875 memcpy(buf, &tmp[offset], size);
876 s->pos += size;
877 return size;
878}
879
881{
882 SharedContext *s = h->priv_data;
883 int64_t res;
884 if (!s->spacemap)
885 return AVERROR(EIO);
886
888 if (filesize < 0)
889 return filesize;
890
891 switch (whence) {
892 case AVSEEK_SIZE:
893 if (filesize)
894 return filesize;
895 res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
896 if (res > 0) {
897 if (set_filesize(h, res) < 0)
898 return AVERROR(EINVAL);
899 } else if (is_ignorable_error(res) && s->ignore_errors) {
900 av_log(h, AV_LOG_WARNING, "Underlying URL failed to get size: %s. "
901 "Continuing with cache file only.\n", av_err2str(res));
902 ffurl_closep(&s->inner);
903 res = AVERROR(ENOSYS);
904 }
905 return res;
906 case SEEK_SET:
907 break;
908 case SEEK_CUR:
909 pos += s->pos;
910 break;
911 case SEEK_END:
912 if (filesize) {
913 pos += filesize;
914 break;
915 }
916
917 /* Defer to underlying protocol if filesize is unknown */
918 res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
919 if (is_ignorable_error(res) && s->ignore_errors) {
920 av_log(h, AV_LOG_WARNING, "Underlying URL failed to seek: %s. "
921 "Continuing with cache file only.\n", av_err2str(res));
922 ffurl_closep(&s->inner);
923 return AVERROR(ENOSYS);
924 } else if (res < 0)
925 return res;
926
927 /* Opportunistically update known filesize */
928 if (set_filesize(h, res - pos) < 0)
929 return AVERROR(EINVAL);
930 av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", res);
931 return s->pos = s->inner_pos = res;
932 default:
933 return AVERROR(EINVAL);
934 }
935
936 if (pos < 0)
937 return AVERROR(EINVAL);
938
939 av_log(h, AV_LOG_DEBUG, "Virtual seek to 0x%"PRIx64"\n", pos);
940 return s->pos = pos;
941}
942
944{
945 SharedContext *s = h->priv_data;
946 return s->inner ? ffurl_get_file_handle(s->inner) : -1;
947}
948
950{
951 SharedContext *s = h->priv_data;
952 int ret = s->inner ? ffurl_get_short_seek(s->inner) : 0;
953 return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size;
954}
955
956#define OFFSET(x) offsetof(SharedContext, x)
957#define D AV_OPT_FLAG_DECODING_PARAM
958
959static const AVOption options[] = {
960 { "cache_dir", "Directory path for shared file cache", OFFSET(cache_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
961 { "metadata_dir", "Directory path for shared file cache metadata", OFFSET(metadata_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
962 { "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
963 { "cache_read_only","Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
964 { "read_only", "Deprecated, use cache_read_only", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D|AV_OPT_FLAG_DEPRECATED },
965 { "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
966 { "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
967 { "ignore_errors", "Continue even if the inner URL failed", OFFSET(ignore_errors), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
968 { "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
969 { "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
970 { "cache_size_max", "Limit the maximum amount of data cached", OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
971 { "offset", "Initial byte offset", OFFSET(offset), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
972 { "cache_key", "Override the hash used to identify the cache file",OFFSET(cache_key), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
973 {0},
974};
975
977 .class_name = "shared",
978 .item_name = av_default_item_name,
979 .option = options,
980 .version = LIBAVUTIL_VERSION_INT,
981};
982
984 .name = "shared",
985 .url_open2 = shared_open,
986 .url_read = shared_read,
987 .url_seek = shared_seek,
988 .url_close = shared_close,
989 .url_get_file_handle = shared_get_file_handle,
990 .url_get_short_seek = shared_get_short_seek,
991 .priv_data_size = sizeof(SharedContext),
992 .priv_data_class = &shared_context_class,
993};
static int read_block(ALSDecContext *ctx, ALSBlockData *bd)
Read the block data.
Definition alsdec.c:1031
static uint8_t hash[HASH_SIZE]
static AVFormatContext * ctx
simple assert() macros that are a bit more flexible than ISO C assert().
#define av_assert1(cond)
assert() equivalent, that does not lie in speed critical code.
Definition avassert.h:58
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition avassert.h:42
#define D
Definition avdct.c:35
int ff_check_interrupt(AVIOInterruptCB *cb)
Check if the user has requested to interrupt a blocking function associated with cb.
Definition avio.c:935
int ffurl_open_whitelist(URLContext **puc, const char *filename, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options, const char *whitelist, const char *blacklist, URLContext *parent)
Create an URLContext for accessing to the resource indicated by url, and open it.
Definition avio.c:467
int64_t ffurl_size(URLContext *h)
Return the filesize of the resource accessed by h, AVERROR(ENOSYS) if the operation is not supported ...
Definition avio.c:880
int ffurl_closep(URLContext **hh)
Close the resource accessed by the URLContext h, and free the memory used by it.
Definition avio.c:664
int ffurl_close(URLContext *h)
Definition avio.c:687
int ffurl_get_short_seek(void *urlcontext)
Return the current short seek threshold value for this URL.
Definition avio.c:919
int ffurl_get_file_handle(URLContext *h)
Return the file descriptor associated with this URL.
Definition avio.c:895
#define AVSEEK_SIZE
Passing this as the "whence" parameter to a seek function causes it to return the filesize without se...
Definition avio.h:481
#define AVIO_FLAG_NONBLOCK
Use non-blocking mode.
Definition avio.h:649
char * av_asprintf(const char *fmt,...)
Definition avstring.c:115
#define flags(name, subs,...)
Definition cbs_h264.c:74
#define i(width, name, range_min, range_max)
Definition cbs_h264.c:63
#define s(width, name)
Definition cbs_vp9.c:198
#define NULL
Definition coverity.c:32
long long int64_t
Definition coverity.c:34
Public header for CRC hash function implementation.
static int16_t block[64]
Definition dct.c:125
error code definitions
static int64_t filesize(AVIOContext *pb)
Definition ffmpeg_mux.c:52
int ret
Definition ffplay.c:391
Misc file utilities.
#define fail
Definition test.h:479
#define AV_OPT_FLAG_DEPRECATED
Set if option is deprecated, users should refer to AVOption.help text for more information.
Definition opt.h:385
@ AV_OPT_TYPE_INT64
Underlying C type is int64_t.
Definition opt.h:262
@ AV_OPT_TYPE_INT
Underlying C type is int.
Definition opt.h:258
@ AV_OPT_TYPE_BOOL
Underlying C type is int.
Definition opt.h:326
@ AV_OPT_TYPE_STRING
Underlying C type is a uint8_t* that is either NULL or points to a C string allocated with the av_mal...
Definition opt.h:275
const AVCRC * av_crc_get_table(AVCRCId crc_id)
Get an initialized standard CRC table.
Definition crc.c:389
uint32_t av_crc(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer, size_t length)
Calculate the CRC of a block.
Definition crc.c:421
@ AV_CRC_32_IEEE
Definition crc.h:52
#define AVERROR_EXIT
Immediate exit was requested; the called function should not be restarted.
Definition error.h:58
#define AVERROR_BUG
Internal bug, also see AVERROR_BUG2.
Definition error.h:52
#define AVERROR_EOF
End of file.
Definition error.h:57
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition error.h:122
#define AVERROR(e)
Definition error.h:45
void av_hash_freep(AVHashContext **ctx)
Free hash context and set hash context pointer to NULL.
Definition hash.c:248
void av_hash_init(AVHashContext *ctx)
Initialize or reset a hash context.
Definition hash.c:151
void av_hash_update(AVHashContext *ctx, const uint8_t *src, size_t len)
Update a hash context with additional data.
Definition hash.c:172
int av_hash_alloc(AVHashContext **ctx, const char *name)
Allocate a hash context for the algorithm specified by name.
Definition hash.c:114
void av_hash_final(AVHashContext *ctx, uint8_t *dst)
Finalize a hash context and compute the actual hash value.
Definition hash.c:193
#define AV_LOG_TRACE
Extremely verbose debugging, useful for libav* development.
Definition log.h:236
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition log.h:231
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition log.h:216
#define AV_LOG_VERBOSE
Detailed information.
Definition log.h:226
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition log.h:210
const char * av_default_item_name(void *ptr)
Return the context name.
Definition log.c:247
int av_strstart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str.
Definition avstring.c:36
#define LIBAVUTIL_VERSION_INT
Definition version.h:85
int av_hash_get_size(const AVHashContext *ctx)
Definition hash.c:109
Generic hashing API.
const VDPAUPixFmtMap * map
unsigned offset
Definition libaomenc.c:763
static int shift(int a, int b)
Definition bonk.c:261
av_cold void CBS_FUNC close(CodedBitstreamContext **ctx_ptr)
Close a context and free all internal state.
Definition cbs.c:118
const char * arg
Definition jacosubdec.c:66
char * ff_data_to_hex(char *buf, const uint8_t *src, int size, int lowercase)
Write hexadecimal string corresponding to given binary data.
Definition utils.c:473
Macro definitions for various function/variable attributes.
#define av_fallthrough
Definition attributes.h:67
int av_file_map_shared(int fd, size_t size, void **bufptr)
Map the beginning of an open file into memory for shared read and write access.
Definition file.c:186
void av_file_unmap_shared(void *bufptr, size_t size)
Unmap the memory mapped by av_file_map_shared().
Definition file.c:198
int avpriv_open(const char *filename, int flags,...)
A wrapper for open() setting O_CLOEXEC.
Definition file_open.c:67
version
Definition libkvazaar.c:313
#define FFMIN(a, b)
Definition macros.h:49
#define FFMAX(a, b)
Definition macros.h:47
#define FFALIGN(x, a)
Definition macros.h:78
Memory handling functions.
#define av_malloc(s)
Definition ops_static.c:52
AVOptions.
miscellaneous OS support macros and functions.
#define O_BINARY
Definition os_support.h:36
const URLProtocol ff_shared_protocol
Definition shared.c:983
static int shared_read(URLContext *h, unsigned char *buf, int size)
Definition shared.c:590
static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
Definition shared.c:552
static int spacemap_grow(URLContext *h, int64_t block)
Definition shared.c:461
static int is_ignorable_error(int64_t err)
Definition shared.c:259
#define HEADER_VERSION
Definition shared.c:70
#define DEF_SET_ONCE(ctype, atype)
Definition shared.c:142
static int set_filesize(URLContext *h, int64_t new_size)
Definition shared.c:240
static int shared_get_short_seek(URLContext *h)
Definition shared.c:949
#define HASH_METHOD
This hash should be resistant against collision attacks, so that an attacker could not generate e....
Definition shared.c:67
static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
Definition shared.c:274
static int64_t shared_seek(URLContext *h, int64_t pos, int whence)
Definition shared.c:880
#define RELEASE_PENDING(block, state)
static int shared_close(URLContext *h)
Definition shared.c:204
static int shared_get_file_handle(URLContext *h)
Definition shared.c:943
static int64_t get_filesize(URLContext *h)
Definition shared.c:226
static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
Definition shared.c:78
static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
Definition shared.c:566
static const AVClass shared_context_class
Definition shared.c:976
static int spacemap_remap(URLContext *h, size_t map_size)
Definition shared.c:394
#define MAX_CORRUPT_BLOCKS
Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume...
Definition shared.c:76
static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
Definition shared.c:110
#define OFFSET(x)
Definition shared.c:956
static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
Definition shared.c:497
#define HASH_SIZE
Definition shared.c:68
BlockState
Definition shared.c:98
@ BLOCK_NONE
block is not cached
Definition shared.c:100
@ BLOCK_PENDING
a thread is currently trying to write this block
Definition shared.c:101
@ BLOCK_FAILED
the underlying I/O source failed to read this block
Definition shared.c:102
#define HEADER_MAGIC
Definition shared.c:69
static int clamp_size(URLContext *h, int size, int64_t pos, int64_t filesize)
Definition shared.c:580
unsigned int pos
Definition spdifenc.c:431
@ memory_order_release
Definition stdatomic.h:32
@ memory_order_relaxed
Definition stdatomic.h:29
@ memory_order_acquire
Definition stdatomic.h:31
#define atomic_fetch_add_explicit(object, operand, order)
Definition stdatomic.h:297
unsigned char atomic_uchar
Definition stdatomic.h:60
FF_ATOMIC_ALIGN64 unsigned long long atomic_ullong
Definition stdatomic.h:68
#define atomic_compare_exchange_strong_explicit(object, expected, desired, success, failure)
Definition stdatomic.h:274
unsigned int atomic_uint
Definition stdatomic.h:64
#define atomic_load_explicit(object, order)
Definition stdatomic.h:247
unsigned short atomic_ushort
Definition stdatomic.h:62
#define atomic_load(object)
Definition stdatomic.h:250
#define atomic_store_explicit(object, desired, order)
Definition stdatomic.h:253
Describe the class of an AVClass context structure.
Definition log.h:76
uint32_t crc
Definition hash.c:70
AVOption.
Definition opt.h:428
atomic_uint state
Definition shared.c:124
char * cache_key
Definition shared.c:169
char * cache_dir
Definition shared.c:167
char * metadata_dir
Definition shared.c:168
int write_err
write error occurred
Definition shared.c:184
int block_size
Definition shared.c:183
int64_t nb_hit
Definition shared.c:200
int block_shift
requested shift; updated on init if it disagrees
Definition shared.c:170
int64_t filesize
once known
Definition shared.c:186
int64_t nb_miss
Definition shared.c:201
Spacemap * spacemap
Definition shared.c:194
uint8_t * tmp_buf
Definition shared.c:182
int64_t timeout
Definition shared.c:172
URLContext * inner
Definition shared.c:163
int retry_errors
Definition shared.c:174
int64_t offset
Definition shared.c:178
int64_t inner_pos
Definition shared.c:164
int num_corrupt
Definition shared.c:185
int ignore_errors
Definition shared.c:173
int64_t blocks_max
maximum number of blocks to cache
Definition shared.c:187
int64_t pos
current logical position
Definition shared.c:181
int64_t cache_size_max
Definition shared.c:177
int read_only
Definition shared.c:171
off_t map_size
Definition shared.c:196
char * cache_path
Definition shared.c:190
char * map_path
Definition shared.c:195
int retry_corrupt
Definition shared.c:175
atomic_uint header_magic
Definition shared.c:128
atomic_ushort version
Definition shared.c:129
Block blocks[]
Definition shared.c:136
atomic_uchar hash[HASH_SIZE]
Definition shared.c:132
atomic_ushort block_shift
Definition shared.c:130
char reserved[72]
Definition shared.c:134
atomic_ullong blocks_cached
Definition shared.c:133
atomic_ullong filesize
Definition shared.c:131
Definition swscale.c:85
#define av_freep(p)
#define av_log(a,...)
static uint8_t tmp[40]
Definition aes_ctr.c:52
static struct @232240247175332074307271266321204261331170211232 state
int av_usleep(unsigned usec)
Sleep for a period of time.
Definition time.c:93
int64_t av_gettime_relative(void)
Get the current time in microseconds since some unspecified starting point.
Definition time.c:57
int size
unbuffered private I/O API
static int64_t ffurl_seek(URLContext *h, int64_t pos, int whence)
Change the position that will be used by the next read/write operation on the resource accessed by h.
Definition url.h:225
static int ffurl_read(URLContext *h, uint8_t *buf, int size)
Read up to size bytes from the resource accessed by h, and store the read bytes in buf.
Definition url.h:184