shared.c File Reference

#include "libavutil/attributes.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/crc.h"
#include "libavutil/error.h"
#include "libavutil/hash.h"
#include "libavutil/file_open.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/time.h"
#include "url.h"
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdatomic.h>
#include <string.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

Go to the source code of this file.

Data Structures
struct	Block
struct	Spacemap
struct	SharedContext

Macros
#define	HASH_METHOD   "SHA512/256"
	This hash should be resistant against collision attacks, so that an attacker could not generate e.g.
#define	HASH_SIZE   32
#define	HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')
#define	HEADER_VERSION   3
#define	MAX_CORRUPT_BLOCKS   10
	Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.
#define	DEF_SET_ONCE(ctype, atype)
#define	OFFSET(x)
#define	D   AV_OPT_FLAG_DECODING_PARAM

Enumerations
enum	BlockState { BLOCK_NONE = 0 , BLOCK_PENDING , BLOCK_FAILED }

Functions
static int	hash_uri (uint8_t hash[HASH_SIZE], const char *uri)
static uint32_t	get_block_crc (const uint8_t *block, size_t block_size)
static int	shared_close (URLContext *h)
static int	cache_map (URLContext *h, int64_t filesize)
static int	spacemap_init (URLContext *h, const uint8_t hash[HASH_SIZE])
static int	spacemap_grow (URLContext *h, int64_t block)
static int64_t	get_filesize (URLContext *h)
static int	set_filesize (URLContext *h, int64_t new_size)
static int	shared_open (URLContext *h, const char *arg, int flags, AVDictionary **options)
static int	spacemap_remap (URLContext *h, size_t map_size)
static int	read_cache (SharedContext *s, uint8_t *buf, size_t size, off_t offset)
static int	write_cache (SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
static size_t	clamp_size (URLContext *h, size_t size, int64_t pos)
static int	shared_read (URLContext *h, unsigned char *buf, int size)
static int64_t	shared_seek (URLContext *h, int64_t pos, int whence)
static int	shared_get_file_handle (URLContext *h)
static int	shared_get_short_seek (URLContext *h)

Variables
static const AVOption	options []
static const AVClass	shared_context_class
const URLProtocol	ff_shared_protocol

Macro Definition Documentation

HASH_METHOD

#define HASH_METHOD   "SHA512/256"

This hash should be resistant against collision attacks, so that an attacker could not generate e.g.

two different URIs that map to the same cache file. This requires at least 64 bits of collision resistance in practice (i.e. 128 bits = 16 bytes of hash size). However, we can be conservative by computing e.g. a 256 bit hash and storing it inside the file header for verification.

Note that due to the way we use atomics, we should avoid zero bytes in the resulting hash; hence we tweak the input slightly to avoid this. The resulting loss in hash strength is negligible, since 32 bytes is already much more than needed.

Definition at line 60 of file shared.c.

Referenced by hash_uri().

HASH_SIZE

#define HASH_SIZE   32

Definition at line 61 of file shared.c.

Referenced by close_out(), hash_uri(), main(), shared_open(), and spacemap_init().

HEADER_MAGIC

#define HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')

Definition at line 62 of file shared.c.

Referenced by spacemap_init().

HEADER_VERSION

#define HEADER_VERSION   3

Definition at line 63 of file shared.c.

Referenced by hash_uri(), and spacemap_init().

MAX_CORRUPT_BLOCKS

#define MAX_CORRUPT_BLOCKS   10

Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.

Definition at line 69 of file shared.c.

Referenced by shared_read().

DEF_SET_ONCE

#define DEF_SET_ONCE	(		ctype,
			atype )

Value:

    static int set_once_##atype(atomic_##atype *const ptr, const ctype value)   \
    {                                                                           \
        ctype prev = 0;                                                         \
        av_assert1(value != 0);                                                 \
        if (atomic_compare_exchange_strong_explicit(                            \
                ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
            return 1;                                                           \
        else if (prev == value)                                                 \
            return 0;                                                           \
        else                                                                    \
            return AVERROR(EINVAL);                                             \
    }

Definition at line 132 of file shared.c.

OFFSET

#define OFFSET

(

x

)

Value:

offsetof(SharedContext, x)

Definition at line 900 of file shared.c.

D

#define D   AV_OPT_FLAG_DECODING_PARAM

Definition at line 901 of file shared.c.

Enumeration Type Documentation

BlockState

enum BlockState

Enumerator
BLOCK_NONE	block is not cached
BLOCK_PENDING	a thread is currently trying to write this block
BLOCK_FAILED	the underlying I/O source failed to read this block

Definition at line 91 of file shared.c.

Function Documentation

hash_uri()

static int hash_uri ( uint8_t hash[HASH_SIZE], const char * uri )

static

Definition at line 71 of file shared.c.

Referenced by shared_open().

get_block_crc()

static uint32_t get_block_crc ( const uint8_t * block, size_t block_size )

static

Definition at line 103 of file shared.c.

Referenced by shared_read().

shared_close()

static int shared_close ( URLContext * h )

static

Definition at line 189 of file shared.c.

Referenced by shared_open().

cache_map()

static int cache_map ( URLContext * h, int64_t filesize )

static

Definition at line 347 of file shared.c.

Referenced by set_filesize(), and shared_open().

spacemap_init()

static int spacemap_init ( URLContext * h, const uint8_t hash[HASH_SIZE] )

static

Definition at line 486 of file shared.c.

Referenced by shared_open().

spacemap_grow()

static int spacemap_grow ( URLContext * h, int64_t block )

static

Definition at line 453 of file shared.c.

Referenced by shared_open(), and shared_read().

get_filesize()

static int64_t get_filesize ( URLContext * h )

static

Definition at line 215 of file shared.c.

Referenced by clamp_size(), shared_open(), shared_seek(), and spacemap_grow().

set_filesize()

static int set_filesize ( URLContext * h, int64_t new_size )

static

Definition at line 221 of file shared.c.

Referenced by shared_open(), shared_read(), and shared_seek().

shared_open()

static int shared_open ( URLContext * h, const char * arg, int flags, AVDictionary ** options )

static

Definition at line 244 of file shared.c.

spacemap_remap()

static int spacemap_remap ( URLContext * h, size_t map_size )

static

Definition at line 383 of file shared.c.

Referenced by spacemap_grow(), and spacemap_init().

read_cache()

static int read_cache ( SharedContext * s, uint8_t * buf, size_t size, off_t offset )

static

Definition at line 539 of file shared.c.

Referenced by shared_read().

write_cache()

static int write_cache ( SharedContext * s, const uint8_t * buf, size_t size, off_t offset )

static

Definition at line 559 of file shared.c.

Referenced by shared_read().

clamp_size()

static size_t clamp_size ( URLContext * h, size_t size, int64_t pos )

static

Definition at line 579 of file shared.c.

Referenced by shared_read().

shared_read()

static int shared_read ( URLContext * h, unsigned char * buf, int size )

static

Definition at line 590 of file shared.c.

shared_seek()

static int64_t shared_seek ( URLContext * h, int64_t pos, int whence )

static

Definition at line 841 of file shared.c.

shared_get_file_handle()

static int shared_get_file_handle ( URLContext * h )

static

Definition at line 887 of file shared.c.

shared_get_short_seek()

static int shared_get_short_seek ( URLContext * h )

static

Definition at line 893 of file shared.c.

Variable Documentation

options

const AVOption options[]

static

Initial value:

= {
    { "cache_dir",      "Directory path for shared file cache",             OFFSET(cache_dir),      AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
    { "block_shift",    "Set the base 2 logarithm of the block size",       OFFSET(block_shift),    AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
    { "read_only",      "Don't write data to the cache, only read from it", OFFSET(read_only),      AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_verify",   "Verify correctness of the cache against the source",   OFFSET(verify),     AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_timeout",  "Time in us to wait before re-fetching pending blocks", OFFSET(timeout),    AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
    { "retry_errors",   "Re-request blocks even if they previously failed", OFFSET(retry_errors),   AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    { "retry_corrupt",  "Re-request blocks that fail the CRC check",        OFFSET(retry_corrupt),  AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    {0},
}

Definition at line 903 of file shared.c.

shared_context_class

const AVClass shared_context_class

static

Initial value:

= {
    .class_name = "shared",
    .item_name  = av_default_item_name,
    .option     = options,
    .version    = LIBAVUTIL_VERSION_INT,
}

Definition at line 914 of file shared.c.

ff_shared_protocol

const URLProtocol ff_shared_protocol

Initial value:

= {
    .name                = "shared",
    .url_open2           = shared_open,
    .url_read            = shared_read,
    .url_seek            = shared_seek,
    .url_close           = shared_close,
    .url_get_file_handle = shared_get_file_handle,
    .url_get_short_seek  = shared_get_short_seek,
    .priv_data_size      = sizeof(SharedContext),
    .priv_data_class     = &shared_context_class,
}

Definition at line 921 of file shared.c.