shared.c File Reference

#include "libavutil/attributes.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/crc.h"
#include "libavutil/error.h"
#include "libavutil/file.h"
#include "libavutil/hash.h"
#include "libavutil/file_open.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/time.h"
#include "internal.h"
#include "os_support.h"
#include "url.h"
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdatomic.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/file.h>

Go to the source code of this file.

Data Structures
struct	Block
struct	Spacemap
struct	SharedContext

Macros
#define	HASH_METHOD   "SHA512/256"
	This hash should be resistant against collision attacks, so that an attacker could not generate e.g.
#define	HASH_SIZE   32
#define	HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')
#define	HEADER_VERSION   3
#define	MAX_CORRUPT_BLOCKS   10
	Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.
#define	DEF_SET_ONCE(ctype, atype)
#define	RELEASE_PENDING(block, state)
#define	OFFSET(x)
#define	D   AV_OPT_FLAG_DECODING_PARAM

Enumerations
enum	BlockState { BLOCK_NONE = 0 , BLOCK_PENDING , BLOCK_FAILED }

Functions
static int	hash_uri (uint8_t hash[HASH_SIZE], const char *uri)
static uint32_t	get_block_crc (const uint8_t *block, size_t block_size)
static int	shared_close (URLContext *h)
static int	cache_map (URLContext *h, int64_t filesize)
static int	spacemap_init (URLContext *h, const uint8_t hash[HASH_SIZE])
static int	spacemap_grow (URLContext *h, int64_t block)
static int64_t	get_filesize (URLContext *h)
static int	set_filesize (URLContext *h, int64_t new_size)
static int	is_ignorable_error (int64_t err)
static int	shared_open (URLContext *h, const char *arg, int flags, AVDictionary **options)
static int	spacemap_remap (URLContext *h, size_t map_size)
static int	read_cache (SharedContext *s, uint8_t *buf, size_t size, off_t offset)
static int	write_cache (SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
static int	clamp_size (URLContext *h, int size, int64_t pos, int64_t filesize)
static int	shared_read (URLContext *h, unsigned char *buf, int size)
static int64_t	shared_seek (URLContext *h, int64_t pos, int whence)
static int	shared_get_file_handle (URLContext *h)
static int	shared_get_short_seek (URLContext *h)

Variables
static const AVOption	options []
static const AVClass	shared_context_class
const URLProtocol	ff_shared_protocol

Macro Definition Documentation

HASH_METHOD

#define HASH_METHOD   "SHA512/256"

This hash should be resistant against collision attacks, so that an attacker could not generate e.g.

two different URIs that map to the same cache file. This requires at least 64 bits of collision resistance in practice (i.e. 128 bits = 16 bytes of hash size). However, we can be conservative by computing e.g. a 256 bit hash and storing it inside the file header for verification.

Note that due to the way we use atomics, we should avoid zero bytes in the resulting hash; hence we tweak the input slightly to avoid this. The resulting loss in hash strength is negligible, since 32 bytes is already much more than needed.

Definition at line 67 of file shared.c.

Referenced by hash_uri().

HASH_SIZE

#define HASH_SIZE   32

Definition at line 68 of file shared.c.

Referenced by close_out(), hash_uri(), main(), shared_open(), and spacemap_init().

HEADER_MAGIC

#define HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')

Definition at line 69 of file shared.c.

Referenced by spacemap_init().

HEADER_VERSION

#define HEADER_VERSION   3

Definition at line 70 of file shared.c.

Referenced by hash_uri(), and spacemap_init().

MAX_CORRUPT_BLOCKS

#define MAX_CORRUPT_BLOCKS   10

Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.

Definition at line 76 of file shared.c.

Referenced by shared_read().

DEF_SET_ONCE

#define DEF_SET_ONCE	(		ctype,
			atype )

Value:

    static int set_once_##atype(atomic_##atype *const ptr, const ctype value)   \
    {                                                                           \
        ctype prev = 0;                                                         \
        av_assert1(value != 0);                                                 \
        if (atomic_compare_exchange_strong_explicit(                            \
                ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
            return 1;                                                           \
        else if (prev == value)                                                 \
            return 0;                                                           \
        else                                                                    \
            return AVERROR(EINVAL);                                             \
    }

Definition at line 142 of file shared.c.

RELEASE_PENDING

#define RELEASE_PENDING	(		block,
			state )

Value:

    do {                                                                       \
        if (acquired) {                                                        \
            av_assert1(state == BLOCK_PENDING);                                \
            atomic_compare_exchange_strong_explicit(                           \
                &block->state, &state, BLOCK_NONE, memory_order_relaxed,       \
                memory_order_relaxed);                                         \
        }                                                                      \
    } while (0)

Referenced by shared_read().

OFFSET

#define OFFSET

(

x

)

Value:

offsetof(SharedContext, x)

Definition at line 999 of file shared.c.

D

#define D   AV_OPT_FLAG_DECODING_PARAM

Definition at line 1000 of file shared.c.

Enumeration Type Documentation

BlockState

enum BlockState

Enumerator
BLOCK_NONE	block is not cached
BLOCK_PENDING	a thread is currently trying to write this block
BLOCK_FAILED	the underlying I/O source failed to read this block

Definition at line 98 of file shared.c.

Function Documentation

hash_uri()

static int hash_uri ( uint8_t hash[HASH_SIZE], const char * uri )

static

Definition at line 78 of file shared.c.

Referenced by shared_open().

get_block_crc()

static uint32_t get_block_crc ( const uint8_t * block, size_t block_size )

static

Definition at line 110 of file shared.c.

Referenced by shared_read().

shared_close()

static int shared_close ( URLContext * h )

static

Definition at line 204 of file shared.c.

Referenced by shared_open().

cache_map()

static int cache_map ( URLContext * h, int64_t filesize )

static

Definition at line 391 of file shared.c.

Referenced by set_filesize(), and shared_open().

spacemap_init()

static int spacemap_init ( URLContext * h, const uint8_t hash[HASH_SIZE] )

static

Definition at line 519 of file shared.c.

Referenced by shared_open().

spacemap_grow()

static int spacemap_grow ( URLContext * h, int64_t block )

static

Definition at line 483 of file shared.c.

Referenced by shared_open(), and shared_read().

get_filesize()

static int64_t get_filesize ( URLContext * h )

static

Definition at line 228 of file shared.c.

Referenced by shared_open(), shared_read(), shared_seek(), and spacemap_grow().

set_filesize()

static int set_filesize ( URLContext * h, int64_t new_size )

static

Definition at line 242 of file shared.c.

Referenced by shared_read(), and shared_seek().

is_ignorable_error()

static int is_ignorable_error ( int64_t err )

static

Definition at line 265 of file shared.c.

Referenced by shared_open(), and shared_seek().

shared_open()

static int shared_open ( URLContext * h, const char * arg, int flags, AVDictionary ** options )

static

Definition at line 280 of file shared.c.

spacemap_remap()

static int spacemap_remap ( URLContext * h, size_t map_size )

static

Definition at line 416 of file shared.c.

Referenced by spacemap_grow(), and spacemap_init().

read_cache()

static int read_cache ( SharedContext * s, uint8_t * buf, size_t size, off_t offset )

static

Definition at line 574 of file shared.c.

Referenced by shared_read().

write_cache()

static int write_cache ( SharedContext * s, const uint8_t * buf, size_t size, off_t offset )

static

Definition at line 594 of file shared.c.

Referenced by shared_read().

clamp_size()

static int clamp_size ( URLContext * h, int size, int64_t pos, int64_t filesize )

static

Definition at line 614 of file shared.c.

Referenced by shared_read().

shared_read()

static int shared_read ( URLContext * h, unsigned char * buf, int size )

static

Definition at line 624 of file shared.c.

shared_seek()

static int64_t shared_seek ( URLContext * h, int64_t pos, int whence )

static

Definition at line 923 of file shared.c.

shared_get_file_handle()

static int shared_get_file_handle ( URLContext * h )

static

Definition at line 986 of file shared.c.

shared_get_short_seek()

static int shared_get_short_seek ( URLContext * h )

static

Definition at line 992 of file shared.c.

Variable Documentation

options

const AVOption options[]

static

Initial value:

= {
    { "cache_dir",      "Directory path for shared file cache",             OFFSET(cache_dir),      AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
    { "metadata_dir",   "Directory path for shared file cache metadata",    OFFSET(metadata_dir),   AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
    { "block_shift",    "Set the base 2 logarithm of the block size",       OFFSET(block_shift),    AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
    { "read_only",      "Don't write data to the cache, only read from it", OFFSET(read_only),      AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_verify",   "Verify correctness of the cache against the source",   OFFSET(verify),     AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_timeout",  "Time in us to wait before re-fetching pending blocks", OFFSET(timeout),    AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
    { "ignore_errors",  "Continue even if the inner URL failed",            OFFSET(ignore_errors),  AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "retry_errors",   "Re-request blocks even if they previously failed", OFFSET(retry_errors),   AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    { "retry_corrupt",  "Re-request blocks that fail the CRC check",        OFFSET(retry_corrupt),  AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    { "cache_size_max", "Limit the maximum amount of data cached",          OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
    {0},
}

Definition at line 1002 of file shared.c.

shared_context_class

const AVClass shared_context_class

static

Initial value:

= {
    .class_name = "shared",
    .item_name  = av_default_item_name,
    .option     = options,
    .version    = LIBAVUTIL_VERSION_INT,
}

Definition at line 1016 of file shared.c.

ff_shared_protocol

const URLProtocol ff_shared_protocol

Initial value:

= {
    .name                = "shared",
    .url_open2           = shared_open,
    .url_read            = shared_read,
    .url_seek            = shared_seek,
    .url_close           = shared_close,
    .url_get_file_handle = shared_get_file_handle,
    .url_get_short_seek  = shared_get_short_seek,
    .priv_data_size      = sizeof(SharedContext),
    .priv_data_class     = &shared_context_class,
}

Definition at line 1023 of file shared.c.