[FFmpeg-devel] [RFC][GSoC][PATCH v1 3/6] avformat/hls: use abr to switch streams
Steven Liu
lingjiujianke at gmail.com
Wed Jul 15 12:24:01 EEST 2020
Hongcheng Zhong <sj.hc_Zhong at sjtu.edu.cn> 于2020年7月15日周三 下午4:38写道：
>
> From: spartazhc <spartazhc at gmail.com>
>
> When abr is enable, it will take over the task to call http to
> download segments, and will return a switch-request for hls to
> switch streams.
> For reason not to waste segments that have been downloaded,
> switch will become effective after old segments is used out.
> Abr cannot work with http_persistent option, and currently use
> http_multiple.
>
> v1 fixed:
> 1. fix memory leak
>
> Signed-off-by: spartazhc <spartazhc at gmail.com>
> ---
> doc/demuxers.texi | 3 +
> libavformat/hls.c | 232 ++++++++++++++++++++++++++++++++++++++++++++--
> 2 files changed, 229 insertions(+), 6 deletions(-)
>
> diff --git a/doc/demuxers.texi b/doc/demuxers.texi
> index 3c15ab9eee..4cdbd95962 100644
> --- a/doc/demuxers.texi
> +++ b/doc/demuxers.texi
> @@ -321,6 +321,9 @@ available in a metadata key named "variant_bitrate".
> It accepts the following options:
>
> @table @option
> + at item abr
> +enable abr to switch streams.
> +
> @item live_start_index
> segment index to start live streams at (negative values are from the end).
>
> diff --git a/libavformat/hls.c b/libavformat/hls.c
> index ba17c4ed96..877bd8c677 100644
> --- a/libavformat/hls.c
> +++ b/libavformat/hls.c
> @@ -189,6 +189,17 @@ struct variant {
> char subtitles_group[MAX_FIELD_LEN];
> };
>
> +struct throughput {
> + int n_throughputs;
> +
> + /* throughputs are in kbps, always record the last 20 times
> + * set record 20 times refer to paper FESTIVE https://doi.org/10.1145/2413176.2413189
> + */
> + float throughput_fifo[20];
> + int head;
> + int tail;
> +};
what about add an #define blablabla_name 20 here if use the record 20
times, and use the blablabla_name at the below code which use the 20
value?
> +
> typedef struct HLSContext {
> AVClass *class;
> AVFormatContext *ctx;
> @@ -213,8 +224,36 @@ typedef struct HLSContext {
> int http_multiple;
> int http_seekable;
> AVIOContext *playlist_pb;
> +
> + int abr;
> + struct throughput *throughputs;
> + int can_switch;
> + int switch_request2;
> + int switch_delay;
> + int64_t switch_timestamp;
> + int64_t delta_timestamp;
> + int cur_pls;
> } HLSContext;
>
> +static struct segment *next_segment(struct playlist *pls);
> +static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in);
> +
> +static void sync_cur_seq(HLSContext *c) {
> + int i;
> + for (i = 0; i < c->n_playlists; i++) {
> + struct playlist *pls = c->playlists[i];
> + pls->cur_seq_no = c->cur_seq_no;
> + }
> +}
> +
> +static struct segment *next2_segment(struct playlist *pls)
> +{
> + int n = pls->cur_seq_no - pls->start_seq_no + 2;
> + if (n >= pls->n_segments)
> + return NULL;
> + return pls->segments[n];
> +}
> +
> static void free_segment_dynarray(struct segment **segments, int n_segments)
> {
> int i;
> @@ -624,6 +663,31 @@ static int open_url_keepalive(AVFormatContext *s, AVIOContext **pb,
> #endif
> }
>
> +static int update_throughputs(struct throughput *thr, float time, int pb_size)
> +{
> + if (pb_size <= 0 || time <= 0)
> + return -1;
EINVAL?
> + if (thr->n_throughputs < 20) {
> + ++thr->n_throughputs;
> + } else {
> + ++thr->head;
> + }
> + thr->throughput_fifo[thr->tail] = (float)(pb_size) / time;
> + thr->tail = (thr->tail + 1) % 20;
> + return 0;
> +}
> +
> +static int64_t get_switch_timestamp(HLSContext *c, struct playlist *pls)
> +{
> + int64_t pos = c->first_timestamp == AV_NOPTS_VALUE ?
> + 0 : c->first_timestamp;
> +
> + for (int i = 0; i < pls->cur_seq_no + 2; i++) {
> + pos += pls->segments[i]->duration;
> + }
> + return pos;
> +}
> +
> static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
> AVDictionary *opts, AVDictionary *opts2, int *is_http_out)
> {
> @@ -639,6 +703,9 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
> } else if (av_strstart(url, "data", NULL)) {
> if (url[4] == '+' || url[4] == ':')
> proto_name = avio_find_protocol_name(url + 5);
> + } else if (av_strstart(url, "ffabr", NULL)) {
> + if (url[5] == '+' || url[5] == ':')
> + proto_name = avio_find_protocol_name(url + 6);
> }
>
> if (!proto_name)
> @@ -669,6 +736,8 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
> ;
> else if (av_strstart(url, "data", NULL) && !strncmp(proto_name, url + 5, strlen(proto_name)) && url[5 + strlen(proto_name)] == ':')
> ;
> + else if (av_strstart(url, "ffabr", NULL) && !strncmp(proto_name, url + 6, strlen(proto_name)) && url[6 + strlen(proto_name)] == ':')
> + ;
> else if (strcmp(proto_name, "file") || !strncmp(url, "file,", 5))
> return AVERROR_INVALIDDATA;
>
> @@ -690,6 +759,43 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
> } else {
> ret = s->io_open(s, pb, url, AVIO_FLAG_READ, &tmp);
> }
> + if (c->abr && ret >= 0) {
> + AVDictionary *abr_ret = NULL;
> + AVDictionaryEntry *en = NULL;
> + struct segment *seg;
> + int pb_size, switch_request;
> + av_opt_get_dict_val(*pb, "abr-metadata", AV_OPT_SEARCH_CHILDREN, &abr_ret);
> + if (abr_ret) {
> + en = av_dict_get(abr_ret, "download_time", NULL, 0);
> + if (en) {
> + pb_size = avio_size(*pb);
> + update_throughputs(c->throughputs, atoi(en->value) / 1000.0, pb_size);
> + av_log(s, AV_LOG_VERBOSE, "[abr] time=%.2fms, size=%.2fkb\n", atoi(en->value) / 1000.0, pb_size / 1000.0);
> + }
> + en = av_dict_get(abr_ret, "switch_request", NULL, 0);
> + if (en) {
> + switch_request = atoi(en->value);
> + av_log(s, AV_LOG_VERBOSE, "[abr] switch request: %s\n", en->value);
> + }
> + if (switch_request != -1) {
> + c->switch_request2 = switch_request;
> + c->switch_delay = 2;
> + c->can_switch = 0;
> + sync_cur_seq(c);
> + seg = next2_segment(c->playlists[c->switch_request2]);
> + if (!seg) {
> + c->switch_request2 = -1;
> + av_log(s, AV_LOG_INFO, "[abr] no more segment need to switch\n");
> + } else {
> + c->switch_timestamp = get_switch_timestamp(c, c->playlists[c->switch_request2]);
> + av_log(s, AV_LOG_VERBOSE, "[abr] switch timestamp: %ld\n", c->switch_timestamp);
> + c->playlists[c->switch_request2]->input_next_requested = 1;
> + ret = open_input(c, c->playlists[c->switch_request2], seg, &c->playlists[c->switch_request2]->input_next);
> + }
> + }
> + av_dict_free(&abr_ret);
> + }
> + }
> if (ret >= 0) {
> // update cookies on http response with setcookies.
> char *new_cookies = NULL;
> @@ -1219,6 +1325,33 @@ static void intercept_id3(struct playlist *pls, uint8_t *buf,
> pls->is_id3_timestamped = (pls->id3_mpegts_timestamp != AV_NOPTS_VALUE);
> }
>
> +static int abrinfo_to_dict(HLSContext *c, char **abr_info)
> +{
> + struct throughput *thr = c->throughputs;
> + char buffer[MAX_URL_SIZE];
> + int size, i;
> + size = snprintf(buffer, sizeof(buffer), "format=hls:");
> + size += snprintf(buffer + size, sizeof(buffer) - size, "cur_pls=%d:", c->cur_pls);
> + size += snprintf(buffer + size, sizeof(buffer) - size, "can_switch=%d:", c->can_switch);
> + size += snprintf(buffer + size, sizeof(buffer) - size, "n_variants=%d:", c->n_variants);
> + for (i = 0; i < c->n_variants; i++) {
> + struct variant *v = c->variants[i];
> + size += snprintf(buffer + size, sizeof(buffer) - size, "variant_bitrate%d=%d:", i, v->bandwidth);
> + }
> + size += snprintf(buffer + size, sizeof(buffer) - size, "n_throughputs=%d:", thr->n_throughputs);
> + if (thr->n_throughputs > 0) {
> + i = thr->head;
> + do {
> + size += snprintf(buffer + size, sizeof(buffer) - size, "throughputs%d=%.2f:", i, thr->throughput_fifo[i]);
> + i = (i + 1) % 20;
> + } while (i != thr->tail);
> + }
> + *abr_info = av_malloc(size);
check the return NULL of av_malloc.
> + snprintf(*abr_info, size, "%s", buffer);
> + av_log(c, AV_LOG_DEBUG, "[abr] abr_info: %s\n", *abr_info);
> + return 0;
> +}
> +
> static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in)
> {
> AVDictionary *opts = NULL;
> @@ -1238,8 +1371,20 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
> av_log(pls->parent, AV_LOG_VERBOSE, "HLS request for url '%s', offset %"PRId64", playlist %d\n",
> seg->url, seg->url_offset, pls->index);
>
> + if (c->abr) {
> + char *abr_opts;
> + abrinfo_to_dict(c, &abr_opts);
> + av_dict_set(&opts, "abr-params", abr_opts, 0); // how to setup flag?
> + av_free(abr_opts);
> + }
> if (seg->key_type == KEY_NONE) {
> - ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> + char url_abr[MAX_URL_SIZE];
> + if (c->abr) {
> + snprintf(url_abr, sizeof(url_abr), "ffabr:%s", seg->url); // + or : tbd
> + ret = open_url(pls->parent, in, url_abr, c->avio_opts, opts, &is_http);
> + } else {
> + ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> + }
> } else if (seg->key_type == KEY_AES_128) {
> char iv[33], key[33], url[MAX_URL_SIZE];
> if (strcmp(seg->key, pls->key_url)) {
> @@ -1273,6 +1418,7 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
> goto cleanup;
> }
> ret = 0;
> + // TODO: Add abr prefix for crypto
> } else if (seg->key_type == KEY_SAMPLE_AES) {
> av_log(pls->parent, AV_LOG_ERROR,
> "SAMPLE-AES encryption is not supported yet\n");
> @@ -1435,7 +1581,17 @@ restart:
> /* Check that the playlist is still needed before opening a new
> * segment. */
> v->needed = playlist_needed(v);
> -
> + if (c->abr) {
> + if (c->switch_request2 == -1)
> + ;
> + else if (v->needed && c->switch_request2 != v->index && c->switch_delay <= 0) {
> + av_log(v->parent, AV_LOG_VERBOSE, "read_data: needed but not download playlist %d ('%s')\n",
> + v->index, v->url);
> + return AVERROR_EOF;
> + }
> + if (!c->can_switch)
> + c->can_switch = 1;
> + }
> if (!v->needed) {
> av_log(v->parent, AV_LOG_INFO, "No longer receiving playlist %d ('%s')\n",
> v->index, v->url);
> @@ -1530,7 +1686,12 @@ reload:
> }
>
> seg = next_segment(v);
> - if (c->http_multiple == 1 && !v->input_next_requested &&
> +
> + // when get switch_request, old stream should stop downloading next seg
> + if (c->abr && c->switch_request2 != v->index && c->switch_timestamp != AV_NOPTS_VALUE
> + && ((c->cur_timestamp + seg->duration * 2) >= c->switch_timestamp) )
> + ;
> + else if (c->http_multiple == 1 && !v->input_next_requested &&
else if (c->http_multiple && !v->input_next_requested &&
> seg && seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
> ret = open_input(c, v, seg, &v->input_next);
> if (ret < 0) {
> @@ -1563,11 +1724,17 @@ reload:
>
> return ret;
> }
> - if (c->http_persistent &&
> + if (c->http_persistent && !c->abr &&
> seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
> v->input_read_done = 1;
> } else {
> ff_format_io_close(v->parent, &v->input);
> + if (c->abr) {
> + if (c->switch_delay > 0)
> + c->switch_delay--;
> + av_log(v->parent, AV_LOG_VERBOSE, "read_data: close pls[%d]->input, v->cur_seq_no=%d, c->switch_delay= %d\n",
> + v->index, v->cur_seq_no, c->switch_delay);
> + }
> }
> v->cur_seq_no++;
>
> @@ -1860,6 +2027,17 @@ static int hls_read_header(AVFormatContext *s)
> c->first_packet = 1;
> c->first_timestamp = AV_NOPTS_VALUE;
> c->cur_timestamp = AV_NOPTS_VALUE;
> + c->switch_request2 = -1;
> + c->switch_delay = 0;
> + c->switch_timestamp = AV_NOPTS_VALUE;
> + c->delta_timestamp = AV_NOPTS_VALUE;
> + c->can_switch = -1;
> +
> + if (c->abr) {
> + c->http_persistent = 0;
> + c->http_multiple = 1;
> + c->throughputs = av_mallocz(sizeof(struct throughput));
check return NULL;
> + }
>
> if ((ret = save_avio_options(s)) < 0)
> goto fail;
> @@ -2049,6 +2227,9 @@ static int hls_read_header(AVFormatContext *s)
> add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE);
> }
>
> + if (c->abr && c->can_switch == -1)
> + c->can_switch = 1;
> +
> update_noheader_flag(s);
>
> return 0;
> @@ -2057,6 +2238,13 @@ fail:
> return ret;
> }
>
> +static void change_discard_flags(struct playlist *pls, int flag)
> +{
> + for (int i = 0; i < pls->n_main_streams; i++) {
> + pls->main_streams[i]->discard = flag;
> + }
> +}
> +
> static int recheck_discard_flags(AVFormatContext *s, int first)
> {
> HLSContext *c = s->priv_data;
> @@ -2067,6 +2255,22 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
> for (i = 0; i < c->n_playlists; i++) {
> struct playlist *pls = c->playlists[i];
>
> + if (c->abr) {
> + if (c->switch_request2 == -1)
> + ;
> + else if (c->switch_request2 == i && c->switch_delay <= 0
> + && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> + && c->switch_timestamp != AV_NOPTS_VALUE) {
> + av_log(s, AV_LOG_VERBOSE, "[switch point] cur_timestamp:%ld\n", c->cur_timestamp);
> + change_discard_flags(pls, AVDISCARD_DEFAULT);
> + c->switch_timestamp = AV_NOPTS_VALUE;
> + } else if (c->switch_request2 != -1 && c->switch_request2 != i && c->switch_delay <= 0
> + && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> + && c->switch_timestamp != AV_NOPTS_VALUE) {
> + change_discard_flags(pls, AVDISCARD_ALL);
> + }
> + }
> +
> cur_needed = playlist_needed(c->playlists[i]);
>
> if (pls->broken) {
> @@ -2077,6 +2281,11 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
> changed = 1;
> pls->cur_seq_no = select_cur_seq_no(c, pls);
> pls->pb.eof_reached = 0;
> + if (c->abr) {
> + pls->cur_seq_no = select_cur_seq_no(c, pls) + 1;
> + avio_flush(&pls->pb);
> + avformat_flush(pls->ctx);
> + }
> if (c->cur_timestamp != AV_NOPTS_VALUE) {
> /* catch up */
> pls->seek_timestamp = c->cur_timestamp;
> @@ -2084,7 +2293,7 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
> pls->seek_stream_index = -1;
> }
> av_log(s, AV_LOG_INFO, "Now receiving playlist %d, segment %d\n", i, pls->cur_seq_no);
> - } else if (first && !cur_needed && pls->needed) {
> + } else if ((first || c->abr) && !cur_needed && pls->needed) {
> ff_format_io_close(pls->parent, &pls->input);
> pls->input_read_done = 0;
> ff_format_io_close(pls->parent, &pls->input_next);
> @@ -2093,6 +2302,9 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
> changed = 1;
> av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);
> }
> +
> + if (c->abr && changed && cur_needed)
> + c->cur_pls = i;
> }
> return changed;
> }
> @@ -2167,7 +2379,13 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
> /* audio elementary streams are id3 timestamped */
> fill_timing_for_id3_timestamped_stream(pls);
> }
> -
> + if (c->abr && c->first_timestamp != AV_NOPTS_VALUE && c->delta_timestamp == AV_NOPTS_VALUE && !i) {
> + c->delta_timestamp = av_rescale_q(pls->pkt.dts,
> + get_timebase(pls), AV_TIME_BASE_Q) - c->first_timestamp + 1;
> + av_log(s, AV_LOG_VERBOSE, "[abr] delta_timestamp=%ld, %ld, %ld\n",
> + c->delta_timestamp, c->first_timestamp,av_rescale_q(pls->pkt.dts,
> + get_timebase(pls), AV_TIME_BASE_Q));
> + }
> if (c->first_timestamp == AV_NOPTS_VALUE &&
> pls->pkt.dts != AV_NOPTS_VALUE)
> c->first_timestamp = av_rescale_q(pls->pkt.dts,
> @@ -2375,6 +2593,8 @@ static int hls_probe(const AVProbeData *p)
> #define OFFSET(x) offsetof(HLSContext, x)
> #define FLAGS AV_OPT_FLAG_DECODING_PARAM
> static const AVOption hls_options[] = {
> + {"abr", "enable abr to switch streams",
> + OFFSET(abr), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS },
> {"live_start_index", "segment index to start live streams at (negative values are from the end)",
> OFFSET(live_start_index), AV_OPT_TYPE_INT, {.i64 = -3}, INT_MIN, INT_MAX, FLAGS},
> {"allowed_extensions", "List of file extensions that hls is allowed to access",
> --
> 2.27.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list