[FFmpeg-devel] [PATCH] avcodec/decode: clean-up if get_hw_frames_parameters fails

[Thomas Guillem]

Ping.

On Mon, Sep 30, 2024, at 13:44, Thomas Guillem via ffmpeg-devel wrote:
> cf. https://trac.ffmpeg.org/ticket/11013
>
>
> On Mon, Sep 30, 2024, at 10:27, Thomas Guillem via ffmpeg-devel wrote:
>> Fixes the following assert:
>>
>> [00007f1df83d17e0] vaapi generic error: 
>> avcodec_get_hw_frames_parameters failed: -22
>> Assertion p_dst->hwaccel_threadsafe || (!dst->hwaccel && 
>> !dst->internal->hwaccel_priv_data) failed at 
>> libavcodec/pthread_frame.c:349
>>
>> Reproduced from VLC with VAAPI, when fallbacking from hw to sw.
>> ---
>>  libavcodec/decode.c | 7 +++++++
>>  1 file changed, 7 insertions(+)
>>
>> diff --git a/libavcodec/decode.c b/libavcodec/decode.c
>> index c331bb8596..e90a6630e1 100644
>> --- a/libavcodec/decode.c
>> +++ b/libavcodec/decode.c
>> @@ -19,6 +19,7 @@
>>   */
>> 
>>  #include <stdint.h>
>> +#include <stdbool.h>
>>  #include <string.h>
>> 
>>  #include "config.h"
>> @@ -1173,6 +1174,7 @@ int 
>> avcodec_get_hw_frames_parameters(AVCodecContext *avctx,
>>      const AVCodecHWConfigInternal *hw_config;
>>      const FFHWAccel *hwa;
>>      int i, ret;
>> +    bool clean_priv_data = false;
>> 
>>      for (i = 0;; i++) {
>>          hw_config = ffcodec(avctx->codec)->hw_configs[i];
>> @@ -1197,6 +1199,7 @@ int 
>> avcodec_get_hw_frames_parameters(AVCodecContext *avctx,
>>              av_buffer_unref(&frames_ref);
>>              return AVERROR(ENOMEM);
>>          }
>> +        clean_priv_data = true;
>>      }
>> 
>>      ret = hwa->frame_params(avctx, frames_ref);
>> @@ -1217,6 +1220,10 @@ int 
>> avcodec_get_hw_frames_parameters(AVCodecContext *avctx,
>> 
>>          *out_frames_ref = frames_ref;
>>      } else {
>> +        if (clean_priv_data) {
>> +            av_freep(avctx->internal->hwaccel_priv_data);
>> +            avctx->internal->hwaccel_priv_data = NULL;
>> +        }
>>          av_buffer_unref(&frames_ref);
>>      }
>>      return ret;
>> -- 
>> 2.45.2
>>
>> _______________________________________________
>> ffmpeg-devel mailing list
>> ffmpeg-devel at ffmpeg.org
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>>
>> To unsubscribe, visit link above, or email
>> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".