[FFmpeg-devel] [PATCH v2 2/2] avutil/hwcontext_vaapi: return ENOSYS for unsupported operation

[Zhao Zhili]

From: Zhao Zhili <zhilizhao at tencent.com>

av_hwframe_transfer_data try with src_ctx first. If the operation
failed with AVERROR(ENOSYS), it will try again with dst_ctx. Return
AVERROR(EINVAL) makes the second step being skipped.
---
 libavutil/hwcontext_vaapi.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 558fed94c6..12bc95119a 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -217,7 +217,7 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
             return 0;
         }
     }
-    return AVERROR(EINVAL);
+    return AVERROR(ENOSYS);
 }
 
 static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
@@ -817,7 +817,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     err = vaapi_get_image_format(hwfc->device_ctx, dst->format, &image_format);
     if (err < 0) {
         // Requested format is not a valid output format.
-        return AVERROR(EINVAL);
+        return err;
     }
 
     map = av_malloc(sizeof(*map));
@@ -992,7 +992,7 @@ static int vaapi_map_to_memory(AVHWFramesContext *hwfc, AVFrame *dst,
     if (dst->format != AV_PIX_FMT_NONE) {
         err = vaapi_get_image_format(hwfc->device_ctx, dst->format, NULL);
         if (err < 0)
-            return AVERROR(ENOSYS);
+            return err;
     }
 
     err = vaapi_map_frame(hwfc, dst, src, flags);
-- 
2.34.1