[FFmpeg-devel] [PATCH] Add wayland support for VAAPI

[David Fort]

Wayland environment became quite popular with gnome 3. This patch adds the ability to
initialize the VAAPI accelerator from a wayland display.

Signed-off-by: David Fort <contact at hardening-consulting.com>
---
 configure                   |  4 ++++
 libavutil/hwcontext_vaapi.c | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/configure b/configure
index 6ca919b..b2e8b1c 100755
--- a/configure
+++ b/configure
@@ -2072,6 +2072,7 @@ HAVE_LIST="
     threads
     vaapi_drm
     vaapi_x11
+    vaapi_wayland
     vdpau_x11
     winrt
 "
@@ -6132,6 +6133,9 @@ enabled vaapi &&
     disable vaapi
 
 enabled vaapi &&
+    check_lib vaapi_wayland "va/va.h va/va_wayland.h" vaGetDisplayWl -lva -lva-wayland -lwayland-client
+
+enabled vaapi &&
     check_lib vaapi_drm "va/va.h va/va_drm.h" vaGetDisplayDRM -lva -lva-drm
 
 enabled vaapi &&
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 3970726..5fb607f 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -24,6 +24,9 @@
 #if HAVE_VAAPI_DRM
 #   include <va/va_drm.h>
 #endif
+#if HAVE_VAAPI_WAYLAND
+#   include <va/va_wayland.h>
+#endif
 
 #include <fcntl.h>
 #if HAVE_UNISTD_H
@@ -47,6 +50,9 @@ typedef struct VAAPIDevicePriv {
 #endif
 
     int drm_fd;
+#if HAVE_VAAPI_WAYLAND
+    struct wl_display *wl_display;
+#endif
 } VAAPIDevicePriv;
 
 typedef struct VAAPISurfaceFormat {
@@ -905,6 +911,10 @@ static void vaapi_device_free(AVHWDeviceContext *ctx)
     if (hwctx->display)
         vaTerminate(hwctx->display);
 
+#if HAVE_VAAPI_WAYLAND
+    if (priv->wl_display)
+        wl_display_disconnect(priv->wl_display);
+#endif
 #if HAVE_VAAPI_X11
     if (priv->x11_display)
         XCloseDisplay(priv->x11_display);
@@ -934,6 +944,29 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     ctx->user_opaque = priv;
     ctx->free        = vaapi_device_free;
 
+#if HAVE_VAAPI_WAYLAND
+    // Try to open the device as a wayland display.
+    if (!display && !(device && device[0] == '/')) {
+		priv->wl_display = wl_display_connect(device);
+		if (!priv->wl_display) {
+			av_log(ctx, AV_LOG_VERBOSE, "Cannot open wayland display "
+							   "%s.\n", device);
+		} else {
+			wl_display_roundtrip (priv->wl_display);
+
+			display = vaGetDisplayWl(priv->wl_display);
+			if (!display) {
+				av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
+					   "from wayland display %s.\n", device);
+				return AVERROR_UNKNOWN;
+			}
+
+			av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+				   "wayland display %s.\n", device);
+		}
+	}
+#endif
+
 #if HAVE_VAAPI_X11
     if (!display && !(device && device[0] == '/')) {
         // Try to open the device as an X11 display.
-- 
2.7.4