[vlc-devel] [PATCH] vaapi: support using DRM in the gl output

[Mathieu Velten]

Fixes #18445
---
 modules/codec/avcodec/vaapi.c                 | 92 ++++++++++++++-------------
 modules/video_output/Makefile.am              |  4 ++
 modules/video_output/opengl/converter_vaapi.c | 39 +++++++++++-
 3 files changed, 89 insertions(+), 46 deletions(-)

diff --git a/modules/codec/avcodec/vaapi.c b/modules/codec/avcodec/vaapi.c
index 4f46802400..408954a7b2 100644
--- a/modules/codec/avcodec/vaapi.c
+++ b/modules/codec/avcodec/vaapi.c
@@ -295,59 +295,61 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     memset(sys, 0, sizeof (*sys));
 
     /* */
-    sys->hw_ctx.display = NULL;
     sys->hw_ctx.config_id = VA_INVALID_ID;
     sys->hw_ctx.context_id = VA_INVALID_ID;
     sys->pool = NULL;
 
-    /* Create a VA display */
+    sys->hw_ctx.display = vlc_vaapi_GetInstance();
+    if (sys->hw_ctx.display == NULL) {
+        /* Create a VA display */
 #ifdef VLC_VA_BACKEND_XLIB
-    sys->p_display_x11 = XOpenDisplay(NULL);
-    if (!sys->p_display_x11)
-    {
-        msg_Err(va, "Could not connect to X server");
-        goto error;
-    }
-
-    sys->hw_ctx.display = vaGetDisplay(sys->p_display_x11);
+        sys->p_display_x11 = XOpenDisplay(NULL);
+        if (!sys->p_display_x11)
+        {
+            msg_Err(va, "Could not connect to X server");
+            goto error;
+        }
+
+        sys->hw_ctx.display = vaGetDisplay(sys->p_display_x11);
 #endif
 #ifdef VLC_VA_BACKEND_DRM
-    static const char drm_device_paths[][20] = {
-        "/dev/dri/renderD128",
-        "/dev/dri/card0"
-    };
-
-    for (int i = 0; ARRAY_SIZE(drm_device_paths); i++)
-    {
-        sys->drm_fd = vlc_open(drm_device_paths[i], O_RDWR);
-        if (sys->drm_fd < 0)
-            continue;
-
-        sys->hw_ctx.display = vaGetDisplayDRM(sys->drm_fd);
-        if (sys->hw_ctx.display)
-            break;
-
-        vlc_close(sys->drm_fd);
-        sys->drm_fd = -1;
-    }
+        static const char drm_device_paths[][20] = {
+            "/dev/dri/renderD128",
+            "/dev/dri/card0"
+        };
+
+        for (size_t i = 0; i < ARRAY_SIZE(drm_device_paths); i++)
+        {
+            sys->drm_fd = vlc_open(drm_device_paths[i], O_RDWR);
+            if (sys->drm_fd < 0)
+                continue;
+
+            sys->hw_ctx.display = vaGetDisplayDRM(sys->drm_fd);
+            if (sys->hw_ctx.display)
+                break;
+
+            vlc_close(sys->drm_fd);
+            sys->drm_fd = -1;
+        }
 #endif
-    if (sys->hw_ctx.display == NULL)
-    {
-        msg_Err(va, "Could not get a VAAPI device");
-        goto error;
-    }
-
-    if (vlc_vaapi_Initialize(o, sys->hw_ctx.display))
-    {
-        sys->hw_ctx.display = NULL;
-        goto error;
-    }
-
-    if (vlc_vaapi_SetInstance(sys->hw_ctx.display))
-    {
-        msg_Err(va, "VAAPI instance already in use");
-        sys->hw_ctx.display = NULL;
-        goto error;
+        if (sys->hw_ctx.display == NULL)
+        {
+            msg_Err(va, "Could not get a VAAPI device");
+            goto error;
+        }
+
+        if (vlc_vaapi_Initialize(o, sys->hw_ctx.display))
+        {
+            sys->hw_ctx.display = NULL;
+            goto error;
+        }
+
+        if (vlc_vaapi_SetInstance(sys->hw_ctx.display))
+        {
+            msg_Err(va, "VAAPI instance already in use");
+            sys->hw_ctx.display = NULL;
+            goto error;
+        }
     }
 
     sys->hw_ctx.config_id =
diff --git a/modules/video_output/Makefile.am b/modules/video_output/Makefile.am
index 2e7c5d8ba7..18406969cf 100644
--- a/modules/video_output/Makefile.am
+++ b/modules/video_output/Makefile.am
@@ -40,6 +40,10 @@ OPENGL_COMMONLIBS += $(LIBVA_X11_LIBS) $(X_LIBS) $(X_PRE_LIBS) -lX11
 OPENGL_COMMONCFLAGS += -DHAVE_VA_X11
 endif
 endif
+if HAVE_VAAPI_DRM
+OPENGL_COMMONLIBS += $(LIBVA_DRM_LIBS)
+OPENGL_COMMONCFLAGS += -DHAVE_VA_DRM
+endif
 endif
 endif
 
diff --git a/modules/video_output/opengl/converter_vaapi.c b/modules/video_output/opengl/converter_vaapi.c
index 73302fc8d9..ade5d539e4 100644
--- a/modules/video_output/opengl/converter_vaapi.c
+++ b/modules/video_output/opengl/converter_vaapi.c
@@ -28,6 +28,8 @@
 
 #include <assert.h>
 
+#include <vlc_fs.h>
+
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 #include <va/va_drmcommon.h>
@@ -41,6 +43,13 @@
 # include <vlc_xlib.h>
 #endif
 
+#ifdef HAVE_VA_DRM
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <va/va_drm.h>
+#endif
+
 #if defined(USE_OPENGL_ES2)
 #   include <GLES2/gl2ext.h>
 #endif
@@ -53,6 +62,7 @@ struct priv
 #ifdef HAVE_VA_X11
     Display *x11dpy;
 #endif
+    int drm_fd;
 
     unsigned fourcc;
     EGLint drm_fourccs[3];
@@ -232,6 +242,9 @@ tc_vaegl_release(const opengl_tex_converter_t *tc)
     if (priv->x11dpy != NULL)
         XCloseDisplay(priv->x11dpy);
 #endif
+    if (priv->drm_fd >= 0)
+        vlc_close(priv->drm_fd);
+
     free(tc->priv);
 }
 
@@ -276,6 +289,7 @@ tc_vaegl_init(opengl_tex_converter_t *tc, VADisplay *vadpy)
 
     priv->vadpy = vadpy;
     priv->fourcc = 0;
+    priv->drm_fd = -1;
 
     if (!HasExtension(tc->glexts, "GL_OES_EGL_image"))
         goto error;
@@ -358,6 +372,30 @@ opengl_tex_converter_vaapi_init(opengl_tex_converter_t *tc)
      || tc->gl->egl.destroyImageKHR == NULL)
         return VLC_EGENERIC;
 
+    tc->pf_get_pool = tc_va_get_pool;
+
+#ifdef HAVE_VA_DRM
+    static const char drm_device_paths[][20] = {
+        "/dev/dri/renderD128",
+        "/dev/dri/card0"
+    };
+
+    for (size_t i = 0; i < ARRAY_SIZE(drm_device_paths); i++)
+    {
+        int drm_fd = vlc_open(drm_device_paths[i], O_RDWR);
+        if (drm_fd < 0)
+            continue;
+
+        if (!tc_vaegl_init(tc, vaGetDisplayDRM(drm_fd))) {
+            struct priv *priv = tc->priv;
+            priv->drm_fd = drm_fd;
+            return VLC_SUCCESS;
+        }
+
+        vlc_close(drm_fd);
+    }
+#endif
+
     switch (tc->gl->surface->type)
     {
 #ifdef HAVE_VA_X11
@@ -388,7 +426,6 @@ opengl_tex_converter_vaapi_init(opengl_tex_converter_t *tc)
         default:
             return VLC_EGENERIC;
     }
-    tc->pf_get_pool = tc_va_get_pool;
 
     return VLC_SUCCESS;
 }
-- 
2.13.0