[vlc-devel] [PATCH] vaapi: refactor display handling

Mathieu Velten matmaul at gmail.com
Wed Jun 28 21:22:06 CEST 2017


The code to initiate the VADisplay has been moved in hw/vaapi.
You can use --vaapi-display-type to force a type of display.

Fixes #18445
---
 modules/codec/Makefile.am                     |  33 ++--
 modules/codec/avcodec/avcodec.c               |  11 ++
 modules/codec/avcodec/avcodec.h               |   4 +
 modules/codec/avcodec/vaapi.c                 | 130 ++++------------
 modules/hw/vaapi/Makefile.am                  |  35 +++--
 modules/hw/vaapi/display.c                    | 216 ++++++++++++++++++++++++++
 modules/hw/vaapi/vlc_vaapi.c                  |  17 +-
 modules/hw/vaapi/vlc_vaapi.h                  |  31 +++-
 modules/video_output/Makefile.am              |  16 +-
 modules/video_output/opengl/converter_vaapi.c |  88 ++++-------
 10 files changed, 359 insertions(+), 222 deletions(-)
 create mode 100644 modules/hw/vaapi/display.c

diff --git a/modules/codec/Makefile.am b/modules/codec/Makefile.am
index c3bb6d207a..fea4de7744 100644
--- a/modules/codec/Makefile.am
+++ b/modules/codec/Makefile.am
@@ -374,33 +374,18 @@ endif
 
 ### avcodec hardware acceleration ###
 
-libvaapi_drm_plugin_la_SOURCES = \
-	codec/avcodec/vaapi.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
-libvaapi_drm_plugin_la_CPPFLAGS = $(AM_CPPFLAGS) -DVLC_VA_BACKEND_DRM
-libvaapi_drm_plugin_la_CFLAGS = $(AM_CFLAGS) \
-	$(LIBVA_DRM_CFLAGS) $(AVCODEC_CFLAGS)
-libvaapi_drm_plugin_la_LIBADD = $(LIBVA_DRM_LIBS) libvlc_vaapi_instance.la
-libvaapi_x11_plugin_la_SOURCES = \
-	codec/avcodec/vaapi.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
-libvaapi_x11_plugin_la_CPPFLAGS = $(AM_CPPFLAGS) -DVLC_VA_BACKEND_XLIB
-libvaapi_x11_plugin_la_CFLAGS = $(AM_CFLAGS) \
-	$(LIBVA_X11_CFLAGS) $(X_CFLAGS) $(AVCODEC_CFLAGS)
-libvaapi_x11_plugin_la_LIBADD = $(LIBVA_X11_LIBS) $(X_LIBS) $(X_PRE_LIBS) \
-	-lX11 libvlc_vaapi_instance.la
-libvaapi_dr_plugin_la_SOURCES = \
-	codec/avcodec/vaapi.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
+libvaapi_plugin_la_SOURCES = \
+	codec/avcodec/vaapi.c
+libvaapi_plugin_la_CPPFLAGS = $(AM_CPPFLAGS)
+libvaapi_plugin_la_CFLAGS = $(AM_CFLAGS) $(AVCODEC_CFLAGS)
+libvaapi_plugin_la_LIBADD = $(LIBVA_LIBS) libvlc_vaapi.la
+libvaapi_dr_plugin_la_SOURCES = $(libvaapi_plugin_la_SOURCES)
 libvaapi_dr_plugin_la_CPPFLAGS = $(AM_CPPFLAGS) -DVLC_VA_BACKEND_DR
 libvaapi_dr_plugin_la_CFLAGS = $(AM_CFLAGS) $(AVCODEC_CFLAGS)
-libvaapi_dr_plugin_la_LIBADD = $(LIBVA_LIBS) libvlc_vaapi_instance.la
-if HAVE_AVCODEC_VAAPI
-if HAVE_VAAPI_DRM
-codec_LTLIBRARIES += libvaapi_drm_plugin.la
-endif
-if HAVE_VAAPI_X11
-codec_LTLIBRARIES += libvaapi_x11_plugin.la
-endif
+libvaapi_dr_plugin_la_LIBADD = $(LIBVA_LIBS) libvlc_vaapi.la
+
+codec_LTLIBRARIES += libvaapi_plugin.la
 codec_LTLIBRARIES += libvaapi_dr_plugin.la
-endif
 
 libdxva2_plugin_la_SOURCES = \
 	codec/avcodec/dxva2.c codec/avcodec/directx_va.c codec/avcodec/directx_va.h \
diff --git a/modules/codec/avcodec/avcodec.c b/modules/codec/avcodec/avcodec.c
index f286a3adaf..17873e0743 100644
--- a/modules/codec/avcodec/avcodec.c
+++ b/modules/codec/avcodec/avcodec.c
@@ -55,6 +55,13 @@ static const char *const enc_hq_list_text[] = {
     N_("rd"), N_("bits"), N_("simple") };
 #endif
 
+static const char * const ppsz_vaapi_display_type[] = {
+    "auto", "drm", "x11", "wl"
+};
+static const char * const ppsz_vaapi_display_type_text[] = {
+    N_("Auto"), N_("DRM"), N_("X11"), N_("Wayland")
+};
+
 #ifdef MERGE_FFMPEG
 # include "../../demux/avformat/avformat.h"
 # include "../../access/avio.h"
@@ -129,6 +136,10 @@ vlc_module_begin ()
     add_string( "avcodec-codec", NULL, CODEC_TEXT, CODEC_LONGTEXT, true )
     add_obsolete_bool( "ffmpeg-hw" ) /* removed since 2.1.0 */
     add_module( "avcodec-hw", "hw decoder", "any", HW_TEXT, HW_LONGTEXT, false )
+    add_string( "vaapi-display-type", "auto",
+                VAAPI_DISPLAY_TYPE_TEXT, VAAPI_DISPLAY_TYPE_LONGTEXT, false )
+        change_string_list( ppsz_vaapi_display_type, ppsz_vaapi_display_type_text )
+        change_safe()
 #if defined(FF_THREAD_FRAME)
     add_obsolete_integer( "ffmpeg-threads" ) /* removed since 2.1.0 */
     add_integer( "avcodec-threads", 0, THREADS_TEXT, THREADS_LONGTEXT, true );
diff --git a/modules/codec/avcodec/avcodec.h b/modules/codec/avcodec/avcodec.h
index aec123dbef..70895e784a 100644
--- a/modules/codec/avcodec/avcodec.h
+++ b/modules/codec/avcodec/avcodec.h
@@ -113,6 +113,10 @@ int ffmpeg_OpenCodec( decoder_t *p_dec, AVCodecContext *, const AVCodec * );
 #define HW_TEXT N_("Hardware decoding")
 #define HW_LONGTEXT N_("This allows hardware decoding when available.")
 
+#define VAAPI_DISPLAY_TYPE_TEXT N_("VAAPI display type")
+#define VAAPI_DISPLAY_TYPE_LONGTEXT N_( \
+    "Type of display used for VAAPI.")
+
 #define THREADS_TEXT N_( "Threads" )
 #define THREADS_LONGTEXT N_( "Number of threads used for decoding, 0 meaning auto" )
 
diff --git a/modules/codec/avcodec/vaapi.c b/modules/codec/avcodec/vaapi.c
index 2a0f1f58ce..8976f18bca 100644
--- a/modules/codec/avcodec/vaapi.c
+++ b/modules/codec/avcodec/vaapi.c
@@ -34,17 +34,6 @@
 #include <vlc_picture.h>
 #include <vlc_picture_pool.h>
 
-#ifdef VLC_VA_BACKEND_XLIB
-# include <vlc_xlib.h>
-# include <va/va_x11.h>
-#endif
-#ifdef VLC_VA_BACKEND_DRM
-# include <sys/types.h>
-# include <sys/stat.h>
-# include <fcntl.h>
-# include <vlc_fs.h>
-# include <va/va_drm.h>
-#endif
 #include <libavcodec/avcodec.h>
 #include <libavcodec/vaapi.h>
 
@@ -54,12 +43,7 @@
 
 struct vlc_va_sys_t
 {
-#ifdef VLC_VA_BACKEND_XLIB
-    Display  *p_display_x11;
-#endif
-#ifdef VLC_VA_BACKEND_DRM
-    int       drm_fd;
-#endif
+    vlc_vaapi_dpy_t vaapi_dpy;
     struct vaapi_context hw_ctx;
 
 #ifndef VLC_VA_BACKEND_DR /* XLIB or DRM */
@@ -147,7 +131,9 @@ static void DeleteDR(vlc_va_t *va, void *hwctx)
 
     vlc_vaapi_DestroyContext(o, sys->hw_ctx.display, sys->hw_ctx.context_id);
     vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
-    vlc_vaapi_ReleaseInstance(sys->hw_ctx.display);
+    vlc_vaapi_ReleaseInstance(sys->vaapi_dpy.p_va_dpy);
+    vlc_vaapi_ReleaseDisplay(sys->vaapi_dpy);
+
     free(sys);
 }
 
@@ -219,6 +205,7 @@ error:
             vlc_vaapi_DestroyContext(o, sys->hw_ctx.display, sys->hw_ctx.context_id);
         if (sys->hw_ctx.config_id != VA_INVALID_ID)
             vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
+        vlc_vaapi_ReleaseDisplay(sys->vaapi_dpy);
         free(sys);
     }
     vlc_vaapi_ReleaseInstance(va_dpy);
@@ -255,12 +242,7 @@ static void Delete(vlc_va_t *va, void **hwctx)
     vlc_vaapi_DestroyContext(o, sys->hw_ctx.display, sys->hw_ctx.context_id);
     vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
     vlc_vaapi_ReleaseInstance(sys->hw_ctx.display);
-#ifdef VLC_VA_BACKEND_XLIB
-    XCloseDisplay(sys->p_display_x11);
-#endif
-#ifdef VLC_VA_BACKEND_DRM
-    vlc_close(sys->drm_fd);
-#endif
+    vlc_vaapi_ReleaseDisplay(sys->vaapi_dpy);
     free(sys);
 }
 
@@ -274,14 +256,6 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     (void) p_sys;
     vlc_object_t *o = VLC_OBJECT(va);
 
-#ifdef VLC_VA_BACKEND_XLIB
-    if (!vlc_xlib_init(o))
-    {
-        msg_Warn(va, "Ignoring VA-X11 API");
-        return VLC_EGENERIC;
-    }
-#endif
-
     VAProfile i_profile;
     unsigned count;
     if (GetVaProfile(ctx, &i_profile, &count) != VLC_SUCCESS)
@@ -300,54 +274,23 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     sys->hw_ctx.context_id = VA_INVALID_ID;
     sys->pool = NULL;
 
-    /* Create a VA display */
-#ifdef VLC_VA_BACKEND_XLIB
-    sys->p_display_x11 = XOpenDisplay(NULL);
-    if (!sys->p_display_x11)
-    {
-        msg_Err(va, "Could not connect to X server");
-        goto error;
-    }
-
-    sys->hw_ctx.display = vaGetDisplay(sys->p_display_x11);
-#endif
-#ifdef VLC_VA_BACKEND_DRM
-    static const char const *drm_device_paths[] = {
-        "/dev/dri/renderD128",
-        "/dev/dri/card0"
-    };
-
-    for (int i = 0; ARRAY_SIZE(drm_device_paths); i++)
-    {
-        sys->drm_fd = vlc_open(drm_device_paths[i], O_RDWR);
-        if (sys->drm_fd < 0)
-            continue;
-
-        sys->hw_ctx.display = vaGetDisplayDRM(sys->drm_fd);
-        if (sys->hw_ctx.display)
-            break;
-
-        vlc_close(sys->drm_fd);
-        sys->drm_fd = -1;
-    }
-#endif
-    if (sys->hw_ctx.display == NULL)
-    {
-        msg_Err(va, "Could not get a VAAPI device");
-        goto error;
-    }
-
-    if (vlc_vaapi_Initialize(o, sys->hw_ctx.display))
-    {
-        sys->hw_ctx.display = NULL;
-        goto error;
-    }
-
-    if (vlc_vaapi_SetInstance(sys->hw_ctx.display))
-    {
-        msg_Err(va, "VAAPI instance already in use");
-        sys->hw_ctx.display = NULL;
-        goto error;
+    sys->hw_ctx.display = vlc_vaapi_GetInstance();
+    if (sys->hw_ctx.display == NULL) {
+        // TODO how to find x11_dpy_name and/or wl_display ?
+        sys->vaapi_dpy = vlc_vaapi_InitializeDisplay(o, NULL, NULL);
+        if (sys->vaapi_dpy.p_va_dpy == NULL)
+        {
+            msg_Err(va, "Could not get a VAAPI device");
+            goto error;
+        }
+        sys->hw_ctx.display = sys->vaapi_dpy.p_va_dpy;
+
+        if (vlc_vaapi_SetInstance(sys->hw_ctx.display))
+        {
+            msg_Err(va, "VAAPI instance already in use");
+            sys->hw_ctx.display = NULL;
+            goto error;
+        }
     }
 
     sys->hw_ctx.config_id =
@@ -393,34 +336,25 @@ error:
         vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
     if (sys->hw_ctx.display != NULL)
         vlc_vaapi_ReleaseInstance(sys->hw_ctx.display);
-#ifdef VLC_VA_BACKEND_XLIB
-    if( sys->p_display_x11 != NULL)
-        XCloseDisplay(sys->p_display_x11);
-#endif
-#ifdef VLC_VA_BACKEND_DRM
-    if( sys->drm_fd != -1 )
-        vlc_close(sys->drm_fd);
-#endif
+
+    vlc_vaapi_ReleaseDisplay(sys->vaapi_dpy);
     free(sys);
     return VLC_EGENERIC;
 }
 #endif
 
 vlc_module_begin ()
-#if defined (VLC_VA_BACKEND_XLIB)
-    set_description( N_("VA-API video decoder via X11") )
-    set_capability( "hw decoder", 0 )
-    set_callbacks( Create, Delete )
-#elif defined (VLC_VA_BACKEND_DRM)
-    set_description( N_("VA-API video decoder via DRM") )
-    set_capability( "hw decoder", 0 )
-    set_callbacks( Create, Delete )
-#elif defined (VLC_VA_BACKEND_DR)
+#ifdef VLC_VA_BACKEND_DR
     set_description( N_("VA-API direct video decoder") )
     set_capability( "hw decoder", 100 )
     set_callbacks( CreateDR, DeleteDR )
+    add_shortcut( "vaapi-dr" )
+#else
+    set_description( N_("VA-API video decoder") )
+    set_capability( "hw decoder", 0 )
+    set_callbacks( Create, Delete )
+    add_shortcut( "vaapi" )
 #endif
     set_category( CAT_INPUT )
     set_subcategory( SUBCAT_INPUT_VCODEC )
-    add_shortcut( "vaapi" )
 vlc_module_end ()
diff --git a/modules/hw/vaapi/Makefile.am b/modules/hw/vaapi/Makefile.am
index d0e595e7c6..0b1a7514bc 100644
--- a/modules/hw/vaapi/Makefile.am
+++ b/modules/hw/vaapi/Makefile.am
@@ -1,23 +1,38 @@
 vaapidir = $(pluginsdir)/vaapi
 
-libvlc_vaapi_instance_la_SOURCES = hw/vaapi/instance.c hw/vaapi/vlc_vaapi.h
-libvlc_vaapi_instance_la_CFLAGS = $(LIBVA_CFLAGS)
-libvlc_vaapi_instance_la_LIBADD = $(LIBVA_LIBS) $(LIBPTHREAD)
-libvlc_vaapi_instance_la_LDFLAGS = -no-undefined	\
-				   -version-info 0:0:0
+if HAVE_VAAPI_WL
+LIBVA_LIBS += $(LIBVA_WL_LIBS)
+LIBVA_CFLAGS += -DHAVE_VA_WL
+endif
+if HAVE_VAAPI_X11
+LIBVA_LIBS += $(LIBVA_X11_LIBS) $(X_LIBS) $(X_PRE_LIBS) -lX11
+LIBVA_CFLAGS += -DHAVE_VA_X11
+endif
+if HAVE_VAAPI_DRM
+LIBVA_LIBS += $(LIBVA_DRM_LIBS)
+LIBVA_CFLAGS += -DHAVE_VA_DRM
+endif
+
+libvlc_vaapi_la_SOURCES = hw/vaapi/vlc_vaapi.c hw/vaapi/instance.c hw/vaapi/display.c hw/vaapi/vlc_vaapi.h
+libvlc_vaapi_la_CFLAGS = $(LIBVA_CFLAGS)
+libvlc_vaapi_la_LIBADD = $(LIBVA_LIBS) $(LTLIBVLCCORE) $(LIBPTHREAD)
+libvlc_vaapi_la_LDFLAGS = \
+	-no-undefined \
+	-export-symbols-regex ^vlc_vaapi_ \
+	-version-info 0:0:0
 
-libvaapi_chroma_plugin_la_SOURCES = hw/vaapi/chroma.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h	\
+libvaapi_chroma_plugin_la_SOURCES = hw/vaapi/chroma.c hw/vaapi/vlc_vaapi.h	\
 				    video_chroma/copy.c video_chroma/copy.h
 libvaapi_chroma_plugin_la_CFLAGS = $(AM_CFLAGS) $(LIBVA_CFLAGS)
-libvaapi_chroma_plugin_la_LIBADD = $(AM_LIBADD) $(LIBVA_LIBS) libvlc_vaapi_instance.la
+libvaapi_chroma_plugin_la_LIBADD = $(AM_LIBADD) $(LIBVA_LIBS) libvlc_vaapi.la
 
-libvaapi_filters_plugin_la_SOURCES = hw/vaapi/filters.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
+libvaapi_filters_plugin_la_SOURCES = hw/vaapi/filters.c hw/vaapi/vlc_vaapi.h
 libvaapi_filters_plugin_la_CPPFLAGS = $(AM_CPPFLAGS)
 libvaapi_filters_plugin_la_CFLAGS = $(AM_CFLAGS) $(LIBVA_CFLAGS)
-libvaapi_filters_plugin_la_LIBADD = $(AM_LIBADD) $(LIBVA_LIBS) libvlc_vaapi_instance.la
+libvaapi_filters_plugin_la_LIBADD = $(AM_LIBADD) $(LIBVA_LIBS) libvlc_vaapi.la
 
 if HAVE_VAAPI
-pkglib_LTLIBRARIES += libvlc_vaapi_instance.la
+pkglib_LTLIBRARIES += libvlc_vaapi.la
 vaapi_LTLIBRARIES = libvaapi_chroma_plugin.la	\
 		    libvaapi_filters_plugin.la
 endif
diff --git a/modules/hw/vaapi/display.c b/modules/hw/vaapi/display.c
new file mode 100644
index 0000000000..c7baaad712
--- /dev/null
+++ b/modules/hw/vaapi/display.c
@@ -0,0 +1,216 @@
+/*****************************************************************************
+ * vlc_vaapi.c: VAAPI helper for VLC
+ *****************************************************************************
+ * Copyright (C) 2017 VLC authors, VideoLAN and VideoLabs
+ *
+ * Authors: Thomas Guillem <thomas at gllm.fr>
+ *          Petri Hintukainen <phintuka at gmail.com>
+ *          Victorien Le Couviour--Tuffet <victorien.lecouviour.tuffet at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <vlc_common.h>
+#include <vlc_fs.h>
+#include "vlc_vaapi.h"
+
+#include <va/va.h>
+
+#ifdef HAVE_VA_DRM
+# include <fcntl.h>
+# include <va/va_drm.h>
+#endif
+
+#ifdef HAVE_VA_X11
+# include <vlc_xlib.h>
+# include <va/va_x11.h>
+#endif
+
+#ifdef HAVE_VA_WL
+# include <va/va_wayland.h>
+#endif
+
+#pragma GCC visibility push(default)
+
+/*****************
+ * VAAPI display *
+ *****************/
+
+struct vlc_vaapi_dpy_priv_t
+{
+    int       drm_fd;
+#ifdef HAVE_VA_X11
+    Display  *p_x11_dpy;
+#endif
+};
+
+int major = 0, minor = 0;
+
+#ifdef HAVE_VA_DRM
+VADisplay
+initializeDisplayDRM(vlc_vaapi_dpy_priv_t* priv) {
+    static const char *drm_device_paths[] = {
+        "/dev/dri/renderD128",
+        "/dev/dri/card0"
+    };
+
+    for (size_t i = 0; i < ARRAY_SIZE(drm_device_paths); i++)
+    {
+        priv->drm_fd = vlc_open(drm_device_paths[i], O_RDWR);
+        if (priv->drm_fd < 0)
+            continue;
+
+        VADisplay p_va_dpy = vaGetDisplayDRM(priv->drm_fd);
+        if (p_va_dpy)
+        {
+            VAStatus s = vaInitialize(p_va_dpy, &major, &minor);
+            if (s == VA_STATUS_SUCCESS)
+            {
+                return p_va_dpy;
+            }
+        }
+        vlc_close(priv->drm_fd);
+    }
+    priv->drm_fd = -1;
+    return NULL;
+}
+#endif
+
+#ifdef HAVE_VA_X11
+VADisplay
+initializeDisplayX11(vlc_object_t *o, vlc_vaapi_dpy_priv_t* priv, char *x11_dpy_name) {
+    if (vlc_xlib_init(o))
+    {
+        priv->p_x11_dpy = XOpenDisplay(x11_dpy_name);
+        if (priv->p_x11_dpy)
+        {
+            VADisplay p_va_dpy = vaGetDisplay(priv->p_x11_dpy);
+            if (p_va_dpy)
+            {
+                VAStatus s = vaInitialize(p_va_dpy, &major, &minor);
+                if (s == VA_STATUS_SUCCESS)
+                {
+                    return p_va_dpy;
+                }
+            }
+            XCloseDisplay(priv->p_x11_dpy);
+        }
+    }
+    priv->p_x11_dpy = NULL;
+    return NULL;
+}
+#endif
+
+#ifdef HAVE_VA_WL
+VADisplay
+initializeDisplayWL(struct wl_display *wl_dpy) {
+    if (wl_dpy != NULL)
+    {
+        VADisplay p_va_dpy = vaGetDisplayWl(wl_dpy);
+        if (p_va_dpy)
+        {
+            VAStatus s = vaInitialize(p_va_dpy, &major, &minor);
+            if (s == VA_STATUS_SUCCESS)
+            {
+                return p_va_dpy;
+            }
+        }
+    }
+    return NULL;
+}
+#endif
+
+vlc_vaapi_dpy_t
+vlc_vaapi_InitializeDisplay(vlc_object_t *o, char *x11_dpy_name, void *wl_dpy)
+{
+    vlc_vaapi_dpy_t vaapi_dpy;
+    vlc_vaapi_dpy_priv_t *priv = NULL;
+    priv = malloc(sizeof *priv);
+    if (unlikely(priv == NULL))
+    {
+        return vaapi_dpy;
+    }
+    memset(priv, 0, sizeof (*priv));
+    priv->drm_fd = -1;
+    vaapi_dpy.priv = priv;
+
+    char *display_type = var_InheritString(o, "vaapi-display-type");
+
+    bool force = false;
+
+    if ((display_type != NULL) && strcmp(display_type, "") && strcmp(display_type, "auto"))
+        force = true;
+
+#ifdef HAVE_VA_DRM
+    if (!force || !strcmp("drm", display_type)) {
+        vaapi_dpy.p_va_dpy = initializeDisplayDRM(priv);
+        if (vaapi_dpy.p_va_dpy != NULL) {
+            msg_Info(o, "using DRM display");
+            return vaapi_dpy;
+        } else if (force) {
+            msg_Err(o, "vaapi: can't use DRM display");
+            return vaapi_dpy;
+        }
+    }
+#endif
+#ifdef HAVE_VA_X11
+    if (!force || !strcmp("x11", display_type)) {
+        vaapi_dpy.p_va_dpy = initializeDisplayX11(o, priv, x11_dpy_name);
+        if (vaapi_dpy.p_va_dpy != NULL) {
+            msg_Info(o, "vaapi: using X11 display");
+            return vaapi_dpy;
+        } else if (force) {
+            msg_Err(o, "vaapi: can't use X11 display");
+            return vaapi_dpy;
+        }
+    }
+#endif
+#ifdef HAVE_VA_WL
+    if (!force || !strcmp("wl", display_type)) {
+        vaapi_dpy.p_va_dpy = initializeDisplayWL((struct wl_display *)wl_dpy);
+        if (vaapi_dpy.p_va_dpy != NULL) {
+            msg_Info(o, "vaapi: using Wayland display");
+            return vaapi_dpy;
+        } else if (force) {
+            msg_Err(o, "vaapi: can't use Wayland display");
+            return vaapi_dpy;
+        }
+    }
+#endif
+
+    free(display_type);
+
+    return vaapi_dpy;
+}
+
+void
+vlc_vaapi_ReleaseDisplay(vlc_vaapi_dpy_t vaapi_dpy)
+{
+    vlc_vaapi_dpy_priv_t *priv = (vlc_vaapi_dpy_priv_t*)vaapi_dpy.priv;
+    if (priv != NULL) {
+#ifdef HAVE_VA_X11
+        if (priv->p_x11_dpy != NULL)
+            XCloseDisplay(priv->p_x11_dpy);
+#endif
+        if (priv->drm_fd >= 0)
+            vlc_close(priv->drm_fd);
+        free(priv);
+        vaapi_dpy.priv = NULL;
+    }
+}
diff --git a/modules/hw/vaapi/vlc_vaapi.c b/modules/hw/vaapi/vlc_vaapi.c
index 04514eb852..020b681b7e 100644
--- a/modules/hw/vaapi/vlc_vaapi.c
+++ b/modules/hw/vaapi/vlc_vaapi.c
@@ -40,6 +40,8 @@
 #include <vlc_fourcc.h>
 #include <vlc_picture_pool.h>
 
+#pragma GCC visibility push(default)
+
 /* This macro is designed to wrap any VA call, and in case of failure,
    display the VA error string then goto the 'error' label (which you must
    define). */
@@ -54,21 +56,6 @@
         }                                               \
     } while (0)
 
-/*****************
- * VAAPI display *
- *****************/
-
-int
-vlc_vaapi_Initialize(vlc_object_t *o, VADisplay dpy)
-{
-    int major = 0, minor = 0;
-    VA_CALL(o, vaInitialize, dpy, &major, &minor);
-    return VLC_SUCCESS;
-error:
-    vaTerminate(dpy);
-    return VLC_EGENERIC;
-}
-
 /**************************
  * VAAPI create & destroy *
  **************************/
diff --git a/modules/hw/vaapi/vlc_vaapi.h b/modules/hw/vaapi/vlc_vaapi.h
index 51dbc69a6c..b54cb6331e 100644
--- a/modules/hw/vaapi/vlc_vaapi.h
+++ b/modules/hw/vaapi/vlc_vaapi.h
@@ -31,6 +31,26 @@
 #include <vlc_fourcc.h>
 #include <vlc_picture_pool.h>
 
+/*****************
+ * VAAPI display *
+ *****************/
+
+typedef struct vlc_vaapi_dpy_t vlc_vaapi_dpy_t;
+typedef struct vlc_vaapi_dpy_priv_t vlc_vaapi_dpy_priv_t;
+
+struct vlc_vaapi_dpy_t
+{
+    VADisplay  *p_va_dpy;
+    vlc_vaapi_dpy_priv_t *priv;
+};
+
+vlc_vaapi_dpy_t
+vlc_vaapi_InitializeDisplay(vlc_object_t *o, char *x11_dpy_name, void *wl_dpy);
+
+void
+vlc_vaapi_ReleaseDisplay(vlc_vaapi_dpy_t);
+
+
 /**************************
  * VA instance management *
  **************************/
@@ -48,13 +68,6 @@ vlc_vaapi_GetInstance(void);
 void
 vlc_vaapi_ReleaseInstance(VADisplay *);
 
-/*****************
- * VAAPI display *
- *****************/
-
-int
-vlc_vaapi_Initialize(vlc_object_t *o, VADisplay va_dpy);
-
 /**************************
  * VAAPI create & destroy *
  **************************/
@@ -144,6 +157,10 @@ vlc_vaapi_QueryVideoProcPipelineCaps(vlc_object_t *o, VADisplay dpy,
                                      unsigned int num_filters,
                                      VAProcPipelineCaps *pipeline_caps);
 
+/*
+VADisplay
+vlc_vaapi_GetDisplay(vlc_object_t *o, Display x_dpy, wl_display * wl_dpy); */
+
 /*******************
  * VAAPI rendering *
  *******************/
diff --git a/modules/video_output/Makefile.am b/modules/video_output/Makefile.am
index 2e7c5d8ba7..cc41390f16 100644
--- a/modules/video_output/Makefile.am
+++ b/modules/video_output/Makefile.am
@@ -24,22 +24,10 @@ OPENGL_COMMONCFLAGS += -DVLCGL_CONV_CVPX
 endif
 if HAVE_EGL
 if HAVE_VAAPI
-OPENGL_COMMONLIBS += $(LIBVA_LIBS) $(LIBVA_EGL_LIBS) libvlc_vaapi_instance.la
+OPENGL_COMMONLIBS += $(LIBVA_LIBS) $(LIBVA_EGL_LIBS) libvlc_vaapi.la
 OPENGL_COMMONSOURCES += video_output/opengl/converter_vaapi.c \
-	hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
+	hw/vaapi/vlc_vaapi.h
 OPENGL_COMMONCFLAGS += -DVLCGL_CONV_VA
-if HAVE_WAYLAND_EGL
-if HAVE_VAAPI_WL
-OPENGL_COMMONLIBS += $(LIBVA_WL_LIBS)
-OPENGL_COMMONCFLAGS += -DHAVE_VA_WL
-endif
-endif
-if HAVE_VAAPI_X11
-if HAVE_XCB
-OPENGL_COMMONLIBS += $(LIBVA_X11_LIBS) $(X_LIBS) $(X_PRE_LIBS) -lX11
-OPENGL_COMMONCFLAGS += -DHAVE_VA_X11
-endif
-endif
 endif
 endif
 
diff --git a/modules/video_output/opengl/converter_vaapi.c b/modules/video_output/opengl/converter_vaapi.c
index 3f2ef14b59..e026fb8c32 100644
--- a/modules/video_output/opengl/converter_vaapi.c
+++ b/modules/video_output/opengl/converter_vaapi.c
@@ -24,6 +24,7 @@
 
 #include "internal.h"
 #include "../../hw/vaapi/vlc_vaapi.h"
+#include <vlc_common.h>
 #include <vlc_vout_window.h>
 
 #include <assert.h>
@@ -32,23 +33,11 @@
 #include <EGL/eglext.h>
 #include <va/va_drmcommon.h>
 
-#ifdef HAVE_VA_WL
-# include <va/va_wayland.h>
-#endif
-
-#ifdef HAVE_VA_X11
-# include <va/va_x11.h>
-# include <vlc_xlib.h>
-#endif
-
 struct priv
 {
-    VADisplay vadpy;
+    vlc_vaapi_dpy_t vadpy;
     VASurfaceID *va_surface_ids;
     PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES;
-#ifdef HAVE_VA_X11
-    Display *x11dpy;
-#endif
 
     unsigned fourcc;
     EGLint drm_fourccs[3];
@@ -70,9 +59,9 @@ vaegl_release_last_pic(const opengl_tex_converter_t *tc, struct priv *priv)
     for (unsigned i = 0; i < priv->last.va_image.num_planes; ++i)
         gl->egl.destroyImageKHR(gl, priv->last.egl_images[i]);
 
-    vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy, priv->last.va_image.buf);
+    vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy.p_va_dpy, priv->last.va_image.buf);
 
-    vlc_vaapi_DestroyImage(o, priv->vadpy, priv->last.va_image.image_id);
+    vlc_vaapi_DestroyImage(o, priv->vadpy.p_va_dpy, priv->last.va_image.image_id);
 
     picture_Release(priv->last.pic);
 }
@@ -144,7 +133,7 @@ tc_vaegl_update(const opengl_tex_converter_t *tc, GLuint *textures,
     }
     else
     {
-        if (vlc_vaapi_DeriveImage(o, priv->vadpy, vlc_vaapi_PicGetSurface(pic),
+        if (vlc_vaapi_DeriveImage(o, priv->vadpy.p_va_dpy, vlc_vaapi_PicGetSurface(pic),
                                   &va_image))
             goto error;
         release_image = true;
@@ -159,7 +148,7 @@ tc_vaegl_update(const opengl_tex_converter_t *tc, GLuint *textures,
         va_buffer_info = (VABufferInfo) {
             .mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME
         };
-        if (vlc_vaapi_AcquireBufferHandle(o, priv->vadpy, va_image.buf,
+        if (vlc_vaapi_AcquireBufferHandle(o, priv->vadpy.p_va_dpy, va_image.buf,
                                           &va_buffer_info))
             goto error;
         release_buffer_info = true;
@@ -204,12 +193,12 @@ error:
     if (release_image)
     {
         if (release_buffer_info)
-            vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy, va_image.buf);
+            vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy.p_va_dpy, va_image.buf);
 
         for (unsigned i = 0; i < 3 && egl_images[i] != NULL; ++i)
             gl->egl.destroyImageKHR(gl, egl_images[i]);
 
-        vlc_vaapi_DestroyImage(o, priv->vadpy, va_image.image_id);
+        vlc_vaapi_DestroyImage(o, priv->vadpy.p_va_dpy, va_image.image_id);
     }
     return VLC_EGENERIC;
 }
@@ -222,12 +211,10 @@ tc_vaegl_release(const opengl_tex_converter_t *tc)
     if (priv->last.pic != NULL)
         vaegl_release_last_pic(tc, priv);
 
-    vlc_vaapi_ReleaseInstance(priv->vadpy);
+    vlc_vaapi_ReleaseInstance(priv->vadpy.p_va_dpy);
+
+    vlc_vaapi_ReleaseDisplay(priv->vadpy);
 
-#ifdef HAVE_VA_X11
-    if (priv->x11dpy != NULL)
-        XCloseDisplay(priv->x11dpy);
-#endif
     free(tc->priv);
 }
 
@@ -261,16 +248,12 @@ tc_va_check_interop_blacklist(opengl_tex_converter_t *tc, VADisplay *vadpy)
 }
 
 static int
-tc_vaegl_init(opengl_tex_converter_t *tc, VADisplay *vadpy)
+tc_vaegl_init(opengl_tex_converter_t *tc, char *x11_dpy_name, struct wl_display *wl_dpy)
 {
-    if (vadpy == NULL)
-        return VLC_EGENERIC;
-
     struct priv *priv = tc->priv = calloc(1, sizeof(struct priv));
     if (unlikely(tc->priv == NULL))
         return VLC_ENOMEM;
 
-    priv->vadpy = vadpy;
     priv->fourcc = 0;
 
     if (!HasExtension(tc->glexts, "GL_OES_EGL_image"))
@@ -291,16 +274,20 @@ tc_vaegl_init(opengl_tex_converter_t *tc, VADisplay *vadpy)
     tc->pf_update  = tc_vaegl_update;
     tc->pf_release = tc_vaegl_release;
 
-    if (vlc_vaapi_Initialize(VLC_OBJECT(tc->gl), priv->vadpy))
+    priv->vadpy = vlc_vaapi_InitializeDisplay(VLC_OBJECT(tc->gl), x11_dpy_name, wl_dpy);
+    if (priv->vadpy.p_va_dpy == NULL)
+    {
+        msg_Err(tc->gl, "Could not get a VAAPI device");
         goto error;
+    }
 
-    if (tc_va_check_interop_blacklist(tc, priv->vadpy))
+    if (tc_va_check_interop_blacklist(tc, priv->vadpy.p_va_dpy))
     {
-        vaTerminate(priv->vadpy);
+        vaTerminate(priv->vadpy.p_va_dpy);
         goto error;
     }
 
-    if (vlc_vaapi_SetInstance(priv->vadpy))
+    if (vlc_vaapi_SetInstance(priv->vadpy.p_va_dpy))
     {
         msg_Err(tc->gl, "VAAPI instance already in use");
         goto error;
@@ -310,12 +297,13 @@ tc_vaegl_init(opengl_tex_converter_t *tc, VADisplay *vadpy)
                                               tc->fmt.space);
     if (tc->fshader == 0)
     {
-        vlc_vaapi_ReleaseInstance(priv->vadpy);
+        vlc_vaapi_ReleaseInstance(priv->vadpy.p_va_dpy);
         goto error;
     }
     return VLC_SUCCESS;
 
 error:
+    vlc_vaapi_ReleaseDisplay(priv->vadpy);
     free(tc->priv);
     return VLC_EGENERIC;
 }
@@ -327,7 +315,7 @@ tc_va_get_pool(const opengl_tex_converter_t *tc, unsigned requested_count)
     struct priv *priv = tc->priv;
 
     picture_pool_t *pool =
-        vlc_vaapi_PoolNew(VLC_OBJECT(tc->gl), priv->vadpy, requested_count,
+        vlc_vaapi_PoolNew(VLC_OBJECT(tc->gl), priv->vadpy.p_va_dpy, requested_count,
                           &priv->va_surface_ids, &tc->fmt, VA_RT_FORMAT_YUV420,
                           VA_FOURCC_NV12);
     if (!pool)
@@ -335,14 +323,14 @@ tc_va_get_pool(const opengl_tex_converter_t *tc, unsigned requested_count)
 
     /* Check if a surface from the pool can be derived */
     VAImage va_image;
-    if (vlc_vaapi_DeriveImage(o, priv->vadpy, priv->va_surface_ids[0],
+    if (vlc_vaapi_DeriveImage(o, priv->vadpy.p_va_dpy, priv->va_surface_ids[0],
                               &va_image))
     {
         picture_pool_Release(pool);
         return NULL;
     }
 
-    vlc_vaapi_DestroyImage(o, priv->vadpy, va_image.image_id);
+    vlc_vaapi_DestroyImage(o, priv->vadpy.p_va_dpy, va_image.image_id);
     return pool;
 }
 
@@ -354,36 +342,28 @@ opengl_tex_converter_vaapi_init(opengl_tex_converter_t *tc)
      || tc->gl->egl.destroyImageKHR == NULL)
         return VLC_EGENERIC;
 
+    struct wl_display *wl_dpy = NULL;
+    char *x11_dpy_name = NULL;
+
     switch (tc->gl->surface->type)
     {
 #ifdef HAVE_VA_X11
         case VOUT_WINDOW_TYPE_XID:
         {
-            if (!vlc_xlib_init(VLC_OBJECT(tc->gl)))
-                return VLC_EGENERIC;
-            Display *x11dpy = XOpenDisplay(tc->gl->surface->display.x11);
-            if (x11dpy == NULL)
-                return VLC_EGENERIC;
-
-            if (tc_vaegl_init(tc, vaGetDisplay(x11dpy)))
-            {
-                XCloseDisplay(x11dpy);
-                return VLC_EGENERIC;
-            }
-            struct priv *priv = tc->priv;
-            priv->x11dpy = x11dpy;
+            x11_dpy_name = tc->gl->surface->display.x11;
             break;
         }
 #endif
 #ifdef HAVE_VA_WL
         case VOUT_WINDOW_TYPE_WAYLAND:
-            if (tc_vaegl_init(tc, vaGetDisplayWl(tc->gl->surface->display.wl)))
-                return VLC_EGENERIC;
+            wl_dpy = tc->gl->surface->display.wl;
             break;
 #endif
-        default:
-            return VLC_EGENERIC;
     }
+
+    if (tc_vaegl_init(tc, x11_dpy_name, wl_dpy))
+        return VLC_EGENERIC;
+
     tc->pf_get_pool = tc_va_get_pool;
 
     return VLC_SUCCESS;
-- 
2.13.0



More information about the vlc-devel mailing list