[vlc-devel] [PATCH 1/6] gl: vaapi: refactor VADisplay creation

[Thomas Guillem]

No functional changes.
---
 modules/video_output/opengl/converter_vaapi.c | 59 ++++++++-----------
 1 file changed, 26 insertions(+), 33 deletions(-)

diff --git a/modules/video_output/opengl/converter_vaapi.c b/modules/video_output/opengl/converter_vaapi.c
index a2f8f34dd8..f4de838b86 100644
--- a/modules/video_output/opengl/converter_vaapi.c
+++ b/modules/video_output/opengl/converter_vaapi.c
@@ -347,51 +347,46 @@ x11_native_destroy_cb(VANativeDisplay native)
     XCloseDisplay(native);
 }
 
-static int
-x11_init_vaapi_instance(opengl_tex_converter_t *tc, struct priv *priv)
+static struct vlc_vaapi_instance *
+x11_init_vaapi_instance(vout_window_t *window, VADisplay *vadpyp)
 {
-    if (!vlc_xlib_init(VLC_OBJECT(tc->gl)))
-        return VLC_EGENERIC;
+    if (!vlc_xlib_init(VLC_OBJECT(window)))
+        return NULL;
 
-    Display *x11dpy = XOpenDisplay(tc->gl->surface->display.x11);
+    Display *x11dpy = XOpenDisplay(window->display.x11);
     if (x11dpy == NULL)
-        return VLC_EGENERIC;
+        return NULL;
 
-    priv->vadpy = vaGetDisplay(x11dpy);
-    if (priv->vadpy == NULL)
+    VADisplay vadpy = *vadpyp = vaGetDisplay(x11dpy);
+    if (vadpy == NULL)
     {
         x11_native_destroy_cb(x11dpy);
-        return VLC_EGENERIC;
+        return NULL;
     }
 
-    priv->vainst = vlc_vaapi_InitializeInstance(VLC_OBJECT(tc->gl), priv->vadpy,
-                                                x11dpy, x11_native_destroy_cb);
-    return priv->vainst != NULL ? VLC_SUCCESS : VLC_EGENERIC;
+    return vlc_vaapi_InitializeInstance(VLC_OBJECT(window), vadpy,
+                                        x11dpy, x11_native_destroy_cb);
 }
 #endif
 
 #ifdef HAVE_VA_DRM
-static int
-drm_init_vaapi_instance(opengl_tex_converter_t *tc, struct priv *priv)
+static struct vlc_vaapi_instance *
+drm_init_vaapi_instance(vout_window_t *window,  VADisplay *vadpyp)
 {
-    priv->vainst =
-        vlc_vaapi_InitializeInstanceDRM(VLC_OBJECT(tc->gl), vaGetDisplayDRM,
-                                        &priv->vadpy, NULL);
-    return priv->vainst != NULL ? VLC_SUCCESS : VLC_EGENERIC;
+    return vlc_vaapi_InitializeInstanceDRM(VLC_OBJECT(window), vaGetDisplayDRM,
+                                           vadpyp, NULL);
 }
 #endif
 
 #ifdef HAVE_VA_WL
-static int
-wl_init_vaapi_instance(opengl_tex_converter_t *tc, struct priv *priv)
+static struct vlc_vaapi_instance *
+wl_init_vaapi_instance(vout_window_t *window, VADisplay *vadpyp)
 {
-    priv->vadpy = vaGetDisplayWl(tc->gl->surface->display.wl);
-    if (priv->vadpy == NULL)
-        return VLC_EGENERIC;
+    VADisplay vadpy = *vadpyp = vaGetDisplayWl(window->display.wl);
+    if (vadpy == NULL)
+        return NULL;
 
-    priv->vainst = vlc_vaapi_InitializeInstance(VLC_OBJECT(tc->gl), priv->vadpy,
-                                                NULL, NULL);
-    return priv->vainst != NULL ? VLC_SUCCESS : VLC_EGENERIC;
+    return vlc_vaapi_InitializeInstance(VLC_OBJECT(window), vadpy, NULL, NULL);
 }
 #endif
 
@@ -444,23 +439,21 @@ Open(vlc_object_t *obj)
     if (priv->glEGLImageTargetTexture2DOES == NULL)
         goto error;
 
-    int ret = VLC_EGENERIC;
 #if defined (HAVE_VA_X11)
     if (tc->gl->surface->type == VOUT_WINDOW_TYPE_XID)
-        ret = x11_init_vaapi_instance(tc, priv);
+        priv->vainst = x11_init_vaapi_instance(tc->gl->surface, &priv->vadpy);
 #elif defined(HAVE_VA_WL)
     if (tc->gl->surface->type == VOUT_WINDOW_TYPE_WAYLAND)
-        ret = wl_init_vaapi_instance(tc, priv);
+        priv->vainst = wl_init_vaapi_instance(tc->gl->surface, &priv->vadpy);
 #elif defined (HAVE_VA_DRM)
-    ret = drm_init_vaapi_instance(tc, priv);
+    priv->vainst = drm_init_vaapi_instance(tc->gl->surface, &priv->vadpy);
 #else
 # error need X11/WL/DRM support
 #endif
 
-    if (ret != VLC_SUCCESS)
+    if (!priv->vainst)
         goto error;
-
-    assert(priv->vadpy != NULL && priv->vainst != NULL);
+    assert(priv->vadpy != NULL);
 
     if (tc_va_check_interop_blacklist(tc, priv->vadpy))
         goto error;
-- 
2.20.1