[vlc-devel] [PATCH] converter_vaapi: check the buffers can be used in tc_va_check_derive_image

[Steve Lhomme]

This is the same check that is done when creating the converter pool.

We must do the tc_va_check_derive_image() after the opengl_fragment_shader_init()
since we check the texture planes.
---
 modules/video_output/opengl/converter_vaapi.c | 40 +++++++++++++++++--
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/modules/video_output/opengl/converter_vaapi.c b/modules/video_output/opengl/converter_vaapi.c
index b4d7fc8bef3..4b8fd06bfbf 100644
--- a/modules/video_output/opengl/converter_vaapi.c
+++ b/modules/video_output/opengl/converter_vaapi.c
@@ -352,6 +352,40 @@ tc_va_check_derive_image(opengl_tex_converter_t *tc,
     VAImage va_image = { .image_id = VA_INVALID_ID };
     int ret = vlc_vaapi_DeriveImage(o, priv->vadpy, va_surface_ids[0],
                                     &va_image);
+    if (ret != VLC_SUCCESS)
+        goto error;
+
+    assert(va_image.format.fourcc == priv->fourcc);
+
+    VABufferInfo va_buffer_info = (VABufferInfo) {
+        .mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME
+    };
+    if (vlc_vaapi_AcquireBufferHandle(o ,priv->vadpy, va_image.buf,
+                                      &va_buffer_info))
+        goto error;
+
+    for (unsigned i = 0; i < va_image.num_planes; ++i)
+    {
+        EGLint w = (va_image.width * tc->texs[i].w.num) / tc->texs[i].w.den;
+        EGLint h = (va_image.height * tc->texs[i].h.num) / tc->texs[i].h.den;
+        EGLImageKHR egl_image =
+            vaegl_image_create(tc, w, h, priv->drm_fourccs[i], va_buffer_info.handle,
+                               va_image.offsets[i], va_image.pitches[i]);
+        if (egl_image == NULL)
+        {
+            msg_Warn(o, "Can't create Image KHR: kernel too old ?");
+            goto error;
+        }
+        vaegl_image_destroy(tc, egl_image);
+    }
+
+error:
+    if (va_image.image_id != VA_INVALID_ID)
+    {
+        if (va_image.buf != VA_INVALID_ID)
+            vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy, va_image.buf);
+        vlc_vaapi_DestroyImage(o, priv->vadpy, va_image.image_id);
+    }
 
     picture_pool_Release(pool);
 
@@ -424,14 +458,14 @@ Open(vlc_object_t *obj)
     if (tc_va_check_interop_blacklist(tc, priv->vadpy))
         goto error;
 
-    if (tc_va_check_derive_image(tc, dec_device))
-        goto error;
-
     tc->fshader = opengl_fragment_shader_init(tc, GL_TEXTURE_2D, vlc_sw_chroma,
                                               tc->fmt.space);
     if (tc->fshader == 0)
         goto error;
 
+    if (tc_va_check_derive_image(tc, dec_device))
+        goto error;
+
     tc->pf_update  = tc_vaegl_update;
     tc->pf_get_pool = tc_vaegl_get_pool;
 
-- 
2.17.1