[vlc-commits] [Git][videolan/vlc][master] opengl: vaapi: check surfaces via the export path

[Steve Lhomme (@robUx4)]

Steve Lhomme pushed to branch master at VideoLAN / VLC


Commits:
706fe761 by Charles Turner at 2026-08-30T05:50:10+00:00
opengl: vaapi: check surfaces via the export path

tc_vaegl_update imports with vaExportSurfaceHandle and the surface's
real drm_format_modifier, but tc_va_check_derive_image validated with
vaDeriveImage and DRM_FORMAT_MOD_INVALID - stricter than the path it
guards. On radeonsi, tiled P010 surfaces at widths that are not a
multiple of 256 are rejected (element pitch vs byte pitch), so the
interop was refused and 10-bit playback silently fell back to software.
NV12 is unaffected (R8, bpe == 1).

Also reject multiplane layers, matching update.

Not codec-specific: any 10-bit 4:2:0 (P010) content at such a width
triggers it - HEVC Main10, VP9 Profile 2, AV1 10-bit. Generate a
reproducer (1920 is not a multiple of 256) and play it via VAAPI:

  ffmpeg -f lavfi -i testsrc2=size=1920x1080:rate=30:duration=3 \
         -pix_fmt yuv420p10le -c:v libvpx-vp9 -profile:v 2 -b:v 2M out.mkv
  vlc -vv --dec-dev decdev_vaapi_wl --vout=gl out.mkv

  glinterop_vaapi gl warning: Can't create Image KHR: kernel too old ?
  gl gl error: Could not create interop
  main video output error: video output display creation failed

after which decoding continues in software. With this change the
interop is accepted and rendering is hardware-accelerated.

Verified on a Radeon 860M (gfx1151, mesa 26.0.8, libva 1.24) with
glinterop_vaapi on native Wayland (xdg_shell), with both HEVC Main10
and VP9 Profile 2.

- - - - -


1 changed file:

- modules/video_output/opengl/interop_vaapi.c


Changes:

=====================================
modules/video_output/opengl/interop_vaapi.c
=====================================
@@ -366,9 +366,55 @@ tc_va_check_derive_image(const struct vlc_gl_interop *interop)
     if (!pool)
         return VLC_EGENERIC;
 
+    int ret;
+
+#if VA_CHECK_VERSION(1, 1, 0)
+    VADRMPRIMESurfaceDescriptor desc;
+
+    ret = vlc_vaapi_ExportSurfaceHandle(o, priv->vadpy, va_surface_ids[0],
+                                        VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
+                                        VA_EXPORT_SURFACE_READ_ONLY |
+                                        VA_EXPORT_SURFACE_SEPARATE_LAYERS,
+                                        &desc);
+    if (ret != VLC_SUCCESS)
+        goto done;
+
+    for (unsigned i = 0; i < desc.num_layers; ++i)
+    {
+        unsigned obj_idx = desc.layers[i].object_index[0];
+
+        if (desc.layers[i].num_planes > 1)
+        {
+            ret = VLC_EGENERIC;
+            goto done_desc;
+        }
+
+        EGLint w = (desc.width * interop->texs[i].w.num) / interop->texs[i].w.den;
+        EGLint h = (desc.height * interop->texs[i].h.num) / interop->texs[i].h.den;
+        EGLImageKHR egl_image =
+            vaegl_image_create(interop, w, h, priv->drm_fourccs[i],
+                               desc.objects[obj_idx].fd,
+                               desc.layers[i].offset[0], desc.layers[i].pitch[0],
+                               desc.objects[obj_idx].drm_format_modifier);
+        if (egl_image == NULL)
+        {
+            msg_Warn(o, "Can't create Image KHR: kernel too old ?");
+            ret = VLC_EGENERIC;
+            goto done_desc;
+        }
+        vaegl_image_destroy(interop, egl_image);
+    }
+
+    ret = VLC_SUCCESS;
+
+done_desc:
+    for (unsigned i = 0; i < desc.num_objects; ++i)
+        close(desc.objects[i].fd);
+    goto done;
+#else
     VAImage va_image = { .image_id = VA_INVALID_ID };
-    int ret = vlc_vaapi_DeriveImage(o, priv->vadpy, va_surface_ids[0],
-                                    &va_image);
+    ret = vlc_vaapi_DeriveImage(o, priv->vadpy, va_surface_ids[0],
+                                &va_image);
     if (ret != VLC_SUCCESS)
         goto done;
     assert(va_image.format.fourcc == priv->fourcc);
@@ -376,10 +422,10 @@ tc_va_check_derive_image(const struct vlc_gl_interop *interop)
     VABufferInfo va_buffer_info = (VABufferInfo) {
         .mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME
     };
-    ret = vlc_vaapi_AcquireBufferHandle(o ,priv->vadpy, va_image.buf,
+    ret = vlc_vaapi_AcquireBufferHandle(o, priv->vadpy, va_image.buf,
                                         &va_buffer_info);
     if (ret != VLC_SUCCESS)
-        goto done;
+        goto done_derive;
 
     for (unsigned i = 0; i < interop->tex_count; ++i)
     {
@@ -393,19 +439,23 @@ tc_va_check_derive_image(const struct vlc_gl_interop *interop)
         {
             msg_Warn(o, "Can't create Image KHR: kernel too old ?");
             ret = VLC_EGENERIC;
-            goto done;
+            goto done_derive;
         }
         vaegl_image_destroy(interop, egl_image);
     }
 
-done:
+    ret = VLC_SUCCESS;
+
+done_derive:
     if (va_image.image_id != VA_INVALID_ID)
     {
         if (va_image.buf != VA_INVALID_ID)
             vlc_vaapi_ReleaseBufferHandle(o, priv->vadpy, va_image.buf);
         vlc_vaapi_DestroyImage(o, priv->vadpy, va_image.image_id);
     }
+#endif
 
+done:
     picture_pool_Release(pool);
 
     return ret;



View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/706fe761898e111c97f917b4957e1809497432c1

-- 
View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/706fe761898e111c97f917b4957e1809497432c1
You're receiving this email because of your account on code.videolan.org. Manage all notifications: https://code.videolan.org/-/profile/notifications | Help: https://code.videolan.org/help