[vlc-commits] [Git][videolan/vlc][3.0.x] 2 commits: avcodec: vaapi: fix the DRM backend's open callback prototype

[Steve Lhomme (@robUx4)]

Steve Lhomme pushed to branch 3.0.x at VideoLAN / VLC


Commits:
a7109938 by Kang Kang at 2026-08-22T05:52:51+00:00
avcodec: vaapi: fix the DRM backend's open callback prototype

vlc_va_Start() invokes the module's open callback with six arguments:

    return open(va, ctx, src_desc, pix_fmt, fmt, p_sys);

Create() matches that, but CreateDRM() was never updated when
529391522a added the AVPixFmtDescriptor parameter, so it still declares
five. Its pix_fmt parameter therefore receives the src_desc pointer,
which never compares equal to AV_PIX_FMT_VAAPI, and CreateDRM() returns
VLC_EGENERIC before doing anything. The module callbacks are typed as
void *, so the compiler cannot diagnose the mismatch.

The result is that vaapi_drm has been dead since 529391522a: every
setup that has no VAAPI-capable vout to allocate the surfaces -- the yuv
and dummy video outputs, transcoding, any headless use -- silently falls
back to software decoding.

- - - - -
20fc39fc by Kang Kang at 2026-08-22T05:52:51+00:00
avcodec: vaapi: give the DRM backend a hw_device_ctx

libavcodec dropped struct vaapi_context with FF_API_STRUCT_VAAPI_CONTEXT
and now builds its own VA config and context, taking the VA display from
AVCodecContext.hw_device_ctx. Create() was adapted for that, CreateDRM()
was not: it only sets the deprecated hwaccel_context, so libavcodec finds
no VAAPI device and hardware decoding fails even once the module opens.

Set up an AVVAAPIDeviceContext the same way Create() does, and skip the
now-pointless vaCreateContext(). DeleteDRM() has to tolerate an invalid
context and config id as a result.

libvaapi_drm_plugin also has to link against libavutil for
av_hwdevice_ctx_alloc(); libvaapi_plugin already pulls it in through
AVCODEC_LIBS.

Tested on radeonsi with an MPEG-2 DVD stream and --vout=yuv, which has
no VAAPI-capable vout and so is the case the DRM backend exists for.

- - - - -


2 changed files:

- modules/codec/Makefile.am
- modules/codec/avcodec/vaapi.c


Changes:

=====================================
modules/codec/Makefile.am
=====================================
@@ -411,7 +411,7 @@ libvaapi_drm_plugin_la_SOURCES = \
 libvaapi_drm_plugin_la_CPPFLAGS = $(AM_CPPFLAGS) -DVLC_VA_BACKEND_DRM
 libvaapi_drm_plugin_la_CFLAGS = $(AM_CFLAGS) \
 	$(LIBVA_DRM_CFLAGS) $(AVCODEC_CFLAGS)
-libvaapi_drm_plugin_la_LIBADD = $(LIBVA_DRM_LIBS)
+libvaapi_drm_plugin_la_LIBADD = $(LIBVA_DRM_LIBS) $(AVCODEC_LIBS)
 libvaapi_plugin_la_SOURCES = \
 	codec/avcodec/vaapi.c hw/vaapi/vlc_vaapi.c hw/vaapi/vlc_vaapi.h
 libvaapi_plugin_la_CPPFLAGS = $(AM_CPPFLAGS)


=====================================
modules/codec/avcodec/vaapi.c
=====================================
@@ -290,15 +290,19 @@ static void DeleteDRM(vlc_va_t *va, void **hwctx)
 
     (void) hwctx;
     picture_pool_Release(sys->pool);
-    vlc_vaapi_DestroyContext(o, sys->hw_ctx.display, sys->hw_ctx.context_id);
-    vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
+    if (sys->hw_ctx.context_id != VA_INVALID_ID)
+        vlc_vaapi_DestroyContext(o, sys->hw_ctx.display, sys->hw_ctx.context_id);
+    if (sys->hw_ctx.config_id != VA_INVALID_ID)
+        vlc_vaapi_DestroyConfig(o, sys->hw_ctx.display, sys->hw_ctx.config_id);
     vlc_vaapi_ReleaseInstance(sys->va_inst);
     free(sys);
 }
 
-static int CreateDRM(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
+static int CreateDRM(vlc_va_t *va, AVCodecContext *ctx,
+                     const AVPixFmtDescriptor *desc, enum PixelFormat pix_fmt,
                      const es_format_t *fmt, picture_sys_t *p_sys)
 {
+    VLC_UNUSED(desc);
     if (pix_fmt != AV_PIX_FMT_VAAPI || p_sys)
         return VLC_EGENERIC;
 
@@ -355,6 +359,7 @@ static int CreateDRM(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt
     if (!sys->pool)
         goto error;
 
+#if FF_API_STRUCT_VAAPI_CONTEXT
     /* Create a context */
     sys->hw_ctx.context_id =
         vlc_vaapi_CreateContext(o, sys->hw_ctx.display, sys->hw_ctx.config_id,
@@ -364,6 +369,27 @@ static int CreateDRM(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt
         goto error;
 
     ctx->hwaccel_context = &sys->hw_ctx;
+#else
+    /* libavcodec creates its own VA config and context and ignores
+     * hwaccel_context; it only needs the VA display. */
+    (void) surfaces;
+
+    AVBufferRef *hwdev_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
+    if (hwdev_ref == NULL)
+        goto error;
+
+    AVHWDeviceContext *hwdev_ctx = (void *) hwdev_ref->data;
+    AVVAAPIDeviceContext *vadev_ctx = hwdev_ctx->hwctx;
+    vadev_ctx->display = sys->hw_ctx.display;
+
+    if (av_hwdevice_ctx_init(hwdev_ref) < 0)
+    {
+        av_buffer_unref(&hwdev_ref);
+        goto error;
+    }
+
+    ctx->hw_device_ctx = hwdev_ref;
+#endif
     va->sys = sys;
     va->description = vaQueryVendorString(sys->hw_ctx.display);
     va->get = GetDRM;



View it on GitLab: https://code.videolan.org/videolan/vlc/-/compare/11065a7e6d33bdd7cb302d49d46c8a0391c7d4ed...20fc39fc9746553742ef1b95857177079934d267

-- 
View it on GitLab: https://code.videolan.org/videolan/vlc/-/compare/11065a7e6d33bdd7cb302d49d46c8a0391c7d4ed...20fc39fc9746553742ef1b95857177079934d267
You're receiving this email because of your account on code.videolan.org. Manage all notifications: https://code.videolan.org/-/profile/notifications | Help: https://code.videolan.org/help