[vlc-devel] [PATCH] DxVA2: fix compatibility with avcodec frame multithreading

[Steve Lhomme]

Fixes #11693
Disable thread_safe_callbacks when we detect DXVA2 might be used

--
the p_va of the decoder is not affected, a dummy p_va is used

the decoding issues appeared on Intel HD Graphics 4000 hardware
---
 modules/codec/avcodec/video.c | 75 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 60 insertions(+), 15 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 381fe62..c4488c0 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -105,6 +105,7 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *, AVFrame * );
 #endif
 static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *,
                                           const enum PixelFormat * );
+static bool ffmpeg_CanHwAccel( AVCodecContext *, const enum PixelFormat * );
 static picture_t *DecodeVideo( decoder_t *, block_t ** );
 
 static uint32_t ffmpeg_CodecTag( vlc_fourcc_t fcc )
@@ -374,7 +375,21 @@ int InitVideoDec( decoder_t *p_dec, AVCodecContext *p_context,
     i_thread_count = __MIN( i_thread_count, 16 );
     msg_Dbg( p_dec, "allowing %d thread(s) for decoding", i_thread_count );
     p_context->thread_count = i_thread_count;
+# if defined(_WIN32)
+    /* Frame multithreading is not thread safe with DXVA2.
+     * When a frame is being copied to host memory, the frame
+     * is locked and cannot be used as a reference frame
+     * simultaneously and thus decoding fails for some frames. This
+     * causes major image corruption. */
+    enum PixelFormat test_formats[] = { AV_PIX_FMT_DXVA2_VLD, AV_PIX_FMT_NONE };
+    if( ffmpeg_CanHwAccel( p_context, &test_formats ) )
+    {
+        msg_Dbg( p_dec, "threaded frame decoding with DXVA2 is not thread safe" );
+        p_context->thread_safe_callbacks = false;
+    }
+#else
     p_context->thread_safe_callbacks = true;
+#endif
 
     switch( p_codec->id )
     {
@@ -396,21 +411,6 @@ int InitVideoDec( decoder_t *p_dec, AVCodecContext *p_context,
             break;
     }
 
-    /* Workaround: frame multithreading is not compatible with
-     * DXVA2. When a frame is being copied to host memory, the frame
-     * is locked and cannot be used as a reference frame
-     * simultaneously and thus decoding fails for some frames. This
-     * causes major image corruption. */
-# if defined(_WIN32)
-    char *avcodec_hw = var_InheritString( p_dec, "avcodec-hw" );
-    if( avcodec_hw == NULL || strcasecmp( avcodec_hw, "none" ) )
-    {
-        msg_Warn( p_dec, "threaded frame decoding is not compatible with DXVA2, disabled" );
-        p_context->thread_type &= ~FF_THREAD_FRAME;
-    }
-    free( avcodec_hw );
-# endif
-
     if( p_context->thread_type & FF_THREAD_FRAME )
         p_dec->i_extra_picture_buffers = 2 * p_context->thread_count;
 #endif
@@ -1305,6 +1305,51 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
 }
 #endif
 
+static bool ffmpeg_CanHwAccel( AVCodecContext *p_codec,
+                               const enum PixelFormat *pi_fmt )
+{
+    decoder_t *p_dec = p_codec->opaque;
+    decoder_sys_t *p_sys = p_dec->p_sys;
+    vlc_va_t *p_va;
+
+    /* Enumerate available formats */
+    bool can_hwaccel = false;
+    for( size_t i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
+    {
+        const AVPixFmtDescriptor *dsc = av_pix_fmt_desc_get(pi_fmt[i]);
+        if (dsc == NULL)
+            continue;
+        if ( dsc->flags & AV_PIX_FMT_FLAG_HWACCEL )
+        {
+            can_hwaccel = true;
+            break;
+        }
+    }
+
+    if (!can_hwaccel)
+        goto end;
+
+    can_hwaccel = false;
+
+    /* Profile and level information is needed now.
+     * TODO: avoid code duplication with avcodec.c */
+    if( p_codec->profile != FF_PROFILE_UNKNOWN)
+        p_dec->fmt_in.i_profile = p_codec->profile;
+    if( p_codec->level != FF_LEVEL_UNKNOWN)
+        p_dec->fmt_in.i_level = p_codec->level;
+
+    p_va = vlc_va_New( VLC_OBJECT(p_dec), p_codec, &p_dec->fmt_in );
+    if( p_va == NULL )
+        goto end;
+
+    can_hwaccel = true;
+
+end:
+    if ( p_va != NULL )
+        vlc_va_Delete( p_va, p_codec );
+    return can_hwaccel;
+}
+
 static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
                                           const enum PixelFormat *pi_fmt )
 {
-- 
2.3.0