[vlc-devel] [PATCH 14/39] d3d11va: don't use the external pool for NVIDIA hardware with too many slices

[Steve Lhomme]

Drivers crash during ID3D11VideoDevice::CreateVideoDecoderOutputView() if a
texture has more than 30 slices.

Fixes #18261
---
 modules/codec/avcodec/d3d11va.c | 13 ++++++++++++-
 modules/video_chroma/dxgi_fmt.c | 16 ++++++++++++++++
 modules/video_chroma/dxgi_fmt.h |  1 +
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/modules/codec/avcodec/d3d11va.c b/modules/codec/avcodec/d3d11va.c
index 8cba76ff6f..f7c8541b4c 100644
--- a/modules/codec/avcodec/d3d11va.c
+++ b/modules/codec/avcodec/d3d11va.c
@@ -110,6 +110,7 @@ struct vlc_va_sys_t
 {
     directx_sys_t                dx_sys;
     vlc_fourcc_t                 i_chroma;
+    UINT                         totalTextureSlices;
 
 #if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
     HINSTANCE                    dxgidebug_dll;
@@ -434,12 +435,12 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
 
             sys->d3dctx = p_sys->context;
             sys->d3dvidctx = d3dvidctx;
-            sys->b_extern_pool = true;
 
             assert(p_sys->texture[KNOWN_DXGI_INDEX] != NULL);
             D3D11_TEXTURE2D_DESC dstDesc;
             ID3D11Texture2D_GetDesc( p_sys->texture[KNOWN_DXGI_INDEX], &dstDesc);
             sys->render = dstDesc.Format;
+            va->sys->totalTextureSlices = dstDesc.ArraySize;
         }
     }
 
@@ -825,6 +826,16 @@ static int DxSetupOutput(vlc_va_t *va, const GUID *input, const video_format_t *
         }
 
         msg_Dbg(va, "Using output format %s for decoder %s", DxgiFormatToStr(processorInput[idx]), psz_decoder_name);
+        if ( va->sys->render == processorInput[idx] )
+        {
+            /* NVIDIA cards crash when calling CreateVideoDecoderOutputView
+             * on more than 30 slices */
+            if (va->sys->totalTextureSlices <= 30 || !isNvidiaHardware(dx_sys->d3ddev))
+                va->sys->b_extern_pool = true;
+            else
+                msg_Warn( va, "NVIDIA GPU with too many slices (%d) detected, use internal pool",
+                          va->sys->totalTextureSlices );
+        }
         va->sys->render = processorInput[idx];
         free(psz_decoder_name);
         return VLC_SUCCESS;
diff --git a/modules/video_chroma/dxgi_fmt.c b/modules/video_chroma/dxgi_fmt.c
index 9fbf702584..f496e98122 100644
--- a/modules/video_chroma/dxgi_fmt.c
+++ b/modules/video_chroma/dxgi_fmt.c
@@ -160,3 +160,19 @@ bool isXboxHardware(ID3D11Device *d3ddev)
     IDXGIAdapter_Release(p_adapter);
     return result;
 }
+
+bool isNvidiaHardware(ID3D11Device *d3ddev)
+{
+    IDXGIAdapter *p_adapter = D3D11DeviceAdapter(d3ddev);
+    if (!p_adapter)
+        return NULL;
+
+    bool result = false;
+    DXGI_ADAPTER_DESC adapterDesc;
+    if (SUCCEEDED(IDXGIAdapter_GetDesc(p_adapter, &adapterDesc))) {
+        result = adapterDesc.VendorId == 0x10DE;
+    }
+
+    IDXGIAdapter_Release(p_adapter);
+    return result;
+}
diff --git a/modules/video_chroma/dxgi_fmt.h b/modules/video_chroma/dxgi_fmt.h
index f20e4888eb..2e4d98704d 100644
--- a/modules/video_chroma/dxgi_fmt.h
+++ b/modules/video_chroma/dxgi_fmt.h
@@ -49,6 +49,7 @@ extern void DxgiFormatMask(DXGI_FORMAT format, video_format_t *);
 
 typedef struct ID3D11Device ID3D11Device;
 bool isXboxHardware(ID3D11Device *d3ddev);
+bool isNvidiaHardware(ID3D11Device *d3ddev);
 IDXGIAdapter *D3D11DeviceAdapter(ID3D11Device *d3ddev);
 
 static inline bool DeviceSupportsFormat(ID3D11Device *d3ddevice,
-- 
2.12.1