[vlc-devel] [PATCH 10/14] d3d11_fmt: move the DXGI debug handling in D3D11_CreateDevice

[Steve Lhomme]

--
replaces https://patches.videolan.org/patch/18757/
- fix inverted DLL logic
- fix uninitialized pointer use
---
 modules/codec/avcodec/d3d11va.c  | 31 -------------------------------
 modules/video_chroma/d3d11_fmt.c | 25 +++++++++++++++++++++++++
 modules/video_chroma/d3d11_fmt.h |  3 +++
 3 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/modules/codec/avcodec/d3d11va.c b/modules/codec/avcodec/d3d11va.c
index 0fcc7b6ab1..93ef39188b 100644
--- a/modules/codec/avcodec/d3d11va.c
+++ b/modules/codec/avcodec/d3d11va.c
@@ -94,10 +94,6 @@ vlc_module_end()
 
 #endif /* __MINGW32__ */
 
-#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
-# include <dxgidebug.h>
-#endif
-
 DEFINE_GUID(DXVA_Intel_H264_NoFGT_ClearVideo,       0x604F8E68, 0x4951, 0x4c54, 0x88, 0xFE, 0xAB, 0xD2, 0x5C, 0x15, 0xB3, 0xD6);
 
 DEFINE_GUID(DXVA2_NoEncrypt,                        0x1b81bed0, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
@@ -112,10 +108,6 @@ struct vlc_va_sys_t
     d3d11_handle_t               hd3d;
     d3d11_device_t               d3d_dev;
 
-#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
-    HINSTANCE                    dxgidebug_dll;
-#endif
-
     /* Video service */
     ID3D11VideoContext           *d3dvidctx;
     DXGI_FORMAT                  render;
@@ -314,11 +306,6 @@ static void Close(vlc_va_t *va, void **ctx)
 
     D3D11_Destroy( &sys->hd3d );
 
-#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
-    if (sys->dxgidebug_dll)
-        FreeLibrary(sys->dxgidebug_dll);
-#endif
-
     free((char *)va->description);
     free(sys);
 }
@@ -338,10 +325,6 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     if (unlikely(sys == NULL))
         return VLC_ENOMEM;
 
-#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
-    sys->dxgidebug_dll = LoadLibrary(TEXT("DXGIDEBUG.DLL"));
-#endif
-
     dx_sys = &sys->dx_sys;
 
     dx_sys->va_pool.pf_create_device           = D3dCreateDevice;
@@ -453,20 +436,6 @@ static int D3dCreateDevice(vlc_va_t *va)
     }
     sys->d3dvidctx = d3dvidctx;
 
-#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
-    HRESULT (WINAPI  * pf_DXGIGetDebugInterface)(const GUID *riid, void **ppDebug);
-    if (sys->dxgidebug_dll) {
-        pf_DXGIGetDebugInterface = (void *)GetProcAddress(sys->dxgidebug_dll, "DXGIGetDebugInterface");
-        if (pf_DXGIGetDebugInterface) {
-            IDXGIDebug *pDXGIDebug = NULL;
-            hr = pf_DXGIGetDebugInterface(&IID_IDXGIDebug, (void**)&pDXGIDebug);
-            if (SUCCEEDED(hr) && pDXGIDebug) {
-                hr = IDXGIDebug_ReportLiveObjects(pDXGIDebug, DXGI_DEBUG_ALL, DXGI_DEBUG_RLO_ALL);
-            }
-        }
-    }
-#endif
-
     return VLC_SUCCESS;
 }
 
diff --git a/modules/video_chroma/d3d11_fmt.c b/modules/video_chroma/d3d11_fmt.c
index 1c99ed4a84..3be3455baa 100644
--- a/modules/video_chroma/d3d11_fmt.c
+++ b/modules/video_chroma/d3d11_fmt.c
@@ -30,6 +30,10 @@
 #define COBJMACROS
 #include <d3d11.h>
 #include <assert.h>
+#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
+# include <initguid.h>
+# include <dxgidebug.h>
+#endif
 
 #include "d3d11_fmt.h"
 
@@ -424,6 +428,22 @@ int D3D11_Create(vlc_object_t *obj, d3d11_handle_t *hd3d)
         msg_Warn(obj, "cannot load d3d11.dll, aborting");
         return VLC_EGENERIC;
     }
+
+# if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
+    if (IsDebuggerPresent())
+    {
+        hd3d->dxgidebug_dll = LoadLibrary(TEXT("DXGIDEBUG.DLL"));
+        HRESULT (WINAPI * pf_DXGIGetDebugInterface)(const GUID *riid, void **ppDebug) = NULL;
+        if (hd3d->dxgidebug_dll)
+            pf_DXGIGetDebugInterface =
+                    (void *)GetProcAddress(hd3d->dxgidebug_dll, "DXGIGetDebugInterface");
+        if (pf_DXGIGetDebugInterface) {
+            IDXGIDebug *pDXGIDebug;
+            if (SUCCEEDED(pf_DXGIGetDebugInterface(&IID_IDXGIDebug, (void**)&pDXGIDebug)))
+                IDXGIDebug_ReportLiveObjects(pDXGIDebug, DXGI_DEBUG_ALL, DXGI_DEBUG_RLO_ALL);
+        }
+    }
+# endif
 #endif
     return VLC_SUCCESS;
 }
@@ -433,5 +453,10 @@ void D3D11_Destroy(d3d11_handle_t *hd3d)
 #if !VLC_WINSTORE_APP
     if (hd3d->hdll)
         FreeLibrary(hd3d->hdll);
+
+#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
+    if (hd3d->dxgidebug_dll)
+        FreeLibrary(hd3d->dxgidebug_dll);
+#endif
 #endif
 }
diff --git a/modules/video_chroma/d3d11_fmt.h b/modules/video_chroma/d3d11_fmt.h
index d5e0848eaf..d639046f50 100644
--- a/modules/video_chroma/d3d11_fmt.h
+++ b/modules/video_chroma/d3d11_fmt.h
@@ -39,6 +39,9 @@ typedef struct
 {
 #if !VLC_WINSTORE_APP
     HINSTANCE                 hdll;       /* handle of the opened d3d11 dll */
+#if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
+    HINSTANCE                 dxgidebug_dll;
+#endif
 #endif
 } d3d11_handle_t;
 
-- 
2.14.2