[vlc-commits] d3d9_fmt: keep the IDirectXVideoDecoder and DXVA2 DLL while pictures are active

[Steve Lhomme]

vlc/vlc-3.0 | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Tue Jan 16 16:28:05 2018 +0100| [b1bd74d2460211316faf14c550932b79c30177f7] | committer: Jean-Baptiste Kempf

d3d9_fmt: keep the IDirectXVideoDecoder and DXVA2 DLL while pictures are active

In some cases the decoder can be freed even though some pictures are still
pending to be displayed/flush in the vout. We need to make sure the DLL and the
decoder module is still available while these pictures are around.

Fixes #19368

(cherry picked from commit bd325fbdcfc6afb2e645f73cfd56903dc6848e65)
Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=b1bd74d2460211316faf14c550932b79c30177f7
---

 modules/codec/avcodec/dxva2.c          |  9 +++++----
 modules/video_chroma/d3d9_fmt.h        | 13 ++++++++++++-
 modules/video_output/win32/direct3d9.c |  1 +
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/modules/codec/avcodec/dxva2.c b/modules/codec/avcodec/dxva2.c
index 029971c488..25de31de83 100644
--- a/modules/codec/avcodec/dxva2.c
+++ b/modules/codec/avcodec/dxva2.c
@@ -177,12 +177,12 @@ static void d3d9_pic_context_destroy(struct picture_context_t *opaque)
     }
 }
 
-static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *);
+static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *, IDirectXVideoDecoder *);
 
 static struct picture_context_t *d3d9_pic_context_copy(struct picture_context_t *ctx)
 {
     struct va_pic_context *src_ctx = (struct va_pic_context*)ctx;
-    struct va_pic_context *pic_ctx = CreatePicContext(src_ctx->picsys.surface);
+    struct va_pic_context *pic_ctx = CreatePicContext(src_ctx->picsys.surface, src_ctx->picsys.decoder);
     if (unlikely(pic_ctx==NULL))
         return NULL;
     pic_ctx->va_surface = src_ctx->va_surface;
@@ -190,7 +190,7 @@ static struct picture_context_t *d3d9_pic_context_copy(struct picture_context_t
     return &pic_ctx->s;
 }
 
-static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *surface)
+static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *surface, IDirectXVideoDecoder *decoder)
 {
     struct va_pic_context *pic_ctx = calloc(1, sizeof(*pic_ctx));
     if (unlikely(pic_ctx==NULL))
@@ -198,6 +198,7 @@ static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *surface)
     pic_ctx->s.destroy = d3d9_pic_context_destroy;
     pic_ctx->s.copy    = d3d9_pic_context_copy;
     pic_ctx->picsys.surface = surface;
+    pic_ctx->picsys.decoder = decoder;
     AcquirePictureSys(&pic_ctx->picsys);
     return pic_ctx;
 }
@@ -205,7 +206,7 @@ static struct va_pic_context *CreatePicContext(IDirect3DSurface9 *surface)
 static struct va_pic_context* NewSurfacePicContext(vlc_va_t *va, int surface_index)
 {
     directx_sys_t *dx_sys = &va->sys->dx_sys;
-    struct va_pic_context *pic_ctx = CreatePicContext(dx_sys->hw_surface[surface_index]);
+    struct va_pic_context *pic_ctx = CreatePicContext(dx_sys->hw_surface[surface_index], dx_sys->decoder);
     if (unlikely(pic_ctx==NULL))
         return NULL;
     /* all the resources are acquired during surfaces init, and a second time in
diff --git a/modules/video_chroma/d3d9_fmt.h b/modules/video_chroma/d3d9_fmt.h
index dcb37676e0..c7242ad525 100644
--- a/modules/video_chroma/d3d9_fmt.h
+++ b/modules/video_chroma/d3d9_fmt.h
@@ -25,14 +25,19 @@
 
 #include <vlc_picture.h>
 
+#define COBJMACROS
 #include <d3d9.h>
+#include <dxva2api.h>
 
 #include "dxgi_fmt.h"
 
 /* owned by the vout for VLC_CODEC_D3D9_OPAQUE */
 struct picture_sys_t
 {
-    LPDIRECT3DSURFACE9 surface;
+    IDirect3DSurface9    *surface;
+    /* decoder only */
+    IDirectXVideoDecoder *decoder; /* keep a reference while the surface exist */
+    HINSTANCE            dxva2_dll;
 };
 
 typedef struct
@@ -69,11 +74,17 @@ picture_sys_t *ActivePictureSys(picture_t *p_pic);
 static inline void AcquirePictureSys(picture_sys_t *p_sys)
 {
     IDirect3DSurface9_AddRef(p_sys->surface);
+    if (p_sys->decoder)
+        IDirectXVideoDecoder_AddRef(p_sys->decoder);
+    p_sys->dxva2_dll = LoadLibrary(TEXT("DXVA2.DLL"));
 }
 
 static inline void ReleasePictureSys(picture_sys_t *p_sys)
 {
     IDirect3DSurface9_Release(p_sys->surface);
+    if (p_sys->decoder)
+        IDirectXVideoDecoder_Release(p_sys->decoder);
+    FreeLibrary(p_sys->dxva2_dll);
 }
 
 HRESULT D3D9_CreateDevice(vlc_object_t *, d3d9_handle_t *, HWND,
diff --git a/modules/video_output/win32/direct3d9.c b/modules/video_output/win32/direct3d9.c
index 84df885972..71b517a59d 100644
--- a/modules/video_output/win32/direct3d9.c
+++ b/modules/video_output/win32/direct3d9.c
@@ -429,6 +429,7 @@ static picture_pool_t *Direct3D9CreatePicturePool(vlc_object_t *o,
         picture_sys_t *picsys = malloc(sizeof(*picsys));
         if (unlikely(picsys == NULL))
             goto error;
+        memset(picsys, 0, sizeof(*picsys));
 
         HRESULT hr = IDirect3DDevice9_CreateOffscreenPlainSurface(p_d3d9_dev->dev,
                                                           fmt->i_width,