[vlc-devel] [PATCH 2/3] direct3d9: fix the black background when the decoder texture is in YUV

[Steve Lhomme]

fixes the bottom line issue in #17420

--
replaces https://patches.videolan.org/patch/16082/
* missing YUV black for the DXVA2 pool
replaces https://patches.videolan.org/patch/16119/
* factorize code
replaces https://patches.videolan.org/patch/16124/
* do it in dxva2 when direct rendering is not used
---
 modules/codec/avcodec/dxva2.c          | 12 ++++++++++++
 modules/video_output/win32/direct3d9.c | 11 ++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/modules/codec/avcodec/dxva2.c b/modules/codec/avcodec/dxva2.c
index 7a9059b..8bfdecd 100644
--- a/modules/codec/avcodec/dxva2.c
+++ b/modules/codec/avcodec/dxva2.c
@@ -689,6 +689,13 @@ static int DxSetupOutput(vlc_va_t *va, const GUID *input, const video_format_t *
     return err;
 }
 
+static D3DCOLOR GetFormatBlack(D3DFORMAT format)
+{
+    if (vlc_fourcc_IsYUV(format))
+        return D3DCOLOR_XYUV(0,0x80,0x80);
+    return D3DCOLOR_ARGB(0xFF, 0, 0, 0);
+}
+
 /**
  * It creates a DXVA2 decoder using the given video format
  */
@@ -828,6 +835,11 @@ static int DxCreateVideoDecoder(vlc_va_t *va, int codec_id, const video_format_t
     }
     sys->decoder = (IUnknown*) decoder;
 
+    for (int i = 0; i < sys->surface_count; i++)
+        IDirect3DDevice9_ColorFill((IDirect3DDevice9*) sys->d3ddev,
+                                   (IDirect3DSurface9*) sys->hw_surface[i],
+                                   NULL, GetFormatBlack(p_sys->render));
+
     msg_Dbg(va, "IDirectXVideoDecoderService_CreateVideoDecoder succeed");
     return VLC_SUCCESS;
 }
diff --git a/modules/video_output/win32/direct3d9.c b/modules/video_output/win32/direct3d9.c
index f190e65..ae0c693 100644
--- a/modules/video_output/win32/direct3d9.c
+++ b/modules/video_output/win32/direct3d9.c
@@ -334,6 +334,13 @@ static void DestroyPicture(picture_t *picture)
     free(picture);
 }
 
+static D3DCOLOR GetFormatBlack(D3DFORMAT format)
+{
+    if (vlc_fourcc_IsYUV(format))
+        return D3DCOLOR_XYUV(0,0x80,0x80);
+    return D3DCOLOR_ARGB(0xFF, 0, 0, 0);
+}
+
 /* */
 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
 {
@@ -367,6 +374,8 @@ static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
            goto error;
         }
 
+        IDirect3DDevice9_ColorFill(vd->sys->d3ddev, picsys->surface, NULL, GetFormatBlack(format));
+
         picture_resource_t resource = {
             .p_sys = picsys,
             .pf_destroy = DestroyPicture,
@@ -1160,7 +1169,7 @@ static int Direct3D9CreatePool(vout_display_t *vd, video_format_t *fmt)
 #endif
 
     /* fill surface with black color */
-    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0));
+    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, GetFormatBlack(d3dfmt->format));
 
     /* Create the associated picture */
     picture_sys_t *picsys = malloc(sizeof(*picsys));
-- 
2.10.1.windows.1