[vlc-devel] [PATCH] direct3d9: fix the black background when the decoder texture is in YUV

[Steve Lhomme]

fixes the bottom line issue in #17420

--
replaces https://patches.videolan.org/patch/16082/
* missing YUV black for the DXVA2 pool
replaces https://patches.videolan.org/patch/16119/
* factorize code
---
 modules/video_output/win32/direct3d9.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/video_output/win32/direct3d9.c b/modules/video_output/win32/direct3d9.c
index f190e65147..ae0c693f66 100644
--- a/modules/video_output/win32/direct3d9.c
+++ b/modules/video_output/win32/direct3d9.c
@@ -334,6 +334,13 @@ static void DestroyPicture(picture_t *picture)
     free(picture);
 }
 
+static D3DCOLOR GetFormatBlack(D3DFORMAT format)
+{
+    if (vlc_fourcc_IsYUV(format))
+        return D3DCOLOR_XYUV(0,0x80,0x80);
+    return D3DCOLOR_ARGB(0xFF, 0, 0, 0);
+}
+
 /* */
 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
 {
@@ -367,6 +374,8 @@ static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
            goto error;
         }
 
+        IDirect3DDevice9_ColorFill(vd->sys->d3ddev, picsys->surface, NULL, GetFormatBlack(format));
+
         picture_resource_t resource = {
             .p_sys = picsys,
             .pf_destroy = DestroyPicture,
@@ -1160,7 +1169,7 @@ static int Direct3D9CreatePool(vout_display_t *vd, video_format_t *fmt)
 #endif
 
     /* fill surface with black color */
-    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0));
+    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, GetFormatBlack(d3dfmt->format));
 
     /* Create the associated picture */
     picture_sys_t *picsys = malloc(sizeof(*picsys));
-- 
2.11.1