[vlc-commits] direct3d9: fix the black background when the decoder texture is in YUV

[Steve Lhomme]

vlc/vlc-2.2 | branch: master | Steve Lhomme <robux4 at videolabs.io> | Wed Mar 22 14:34:36 2017 +0100| [b34d49b85b8c5332309df1e49ea8bfd72ecaf89b] | committer: Hugo Beauzée-Luyssen

direct3d9: fix the black background when the decoder texture is in YUV

fixes the bottom line issue in #17420

Manually cherry-picked from: 6b236169c5f1bf6c131f8e7f697552523bcacdd9

Signed-off-by: Hugo Beauzée-Luyssen <hugo at beauzee.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-2.2.git/?a=commit;h=b34d49b85b8c5332309df1e49ea8bfd72ecaf89b
---

 modules/codec/avcodec/dxva2.c       | 8 ++++++++
 modules/video_output/msw/direct3d.c | 9 ++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/modules/codec/avcodec/dxva2.c b/modules/codec/avcodec/dxva2.c
index 3720319..e7f10f1 100644
--- a/modules/codec/avcodec/dxva2.c
+++ b/modules/codec/avcodec/dxva2.c
@@ -836,6 +836,13 @@ static int DxFindVideoServiceConversion(vlc_va_t *va, GUID *input, D3DFORMAT *ou
     return VLC_EGENERIC;
 }
 
+static D3DCOLOR GetFormatBlack(D3DFORMAT format)
+{
+    if (vlc_fourcc_IsYUV(format))
+        return D3DCOLOR_XYUV(0,0x80,0x80);
+    return D3DCOLOR_ARGB(0xFF, 0, 0, 0);
+}
+
 /**
  * It creates a DXVA2 decoder using the given video format
  */
@@ -887,6 +894,7 @@ static int DxCreateVideoDecoder(vlc_va_t *va,
     for (unsigned i = 0; i < sys->surface_count; i++) {
         vlc_va_surface_t *surface = &sys->surface[i];
         surface->d3d = sys->hw_surface[i];
+        IDirect3DDevice9_ColorFill(sys->d3ddev, surface->d3d, NULL, GetFormatBlack(sys->render));
         surface->refcount = 0;
         surface->order = 0;
     }
diff --git a/modules/video_output/msw/direct3d.c b/modules/video_output/msw/direct3d.c
index 67f9921..acb0fa0 100644
--- a/modules/video_output/msw/direct3d.c
+++ b/modules/video_output/msw/direct3d.c
@@ -270,6 +270,13 @@ static void Close(vlc_object_t *object)
     free(vd->sys);
 }
 
+static D3DCOLOR GetFormatBlack(D3DFORMAT format)
+{
+    if (vlc_fourcc_IsYUV(format))
+        return D3DCOLOR_XYUV(0,0x80,0x80);
+    return D3DCOLOR_ARGB(0xFF, 0, 0, 0);
+}
+
 /* */
 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
 {
@@ -982,7 +989,7 @@ static int Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt)
         return VLC_EGENERIC;
     }
     /* fill surface with black color */
-    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0));
+    IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, GetFormatBlack(d3dfmt->format));
 
     /* Create the associated picture */
     picture_sys_t *picsys = malloc(sizeof(*picsys));