[vlc-devel] [PATCH 2/3] d3d11_fmt: add the possibility to select an RGB format

[Steve Lhomme]

By default YUV formats are favored.
---
 modules/hw/d3d11/d3d11_surface.c        |  2 +-
 modules/video_chroma/d3d11_fmt.c        |  3 +++
 modules/video_chroma/d3d11_fmt.h        |  9 +++++----
 modules/video_output/win32/direct3d11.c | 17 ++++++++++-------
 4 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/modules/hw/d3d11/d3d11_surface.c b/modules/hw/d3d11/d3d11_surface.c
index e8e34b7fe6..e9d601fe8b 100644
--- a/modules/hw/d3d11/d3d11_surface.c
+++ b/modules/hw/d3d11/d3d11_surface.c
@@ -217,7 +217,7 @@ static int assert_staging(filter_t *p_filter, picture_sys_t *p_sys)
         /* failed with the this format, try a different one */
         UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
         const d3d_format_t *new_fmt =
-                FindD3D11Format( p_device, 0, 0, false, supportFlags );
+                FindD3D11Format( p_device, 0, false, 0, false, supportFlags );
         if (new_fmt && texDesc.Format != new_fmt->formatTexture)
         {
             DXGI_FORMAT srcFormat = texDesc.Format;
diff --git a/modules/video_chroma/d3d11_fmt.c b/modules/video_chroma/d3d11_fmt.c
index 8d3b3bd848..cce11fec5b 100644
--- a/modules/video_chroma/d3d11_fmt.c
+++ b/modules/video_chroma/d3d11_fmt.c
@@ -355,6 +355,7 @@ int D3D11CheckDriverVersion(ID3D11Device *d3ddev, UINT vendorId, const struct wd
 
 const d3d_format_t *FindD3D11Format(ID3D11Device *d3ddevice,
                                     vlc_fourcc_t i_src_chroma,
+                                    bool rgb_only,
                                     uint8_t bits_per_channel,
                                     bool allow_opaque,
                                     UINT supportFlags)
@@ -369,6 +370,8 @@ const d3d_format_t *FindD3D11Format(ID3D11Device *d3ddevice,
             continue;
         if (!allow_opaque && is_d3d11_opaque(output_format->fourcc))
             continue;
+        if (rgb_only && vlc_fourcc_IsYUV(output_format->fourcc))
+            continue;
 
         DXGI_FORMAT textureFormat;
         if (output_format->formatTexture == DXGI_FORMAT_UNKNOWN)
diff --git a/modules/video_chroma/d3d11_fmt.h b/modules/video_chroma/d3d11_fmt.h
index 68d02b9fe5..d8687421eb 100644
--- a/modules/video_chroma/d3d11_fmt.h
+++ b/modules/video_chroma/d3d11_fmt.h
@@ -119,10 +119,11 @@ static inline bool DeviceSupportsFormat(ID3D11Device *d3ddevice,
 }
 
 const d3d_format_t *FindD3D11Format(ID3D11Device *d3ddevice,
-                                                  vlc_fourcc_t i_src_chroma,
-                                                  uint8_t bits_per_channel,
-                                                  bool allow_opaque,
-                                                  UINT supportFlags);
+                                    vlc_fourcc_t i_src_chroma,
+                                    bool rgb_only,
+                                    uint8_t bits_per_channel,
+                                    bool allow_opaque,
+                                    UINT supportFlags);
 
 int AllocateTextures(vlc_object_t *obj, d3d11_device_t *d3d_dev,
                      const d3d_format_t *cfg, const video_format_t *fmt,
diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index 8985d3b6f1..7d2efceb2d 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -1445,27 +1445,27 @@ static const d3d_format_t *GetDirectRenderingFormat(vout_display_t *vd, vlc_four
     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
     if (is_d3d11_opaque(i_src_chroma))
         supportFlags |= D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
-    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
+    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
 }
 
 static const d3d_format_t *GetDirectDecoderFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
 {
     UINT supportFlags = D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
-    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
+    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
 }
 
-static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor)
+static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor, bool rgb_only)
 {
     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
     if (from_processor)
         supportFlags |= D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
-    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, 0, bit_depth, false, supportFlags );
+    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, 0, rgb_only, bit_depth, false, supportFlags );
 }
 
 static const d3d_format_t *GetBlendableFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
 {
     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_BLENDABLE;
-    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, 0, false, supportFlags );
+    return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, false, supportFlags );
 }
 
 static int Direct3D11Open(vout_display_t *vd)
@@ -1603,12 +1603,15 @@ static int SetupOutputFormat(vout_display_t *vd, video_format_t *fmt)
             break;
         }
 
-        sys->picQuadConfig = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL);
+        bool is_rgb = !vlc_fourcc_IsYUV(fmt->i_chroma);
+        sys->picQuadConfig = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, is_rgb);
+        if (!sys->picQuadConfig && is_rgb)
+            sys->picQuadConfig = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, false);
     }
 
     // look for any pixel format that we can handle
     if ( !sys->picQuadConfig )
-        sys->picQuadConfig = GetDisplayFormatByDepth(vd, 0, false);
+        sys->picQuadConfig = GetDisplayFormatByDepth(vd, 0, false, false);
 
     if ( !sys->picQuadConfig )
     {
-- 
2.14.2