[vlc-commits] direct3d9: add an option to use DXVA-HD rather than StretchRect by default

[Steve Lhomme]

vlc | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Thu Mar 12 11:46:05 2020 +0100| [e9e46687f353929e66a458b90f915d3fd328888b] | committer: Steve Lhomme

direct3d9: add an option to use DXVA-HD rather than StretchRect by default

The driver may lie about its conversion capabilities (like Y410 to X8R8G8B8 on
Intel) when DXVA-HD works correctly. So we use this option by default.

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=e9e46687f353929e66a458b90f915d3fd328888b
---

 modules/video_chroma/d3d9_fmt.h             |  3 +++
 modules/video_output/opengl/interop_dxva2.c | 12 ++++++-----
 modules/video_output/win32/direct3d9.c      | 31 +++++++++++++++++------------
 3 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/modules/video_chroma/d3d9_fmt.h b/modules/video_chroma/d3d9_fmt.h
index 34c9fd7514..e0772fa894 100644
--- a/modules/video_chroma/d3d9_fmt.h
+++ b/modules/video_chroma/d3d9_fmt.h
@@ -31,6 +31,9 @@
 
 #include "dxgi_fmt.h"
 
+#define DXVAHD_TEXT N_("Use DXVA-HD for color conversion")
+#define DXVAHD_LONGTEXT N_("Use DXVA-HD for color conversion")
+
 /* owned by the vout for VLC_CODEC_D3D9_OPAQUE */
 typedef struct
 {
diff --git a/modules/video_output/opengl/interop_dxva2.c b/modules/video_output/opengl/interop_dxva2.c
index 9da5b7be46..cf2a7a7563 100644
--- a/modules/video_output/opengl/interop_dxva2.c
+++ b/modules/video_output/opengl/interop_dxva2.c
@@ -66,6 +66,8 @@ vlc_module_begin ()
     set_description("DXVA2 surface converter")
     set_capability("glinterop", 1)
     set_callbacks(GLConvOpen, GLConvClose)
+
+    add_bool("direct3d9-dxvahd", true, DXVAHD_TEXT, DXVAHD_LONGTEXT, true)
 vlc_module_end ()
 
 struct wgl_vt {
@@ -453,15 +455,15 @@ GLConvOpen(vlc_object_t *obj)
     priv->vt = vt;
 
     HRESULT hr;
-    bool force_dxva_hd = false;
-    if ( interop->fmt.color_range != COLOR_RANGE_FULL &&
-         d3d9_decoder->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA )
+    bool force_dxva_hd = var_InheritBool(interop, "direct3d9-dxvahd");
+    if (force_dxva_hd || (interop->fmt.color_range != COLOR_RANGE_FULL &&
+                          d3d9_decoder->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA))
     {
         // NVIDIA bug, YUV to RGB internal conversion in StretchRect always converts from limited to limited range
         msg_Dbg(interop->gl, "init DXVA-HD processor from %4.4s to RGB", (const char*)&vctx_sys->format);
         int err = InitRangeProcessor(interop, d3d9_decoder->d3ddev.devex, vctx_sys->format);
-        if (err == VLC_SUCCESS)
-            force_dxva_hd = true;
+        if (err != VLC_SUCCESS)
+            force_dxva_hd = false;
     }
     if (!force_dxva_hd)
     {
diff --git a/modules/video_output/win32/direct3d9.c b/modules/video_output/win32/direct3d9.c
index b72033b170..ef0d83ef01 100644
--- a/modules/video_output/win32/direct3d9.c
+++ b/modules/video_output/win32/direct3d9.c
@@ -106,6 +106,8 @@ vlc_module_begin ()
     add_loadfile("direct3d9-shader-file", NULL,
                  PIXEL_SHADER_FILE_TEXT, PIXEL_SHADER_FILE_LONGTEXT)
 
+    add_bool("direct3d9-dxvahd", true, DXVAHD_TEXT, DXVAHD_LONGTEXT, true)
+
     add_shortcut("direct3d9", "direct3d")
     set_callback_display(Open, 280)
 vlc_module_end ()
@@ -1625,20 +1627,9 @@ static int Direct3D9Open(vout_display_t *vd, video_format_t *fmt, vlc_video_cont
         msg_Err(vd, "unsupported source pixel format %4.4s", &vd->source.i_chroma);
         goto error;
     }
-    msg_Dbg(vd, "found input surface format %s", d3dfmt->name);
-
-    bool force_dxva_hd = false;
-    // test whether device can perform color-conversion from that format to target format
-    HRESULT hr = IDirect3D9_CheckDeviceFormatConversion(sys->d3d9_device->hd3d.obj,
-                                                sys->d3d9_device->d3ddev.adapterId,
-                                                D3DDEVTYPE_HAL,
-                                                d3dfmt->format, sys->BufferFormat);
-    if (FAILED(hr))
-    {
-        msg_Dbg(vd, "Unsupported conversion trying with DXVA-HD");
-        force_dxva_hd = true;
-    }
+    msg_Dbg(vd, "found input surface format %s for source %4.4s", d3dfmt->name, (const char *)&vd->source.i_chroma);
 
+    bool force_dxva_hd = var_InheritBool(vd, "direct3d9-dxvahd");
     if (force_dxva_hd || (dst_format && vd->source.color_range != COLOR_RANGE_FULL && dst_format->rmask && !d3dfmt->rmask &&
                           sys->d3d9_device->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA))
     {
@@ -1646,7 +1637,21 @@ static int Direct3D9Open(vout_display_t *vd, video_format_t *fmt, vlc_video_cont
         msg_Dbg(vd, "init DXVA-HD processor from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown");
         int err = InitRangeProcessor( vd, d3dfmt, &render_out );
         if (err != VLC_SUCCESS)
+            force_dxva_hd = false;
+    }
+    if (!force_dxva_hd)
+    {
+        // test whether device can perform color-conversion from that format to target format
+        HRESULT hr = IDirect3D9_CheckDeviceFormatConversion(sys->d3d9_device->hd3d.obj,
+                                                    sys->d3d9_device->d3ddev.adapterId,
+                                                    D3DDEVTYPE_HAL,
+                                                    d3dfmt->format, sys->BufferFormat);
+        if (FAILED(hr))
+        {
+            msg_Dbg(vd, "Unsupported conversion from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown" );
             goto error;
+        }
+        msg_Dbg(vd, "using StrecthRect from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown" );
     }
 
     /* */