[vlc-devel] [PATCH v2] direct3d9: add an option to use DXVA-HD rather than StretchRect by default

[Steve Lhomme]

The driver may lie about its conversion capabilities (like Y410 to X8R8G8B8 on
Intel) when DXVA-HD works correctly. So we use this option by default.

--
v2 adds the same option for the OpenGL DXVA2 Interop module which could benefit
from this too.

---
 modules/video_output/opengl/interop_dxva2.c | 15 ++++++---
 modules/video_output/win32/direct3d9.c      | 34 +++++++++++++--------
 2 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/modules/video_output/opengl/interop_dxva2.c b/modules/video_output/opengl/interop_dxva2.c
index 9da5b7be468..fedb15710de 100644
--- a/modules/video_output/opengl/interop_dxva2.c
+++ b/modules/video_output/opengl/interop_dxva2.c
@@ -59,6 +59,9 @@
 static int  GLConvOpen(vlc_object_t *);
 static void GLConvClose(vlc_object_t *);
 
+#define DXVAHD_TEXT N_("Use DXVA-HD conversion")
+#define DXVAHD_LONGTEXT N_("Use DXVA-HD to render pictures instead of legacy StrecthRect.")
+
 vlc_module_begin ()
     set_shortname("dxva2")
     set_category(CAT_VIDEO)
@@ -66,6 +69,8 @@ vlc_module_begin ()
     set_description("DXVA2 surface converter")
     set_capability("glinterop", 1)
     set_callbacks(GLConvOpen, GLConvClose)
+
+    add_bool("direct3d9-dxvahd", true, DXVAHD_TEXT, DXVAHD_LONGTEXT, true)
 vlc_module_end ()
 
 struct wgl_vt {
@@ -453,15 +458,15 @@ GLConvOpen(vlc_object_t *obj)
     priv->vt = vt;
 
     HRESULT hr;
-    bool force_dxva_hd = false;
-    if ( interop->fmt.color_range != COLOR_RANGE_FULL &&
-         d3d9_decoder->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA )
+    bool force_dxva_hd = var_InheritBool(interop, "direct3d9-dxvahd");
+    if (force_dxva_hd || (interop->fmt.color_range != COLOR_RANGE_FULL &&
+                          d3d9_decoder->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA))
     {
         // NVIDIA bug, YUV to RGB internal conversion in StretchRect always converts from limited to limited range
         msg_Dbg(interop->gl, "init DXVA-HD processor from %4.4s to RGB", (const char*)&vctx_sys->format);
         int err = InitRangeProcessor(interop, d3d9_decoder->d3ddev.devex, vctx_sys->format);
-        if (err == VLC_SUCCESS)
-            force_dxva_hd = true;
+        if (err != VLC_SUCCESS)
+            force_dxva_hd = false;
     }
     if (!force_dxva_hd)
     {
diff --git a/modules/video_output/win32/direct3d9.c b/modules/video_output/win32/direct3d9.c
index b72033b170b..d256044037d 100644
--- a/modules/video_output/win32/direct3d9.c
+++ b/modules/video_output/win32/direct3d9.c
@@ -90,6 +90,9 @@ static void Close(vout_display_t *);
 /* The latest option in the selection list: used for loading a shader file. */
 #define SELECTED_SHADER_FILE N_("HLSL File")
 
+#define DXVAHD_TEXT N_("Use DXVA-HD conversion")
+#define DXVAHD_LONGTEXT N_("Use DXVA-HD to render pictures instead of legacy StrecthRect.")
+
 #define D3D9_HELP N_("Recommended video output for Windows Vista and later versions")
 
 vlc_module_begin ()
@@ -106,6 +109,8 @@ vlc_module_begin ()
     add_loadfile("direct3d9-shader-file", NULL,
                  PIXEL_SHADER_FILE_TEXT, PIXEL_SHADER_FILE_LONGTEXT)
 
+    add_bool("direct3d9-dxvahd", true, DXVAHD_TEXT, DXVAHD_LONGTEXT, true)
+
     add_shortcut("direct3d9", "direct3d")
     set_callback_display(Open, 280)
 vlc_module_end ()
@@ -1625,20 +1630,9 @@ static int Direct3D9Open(vout_display_t *vd, video_format_t *fmt, vlc_video_cont
         msg_Err(vd, "unsupported source pixel format %4.4s", &vd->source.i_chroma);
         goto error;
     }
-    msg_Dbg(vd, "found input surface format %s", d3dfmt->name);
-
-    bool force_dxva_hd = false;
-    // test whether device can perform color-conversion from that format to target format
-    HRESULT hr = IDirect3D9_CheckDeviceFormatConversion(sys->d3d9_device->hd3d.obj,
-                                                sys->d3d9_device->d3ddev.adapterId,
-                                                D3DDEVTYPE_HAL,
-                                                d3dfmt->format, sys->BufferFormat);
-    if (FAILED(hr))
-    {
-        msg_Dbg(vd, "Unsupported conversion trying with DXVA-HD");
-        force_dxva_hd = true;
-    }
+    msg_Dbg(vd, "found input surface format %s for source %4.4s", d3dfmt->name, (const char *)&vd->source.i_chroma);
 
+    bool force_dxva_hd = var_InheritBool(vd, "direct3d9-dxvahd");
     if (force_dxva_hd || (dst_format && vd->source.color_range != COLOR_RANGE_FULL && dst_format->rmask && !d3dfmt->rmask &&
                           sys->d3d9_device->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA))
     {
@@ -1646,7 +1640,21 @@ static int Direct3D9Open(vout_display_t *vd, video_format_t *fmt, vlc_video_cont
         msg_Dbg(vd, "init DXVA-HD processor from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown");
         int err = InitRangeProcessor( vd, d3dfmt, &render_out );
         if (err != VLC_SUCCESS)
+            force_dxva_hd = false;
+    }
+    if (!force_dxva_hd)
+    {
+        // test whether device can perform color-conversion from that format to target format
+        HRESULT hr = IDirect3D9_CheckDeviceFormatConversion(sys->d3d9_device->hd3d.obj,
+                                                    sys->d3d9_device->d3ddev.adapterId,
+                                                    D3DDEVTYPE_HAL,
+                                                    d3dfmt->format, sys->BufferFormat);
+        if (FAILED(hr))
+        {
+            msg_Dbg(vd, "Unsupported conversion from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown" );
             goto error;
+        }
+        msg_Dbg(vd, "using StrecthRect from %s to %s", d3dfmt->name, dst_format?dst_format->name:"unknown" );
     }
 
     /* */
-- 
2.17.1