[vlc-commits] opengl: dxva2: make sure we can handle the conversion in StretchRect

[Steve Lhomme]

vlc | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Mon Mar 16 13:18:32 2020 +0100| [b7a56bc889e4cc02f3bc626c20abb01ee5b394d8] | committer: Steve Lhomme

opengl: dxva2: make sure we can handle the conversion in StretchRect

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=b7a56bc889e4cc02f3bc626c20abb01ee5b394d8
---

 modules/video_output/opengl/interop_dxva2.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/video_output/opengl/interop_dxva2.c b/modules/video_output/opengl/interop_dxva2.c
index a8b37a0835..b42ab8c6eb 100644
--- a/modules/video_output/opengl/interop_dxva2.c
+++ b/modules/video_output/opengl/interop_dxva2.c
@@ -83,6 +83,8 @@ struct glpriv
     HANDLE gl_handle_d3d;
     HANDLE gl_render;
     IDirect3DSurface9 *dx_render;
+
+    D3DFORMAT OutputFormat;
 };
 
 static int
@@ -230,14 +232,26 @@ GLConvOpen(vlc_object_t *obj)
     if (!priv)
         return VLC_ENOMEM;
     interop->priv = priv;
+    priv->OutputFormat = D3DFMT_X8R8G8B8;
     priv->vt = vt;
 
     HRESULT hr;
+    // test whether device can perform color-conversion from that format to target format
+    hr = IDirect3D9_CheckDeviceFormatConversion(d3d9_decoder->hd3d.obj,
+                                                d3d9_decoder->d3ddev.adapterId,
+                                                D3DDEVTYPE_HAL,
+                                                vctx_sys->format, priv->OutputFormat);
+    if (FAILED(hr))
+    {
+        msg_Dbg(interop->gl, "Unsupported conversion from %4.4s to RGB", (const char*)&vctx_sys->format );
+        goto error;
+    }
+
     HANDLE shared_handle = NULL;
     hr = IDirect3DDevice9Ex_CreateRenderTarget(d3d9_decoder->d3ddev.devex,
                                                interop->fmt.i_visible_width,
                                                interop->fmt.i_visible_height,
-                                               D3DFMT_X8R8G8B8,
+                                               priv->OutputFormat,
                                                D3DMULTISAMPLE_NONE, 0, FALSE,
                                                &priv->dx_render, &shared_handle);
     if (FAILED(hr))