[vlc-commits] d3d11va: don't use the external pool on Qualcomm devices if the sizes don't match

[Steve Lhomme]

vlc | branch: master | Steve Lhomme <robUx4 at videolabs.io> | Wed Jul 26 15:06:39 2017 +0200| [e1be6b1563d16e9fdb77d75d4a2d2f6cc1e4ffcf] | committer: Jean-Baptiste Kempf

d3d11va: don't use the external pool on Qualcomm devices if the sizes don't match

On such devices the decoder needs to have the exact size with the required DXVA
padding but no more and no less. So we can't anything that doesn't match these
criteria for decoding.

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=e1be6b1563d16e9fdb77d75d4a2d2f6cc1e4ffcf
---

 modules/codec/avcodec/d3d11va.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/modules/codec/avcodec/d3d11va.c b/modules/codec/avcodec/d3d11va.c
index 12ee88fcb8..0a874e48e6 100644
--- a/modules/codec/avcodec/d3d11va.c
+++ b/modules/codec/avcodec/d3d11va.c
@@ -110,6 +110,8 @@ struct vlc_va_sys_t
 {
     directx_sys_t                dx_sys;
     UINT                         totalTextureSlices;
+    unsigned                     textureWidth;
+    unsigned                     textureHeight;
 
 #if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
     HINSTANCE                    dxgidebug_dll;
@@ -373,6 +375,8 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
             D3D11_TEXTURE2D_DESC dstDesc;
             ID3D11Texture2D_GetDesc( p_sys->texture[KNOWN_DXGI_INDEX], &dstDesc);
             sys->render = dstDesc.Format;
+            va->sys->textureWidth = dstDesc.Width;
+            va->sys->textureHeight = dstDesc.Height;
             va->sys->totalTextureSlices = dstDesc.ArraySize;
         }
     }
@@ -698,6 +702,23 @@ static int DxSetupOutput(vlc_va_t *va, const GUID *input, const video_format_t *
     return VLC_EGENERIC;
 }
 
+static bool CanUseDecoderPadding(directx_sys_t *dx_sys)
+{
+    IDXGIAdapter *pAdapter = D3D11DeviceAdapter(dx_sys->d3ddev);
+    if (!pAdapter)
+        return false;
+
+    DXGI_ADAPTER_DESC adapterDesc;
+    HRESULT hr = IDXGIAdapter_GetDesc(pAdapter, &adapterDesc);
+    IDXGIAdapter_Release(pAdapter);
+    if (FAILED(hr))
+        return false;
+
+    /* Qualcomm hardware has issues with textures and pixels that should not be
+    * part of the decoded area */
+    return adapterDesc.VendorId != 0x4D4F4351;
+}
+
 /**
  * It creates a Direct3D11 decoder using the given video format
  */
@@ -726,6 +747,13 @@ static int DxCreateDecoderSurfaces(vlc_va_t *va, int codec_id,
         return VLC_EGENERIC;
     }
 #endif
+    if ((sys->textureWidth != fmt->i_width || sys->textureHeight != fmt->i_height) &&
+        !CanUseDecoderPadding(dx_sys))
+    {
+        msg_Dbg(va, "mismatching external pool sizes use the internal one %dx%d vs %dx%d",
+                sys->textureWidth, sys->textureHeight, fmt->i_width, fmt->i_height);
+        dx_sys->can_extern_pool = false;
+    }
 
     D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC viewDesc;
     ZeroMemory(&viewDesc, sizeof(viewDesc));