[vlc-devel] [PATCH] d3d11va: don't use the external pool on Qualcomm devices if the sizes don't match

[Steve Lhomme]

On such devices the decoder needs to have the exact size with the required DXVA
padding but no more and no less. So we can't anything that doesn't match these
criteria for decoding.

--
replaces https://patches.videolan.org/patch/17624/
- belongs where the size requirements can be checked
---
 modules/codec/avcodec/d3d11va.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/modules/codec/avcodec/d3d11va.c b/modules/codec/avcodec/d3d11va.c
index 699277d31c..d20e5ffdb6 100644
--- a/modules/codec/avcodec/d3d11va.c
+++ b/modules/codec/avcodec/d3d11va.c
@@ -110,6 +110,8 @@ struct vlc_va_sys_t
 {
     directx_sys_t                dx_sys;
     UINT                         totalTextureSlices;
+    unsigned                     textureWidth;
+    unsigned                     textureHeight;
 
 #if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
     HINSTANCE                    dxgidebug_dll;
@@ -373,6 +375,8 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
             D3D11_TEXTURE2D_DESC dstDesc;
             ID3D11Texture2D_GetDesc( p_sys->texture[KNOWN_DXGI_INDEX], &dstDesc);
             sys->render = dstDesc.Format;
+            va->sys->textureWidth = dstDesc.Width;
+            va->sys->textureHeight = dstDesc.Height;
             va->sys->totalTextureSlices = dstDesc.ArraySize;
         }
     }
@@ -689,6 +693,23 @@ static int DxSetupOutput(vlc_va_t *va, const GUID *input, const video_format_t *
     return VLC_EGENERIC;
 }
 
+static bool CanUseDecoderPadding(directx_sys_t *dx_sys)
+{
+    IDXGIAdapter *pAdapter = D3D11DeviceAdapter(dx_sys->d3ddev);
+    if (!pAdapter)
+        return false;
+
+    DXGI_ADAPTER_DESC adapterDesc;
+    HRESULT hr = IDXGIAdapter_GetDesc(pAdapter, &adapterDesc);
+    IDXGIAdapter_Release(pAdapter);
+    if (FAILED(hr))
+        return false;
+
+    /* Qualcomm hardware has issues with textures and pixels that should not be
+    * part of the decoded area */
+    return adapterDesc.VendorId != 0x4D4F4351;
+}
+
 /**
  * It creates a Direct3D11 decoder using the given video format
  */
@@ -717,6 +738,13 @@ static int DxCreateDecoderSurfaces(vlc_va_t *va, int codec_id,
         return VLC_EGENERIC;
     }
 #endif
+    if ((sys->textureWidth != fmt->i_width || sys->textureHeight != fmt->i_height) &&
+        !CanUseDecoderPadding(dx_sys))
+    {
+        msg_Dbg(va, "mismatching external pool sizes use the internal one %dx%d vs %dx%d",
+                sys->textureWidth, sys->textureHeight, fmt->i_width, fmt->i_height);
+        dx_sys->can_extern_pool = false;
+    }
 
     D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC viewDesc;
     ZeroMemory(&viewDesc, sizeof(viewDesc));
-- 
2.12.1