[vlc-devel] [PATCH] direct3d11: don't allocate NV12/P010 textures with odd sizes

[Steve Lhomme]

fixes #17419

--
replaces https://patches.videolan.org/patch/14572/
* also crop the output from D3D11VA according to the NV12/P010 constraints
---
 modules/video_output/win32/direct3d11.c | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index 183984e..3167e60 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -861,14 +861,26 @@ static void Prepare(vout_display_t *vd, picture_t *picture, subpicture_t *subpic
             WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
         }
         D3D11_BOX box;
-        box.left   = picture->format.i_x_offset;
-        box.right  = picture->format.i_x_offset + picture->format.i_visible_width;
-        box.top    = picture->format.i_y_offset;
-        box.bottom = picture->format.i_y_offset + picture->format.i_visible_height;
+        picture_sys_t *p_sys = picture->p_sys;
+        D3D11_TEXTURE2D_DESC texDesc;
+        ID3D11Texture2D_GetDesc( p_sys->texture, &texDesc );
+        if (texDesc.Format == DXGI_FORMAT_NV12 || texDesc.Format == DXGI_FORMAT_P010)
+        {
+            box.left   = (picture->format.i_x_offset + 1) & ~1;
+            box.right  = (picture->format.i_x_offset + picture->format.i_visible_width) & ~1;
+            box.top    = (picture->format.i_y_offset + 1) & ~1;
+            box.bottom = (picture->format.i_y_offset + picture->format.i_visible_height) & ~1;
+        }
+        else
+        {
+            box.left   = picture->format.i_x_offset;
+            box.right  = picture->format.i_x_offset + picture->format.i_visible_width;
+            box.top    = picture->format.i_y_offset;
+            box.bottom = picture->format.i_y_offset + picture->format.i_visible_height;
+        }
         box.back = 1;
         box.front = 0;
 
-        picture_sys_t *p_sys = picture->p_sys;
         ID3D11DeviceContext_CopySubresourceRegion(sys->d3dcontext,
                                                   (ID3D11Resource*) sys->picQuad.pTexture,
                                                   0, 0, 0, 0,
@@ -1714,6 +1726,11 @@ static int AllocQuad(vout_display_t *vd, const video_format_t *fmt, d3d_quad_t *
         if ( i_extra )
             texDesc.Height -= p_chroma_desc->p[plane].h.den / p_chroma_desc->p[plane].h.num - i_extra;
     }
+    if (texDesc.Format == DXGI_FORMAT_NV12 || texDesc.Format == DXGI_FORMAT_P010)
+    {
+        texDesc.Width  &= ~1;
+        texDesc.Height &= ~1;
+    }
 
     hr = ID3D11Device_CreateTexture2D(sys->d3ddevice, &texDesc, NULL, &quad->pTexture);
     if (FAILED(hr)) {
-- 
2.8.2