[vlc-devel] [PATCH 1/2] avcodec: move the DXVA surface constraints early in the code

[Steve Lhomme]

It needs to be done before the surface are created in the vout. It was OK when
the decoder was creating its own surfaces but now that it's done in the vout it
needs to have the proper sizes known by the vout.

The constraints are only valid for DXVA2/D3D11VA. Other hardware decoders used
by libavcodec may have different constraints.

Fixes #17856
---
 modules/codec/avcodec/directx_va.c | 15 ++-------------
 modules/codec/avcodec/video.c      | 26 ++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/modules/codec/avcodec/directx_va.c b/modules/codec/avcodec/directx_va.c
index 1c26edcd6e..f588bb0640 100644
--- a/modules/codec/avcodec/directx_va.c
+++ b/modules/codec/avcodec/directx_va.c
@@ -272,7 +272,6 @@ char *directx_va_GetDecoderName(const GUID *guid)
 /* */
 int directx_va_Setup(vlc_va_t *va, directx_sys_t *dx_sys, AVCodecContext *avctx)
 {
-    int surface_alignment = 16;
     int surface_count = 2;
 
     if (dx_sys->width == avctx->coded_width && dx_sys->height == avctx->coded_height
@@ -291,16 +290,7 @@ int directx_va_Setup(vlc_va_t *va, directx_sys_t *dx_sys, AVCodecContext *avctx)
 
     switch ( dx_sys->codec_id )
     {
-    case AV_CODEC_ID_MPEG2VIDEO:
-        /* decoding MPEG-2 requires additional alignment on some Intel GPUs,
-           but it causes issues for H.264 on certain AMD GPUs..... */
-        surface_alignment = 32;
-        surface_count += 2;
-        break;
     case AV_CODEC_ID_HEVC:
-        /* the HEVC DXVA2 spec asks for 128 pixel aligned surfaces to ensure
-           all coding features have enough room to work with */
-        surface_alignment = 128;
         surface_count += 16;
         break;
     case AV_CODEC_ID_H264:
@@ -318,11 +308,10 @@ int directx_va_Setup(vlc_va_t *va, directx_sys_t *dx_sys, AVCodecContext *avctx)
 
     dx_sys->surface_count = surface_count;
 
-#define ALIGN(x, y) (((x) + ((y) - 1)) & ~((y) - 1))
     dx_sys->width  = avctx->coded_width;
     dx_sys->height = avctx->coded_height;
-    dx_sys->surface_width  = ALIGN(dx_sys->width, surface_alignment);
-    dx_sys->surface_height = ALIGN(dx_sys->height, surface_alignment);
+    dx_sys->surface_width  = dx_sys->width;
+    dx_sys->surface_height = dx_sys->height;
 
     /* FIXME transmit a video_format_t by VaSetup directly */
     video_format_t fmt;
diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index f49e82db2f..e559c9f057 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -140,7 +140,33 @@ static int lavc_GetVideoFormat(decoder_t *dec, video_format_t *restrict fmt,
         avcodec_align_dimensions2(ctx, &width, &height, aligns);
     }
     else /* hardware decoding */
+    {
+        if (pix_fmt == AV_PIX_FMT_DXVA2_VLD || pix_fmt == AV_PIX_FMT_D3D11VA_VLD)
+        {
+            int surface_alignment;
+            switch (ctx->codec_id)
+            {
+            case AV_CODEC_ID_MPEG2VIDEO:
+                /* decoding MPEG-2 requires additional alignment on some Intel GPUs,
+                   but it causes issues for H.264 on certain AMD GPUs..... */
+                surface_alignment = 32;
+                break;
+            case AV_CODEC_ID_HEVC:
+                /* the HEVC DXVA2 spec asks for 128 pixel aligned surfaces to ensure
+                   all coding features have enough room to work with */
+                surface_alignment = 128;
+                break;
+            default:
+                surface_alignment = 16;
+                break;
+            }
+#define ALIGN(x, y) (((x) + ((y) - 1)) & ~((y) - 1))
+            width  = ALIGN(width, surface_alignment);
+            height = ALIGN(height, surface_alignment);
+        }
+
         fmt->i_chroma = vlc_va_GetChroma(pix_fmt, sw_pix_fmt);
+    }
 
     if( width == 0 || height == 0 || width > 8192 || height > 8192 )
     {
-- 
2.11.1