[vlc-devel] [PATCH 2/2] nvdec: fix copy heights

[quentin.chateau at deepskycorp.com]

From: Quentin Chateau <quentin.chateau at deepskycorp.com>

In the picture context, bufferHeight contains
the full buffer height, not the height of one plane
as it used to. This commit fixed the memory copies
to handle this change.

The device-device copy in nvdec.c has been greatly
simplified as neither the source nor the destination
use planes, the whole copy can be made with a single
cuMemcpy call.
---
 modules/hw/nvdec/chroma.c   |  6 ++--
 modules/hw/nvdec/nvdec.c    | 63 ++++++++-----------------------------
 modules/hw/nvdec/nvdec_gl.c |  2 +-
 3 files changed, 18 insertions(+), 53 deletions(-)

diff --git a/modules/hw/nvdec/chroma.c b/modules/hw/nvdec/chroma.c
index 88353f5cbe..efc9f789d7 100644
--- a/modules/hw/nvdec/chroma.c
+++ b/modules/hw/nvdec/chroma.c
@@ -77,6 +77,8 @@ static picture_t * FilterCUDAToCPU( filter_t *p_filter, picture_t *src )
             memset(plane.p_pixels, 0xFF, plane.i_pitch * plane.i_visible_lines);
             continue;
         }
+
+        size_t copyHeight = __MIN(srcpic->bufferHeight - srcY, (unsigned)plane.i_visible_lines);
         CUDA_MEMCPY2D cu_cpy = {
             .srcMemoryType  = CU_MEMORYTYPE_DEVICE,
             .srcDevice      = srcpic->devicePtr,
@@ -86,12 +88,12 @@ static picture_t * FilterCUDAToCPU( filter_t *p_filter, picture_t *src )
             .dstHost        = plane.p_pixels,
             .dstPitch       = plane.i_pitch,
             .WidthInBytes   = __MIN(srcpic->bufferPitch, (unsigned)dst->p[0].i_pitch),
-            .Height         = __MIN(srcpic->bufferHeight, (unsigned)plane.i_visible_lines),
+            .Height         = copyHeight,
         };
         result = CALL_CUDA(cuMemcpy2DAsync, &cu_cpy, 0);
         if (result != VLC_SUCCESS)
             goto done;
-        srcY += srcpic->bufferHeight;
+        srcY += copyHeight;
     }
     picture_CopyProperties(dst, src);
 
diff --git a/modules/hw/nvdec/nvdec.c b/modules/hw/nvdec/nvdec.c
index 8bfea6aa4a..387a5503d9 100644
--- a/modules/hw/nvdec/nvdec.c
+++ b/modules/hw/nvdec/nvdec.c
@@ -327,56 +327,19 @@ static int CUDAAPI HandlePictureDisplay(void *p_opaque, CUVIDPARSERDISPINFO *p_d
         if (result != VLC_SUCCESS)
             goto error;
 
-        size_t srcY = 0;
-        size_t dstY = 0;
-        if (p_pic->format.i_chroma == VLC_CODEC_NVDEC_OPAQUE_444 || p_pic->format.i_chroma == VLC_CODEC_NVDEC_OPAQUE_444_16B)
-        {
-            for (int i_plane = 0; i_plane < 3; i_plane++) {
-                CUDA_MEMCPY2D cu_cpy = {
-                    .srcMemoryType  = CU_MEMORYTYPE_DEVICE,
-                    .srcDevice      = frameDevicePtr,
-                    .srcY           = srcY,
-                    .srcPitch       = i_pitch,
-                    .dstMemoryType  = CU_MEMORYTYPE_DEVICE,
-                    .dstDevice      = picctx->devicePtr,
-                    .dstPitch       = picctx->bufferPitch,
-                    .dstY           = dstY,
-                    .WidthInBytes   = i_pitch,
-                    .Height         = __MIN(picctx->bufferHeight, p_dec->fmt_out.video.i_y_offset + p_dec->fmt_out.video.i_visible_height),
-                };
-                result = CALL_CUDA_DEC(cuMemcpy2DAsync, &cu_cpy, 0);
-                if (unlikely(result != VLC_SUCCESS))
-                    goto error;
-
-                srcY += picctx->bufferHeight;
-                dstY += p_sys->decoderHeight;
-            }
-        }
-        else
-        {
-            for (int i_plane = 0; i_plane < 2; i_plane++) {
-                CUDA_MEMCPY2D cu_cpy = {
-                    .srcMemoryType  = CU_MEMORYTYPE_DEVICE,
-                    .srcDevice      = frameDevicePtr,
-                    .srcY           = srcY,
-                    .srcPitch       = i_pitch,
-                    .dstMemoryType  = CU_MEMORYTYPE_DEVICE,
-                    .dstDevice      = picctx->devicePtr,
-                    .dstPitch       = picctx->bufferPitch,
-                    .dstY           = dstY,
-                    .WidthInBytes   = i_pitch,
-                    .Height         = __MIN(picctx->bufferHeight, p_dec->fmt_out.video.i_y_offset + p_dec->fmt_out.video.i_visible_height),
-                };
-                if (i_plane == 1)
-                    cu_cpy.Height >>= 1;
-                result = CALL_CUDA_DEC(cuMemcpy2DAsync, &cu_cpy, 0);
-                if (unlikely(result != VLC_SUCCESS))
-                    goto error;
-
-                srcY += picctx->bufferHeight;
-                dstY += p_sys->decoderHeight;
-            }
-        }
+        CUDA_MEMCPY2D cu_cpy = {
+            .srcMemoryType  = CU_MEMORYTYPE_DEVICE,
+            .srcDevice      = frameDevicePtr,
+            .srcPitch       = i_pitch,
+            .dstMemoryType  = CU_MEMORYTYPE_DEVICE,
+            .dstDevice      = picctx->devicePtr,
+            .dstPitch       = picctx->bufferPitch,
+            .WidthInBytes   = i_pitch,
+            .Height         = picctx->bufferHeight,
+        };
+        result = CALL_CUDA_DEC(cuMemcpy2DAsync, &cu_cpy, 0);
+        if (unlikely(result != VLC_SUCCESS))
+            goto error;
     }
     else
     {
diff --git a/modules/hw/nvdec/nvdec_gl.c b/modules/hw/nvdec/nvdec_gl.c
index 137e730028..dc69179aea 100644
--- a/modules/hw/nvdec/nvdec_gl.c
+++ b/modules/hw/nvdec/nvdec_gl.c
@@ -138,7 +138,7 @@ tc_nvdec_gl_update(const struct vlc_gl_interop *interop, GLuint textures[],
         result = CALL_CUDA(cuMemcpy2DAsync, &cu_cpy, 0);
         if (result != VLC_SUCCESS)
             goto error;
-        srcY += srcpic->bufferHeight;
+        srcY += tex_heights[i];
     }
 
 error:
-- 
2.17.1