[vlc-commits] nvdec: fix use-after free in chroma filter

[Quentin Chateau]

vlc | branch: master | Quentin Chateau <quentin.chateau at deepskycorp.com> | Tue Feb 25 12:41:13 2020 +0100| [a47ad04997cffbcca0ce511a97b7dd320872d79f] | committer: Thomas Guillem

nvdec: fix use-after free in chroma filter

Signed-off-by: Thomas Guillem <thomas at gllm.fr>

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=a47ad04997cffbcca0ce511a97b7dd320872d79f
---

 modules/hw/nvdec/chroma.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/modules/hw/nvdec/chroma.c b/modules/hw/nvdec/chroma.c
index c8952624d4..88353f5cbe 100644
--- a/modules/hw/nvdec/chroma.c
+++ b/modules/hw/nvdec/chroma.c
@@ -68,6 +68,7 @@ static picture_t * FilterCUDAToCPU( filter_t *p_filter, picture_t *src )
         return NULL;
     }
 
+    int sync_result;
     size_t srcY = 0;
     for (int i_plane = 0; i_plane < dst->i_planes; i_plane++) {
         plane_t plane = dst->p[i_plane];
@@ -89,17 +90,28 @@ static picture_t * FilterCUDAToCPU( filter_t *p_filter, picture_t *src )
         };
         result = CALL_CUDA(cuMemcpy2DAsync, &cu_cpy, 0);
         if (result != VLC_SUCCESS)
-        {
-            picture_Release(dst);
-            dst = NULL;
             goto done;
-        }
         srcY += srcpic->bufferHeight;
     }
     picture_CopyProperties(dst, src);
 
 done:
+    // Always synchronize the cuda stream before releasing src:
+    // there may be pending async copies even if one of them
+    // returned an error
+    sync_result = CALL_CUDA(cuStreamSynchronize, 0);
+    // Keep result as it was if it was an error
+    // Otherwise use the result of cuStreamSynchronize, which
+    // may return an error related to the async copies as well
+    result = result != VLC_SUCCESS ? result : sync_result;
+
     CALL_CUDA(cuCtxPopCurrent, NULL);
+
+    if (result != VLC_SUCCESS)
+    {
+        picture_Release(dst);
+        dst = NULL;
+    }
     picture_Release(src);
     vlc_decoder_device_Release(dec_dev);
     return dst;