[vlc-commits] chroma: cvpx: fix possible buffer overflow

[Thomas Guillem]

vlc/vlc-3.0 | branch: master | Thomas Guillem <thomas at gllm.fr> | Thu Aug  9 13:30:13 2018 +0200| [d9e27703851d3e2b3db300b1f6727569919507ca] | committer: Thomas Guillem

chroma: cvpx: fix possible buffer overflow

cf. https://code.videolan.org/videolan/vlc-ios/issues/261

A lot of crashes happen when doing the CVPX HW -> SW conversion. Unfortunately,
I was unable to reproduce it.

My best guess is that this buffer overflow is caused by the i_visible_height
that is too big.  This value should be correct since it comes from the hxxx
packetizer, but it may be different from the actual height of the cvpx buffer.

(cherry picked from commit d0de1ecd59c07bb5c3d857c3e1bed9616a1e09f3)
Signed-off-by: Thomas Guillem <thomas at gllm.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=d9e27703851d3e2b3db300b1f6727569919507ca
---

 modules/video_chroma/cvpx.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/modules/video_chroma/cvpx.c b/modules/video_chroma/cvpx.c
index 9c586b63c7..fe4aee62d0 100644
--- a/modules/video_chroma/cvpx.c
+++ b/modules/video_chroma/cvpx.c
@@ -78,7 +78,8 @@ vlc_module_end ()
  * CVPX to/from I420 conversion *
  ********************************/
 
-static void Copy(filter_t *p_filter, picture_t *dst, picture_t *src)
+static void Copy(filter_t *p_filter, picture_t *dst, picture_t *src,
+                 unsigned height)
 {
     filter_sys_t *p_sys = p_filter->p_sys;
 
@@ -90,11 +91,11 @@ static void Copy(filter_t *p_filter, picture_t *dst, picture_t *src)
                                     src->p[2].i_pitch };
 
 #define DO(x) \
-    x(dst, src_planes, src_pitches, src->format.i_visible_height, &p_sys->sw.cache)
+    x(dst, src_planes, src_pitches, height, &p_sys->sw.cache)
 #define DO_S(x, shift) \
-    x(dst, src_planes, src_pitches, src->format.i_visible_height, shift, &p_sys->sw.cache)
+    x(dst, src_planes, src_pitches, height, shift, &p_sys->sw.cache)
 #define DO_P(x) \
-    x(dst, src_planes[0], src_pitches[0], src->format.i_visible_height, &p_sys->sw.cache)
+    x(dst, src_planes[0], src_pitches[0], height, &p_sys->sw.cache)
 
     const vlc_fourcc_t infcc = src->format.i_chroma;
     const vlc_fourcc_t outfcc = dst->format.i_chroma;
@@ -153,8 +154,8 @@ static picture_t *CVPX_TO_SW_Filter(filter_t *p_filter, picture_t *src)
 {
     filter_sys_t *p_sys = p_filter->p_sys;
 
-    picture_t *src_sw =
-        cvpxpic_create_mapped(&p_sys->sw.fmt, cvpxpic_get_ref(src), true);
+    CVPixelBufferRef cvpx = cvpxpic_get_ref(src);
+    picture_t *src_sw = cvpxpic_create_mapped(&p_sys->sw.fmt, cvpx, true);
     if (!src_sw)
     {
         picture_Release(src);
@@ -169,7 +170,9 @@ static picture_t *CVPX_TO_SW_Filter(filter_t *p_filter, picture_t *src)
         return NULL;
     }
 
-    Copy(p_filter, dst, src_sw);
+    size_t height = CVPixelBufferGetHeight(cvpx);
+    Copy(p_filter, dst, src_sw, __MIN(height, dst->format.i_visible_height));
+
     picture_Release(src_sw);
 
     picture_CopyProperties(dst, src);
@@ -208,7 +211,8 @@ static picture_t *SW_TO_CVPX_Filter(filter_t *p_filter, picture_t *src)
         return NULL;
     }
 
-    Copy(p_filter, mapped_dst, src);
+    size_t height = CVPixelBufferGetHeight(cvpx);
+    Copy(p_filter, mapped_dst, src, __MIN(height, src->format.i_visible_height));
 
     /* Attach the CVPX to a new opaque picture */
     cvpxpic_attach(dst, cvpxpic_get_ref(mapped_dst));