[vlc-devel] commit: __vout_AllocatePicture() : check all possibilities of integer overflow ( Rafaël Carré )

[git version control]

vlc | branch: master | Rafaël Carré <rafael.carre at gmail.com> | Fri Oct 30 13:35:52 2009 +0100| [4e541fa9cad02b10e3cc7bb237695288189b1e13] | committer: Rafaël Carré 

__vout_AllocatePicture() : check all possibilities of integer overflow

Thanks to nefrir and mfwitten on irc for their help

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=4e541fa9cad02b10e3cc7bb237695288189b1e13
---

 src/video_output/vout_pictures.c |   27 +++++++++++++++++++++++++--
 1 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/video_output/vout_pictures.c b/src/video_output/vout_pictures.c
index 68656bc..fe56010 100644
--- a/src/video_output/vout_pictures.c
+++ b/src/video_output/vout_pictures.c
@@ -570,8 +570,31 @@ int __vout_AllocatePicture( vlc_object_t *p_this, picture_t *p_pic,
     }
 
     /* Calculate how big the new image should be */
-    size_t i_bytes = (size_t)p_pic->format.i_bits_per_pixel *
-        i_width_aligned * i_height_aligned / 8;
+
+    /*
+     * bytes = width_aligned * height_aligned * bpp / 8
+     * We need to check for an integer overflow at each multiplication since
+     * height & width (and bpp?) could be arbitrary large
+     */
+
+    size_t i_bytes = 0;
+    /* i_width_aligned is a multiple of 16, so we can divide by 8 now */
+    size_t i_width_aligned_divided = i_width_aligned / 8;
+    if( i_width_aligned_divided <= (SIZE_MAX/i_height_aligned) )
+    {
+        size_t i_pixels_divided = i_width_aligned_divided * i_height_aligned;
+        size_t i_bpp = p_pic->format.i_bits_per_pixel;
+        if( i_pixels_divided <= (SIZE_MAX/i_bpp) )
+        {
+            i_bytes = i_pixels_divided * i_bpp;
+        }
+    }
+
+    if( i_bytes == 0 )
+    {
+        p_pic->i_planes = 0;
+        return VLC_ENOMEM;
+    }
 
     p_pic->p_data = vlc_memalign( &p_pic->p_data_orig, 16, i_bytes );