[vlc-commits] picture: check for overflow in picture_Setup()

[Rémi Denis-Courmont]

vlc | branch: master | Rémi Denis-Courmont <remi at remlab.net> | Sat Feb 10 20:57:14 2018 +0200| [50251bfbaee96d061f950dfbfc39b63797e50917] | committer: Rémi Denis-Courmont

picture: check for overflow in picture_Setup()

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=50251bfbaee96d061f950dfbfc39b63797e50917
---

 src/misc/picture.c | 48 ++++++++++++++++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/src/misc/picture.c b/src/misc/picture.c
index a35ff0ff15..490b4a8300 100644
--- a/src/misc/picture.c
+++ b/src/misc/picture.c
@@ -32,6 +32,7 @@
 # include "config.h"
 #endif
 #include <assert.h>
+#include <limits.h>
 
 #include <vlc_common.h>
 #include "picture.h"
@@ -161,9 +162,10 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt )
        Which is respected if you have
        V % lcm( p_dsc->p[0..planes].w.i_den * 16) == 0
     */
-    int i_modulo_w = 1;
-    int i_modulo_h = 1;
-    unsigned int i_ratio_h  = 1;
+    unsigned i_modulo_w = 1;
+    unsigned i_modulo_h = 1;
+    unsigned i_ratio_h  = 1;
+
     for( unsigned i = 0; i < p_dsc->plane_count; i++ )
     {
         i_modulo_w = LCM( i_modulo_w, 16 * p_dsc->p[i].w.den );
@@ -173,22 +175,44 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt )
     }
     i_modulo_h = LCM( i_modulo_h, 32 );
 
-    const int i_width_aligned  = ( fmt->i_width  + i_modulo_w - 1 ) / i_modulo_w * i_modulo_w;
-    const int i_height_aligned = ( fmt->i_height + i_modulo_h - 1 ) / i_modulo_h * i_modulo_h;
-    const int i_height_extra   = 2 * i_ratio_h; /* This one is a hack for some ASM functions */
+    unsigned width, height;
+
+    if (unlikely(add_overflow(fmt->i_width, i_modulo_w - 1, &width))
+     || unlikely(add_overflow(fmt->i_height, i_modulo_h - 1, &height)))
+        return VLC_EGENERIC;
+
+    width = width / i_modulo_w * i_modulo_w;
+    height = height / i_modulo_h * i_modulo_h;
+
+    /* Hack: append two scan lines for some SIMD assembler */
+    if (unlikely(add_overflow(height, 2 * i_ratio_h, &height)))
+        return VLC_EGENERIC;
+
+    /* plane_t uses 'int'. */
+    if (unlikely(width > INT_MAX) || unlikely(height > INT_MAX))
+        return VLC_EGENERIC;
+
     for( unsigned i = 0; i < p_dsc->plane_count; i++ )
     {
         plane_t *p = &p_picture->p[i];
+        const vlc_rational_t *h = &p_dsc->p[i].h;
+        const vlc_rational_t *w = &p_dsc->p[i].w;
+
+        /* A plane cannot be over-sampled. This could lead to overflow. */
+        assert(h->den >= h->num);
+        assert(w->den >= w->num);
+
+        p->i_lines = height * h->num / h->den;
+        p->i_visible_lines = fmt->i_visible_height * h->num / h->den;
 
-        p->i_lines         = (i_height_aligned + i_height_extra ) * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
-        p->i_visible_lines = fmt->i_visible_height * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
-        p->i_pitch         = i_width_aligned * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
-        p->i_visible_pitch = fmt->i_visible_width * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
-        p->i_pixel_pitch   = p_dsc->pixel_size;
+        p->i_pitch = width * w->num / w->den * p_dsc->pixel_size;
+        p->i_visible_pitch = fmt->i_visible_width * w->num / w->den
+                             * p_dsc->pixel_size;
+        p->i_pixel_pitch = p_dsc->pixel_size;
 
         assert( (p->i_pitch % 16) == 0 );
     }
-    p_picture->i_planes  = p_dsc->plane_count;
+    p_picture->i_planes = p_dsc->plane_count;
 
     return VLC_SUCCESS;
 }