[vlc-devel] commit: Fixed segfault with invalid vout input aspect ratio. ( Laurent Aimar )

[git version control]

vlc | branch: master | Laurent Aimar <fenrir at videolan.org> | Wed Aug 27 13:31:26 2008 +0200| [52b77526210ff7b1aeb9f30b4ab023e393ffdcbd] | committer: Laurent Aimar 

Fixed segfault with invalid vout input aspect ratio.

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=52b77526210ff7b1aeb9f30b4ab023e393ffdcbd
---

 src/video_output/video_output.c  |   10 ++++++++--
 src/video_output/vout_pictures.c |   24 +++++++++++++-----------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index a68d7fd..d2d4719 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -227,6 +227,14 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
     char *psz_parser;
     char *psz_name;
 
+    if( i_width <= 0 || i_height <= 0 || i_aspect <= 0 )
+        return NULL;
+
+    vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
+                 p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
+    if( p_fmt->i_sar_num <= 0 || p_fmt->i_sar_den <= 0 )
+        return NULL;
+
     /* Allocate descriptor */
     static const char typename[] = "video output";
     p_vout = vlc_custom_create( p_parent, sizeof( *p_vout ), VLC_OBJECT_VOUT,
@@ -251,8 +259,6 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
     /* Initialize the rendering heap */
     I_RENDERPICTURES = 0;
 
-    vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
-                 p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
     p_vout->fmt_render        = *p_fmt;   /* FIXME palette */
     p_vout->fmt_in            = *p_fmt;   /* FIXME palette */
 
diff --git a/src/video_output/vout_pictures.c b/src/video_output/vout_pictures.c
index 8c63e33..9c62089 100644
--- a/src/video_output/vout_pictures.c
+++ b/src/video_output/vout_pictures.c
@@ -471,20 +471,22 @@ void vout_PlacePicture( vout_thread_t *p_vout,
         *pi_height = __MIN( i_height, p_vout->fmt_in.i_visible_height );
     }
 
-    if( p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
-        *pi_height / p_vout->fmt_in.i_visible_height /
-        p_vout->fmt_in.i_sar_den > *pi_width )
+     int64_t i_scaled_width = p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
+                              *pi_height / p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
+     int64_t i_scaled_height = p_vout->fmt_in.i_visible_height * (int64_t)p_vout->fmt_in.i_sar_den *
+                               *pi_width / p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+
+    if( i_scaled_width <= 0 || i_scaled_height <= 0 )
     {
-        *pi_height = p_vout->fmt_in.i_visible_height *
-            (int64_t)p_vout->fmt_in.i_sar_den * *pi_width /
-            p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+        msg_Warn( p_vout, "ignoring broken aspect ratio" );
+        i_scaled_width = *pi_width;
+        i_scaled_height = *pi_height;
     }
+
+    if( i_scaled_width > *pi_width )
+        *pi_height = i_scaled_height;
     else
-    {
-        *pi_width = p_vout->fmt_in.i_visible_width *
-            (int64_t)p_vout->fmt_in.i_sar_num * *pi_height /
-            p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
-    }
+        *pi_width = i_scaled_width;
 
     switch( p_vout->i_alignment & VOUT_ALIGN_HMASK )
     {