[vlc-commits] Avoid potential integer overflow when computing AR when transcoding.

[Laurent Aimar]

vlc | branch: master | Laurent Aimar <fenrir at videolan.org> | Tue Jan 10 20:58:15 2012 +0100| [0c0c1a06aedab0ddd73c7a9bf5290527deb07da0] | committer: Laurent Aimar

Avoid potential integer overflow when computing AR when transcoding.

It was introduced by [45ee0e5c08f3ace0cb314f9108839ce856ba3f4e].

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=0c0c1a06aedab0ddd73c7a9bf5290527deb07da0
---

 modules/stream_out/transcode/video.c |   24 ++++++++++++------------
 1 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index d1aa9c4..f4d71fd 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -474,20 +474,20 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( id->p_encoder->fmt_out.video.i_sar_num <= 0 ||
         id->p_encoder->fmt_out.video.i_sar_den <= 0 )
     {
-        vlc_ureduce( &id->p_decoder->fmt_out.video.i_sar_num,
-                     &id->p_decoder->fmt_out.video.i_sar_den,
-                     id->p_decoder->fmt_out.video.i_sar_num,
-                     id->p_decoder->fmt_out.video.i_sar_den,
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     0 );
+    }
+    else
+    {
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     id->p_encoder->fmt_out.video.i_sar_num,
+                     id->p_encoder->fmt_out.video.i_sar_den,
                      0 );
-
-        id->p_encoder->fmt_out.video.i_sar_num = id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height;
-        id->p_encoder->fmt_out.video.i_sar_den = id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width;
     }
-    vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
-                 &id->p_encoder->fmt_out.video.i_sar_den,
-                 id->p_encoder->fmt_out.video.i_sar_num,
-                 id->p_encoder->fmt_out.video.i_sar_den,
-                 0 );
 
     id->p_encoder->fmt_in.video.i_sar_num =
         id->p_encoder->fmt_out.video.i_sar_num;