[vlc-commits] Avoid potential integer overflow when computing AR when transcoding.

[Laurent Aimar]

vlc/vlc-1.2 | branch: master | Laurent Aimar <fenrir at videolan.org> | Tue Jan 10 20:58:15 2012 +0100| [b6d998df7ae0f74b884e6989a0de06b49c2a92ec] | committer: Jean-Baptiste Kempf

Avoid potential integer overflow when computing AR when transcoding.

It was introduced by [99e262c40f7de62aa4969b1584cce3390f80cc23].
(cherry picked from commit 0c0c1a06aedab0ddd73c7a9bf5290527deb07da0)

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-1.2.git/?a=commit;h=b6d998df7ae0f74b884e6989a0de06b49c2a92ec
---

 modules/stream_out/transcode/video.c |   24 ++++++++++++------------
 1 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 9bb4a5f..4278d59 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -493,20 +493,20 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( id->p_encoder->fmt_out.video.i_sar_num <= 0 ||
         id->p_encoder->fmt_out.video.i_sar_den <= 0 )
     {
-        vlc_ureduce( &id->p_decoder->fmt_out.video.i_sar_num,
-                     &id->p_decoder->fmt_out.video.i_sar_den,
-                     id->p_decoder->fmt_out.video.i_sar_num,
-                     id->p_decoder->fmt_out.video.i_sar_den,
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     0 );
+    }
+    else
+    {
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     id->p_encoder->fmt_out.video.i_sar_num,
+                     id->p_encoder->fmt_out.video.i_sar_den,
                      0 );
-
-        id->p_encoder->fmt_out.video.i_sar_num = id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height;
-        id->p_encoder->fmt_out.video.i_sar_den = id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width;
     }
-    vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
-                 &id->p_encoder->fmt_out.video.i_sar_den,
-                 id->p_encoder->fmt_out.video.i_sar_num,
-                 id->p_encoder->fmt_out.video.i_sar_den,
-                 0 );
 
     id->p_encoder->fmt_in.video.i_sar_num =
         id->p_encoder->fmt_out.video.i_sar_num;