[vlc-commits] encoder: dirac: fix heap overflow

[Francois Cartegnie]

vlc | branch: master | Francois Cartegnie <fcvlcdev at free.fr> | Wed Jan 15 19:52:26 2014 +0100| [cf77bbb044a9e58959dadb81adcd818c61a9b10a] | committer: Francois Cartegnie

encoder: dirac: fix heap overflow

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=cf77bbb044a9e58959dadb81adcd818c61a9b10a
---

 modules/codec/dirac.c |   24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/modules/codec/dirac.c b/modules/codec/dirac.c
index 36d8262..94319a3 100644
--- a/modules/codec/dirac.c
+++ b/modules/codec/dirac.c
@@ -41,6 +41,7 @@
 #include <vlc_plugin.h>
 #include <vlc_codec.h>
 #include <vlc_sout.h>
+#include <vlc_picture.h>
 
 #include <libdirac_encoder/dirac_encoder.h>
 
@@ -519,19 +520,16 @@ static int OpenEncoder( vlc_object_t *p_this )
         p_enc->fmt_in.i_codec = VLC_CODEC_I420;
         p_enc->fmt_in.video.i_bits_per_pixel = 12;
         p_sys->ctx.src_params.chroma = format420;
-        p_sys->i_buffer_in = p_enc->fmt_in.video.i_visible_width * p_enc->fmt_in.video.i_visible_height * 3 / 2;
     }
     else if( !strcmp( psz_tmp, "422" ) ) {
         p_enc->fmt_in.i_codec = VLC_CODEC_I422;
         p_enc->fmt_in.video.i_bits_per_pixel = 16;
         p_sys->ctx.src_params.chroma = format422;
-        p_sys->i_buffer_in = p_enc->fmt_in.video.i_visible_width * p_enc->fmt_in.video.i_visible_height * 2;
     }
     else if( !strcmp( psz_tmp, "444" ) ) {
         p_enc->fmt_in.i_codec = VLC_CODEC_I444;
         p_enc->fmt_in.video.i_bits_per_pixel = 24;
         p_sys->ctx.src_params.chroma = format444;
-        p_sys->i_buffer_in = p_enc->fmt_in.video.i_visible_width * p_enc->fmt_in.video.i_visible_height * 3;
     }
     else {
         msg_Err( p_enc, "Invalid chroma format: %s", psz_tmp );
@@ -540,6 +538,26 @@ static int OpenEncoder( vlc_object_t *p_this )
     }
     free( psz_tmp );
 
+    picture_t picture;
+    picture_Setup( &picture, p_enc->fmt_in.i_codec,
+                   p_enc->fmt_in.video.i_width,
+                   p_enc->fmt_in.video.i_height,
+                   p_enc->fmt_in.video.i_sar_num,
+                   p_enc->fmt_in.video.i_sar_den );
+
+    switch( p_enc->fmt_in.i_codec )
+    {
+    case VLC_CODEC_I420:
+        p_sys->i_buffer_in = picture.format.i_visible_width * picture.format.i_visible_height * 3 / 2;
+        break;
+    case VLC_CODEC_I422:
+        p_sys->i_buffer_in = picture.format.i_visible_width * picture.format.i_visible_height * 2;
+        break;
+    case VLC_CODEC_I444:
+        p_sys->i_buffer_in = picture.format.i_visible_width * picture.format.i_visible_height * 3;
+        break;
+    }
+
     p_sys->ctx.enc_params.qf = var_GetFloat( p_enc, ENC_CFG_PREFIX ENC_QUALITY_FACTOR );
 
     /* use bitrate from sout-transcode-vb in kbps */