[vlc-devel] [PATCH 2/2] codec: x264: fix memory issue

[Zhao Zhili]

1. potential of invalid alloc when x264_encoder_headers returns negative
on error
2. potential of free invalid pointer or double-free when p_sys->p_sei
allocation failed
---
 modules/codec/x264.c | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/modules/codec/x264.c b/modules/codec/x264.c
index eca3ba0..049ae92 100644
--- a/modules/codec/x264.c
+++ b/modules/codec/x264.c
@@ -1387,25 +1387,37 @@ static int  Open ( vlc_object_t *p_this )
     }
 
     /* get the globals headers */
-    size_t i_extra = x264_encoder_headers( p_sys->h, &nal, &i_nal );
-    uint8_t *p_extra = p_enc->fmt_out.p_extra = malloc( i_extra );
-    if( !p_extra )
+    int i_extra = x264_encoder_headers( p_sys->h, &nal, &i_nal );
+    if( i_extra < 0 )
     {
+        msg_Err( p_enc, "encoder headers failed" );
         Close( VLC_OBJECT(p_enc) );
-        return VLC_ENOMEM;
+        return VLC_EGENERIC;
+    }
+
+    uint8_t *p_extra = NULL;
+    if( i_extra > 0 )
+    {
+        p_extra = malloc( i_extra );
+        if( !p_extra )
+        {
+            Close( VLC_OBJECT(p_enc) );
+            return VLC_ENOMEM;
+        }
     }
 
+    uint8_t *p_extra_tmp = p_extra;
     for( i = 0; i < i_nal; i++ )
     {
         if( nal[i].i_type != NAL_SEI )
         {
-            memcpy( p_extra, nal[i].p_payload, nal[i].i_payload );
-            p_extra += nal[i].i_payload;
+            memcpy( p_extra_tmp, nal[i].p_payload, nal[i].i_payload );
+            p_extra_tmp += nal[i].i_payload;
             continue; /* next NAL */
         }
 
         /* we won't store this NAL in p_extra */
-        assert( i_extra >= (size_t)nal[i].i_payload );
+        assert( i_extra >= nal[i].i_payload );
         i_extra -= nal[i].i_payload;
 
         /* Make sure we only have one SEI NAL in the headers */
@@ -1421,7 +1433,7 @@ static int  Open ( vlc_object_t *p_this )
         }
         memcpy( p_sys->p_sei, nal[i].p_payload, nal[i].i_payload );
     }
-
+    p_enc->fmt_out.p_extra = p_extra;
     p_enc->fmt_out.i_extra = i_extra;
 
     return VLC_SUCCESS;
-- 
2.9.5