[vlc-commits] demux:mkv: clean the usage of es_format_t i_extra

[Steve Lhomme]

vlc/vlc-3.0 | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Mon Feb 11 08:57:40 2019 +0100| [77dc7898520c141b26623311aedd4fe5146efdb9] | committer: Hugo Beauzée-Luyssen

demux:mkv: clean the usage of es_format_t i_extra

Make sure we don't use negative values or a value when the extra buffer
allocation failed.

https://hackerone.com/reports/493436

Signed-off-by: Hugo Beauzée-Luyssen <hugo at beauzee.fr>
(cherry picked from commit b4f6b391594c5321bef8e2d661b3dde51d88151a)
Signed-off-by: Hugo Beauzée-Luyssen <hugo at beauzee.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=77dc7898520c141b26623311aedd4fe5146efdb9
---

 modules/demux/mkv/matroska_segment_parse.cpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/modules/demux/mkv/matroska_segment_parse.cpp b/modules/demux/mkv/matroska_segment_parse.cpp
index 8f12262ebc..5b8b1a78d9 100644
--- a/modules/demux/mkv/matroska_segment_parse.cpp
+++ b/modules/demux/mkv/matroska_segment_parse.cpp
@@ -40,6 +40,7 @@ extern "C" {
 #include <vlc_codecs.h>
 #include <stdexcept>
 #include <limits>
+#include <algorithm>
 
 /* GetFourCC helper */
 #define GetFOURCC( p )  __GetFOURCC( (uint8_t*)p )
@@ -1496,16 +1497,16 @@ bool matroska_segment_c::TrackInit( mkv_track_t * p_tk )
                 vars.p_fmt->video.i_height= GetDWLE( &p_bih->biHeight );
                 vars.p_fmt->i_codec       = GetFOURCC( &p_bih->biCompression );
 
-                vars.p_fmt->i_extra       = GetDWLE( &p_bih->biSize ) - sizeof( VLC_BITMAPINFOHEADER );
-                if( vars.p_fmt->i_extra > 0 )
+                /* Very unlikely yet possible: bug #5659*/
+                const unsigned int min_extra = std::min(GetDWLE( &p_bih->biSize ), vars.p_tk->i_extra_data);
+                if ( min_extra > sizeof( VLC_BITMAPINFOHEADER ))
                 {
-                    /* Very unlikely yet possible: bug #5659*/
-                    size_t maxlen = vars.p_tk->i_extra_data - sizeof( VLC_BITMAPINFOHEADER );
-                    vars.p_fmt->i_extra = ( (unsigned)vars.p_fmt->i_extra < maxlen )?
-                        vars.p_fmt->i_extra : maxlen;
-
+                    vars.p_fmt->i_extra = min_extra - sizeof( VLC_BITMAPINFOHEADER );
                     vars.p_fmt->p_extra = xmalloc( vars.p_fmt->i_extra );
-                    memcpy( vars.p_fmt->p_extra, &p_bih[1], vars.p_fmt->i_extra );
+                    if (likely(vars.p_fmt->p_extra != NULL))
+                        memcpy( vars.p_fmt->p_extra, &p_bih[1], vars.p_fmt->i_extra );
+                    else
+                        vars.p_fmt->i_extra = 0;
                 }
                 else if( vars.p_fmt->i_codec == VLC_FOURCC('W','V','C','1') )
                 {
@@ -1676,7 +1677,7 @@ bool matroska_segment_c::TrackInit( mkv_track_t * p_tk )
                 p_tk->fmt.audio.i_bitspersample = GetWLE( &p_wf->wBitsPerSample );
 
                 p_tk->fmt.i_extra            = GetWLE( &p_wf->cbSize );
-                if( p_tk->fmt.i_extra > 0 )
+                if( p_tk->fmt.i_extra != 0 )
                 {
                     p_tk->fmt.p_extra = xmalloc( p_tk->fmt.i_extra );
                     if( p_tk->fmt.p_extra )