[vlc-commits] codec: flac: fix heap write overflow on frame format change

[Francois Cartegnie]

vlc | branch: master | Francois Cartegnie <fcvlcdev at free.fr> | Wed May 31 13:02:29 2017 +0200| [83b646f1e8fb89f99064d9aaef3754ccc77bbeac] | committer: Francois Cartegnie

codec: flac: fix heap write overflow on frame format change

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=83b646f1e8fb89f99064d9aaef3754ccc77bbeac
---

 modules/codec/flac.c | 63 +++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 18 deletions(-)

diff --git a/modules/codec/flac.c b/modules/codec/flac.c
index b53dc8406f..7d2913a40d 100644
--- a/modules/codec/flac.c
+++ b/modules/codec/flac.c
@@ -195,6 +195,29 @@ static void Interleave( int32_t *p_out, const int32_t * const *pp_in,
 }
 
 /*****************************************************************************
+ * DecoderSetOutputFormat: helper function to convert and check frame format
+ *****************************************************************************/
+static int DecoderSetOutputFormat( unsigned i_channels, unsigned i_rate,
+                                   unsigned i_streaminfo_rate,
+                                   unsigned i_bitspersample,
+                                   audio_format_t *fmt,
+                                   uint8_t *pi_channels_reorder )
+{
+    if( i_channels == 0 || i_channels > FLAC__MAX_CHANNELS ||
+        i_bitspersample == 0 || (i_rate == 0 && i_streaminfo_rate == 0) )
+        return VLC_EGENERIC;
+
+    fmt->i_channels = i_channels;
+    fmt->i_rate = (i_rate > 0 ) ? i_rate : i_streaminfo_rate;
+    fmt->i_physical_channels =
+    fmt->i_original_channels = pi_channels_maps[i_channels];
+    memcpy( pi_channels_reorder, ppi_reorder[i_channels], i_channels );
+    fmt->i_bitspersample = i_bitspersample;
+
+    return VLC_SUCCESS;
+}
+
+/*****************************************************************************
  * DecoderWriteCallback: called by libflac to output decoded samples
  *****************************************************************************/
 static FLAC__StreamDecoderWriteStatus
@@ -206,15 +229,28 @@ DecoderWriteCallback( const FLAC__StreamDecoder *decoder,
     decoder_t *p_dec = (decoder_t *)client_data;
     decoder_sys_t *p_sys = p_dec->p_sys;
 
-    if( p_dec->fmt_out.audio.i_channels == 0 ||
-        p_dec->fmt_out.audio.i_channels > 8 )
-        return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE;
-    if( date_Get( &p_sys->end_date ) <= VLC_TS_INVALID )
+    if( DecoderSetOutputFormat( frame->header.channels,
+                                frame->header.sample_rate,
+                                p_sys->b_stream_info ? p_sys->stream_info.sample_rate : 0,
+                                frame->header.bits_per_sample,
+                                &p_dec->fmt_out.audio,
+                                p_sys->rgi_channels_reorder ) )
         return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE;
 
+    if( p_sys->end_date.i_divider_num != p_dec->fmt_out.audio.i_rate )
+    {
+        if( p_sys->end_date.i_divider_num > 0 )
+            date_Change( &p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1 );
+        else
+            date_Init( &p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1 );
+    }
+
     if( decoder_UpdateAudioFormat( p_dec ) )
         return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE;
 
+    if( date_Get( &p_sys->end_date ) <= VLC_TS_INVALID )
+        return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE;
+
     p_sys->p_aout_buffer =
         decoder_NewAudioBuffer( p_dec, frame->header.blocksize );
 
@@ -277,20 +313,11 @@ static void DecoderMetadataCallback( const FLAC__StreamDecoder *decoder,
     {
         case FLAC__METADATA_TYPE_STREAMINFO:
             /* Setup the format */
-            p_dec->fmt_out.audio.i_rate     = metadata->data.stream_info.sample_rate;
-            p_dec->fmt_out.audio.i_channels = metadata->data.stream_info.channels;
-            if( metadata->data.stream_info.channels < 9 )
-            {
-                p_dec->fmt_out.audio.i_physical_channels =
-                p_dec->fmt_out.audio.i_original_channels =
-                    pi_channels_maps[metadata->data.stream_info.channels];
-                memcpy( p_sys->rgi_channels_reorder,
-                        ppi_reorder[metadata->data.stream_info.channels],
-                        metadata->data.stream_info.channels );
-            }
-            if (!p_dec->fmt_out.audio.i_bitspersample)
-                p_dec->fmt_out.audio.i_bitspersample =
-                    metadata->data.stream_info.bits_per_sample;
+            DecoderSetOutputFormat( metadata->data.stream_info.channels,
+                                    metadata->data.stream_info.sample_rate,
+                                    metadata->data.stream_info.sample_rate,
+                                    metadata->data.stream_info.bits_per_sample,
+                                    &p_dec->fmt_out.audio, p_sys->rgi_channels_reorder );
 
             msg_Dbg( p_dec, "channels:%d samplerate:%d bitspersamples:%d",
                      p_dec->fmt_out.audio.i_channels, p_dec->fmt_out.audio.i_rate,