[vlc-commits] avcodec: encoder: fix buffer over-read

[Thomas Guillem]

vlc/vlc-3.0 | branch: master | Thomas Guillem <thomas at gllm.fr> | Wed May 23 18:59:07 2018 +0200| [cdab0fc5fcc23b9037391a9cdd789cdd23fb4b18] | committer: Thomas Guillem

avcodec: encoder: fix buffer over-read

It happened when the mask was not the default one.

(cherry picked from commit a0f4ae67a0c34691c851f7d6aef2554f17021950)
Signed-off-by: Thomas Guillem <thomas at gllm.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=cdab0fc5fcc23b9037391a9cdd789cdd23fb4b18
---

 modules/codec/avcodec/encoder.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/codec/avcodec/encoder.c b/modules/codec/avcodec/encoder.c
index f24bdd801b..c587e2b243 100644
--- a/modules/codec/avcodec/encoder.c
+++ b/modules/codec/avcodec/encoder.c
@@ -745,6 +745,7 @@ int InitVideoEnc( vlc_object_t *p_this )
          */
         const unsigned i_order_max = 8 * sizeof(p_context->channel_layout);
         uint32_t pi_order_dst[AOUT_CHAN_MAX] = { };
+        uint32_t order_mask = 0;
         int i_channels_src = 0;
 
         if( p_context->channel_layout )
@@ -756,6 +757,7 @@ int InitVideoEnc( vlc_object_t *p_this )
                 {
                     msg_Dbg( p_enc, "%d %"PRIx64" mapped to %"PRIx64"", i_channels_src, pi_channels_map[i][0], pi_channels_map[i][1]);
                     pi_order_dst[i_channels_src++] = pi_channels_map[i][1];
+                    order_mask |= pi_channels_map[i][1];
                 }
             }
         }
@@ -769,14 +771,16 @@ int InitVideoEnc( vlc_object_t *p_this )
                 {
                     msg_Dbg( p_enc, "%d channel is %"PRIx64"", i_channels_src, pi_channels_map[i][1]);
                     pi_order_dst[i_channels_src++] = pi_channels_map[i][1];
+                    order_mask |= pi_channels_map[i][1];
                 }
             }
         }
         if( i_channels_src != p_context->channels )
             msg_Err( p_enc, "Channel layout not understood" );
 
-        p_sys->i_channels_to_reorder = aout_CheckChannelReorder( NULL, pi_order_dst,
-            channel_mask[p_context->channels][0], p_sys->pi_reorder_layout );
+        p_sys->i_channels_to_reorder =
+            aout_CheckChannelReorder( NULL, pi_order_dst, order_mask,
+                                      p_sys->pi_reorder_layout );
 #endif
 
         if ( p_enc->fmt_out.i_codec == VLC_CODEC_MP4A )