[vlc-commits] gradient: fix integer overflow

[Rémi Denis-Courmont]

vlc | branch: master | Rémi Denis-Courmont <remi at remlab.net> | Thu Sep  7 21:54:37 2017 +0300| [0a0690cc4a4b009de856143378cfebfe2b7a680f] | committer: Rémi Denis-Courmont

gradient: fix integer overflow

If a and/or b is unsigned, a - b is unsigned.
Then if a < b, a - b > INT_MAX.
And then abs(a - b) performs an undefined implicit conversion to int.

This converts to int before computing the difference. Since all samples
are 8-bits unsigned, this cannot overflow.

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=0a0690cc4a4b009de856143378cfebfe2b7a680f
---

 modules/video_filter/gradient.c | 65 +++++++++++++++++------------------------
 1 file changed, 26 insertions(+), 39 deletions(-)

diff --git a/modules/video_filter/gradient.c b/modules/video_filter/gradient.c
index 55a84e9990..7479377d8d 100644
--- a/modules/video_filter/gradient.c
+++ b/modules/video_filter/gradient.c
@@ -382,26 +382,20 @@ static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
     {                                                           \
         for( int x = 1; x < i_src_visible - 1; x++ )            \
         {                                                       \
-            const uint32_t a =                                  \
-            (                                                   \
-              abs(                                              \
-                 ( p_smooth[(y-1)*i_src_visible+x-1]            \
-                   - p_smooth[(y+1)*i_src_visible+x-1] )        \
-               + ( ( p_smooth[(y-1)*i_src_visible+x]            \
-                    - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
-               + ( p_smooth[(y-1)*i_src_visible+x+1]            \
-                   - p_smooth[(y+1)*i_src_visible+x+1] )        \
-              )                                                 \
-            +                                                   \
-              abs(                                              \
-                 ( p_smooth[(y-1)*i_src_visible+x-1]            \
-                   - p_smooth[(y-1)*i_src_visible+x+1] )        \
-               + ( ( p_smooth[y*i_src_visible+x-1]              \
-                    - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
-               + ( p_smooth[(y+1)*i_src_visible+x-1]            \
-                   - p_smooth[(y+1)*i_src_visible+x+1] )        \
-              )                                                 \
-            );
+            const uint32_t a = \
+                  abs(((int)p_smooth[(y - 1) * i_src_visible + x - 1] \
+                     - (int)p_smooth[(y + 1) * i_src_visible + x - 1]) \
+                   + (((int)p_smooth[(y - 1) * i_src_visible + x] \
+                     - (int)p_smooth[(y + 1) * i_src_visible + x]) * 2) \
+                    + ((int)p_smooth[(y - 1) * i_src_visible + x + 1] \
+                     - (int)p_smooth[(y + 1) * i_src_visible + x + 1])) \
+                + abs(((int)p_smooth[(y - 1) * i_src_visible + x - 1] \
+                     - (int)p_smooth[(y - 1) * i_src_visible + x + 1]) \
+                   + (((int)p_smooth[y       * i_src_visible + x - 1] \
+                     - (int)p_smooth[y       * i_src_visible + x + 1]) * 2) \
+                    + ((int)p_smooth[(y + 1) * i_src_visible + x - 1] \
+                     - (int)p_smooth[(y + 1) * i_src_visible + x + 1]));
+
     if( p_filter->p_sys->i_gradient_type )
     {
         if( p_filter->p_sys->b_cartoon )
@@ -694,25 +688,18 @@ static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
         for( int x = 4; x < i_src_visible - 4; x++ )
         {
             uint32_t a =
-            (
-              abs(
-                ( ( p_smooth[(y-1)*i_src_visible+x]
-                    - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
-               + ( p_smooth[(y-1)*i_src_visible+x-1]
-                   - p_smooth[(y+1)*i_src_visible+x-1] )
-               + ( p_smooth[(y-1)*i_src_visible+x+1]
-                   - p_smooth[(y+1)*i_src_visible+x+1] )
-              )
-            +
-              abs(
-                ( ( p_smooth[y*i_src_visible+x-1]
-                    - p_smooth[y*i_src_visible+x+1] ) <<1 )
-               + ( p_smooth[(y-1)*i_src_visible+x-1]
-                   - p_smooth[(y-1)*i_src_visible+x+1] )
-               + ( p_smooth[(y+1)*i_src_visible+x-1]
-                   - p_smooth[(y+1)*i_src_visible+x+1] )
-              )
-            );
+                  abs((((int)p_smooth[(y - 1) * i_src_visible + x]
+                      - (int)p_smooth[(y + 1) * i_src_visible + x]) * 2)
+                     + ((int)p_smooth[(y - 1) * i_src_visible + x - 1]
+                      - (int)p_smooth[(y + 1) * i_src_visible + x - 1])
+                     + ((int)p_smooth[(y - 1) * i_src_visible + x + 1]
+                      - (int)p_smooth[(y + 1) * i_src_visible + x + 1]))
+                + abs((((int)p_smooth[y * i_src_visible + x - 1]
+                      - (int)p_smooth[y * i_src_visible + x + 1]) * 2)
+                     + ((int)p_smooth[(y - 1) * i_src_visible + x - 1]
+                      - (int)p_smooth[(y - 1) * i_src_visible + x + 1])
+                     + ((int)p_smooth[(y + 1) * i_src_visible + x - 1]
+                      - (int)p_smooth[(y + 1) * i_src_visible + x + 1]));
             if( a>>8 )
             {
                 for( int i = 0; i < i_nb_steps; i++ )