[vlc-commits] araw: avoid underflow in conversion

[Rémi Denis-Courmont]

vlc/vlc-3.0 | branch: master | Rémi Denis-Courmont <remi at remlab.net> | Sun Jan  7 15:23:07 2018 +0200| [99690919958b0f93f9d9127d6f0d04eb90acf152] | committer: Jean-Baptiste Kempf

araw: avoid underflow in conversion

Cast to positive numbers to int explicitly before computing the
difference, so the result will be signed correctly.

(cherry picked from commit a8fdf9068bdbfdcd1ebf5e49cfe0f495b009dbea)
Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=99690919958b0f93f9d9127d6f0d04eb90acf152
---

 modules/codec/araw.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/codec/araw.c b/modules/codec/araw.c
index 460e09b9f6..0d8145527a 100644
--- a/modules/codec/araw.c
+++ b/modules/codec/araw.c
@@ -609,18 +609,20 @@ static void F64IDecode( void *outp, const uint8_t *in, unsigned samples )
     }
 }
 
-static int16_t dat12tos16( uint_fast16_t y )
+static int_fast16_t dat12tos16( uint_fast16_t y )
 {
-    static const uint16_t diff[16] = {
+    static const int16_t diff[16] = {
        0x0000, 0x0000, 0x0100, 0x0200, 0x0300, 0x0400, 0x0500, 0x0600,
-       0x0A00, 0x0B00, 0x0C00, 0x0D00, 0x0E00, 0x0F00, 0x1000, 0x1000 };
+       0x0A00, 0x0B00, 0x0C00, 0x0D00, 0x0E00, 0x0F00, 0x1000, 0x1000,
+    };
     static const uint8_t shift[16] = {
-        0, 0, 1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1, 0, 0 };
+        0, 0, 1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1, 0, 0
+    };
 
     assert(y < 0x1000);
 
     int d = y >> 8;
-    return (y - diff[d]) << shift[d];
+    return ((int)y - diff[d]) << shift[d];
 }
 
 static void DAT12Decode( void *outp, const uint8_t *in, unsigned samples )