[x265] [PATCH 2 of 2] check: add checks for integer overflow in rdcost functions

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1399352771 18000
#      Tue May 06 00:06:11 2014 -0500
# Node ID 2577eb2cbd205063477dbf3e800fdc09d17c20aa
# Parent  5c590a31734b50d357651cd0536d21c56985d764
check: add checks for integer overflow in rdcost functions

diff -r 5c590a31734b -r 2577eb2cbd20 source/Lib/TLibCommon/TComRdCost.h
--- a/source/Lib/TLibCommon/TComRdCost.h	Mon May 05 23:26:29 2014 -0500
+++ b/source/Lib/TLibCommon/TComRdCost.h	Tue May 06 00:06:11 2014 -0500
@@ -80,15 +80,39 @@
         m_crDistortionWeight = (uint64_t)floor(256.0 * crDistortionWeight);
     }
 
-    inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits) { return distortion + ((bits * m_lambdaMotionSSE + 128) >> 8); }
+    inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits)
+    {
+        X265_CHECK(abs(distortion + ((bits * m_lambdaMotionSSE + 128) >> 8)) -
+                      (distortion + (float)bits * m_lambdaMotionSSE / 256.0) < 2,
+                   "calcRdCost wrap detected dist: %d, bits %d, lambda: %d\n", distortion, bits, (int)m_lambdaMotionSSE);
+        return distortion + ((bits * m_lambdaMotionSSE + 128) >> 8);
+    }
 
-    inline uint64_t calcRdSADCost(uint32_t sadCost, uint32_t bits) { return sadCost + ((bits * m_lambdaMotionSAD + 128) >> 8); }
+    inline uint64_t calcRdSADCost(uint32_t sadCost, uint32_t bits)
+    {
+        X265_CHECK(abs(sadCost + ((bits * m_lambdaMotionSAD + 128) >> 8)) -
+                      (sadCost + (float)bits * m_lambdaMotionSAD / 256.0) < 2,
+                   "calcRdSADCost wrap detected dist: %d, bits %d, lambda: %d\n", sadCost, bits, (int)m_lambdaMotionSAD);
+        return sadCost + ((bits * m_lambdaMotionSAD + 128) >> 8);
+    }
 
     inline uint32_t getCost(uint32_t bits)                     { return (uint32_t)((bits * m_lambdaMotionSAD + 128) >> 8); }
 
-    inline uint32_t scaleChromaDistCb(uint32_t dist)           { return (uint32_t)(((dist * m_cbDistortionWeight) + 128) >> 8); }
+    inline uint32_t scaleChromaDistCb(uint32_t dist)
+    {
+        X265_CHECK(abs(((dist * m_cbDistortionWeight + 128) >> 8) -
+                      (float)dist * m_cbDistortionWeight / 256.0) < 2,
+                   "scaleChromaDistCb wrap detected dist: %d, lambda: %d\n", dist, (int)m_cbDistortionWeight);
+        return (uint32_t)(((dist * m_cbDistortionWeight) + 128) >> 8);
+    }
 
-    inline uint32_t scaleChromaDistCr(uint32_t dist)           { return (uint32_t)(((dist * m_crDistortionWeight) + 128) >> 8); }
+    inline uint32_t scaleChromaDistCr(uint32_t dist)
+    {
+        X265_CHECK(abs(((dist * m_crDistortionWeight + 128) >> 8) -
+                      (float)dist * m_crDistortionWeight / 256.0) < 2,
+                   "scaleChromaDistCr wrap detected dist: %d, lambda: %d\n", dist, (int)m_crDistortionWeight);
+        return (uint32_t)(((dist * m_crDistortionWeight) + 128) >> 8);
+    }
 };
 }
 //! \}