[x265] rdcost: overflow check by integer

[Satoshi Nakagawa]

# HG changeset patch
# User Satoshi Nakagawa <nakagawa424 at oki.com>
# Date 1400857636 -32400
#      Sat May 24 00:07:16 2014 +0900
# Node ID 61ad3e4d5b3264f7d6b12f2820292f53cb98cbbd
# Parent  5134e76aa729b6fece18701fdc00390c2f2ffb32
rdcost: overflow check by integer

diff -r 5134e76aa729 -r 61ad3e4d5b32 source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Thu May 22 21:46:21 2014 -0500
+++ b/source/encoder/rdcost.h	Sat May 24 00:07:16 2014 +0900
@@ -26,6 +26,11 @@
 
 #include "common.h"
 
+#if !defined(UINT64_MAX)
+#include <limits>
+#define UINT64_MAX (std::numeric_limits<uint64_t>::max())
+#endif
+
 namespace x265 {
 // private namespace
 
@@ -75,8 +80,7 @@
 
     inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits)
     {
-        X265_CHECK(abs((float)((bits * m_lambdaSSE + 128) >> 8) -
-                       (float)bits * m_lambdaSSE / 256.0) < 2,
+        X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSSE,
                    "calcRdCost wrap detected dist: %d, bits %d, lambda: %d\n", distortion, bits, (int)m_lambdaSSE);
         return distortion + ((bits * m_lambdaSSE + 128) >> 8);
     }
@@ -105,8 +109,7 @@
 
     inline uint64_t calcRdSADCost(uint32_t sadCost, uint32_t bits)
     {
-        X265_CHECK(abs((float)((bits * m_lambdaSAD + 128) >> 8) -
-                       (float)bits * m_lambdaSAD / 256.0) < 2,
+        X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSAD,
                    "calcRdSADCost wrap detected dist: %d, bits %d, lambda: "X265_LL"\n", sadCost, bits, m_lambdaSAD);
         return sadCost + ((bits * m_lambdaSAD + 128) >> 8);
     }
@@ -118,16 +121,14 @@
 
     inline uint32_t scaleChromaDistCb(uint32_t dist)
     {
-        X265_CHECK(abs((float)((dist * m_cbDistortionWeight + 128) >> 8) -
-                       (float)dist * m_cbDistortionWeight / 256.0) < 2,
+        X265_CHECK(dist <= (UINT64_MAX - 128) / m_cbDistortionWeight,
                    "scaleChromaDistCb wrap detected dist: %d, lambda: "X265_LL"\n", dist, m_cbDistortionWeight);
         return (uint32_t)(((dist * m_cbDistortionWeight) + 128) >> 8);
     }
 
     inline uint32_t scaleChromaDistCr(uint32_t dist)
     {
-        X265_CHECK(abs((float)((dist * m_crDistortionWeight + 128) >> 8) -
-                       (float)dist * m_crDistortionWeight / 256.0) < 2,
+        X265_CHECK(dist <= (UINT64_MAX - 128) / m_crDistortionWeight,
                    "scaleChromaDistCr wrap detected dist: %d, lambda: "X265_LL"\n", dist, m_crDistortionWeight);
         return (uint32_t)(((dist * m_crDistortionWeight) + 128) >> 8);
     }