[x265] [PATCH] cost: add extra overflow checks in cost

[Divya Manivannan]

# HG changeset patch
# User Divya Manivannan <divya at multicorewareinc.com>
# Date 1443610499 -19800
#      Wed Sep 30 16:24:59 2015 +0530
# Node ID 28bd3020855e0e2329738b82309d4113d091dd3f
# Parent  4a54caf900a969cd94185a865b2593db1f19813f
cost: add extra overflow checks in cost

diff -r 4a54caf900a9 -r 28bd3020855e source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Wed Sep 30 16:24:57 2015 +0530
+++ b/source/encoder/rdcost.h	Wed Sep 30 16:24:59 2015 +0530
@@ -118,6 +118,15 @@
     /* return the RD cost of this prediction, including the effect of psy-rd */
     inline uint64_t calcPsyRdCost(sse_ret_t distortion, uint32_t bits, uint32_t psycost) const
     {
+#if X265_DEPTH < 10
+        X265_CHECK((bits <= (UINT64_MAX / m_lambda2)) && (psycost <= UINT64_MAX / (m_lambda * m_psyRd)),
+                   "calcPsyRdCost wrap detected dist: %u, bits: %u, lambda: " X265_LL ", lambda2: " X265_LL "\n",
+                   distortion, bits, m_lambda, m_lambda2);
+#else
+        X265_CHECK((bits <= (UINT64_MAX / m_lambda2)) && (psycost <= UINT64_MAX / (m_lambda * m_psyRd)),
+                   "calcPsyRdCost wrap detected dist: " X265_LL ", bits: %u, lambda: " X265_LL ", lambda2: " X265_LL "\n",
+                   distortion, bits, m_lambda, m_lambda2);
+#endif
         return distortion + ((m_lambda * m_psyRd * psycost) >> 24) + ((bits * m_lambda2) >> 8);
     }
 
@@ -144,6 +153,8 @@
 
     inline uint32_t getCost(uint32_t bits) const
     {
+        X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambda,
+                   "getCost wrap detected bits: %u, lambda: " X265_LL "\n", bits, m_lambda);
         return (uint32_t)((bits * m_lambda + 128) >> 8);
     }
 };