[x265] rdcost: overflow check by integer

[Steve Borho]

On Fri, May 23, 2014 at 10:10 AM, Satoshi Nakagawa <nakagawa424 at oki.com> wrote:
> # HG changeset patch
> # User Satoshi Nakagawa <nakagawa424 at oki.com>
> # Date 1400857636 -32400
> #      Sat May 24 00:07:16 2014 +0900
> # Node ID 61ad3e4d5b3264f7d6b12f2820292f53cb98cbbd
> # Parent  5134e76aa729b6fece18701fdc00390c2f2ffb32
> rdcost: overflow check by integer
>
> diff -r 5134e76aa729 -r 61ad3e4d5b32 source/encoder/rdcost.h
> --- a/source/encoder/rdcost.h   Thu May 22 21:46:21 2014 -0500
> +++ b/source/encoder/rdcost.h   Sat May 24 00:07:16 2014 +0900
> @@ -26,6 +26,11 @@
>
>  #include "common.h"
>
> +#if !defined(UINT64_MAX)
> +#include <limits>
> +#define UINT64_MAX (std::numeric_limits<uint64_t>::max())
> +#endif

this patch is great except this bit.  UINT64_MAX is supposed to be
defined by stdint.h, which we include in common.h

http://tigcc.ticalc.org/doc/stdint.html

If it was missing for you, it likely means we need to add this hack to
compat/msvc/stdint.h

>  namespace x265 {
>  // private namespace
>
> @@ -75,8 +80,7 @@
>
>      inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits)
>      {
> -        X265_CHECK(abs((float)((bits * m_lambdaSSE + 128) >> 8) -
> -                       (float)bits * m_lambdaSSE / 256.0) < 2,
> +        X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSSE,
>                     "calcRdCost wrap detected dist: %d, bits %d, lambda: %d\n", distortion, bits, (int)m_lambdaSSE);
>          return distortion + ((bits * m_lambdaSSE + 128) >> 8);
>      }
> @@ -105,8 +109,7 @@
>
>      inline uint64_t calcRdSADCost(uint32_t sadCost, uint32_t bits)
>      {
> -        X265_CHECK(abs((float)((bits * m_lambdaSAD + 128) >> 8) -
> -                       (float)bits * m_lambdaSAD / 256.0) < 2,
> +        X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSAD,
>                     "calcRdSADCost wrap detected dist: %d, bits %d, lambda: "X265_LL"\n", sadCost, bits, m_lambdaSAD);
>          return sadCost + ((bits * m_lambdaSAD + 128) >> 8);
>      }
> @@ -118,16 +121,14 @@
>
>      inline uint32_t scaleChromaDistCb(uint32_t dist)
>      {
> -        X265_CHECK(abs((float)((dist * m_cbDistortionWeight + 128) >> 8) -
> -                       (float)dist * m_cbDistortionWeight / 256.0) < 2,
> +        X265_CHECK(dist <= (UINT64_MAX - 128) / m_cbDistortionWeight,
>                     "scaleChromaDistCb wrap detected dist: %d, lambda: "X265_LL"\n", dist, m_cbDistortionWeight);
>          return (uint32_t)(((dist * m_cbDistortionWeight) + 128) >> 8);
>      }
>
>      inline uint32_t scaleChromaDistCr(uint32_t dist)
>      {
> -        X265_CHECK(abs((float)((dist * m_crDistortionWeight + 128) >> 8) -
> -                       (float)dist * m_crDistortionWeight / 256.0) < 2,
> +        X265_CHECK(dist <= (UINT64_MAX - 128) / m_crDistortionWeight,
>                     "scaleChromaDistCr wrap detected dist: %d, lambda: "X265_LL"\n", dist, m_crDistortionWeight);
>          return (uint32_t)(((dist * m_crDistortionWeight) + 128) >> 8);
>      }
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel



-- 
Steve Borho