[x265] [PATCH 2 of 5] fix PSYVALUE shift overflow (fixes #180) [OUTPUT CHANGE on 12bpp]

[Min Chen]

# HG changeset patch
# User Min Chen <chenm003 at 163.com>
# Date 1443138811 18000
# Node ID bcd926000cd8fecf2c2c52af49e94d4405b66280
# Parent  58128ae6e4d8ec857a52537c75602421d67ff549
fix PSYVALUE shift overflow (fixes #180) [OUTPUT CHANGE on 12bpp]
---
 source/common/quant.cpp |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff -r 58128ae6e4d8 -r bcd926000cd8 source/common/quant.cpp
--- a/source/common/quant.cpp	Thu Sep 24 18:53:28 2015 -0500
+++ b/source/common/quant.cpp	Thu Sep 24 18:53:31 2015 -0500
@@ -589,7 +589,7 @@
 #define UNQUANT(lvl)    (((lvl) * (unquantScale[blkPos] << per) + unquantRound) >> unquantShift)
 #define SIGCOST(bits)   ((lambda2 * (bits)) >> 8)
 #define RDCOST(d, bits) ((((int64_t)d * d) << scaleBits) + SIGCOST(bits))
-#define PSYVALUE(rec)   ((psyScale * (rec)) >> (2 * transformShift + 1))
+#define PSYVALUE(rec)   ((psyScale * (rec)) >> X265_MAX(0, (2 * transformShift + 1)))
 
     int64_t costCoeff[trSize * trSize];   /* d*d + lambda * bits */
     int64_t costUncoded[trSize * trSize]; /* d*d + lambda * 0    */