[x265] [PATCH 2 of 5] quant: avoid right-shift overflow and undefined behavior trap

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1424891691 21600
#      Wed Feb 25 13:14:51 2015 -0600
# Node ID 603de2f3d4380127f77462bb5497fd8cf232c332
# Parent  515a229245f687815b0f1c9d6973daa56219ba6b
quant: avoid right-shift overflow and undefined behavior trap

diff -r 515a229245f6 -r 603de2f3d438 source/common/quant.cpp
--- a/source/common/quant.cpp	Wed Feb 25 12:08:52 2015 -0600
+++ b/source/common/quant.cpp	Wed Feb 25 13:14:51 2015 -0600
@@ -1105,7 +1105,8 @@
 
     const uint32_t trSizeCG = 1 << log2TrSizeCG;
     X265_CHECK(trSizeCG <= 8, "transform CG is too large\n");
-    const uint32_t sigPos = (uint32_t)(sigCoeffGroupFlag64 >> (1 + (cgPosY << log2TrSizeCG) + cgPosX));
+    const uint32_t shift = (cgPosY << log2TrSizeCG) + cgPosX + 1;
+    const uint32_t sigPos = (uint32_t)(shift >= 64 ? 0 : sigCoeffGroupFlag64 >> shift);
     const uint32_t sigRight = ((int32_t)(cgPosX - (trSizeCG - 1)) >> 31) & (sigPos & 1);
     const uint32_t sigLower = ((int32_t)(cgPosY - (trSizeCG - 1)) >> 31) & (sigPos >> (trSizeCG - 2)) & 2;