[x265-commits] [x265] quant: remove extra upshift by 4 in s_invQuantScales

[Deepthi Nandakumar]

details:   http://hg.videolan.org/x265/rev/1921583cf797
branches:  
changeset: 7782:1921583cf797
user:      Deepthi Nandakumar <deepthi at multicorewareinc.com>
date:      Wed Aug 13 14:19:28 2014 +0530
description:
quant: remove extra upshift by 4 in s_invQuantScales

Adding 4 to unQuantPer was a bug, since unQuantScale was already upshifted by 4.
Subject: [x265] quant: simplify UNQUANT() variables

details:   http://hg.videolan.org/x265/rev/9ad7668756f7
branches:  
changeset: 7783:9ad7668756f7
user:      Steve Borho <steve at borho.org>
date:      Wed Aug 13 15:46:39 2014 -0500
description:
quant: simplify UNQUANT() variables

diffstat:

 source/common/quant.cpp       |  18 +++---------------
 source/common/scalinglist.cpp |   2 +-
 2 files changed, 4 insertions(+), 16 deletions(-)

diffs (41 lines):

diff -r d66e257ace32 -r 9ad7668756f7 source/common/quant.cpp
--- a/source/common/quant.cpp	Wed Aug 13 17:29:18 2014 -0700
+++ b/source/common/quant.cpp	Wed Aug 13 15:46:39 2014 -0500
@@ -500,22 +500,10 @@ uint32_t Quant::rdoQuant(TComDataCU* cu,
      * some shifts around. To avoid this we add an addition shift factor to the dequant coeff.  Note
      * that in real dequant there is clipping at several stages. We skip the clipping when measuring
      * RD cost */
-#define UNQUANT(lvl) (((lvl) * (unquantScale[blkPos] << unquantPer) + unquantRound) >> unquantShift)
+#define UNQUANT(lvl) (((lvl) * (unquantScale[blkPos] << per) + unquantRound) >> unquantShift)
     int32_t *unquantScale = m_scalingList->m_dequantCoef[log2TrSize - 2][scalingListType][rem];
-    int unquantShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - transformShift;
-    int unquantRound, unquantPer;
-    unquantShift += 4;
-    if (unquantShift > per)
-    {
-        unquantRound = 1 << (unquantShift - per - 1);
-        unquantPer = per;
-    }
-    else
-    {
-        unquantPer = per + 4;
-        unquantShift += 4;
-        unquantRound = 0;
-    }
+    int unquantShift = QUANT_IQUANT_SHIFT - QUANT_SHIFT - transformShift + m_scalingList->m_bEnabled ? 4 : 0;
+    int unquantRound = (unquantShift > per) ? 1 << (unquantShift - per - 1) : 0;
 
 #define SIGCOST(bits)   ((lambda2 * (bits)) >> 8)
 #define RDCOST(d, bits) ((((int64_t)d * d) << scaleBits) + ((lambda2 * (bits)) >> 8))
diff -r d66e257ace32 -r 9ad7668756f7 source/common/scalinglist.cpp
--- a/source/common/scalinglist.cpp	Wed Aug 13 17:29:18 2014 -0700
+++ b/source/common/scalinglist.cpp	Wed Aug 13 15:46:39 2014 -0500
@@ -347,7 +347,7 @@ void ScalingList::setupQuantMatrices()
                     for (int i = 0; i < count; i++)
                     {
                         quantCoeff[i] = s_quantScales[rem];
-                        dequantCoeff[i] = s_invQuantScales[rem] << 4;
+                        dequantCoeff[i] = s_invQuantScales[rem];
                     }
                 }
             }