[x265] [PATCH] quant: change how RDOQ measures distortion [CHANGES OUTPUTS]
Steve Borho
steve at borho.org
Tue Aug 5 03:59:36 CEST 2014
# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1407200851 18000
# Mon Aug 04 20:07:31 2014 -0500
# Node ID db7bd7243e3577b15c6b7f94d84b479952232460
# Parent 08304a2980656feb93165f4c78c8047743419182
quant: change how RDOQ measures distortion [CHANGES OUTPUTS]
RDOQ, as it was written in the HM, expects scaled level values to be output by
quant; these are the output levels multiplied by the quantizing coeffificient
but without the rounding factor and without the downshift. It would then
measure distortion as the difference between this scaled level and level <<
qbits (a rough unquant). To make this math work, it was pre-calculating an
error scale factor (per block position, since the quantization coefficients can
vary) which divided the result by the squared scale factor and upshifting to
simultaneously account for the FIX15 nature of the signaling costs and the
uniform scaling of the forward transform. To roughly summarize:
errScale = (1 << (15 - 2 * transformShift)) / (quantCoeff[i] * quantCoeff[i])
levelScaled = level * quantCoeff[i]
distortion = levelScaled - (level << qbits);
cost = distortion * distortion * errScale + lambda2 * bitsFix15
It was forced to use floating point math for the errScale and distortion
calculations, and thus did not bother with fixed point math for lambda2.
This commit changes the distortion measurement to be the difference between the
original (pre-quantization) DCT coefficient and the unquantized level.
unquantAbsLevel = (level * quantCoeff[i] + pad) >> shift;
distortion = unquantAbsLevel - abs(signCoef);
distScale = 1 << (15 - 2 * transformShift);
cost = distortion * distortion << distScale + lambda2 * bitsFix15
Note that the same scale factor is still required to account for the FIX15 bit
cost and the forward DCT scale but now it is a simple shift operation.
This commit does not change the data types; that will be a later commit once the
dynamic ranges have been properly evaluated. After this change, the nquant
primitive can be simplified to no longer output scaled levels.
diff -r 08304a298065 -r db7bd7243e35 source/common/quant.cpp
--- a/source/common/quant.cpp Mon Aug 04 16:22:46 2014 -0500
+++ b/source/common/quant.cpp Mon Aug 04 20:07:31 2014 -0500
@@ -532,7 +532,6 @@
int scaleBits = SCALE_BITS - 2 * transformShift;
double lambda2 = m_lambdas[ttype];
- double *errScale = m_scalingList->m_errScale[log2TrSize - 2][scalingListType][rem];
bool bIsLuma = ttype == TEXT_LUMA;
double totalUncodedCost = 0;
@@ -582,18 +581,16 @@
{
scanPos = (cgScanPos << MLS_CG_SIZE) + scanPosinCG;
uint32_t blkPos = codingParameters.scan[scanPos];
- double scaleFactor = errScale[blkPos]; /* (1 << scaleBits) / (quantCoef * quantCoef) */
- int levelScaled = scaledCoeff[blkPos]; /* abs(coef) * quantCoef */
- uint32_t maxAbsLevel = abs(dstCoeff[blkPos]); /* abs(coef) */
- int signCoef = m_resiDctCoeff[blkPos];
- int predictedCoef = m_fencDctCoeff[blkPos] - signCoef;
+ uint32_t maxAbsLevel = abs(dstCoeff[blkPos]); /* abs(quantized coeff) */
+ int signCoef = m_resiDctCoeff[blkPos]; /* pre-quantization DCT coeff */
+ int predictedCoef = m_fencDctCoeff[blkPos] - signCoef; /* predicted DCT = source DCT - residual DCT*/
- /* RDOQ measures distortion as the scaled level squared times a
- * scale factor which tries to remove the quantCoef back out, but
- * adds scaleBits to account for IEP_RATE which is 32k (1 << SCALE_BITS) */
+ /* RDOQ measures distortion as the squared difference between the unquantized coded level
+ * and the original DCT coefficient. The result is shifted scaleBits to account for the
+ * FIX15 nature of the CABAC cost tables minus the forward transform scale */
- /* cost of not coding this coefficient (no signal bits) */
- costUncoded[scanPos] = ((uint64_t)levelScaled * levelScaled) * scaleFactor;
+ /* cost of not coding this coefficient (all distortion, no signal bits) */
+ costUncoded[scanPos] = (signCoef * signCoef) << scaleBits;
if (usePsy && blkPos)
/* when no coefficient is coded, predicted coef == recon coef */
costUncoded[scanPos] -= (int)(((m_psyRdoqScale * predictedCoef) << scaleBits) >> 8);
@@ -616,7 +613,7 @@
costCoeff[scanPos] = 0;
baseCost += costUncoded[scanPos];
- /* coeff in unsignaled coeff groups have no signal cost */
+ /* coefficients after lastNZ have no signal cost */
costSig[scanPos] = 0;
}
else
@@ -654,19 +651,19 @@
}
if (maxAbsLevel)
{
- const int64_t err1 = levelScaled - ((int64_t)maxAbsLevel << qbits);
- double err2 = (double)(err1 * err1);
-
uint32_t minAbsLevel = X265_MAX(maxAbsLevel - 1, 1);
for (uint32_t lvl = maxAbsLevel; lvl >= minAbsLevel; lvl--)
{
uint32_t rateCost = getICRateCost(lvl, lvl - baseLevel, greaterOneBits, levelAbsBits, goRiceParam, c1c2Idx);
- double curCost = err2 * scaleFactor + lambda2 * (codedSigBits + rateCost + IEP_RATE);
+
+ int unquantAbsLevel = ((lvl << unquantPreshift) * (unquantScale[blkPos] << per) + unquantRound) >> unquantShift;
+ int d = abs(unquantAbsLevel - abs(signCoef));
+ int64_t distortion = ((int64_t)d * d) << scaleBits;
+ double curCost = distortion + lambda2 * (codedSigBits + rateCost + IEP_RATE);
// Psy RDOQ: bias in favor of higher AC coefficients in the reconstructed frame
if (usePsy && blkPos)
{
- int unquantAbsLevel = ((lvl << unquantPreshift) * (unquantScale[blkPos] << per) + unquantRound) >> unquantShift;
int reconCoef = abs(unquantAbsLevel + SIGN(predictedCoef, signCoef));
curCost -= (int)(((m_psyRdoqScale * reconCoef) << scaleBits) >> 8);
}
@@ -676,21 +673,15 @@
level = lvl;
costCoeff[scanPos] = curCost;
costSig[scanPos] = lambda2 * codedSigBits;
- }
-
- if (lvl > minAbsLevel)
- {
- // add deltas to get squared distortion at minAbsLevel
- int64_t err3 = (int64_t)2 * err1 * ((int64_t)1 << qbits);
- int64_t err4 = ((int64_t)1 << qbits) * ((int64_t)1 << qbits);
- err2 += err3 + err4;
+ deltaU[blkPos] = d << 8;
}
}
}
- deltaU[blkPos] = (levelScaled - ((int)level << qbits)) >> (qbits - 8);
dstCoeff[blkPos] = level;
baseCost += costCoeff[scanPos];
+ if (!level)
+ deltaU[blkPos] = -signCoef << 8;
/* record costs for sign-hiding performed at the end */
if (level)
diff -r 08304a298065 -r db7bd7243e35 source/common/scalinglist.cpp
--- a/source/common/scalinglist.cpp Mon Aug 04 16:22:46 2014 -0500
+++ b/source/common/scalinglist.cpp Mon Aug 04 20:07:31 2014 -0500
@@ -128,7 +128,6 @@
{
memset(m_quantCoef, 0, sizeof(m_quantCoef));
memset(m_dequantCoef, 0, sizeof(m_dequantCoef));
- memset(m_errScale, 0, sizeof(m_errScale));
memset(m_scalingListCoef, 0, sizeof(m_scalingListCoef));
}
@@ -145,8 +144,7 @@
{
m_quantCoef[sizeId][listId][rem] = X265_MALLOC(int32_t, s_numCoefPerSize[sizeId]);
m_dequantCoef[sizeId][listId][rem] = X265_MALLOC(int32_t, s_numCoefPerSize[sizeId]);
- m_errScale[sizeId][listId][rem] = X265_MALLOC(double, s_numCoefPerSize[sizeId]);
- ok &= m_quantCoef[sizeId][listId][rem] && m_dequantCoef[sizeId][listId][rem] && m_errScale[sizeId][listId][rem];
+ ok &= m_quantCoef[sizeId][listId][rem] && m_dequantCoef[sizeId][listId][rem];
}
}
}
@@ -164,7 +162,6 @@
{
X265_FREE(m_quantCoef[sizeId][listId][rem]);
X265_FREE(m_dequantCoef[sizeId][listId][rem]);
- X265_FREE(m_errScale[sizeId][listId][rem]);
}
}
}
@@ -331,11 +328,6 @@
int stride = X265_MIN(MAX_MATRIX_SIZE_NUM, width);
int count = s_numCoefPerSize[size];
- // Error scale constants
- int log2TrSize = size + 2;
- int transformShift = MAX_TR_DYNAMIC_RANGE - X265_DEPTH - log2TrSize; // Represents scaling through forward transform
- int scalingBits = 1 << (SCALE_BITS - 2 * transformShift); // Compensate for scaling of bitcount in Lagrange cost function
-
for (int list = 0; list < s_numListsAtSize[size]; list++)
{
int32_t *coeff = m_scalingListCoef[size][list];
@@ -345,7 +337,6 @@
{
int32_t *quantCoeff = m_quantCoef[size][list][rem];
int32_t *dequantCoeff = m_dequantCoef[size][list][rem];
- double *errScale = m_errScale[size][list][rem];
if (m_bEnabled)
{
@@ -361,9 +352,6 @@
dequantCoeff[i] = s_invQuantScales[rem] << 4;
}
}
-
- for (int i = 0; i < count; i++)
- errScale[i] = (double)scalingBits / (quantCoeff[i] * quantCoeff[i]);
}
}
}
diff -r 08304a298065 -r db7bd7243e35 source/common/scalinglist.h
--- a/source/common/scalinglist.h Mon Aug 04 16:22:46 2014 -0500
+++ b/source/common/scalinglist.h Mon Aug 04 20:07:31 2014 -0500
@@ -49,7 +49,6 @@
int32_t* m_quantCoef[NUM_SIZES][NUM_LISTS][NUM_REM]; // array of quantization matrix coefficient 4x4
int32_t* m_dequantCoef[NUM_SIZES][NUM_LISTS][NUM_REM]; // array of dequantization matrix coefficient 4x4
- double* m_errScale[NUM_SIZES][NUM_LISTS][NUM_REM];
bool m_bEnabled;
bool m_bDataPresent; // non-default scaling lists must be signaled
More information about the x265-devel
mailing list