[x265] TComTrQuant: lambda for each Cb and Cr
Satoshi Nakagawa
nakagawa424 at oki.com
Sun Jun 8 04:01:38 CEST 2014
# HG changeset patch
# User Satoshi Nakagawa <nakagawa424 at oki.com>
# Date 1402192642 -32400
# Sun Jun 08 10:57:22 2014 +0900
# Node ID 188e115f07427c759f47154a864467be21b5b6a1
# Parent e5656f1e190453efa84732269b259a6dee608ff9
TComTrQuant: lambda for each Cb and Cr
diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibCommon/TComTrQuant.cpp
--- a/source/Lib/TLibCommon/TComTrQuant.cpp Thu Jun 05 22:45:25 2014 -0500
+++ b/source/Lib/TLibCommon/TComTrQuant.cpp Sun Jun 08 10:57:22 2014 +0900
@@ -262,49 +262,35 @@
}
uint32_t TComTrQuant::xQuant(TComDataCU* cu, int32_t* coef, coeff_t* qCoef, int trSize,
- TextType ttype, uint32_t absPartIdx, int32_t *lastPos, bool curUseRDOQ)
+ TextType ttype, uint32_t absPartIdx, int32_t *lastPos)
{
- uint32_t acSum = 0;
- int add = 0;
- bool useRDOQ = m_useRDOQ && curUseRDOQ;
+ const uint32_t log2TrSize = g_convertToBit[trSize] + 2;
+ TUEntropyCodingParameters codingParameters;
+ getTUEntropyCodingParameters(cu, codingParameters, absPartIdx, log2TrSize, ttype);
+ int deltaU[32 * 32];
- if (useRDOQ)
+ int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
+ X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
+ int32_t *quantCoeff = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
+
+ int transformShift = MAX_TR_DYNAMIC_RANGE - X265_DEPTH - log2TrSize; // Represents scaling through forward transform
+
+ int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift;
+ int add = (cu->getSlice()->getSliceType() == I_SLICE ? 171 : 85) << (qbits - 9);
+
+ int numCoeff = 1 << log2TrSize * 2;
+ uint32_t acSum = primitives.quant(coef, quantCoeff, deltaU, qCoef, qbits, add, numCoeff, lastPos);
+
+ if (acSum >= 2 && cu->getSlice()->getPPS()->getSignHideFlag())
{
- acSum = xRateDistOptQuant(cu, coef, qCoef, trSize, ttype, absPartIdx, lastPos);
- }
- else
- {
- const uint32_t log2TrSize = g_convertToBit[trSize] + 2;
- TUEntropyCodingParameters codingParameters;
- getTUEntropyCodingParameters(cu, codingParameters, absPartIdx, log2TrSize, ttype);
- int deltaU[32 * 32];
-
- int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
- X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
- int32_t *quantCoeff = 0;
- quantCoeff = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
-
- int transformShift = MAX_TR_DYNAMIC_RANGE - X265_DEPTH - log2TrSize; // Represents scaling through forward transform
-
- int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift;
- add = (cu->getSlice()->getSliceType() == I_SLICE ? 171 : 85) << (qbits - 9);
-
- int numCoeff = 1 << log2TrSize * 2;
- acSum += primitives.quant(coef, quantCoeff, deltaU, qCoef, qbits, add, numCoeff, lastPos);
-
- if (cu->getSlice()->getPPS()->getSignHideFlag() && acSum >= 2)
- {
- signBitHidingHDQ(qCoef, coef, deltaU, codingParameters);
- }
+ signBitHidingHDQ(qCoef, coef, deltaU, codingParameters);
}
return acSum;
}
-void TComTrQuant::init(uint32_t maxTrSize, bool useRDOQ, bool useTransformSkipFast)
+void TComTrQuant::init(bool useRDOQ)
{
- m_maxTrSize = maxTrSize;
- m_useRDOQ = useRDOQ;
- m_useTransformSkipFast = useTransformSkipFast;
+ m_useRDOQ = useRDOQ;
}
uint32_t TComTrQuant::transformNxN(TComDataCU* cu,
@@ -363,7 +349,12 @@
}
}
}
- return xQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos, curUseRDOQ);
+
+ if (m_useRDOQ && curUseRDOQ)
+ {
+ return xRateDistOptQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos);
+ }
+ return xQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos);
}
void TComTrQuant::invtransformNxN(bool transQuantBypass, uint32_t mode, int16_t* residual, uint32_t stride, coeff_t* coeff, uint32_t trSize, int scalingListType, bool useTransformSkip, int lastPos)
@@ -525,10 +516,9 @@
X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift; // Right shift of non-RDOQ quantizer; level = (coeff*Q + offset)>>q_bits
- double *errScaleOrg = getErrScaleCoeff(scalingListType, log2TrSize - 2, m_qpParam.m_rem);
- int32_t *qCoefOrg = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
- int32_t *qCoef = qCoefOrg;
- double *errScale = errScaleOrg;
+ int add = (1 << (qbits - 1));
+ double *errScale = getErrScaleCoeff(scalingListType, log2TrSize - 2, m_qpParam.m_rem);
+ int32_t *qCoef = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
double costCoeff[32 * 32];
double costSig[32 * 32];
@@ -575,9 +565,9 @@
int Q = qCoef[blkPos];
double scaleFactor = errScale[blkPos];
int levelDouble = srcCoeff[blkPos];
- levelDouble = (int)std::min<int64_t>((int64_t)abs((int)levelDouble) * Q, MAX_INT - (1 << (qbits - 1)));
+ levelDouble = (int)std::min<int64_t>((int64_t)abs((int)levelDouble) * Q, MAX_INT - add);
- uint32_t maxAbsLevel = (levelDouble + (1 << (qbits - 1))) >> qbits;
+ uint32_t maxAbsLevel = (levelDouble + add) >> qbits;
costCoeff0[scanPos] = ((uint64_t)levelDouble * levelDouble) * scaleFactor;
blockUncodedCost += costCoeff0[scanPos];
@@ -599,11 +589,6 @@
X265_CHECK(!!(c2Idx == 0) == ((-(int)c2Idx) >> (sizeof(int) * CHAR_BIT - 1)) + 1, "scan validation 3\n");
X265_CHECK(baseLevel == ((c1Idx < C1FLAG_NUMBER) ? (2 + (c2Idx == 0)) : 1), "scan validation 4\n");
- rateIncUp[blkPos] = 0;
- rateIncDown[blkPos] = 0;
- deltaU[blkPos] = 0;
- sigRateDelta[blkPos] = 0;
-
//===== coefficient level estimation =====
uint32_t level;
const uint32_t oneCtx = 4 * ctxSet + c1;
@@ -618,6 +603,7 @@
level = xGetCodedLevel(costCoeff[scanPos], curCostSig, costSig[scanPos],
levelDouble, maxAbsLevel, baseLevel, greaterOneBits, levelAbsBits, goRiceParam,
c1c2Idx, qbits, scaleFactor, 1);
+ sigRateDelta[blkPos] = 0;
}
else
{
@@ -651,6 +637,7 @@
else // level == 0
{
rateIncUp[blkPos] = greaterOneBits[0];
+ rateIncDown[blkPos] = 0;
}
dstCoeff[blkPos] = level;
baseCost += costCoeff[scanPos];
@@ -869,7 +856,7 @@
{
int64_t rdFactor = (int64_t)(
g_invQuantScales[m_qpParam.rem()] * g_invQuantScales[m_qpParam.rem()] * (1 << (2 * m_qpParam.m_per))
- / m_lambda / 16 / (1 << DISTORTION_PRECISION_ADJUSTMENT(2 * (X265_DEPTH - 8)))
+ / (m_lambda * (16 << DISTORTION_PRECISION_ADJUSTMENT(2 * (X265_DEPTH - 8))))
+ 0.5);
int lastCG = 1;
diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibCommon/TComTrQuant.h
--- a/source/Lib/TLibCommon/TComTrQuant.h Thu Jun 05 22:45:25 2014 -0500
+++ b/source/Lib/TLibCommon/TComTrQuant.h Sun Jun 08 10:57:22 2014 +0900
@@ -124,7 +124,7 @@
~TComTrQuant();
// initialize class
- void init(uint32_t maxTrSize, bool useRDOQ, bool useTransformSkipFast);
+ void init(bool useRDOQ);
// transform & inverse transform functions
uint32_t transformNxN(TComDataCU* cu, int16_t* residual, uint32_t stride, coeff_t* coeff, uint32_t trSize,
@@ -134,9 +134,9 @@
// Misc functions
void setQPforQuant(int qpy, TextType ttype, int qpBdOffset, int chromaQPOffset, int chFmt);
- void setLambda(double lambdaLuma, double lambdaChroma) { m_lumaLambda = lambdaLuma; m_chromaLambda = lambdaChroma; }
+ void setLambdas(double lambdaY, double lambdaCb, double lambdaCr) { m_lambdas[0] = lambdaY; m_lambdas[1] = lambdaCb; m_lambdas[2] = lambdaCr; }
- void selectLambda(TextType ttype) { m_lambda = (ttype == TEXT_LUMA) ? m_lumaLambda : m_chromaLambda; }
+ void selectLambda(TextType ttype) { m_lambda = m_lambdas[ttype]; }
void initScalingList();
void destroyScalingList();
@@ -206,12 +206,9 @@
QpParam m_qpParam;
double m_lambda;
- double m_lumaLambda;
- double m_chromaLambda;
+ double m_lambdas[3];
- uint32_t m_maxTrSize;
bool m_useRDOQ;
- bool m_useTransformSkipFast;
bool m_scalingListEnabledFlag;
int32_t* m_tmpCoeff;
@@ -224,7 +221,7 @@
void xTransformSkip(int16_t* resiBlock, uint32_t stride, int32_t* coeff, int trSize);
void signBitHidingHDQ(coeff_t* qcoeff, coeff_t* coeff, int32_t* deltaU, const TUEntropyCodingParameters &codingParameters);
- uint32_t xQuant(TComDataCU* cu, int32_t* src, coeff_t* dst, int trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos, bool curUseRDOQ = true);
+ uint32_t xQuant(TComDataCU* cu, int32_t* src, coeff_t* dst, int trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos);
// RDOQ functions
uint32_t xRateDistOptQuant(TComDataCU* cu, int32_t* srcCoeff, coeff_t* dstCoeff, uint32_t trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos);
diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibEncoder/TEncSearch.cpp
--- a/source/Lib/TLibEncoder/TEncSearch.cpp Thu Jun 05 22:45:25 2014 -0500
+++ b/source/Lib/TLibEncoder/TEncSearch.cpp Sun Jun 08 10:57:22 2014 +0900
@@ -143,13 +143,12 @@
return false;
}
-void TEncSearch::setQP(int qp, double crWeight, double cbWeight)
+void TEncSearch::setQP(int qp, double cbWeight, double crWeight)
{
double lambda2 = x265_lambda2_tab[qp];
- double chromaLambda = lambda2 / crWeight;
m_me.setQP(qp);
- m_trQuant->setLambda(lambda2, chromaLambda);
+ m_trQuant->setLambdas(lambda2, lambda2 / cbWeight, lambda2 / crWeight);
m_rdCost->setLambda(lambda2, x265_lambda_tab[qp]);
m_rdCost->setCbDistortionWeight(cbWeight);
m_rdCost->setCrDistortionWeight(crWeight);
@@ -543,7 +542,7 @@
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
}
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
- m_trQuant->selectLambda(TEXT_CHROMA);
+ m_trQuant->selectLambda(ttype);
absSum = m_trQuant->transformNxN(cu, residual, stride, coeff, tuSize, ttype, absPartIdx, &lastPos, useTransformSkipChroma);
@@ -1475,8 +1474,7 @@
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
}
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
-
- m_trQuant->selectLambda(TEXT_CHROMA);
+ m_trQuant->selectLambda(ttype);
absSum = m_trQuant->transformNxN(cu, residual, stride, coeff, tuSize, ttype, absPartIdxC, &lastPos, useTransformSkipChroma);
@@ -2930,14 +2928,13 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
-
- m_trQuant->selectLambda(TEXT_CHROMA);
-
+ m_trQuant->selectLambda(TEXT_CHROMA_U);
absSumU = m_trQuant->transformNxN(cu, curResiU, strideResiC, coeffCurU + subTUBufferOffset,
trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPosU, false, curuseRDOQ);
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
+ m_trQuant->selectLambda(TEXT_CHROMA_V);
absSumV = m_trQuant->transformNxN(cu, curResiV, strideResiC, coeffCurV + subTUBufferOffset,
trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPosV, false, curuseRDOQ);
@@ -3131,14 +3128,13 @@
//Cb transform
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
-
- m_trQuant->selectLambda(TEXT_CHROMA);
-
+ m_trQuant->selectLambda(TEXT_CHROMA_U);
absSum[TEXT_CHROMA_U][tuIterator.m_section] = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurU + subTUBufferOffset,
trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPos[TEXT_CHROMA_U][tuIterator.m_section], false, curuseRDOQ);
//Cr transform
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
+ m_trQuant->selectLambda(TEXT_CHROMA_V);
absSum[TEXT_CHROMA_V][tuIterator.m_section] = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurV + subTUBufferOffset,
trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPos[TEXT_CHROMA_V][tuIterator.m_section], false, curuseRDOQ);
@@ -3415,7 +3411,6 @@
}
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, QP_BD_OFFSET, 0, chFmt);
-
m_trQuant->selectLambda(TEXT_LUMA);
absSumTransformSkipY = m_trQuant->transformNxN(cu, resiYuv->getLumaAddr(absPartIdx), resiYuv->m_width, coeffCurY,
trSize, TEXT_LUMA, absPartIdx, &lastPosTransformSkip[TEXT_LUMA][0], true, curuseRDOQ);
@@ -3496,12 +3491,12 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
- m_trQuant->selectLambda(TEXT_CHROMA);
-
+ m_trQuant->selectLambda(TEXT_CHROMA_U);
absSumTransformSkipU = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurU + subTUBufferOffset,
trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPosTransformSkip[TEXT_CHROMA_U][tuIterator.m_section], true, curuseRDOQ);
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
+ m_trQuant->selectLambda(TEXT_CHROMA_V);
absSumTransformSkipV = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurV + subTUBufferOffset,
trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPosTransformSkip[TEXT_CHROMA_V][tuIterator.m_section], true, curuseRDOQ);
diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibEncoder/TEncSearch.h
--- a/source/Lib/TLibEncoder/TEncSearch.h Thu Jun 05 22:45:25 2014 -0500
+++ b/source/Lib/TLibEncoder/TEncSearch.h Sun Jun 08 10:57:22 2014 +0900
@@ -142,7 +142,7 @@
void setRDGoOnSbacCoder(TEncSbac* rdGoOnSbacCoder) { m_rdGoOnSbacCoder = rdGoOnSbacCoder; }
- void setQP(int QP, double crWeight, double cbWeight);
+ void setQP(int QP, double cbWeight, double crWeight);
TEncSearch();
virtual ~TEncSearch();
diff -r e5656f1e1904 -r 188e115f0742 source/encoder/cturow.cpp
--- a/source/encoder/cturow.cpp Thu Jun 05 22:45:25 2014 -0500
+++ b/source/encoder/cturow.cpp Sun Jun 08 10:57:22 2014 +0900
@@ -33,7 +33,7 @@
{
m_rdGoOnSbacCoder.init(&m_rdGoOnBinCodersCABAC);
m_sbacCoder.init(&m_binCoderCABAC);
- m_trQuant.init(1 << top->m_quadtreeTULog2MaxSize, top->m_bEnableRDOQ, !!top->m_param->bEnableTSkipFast);
+ m_trQuant.init(top->m_bEnableRDOQ);
m_rdCost.setPsyRdScale(top->m_param->rdLevel >= 5 ? top->m_param->psyRd : 0);
m_rdSbacCoders = new TEncSbac * *[g_maxCUDepth + 1];
m_binCodersCABAC = new TEncBinCABAC * *[g_maxCUDepth + 1];
diff -r e5656f1e1904 -r 188e115f0742 source/encoder/frameencoder.cpp
--- a/source/encoder/frameencoder.cpp Thu Jun 05 22:45:25 2014 -0500
+++ b/source/encoder/frameencoder.cpp Sun Jun 08 10:57:22 2014 +0900
@@ -392,7 +392,7 @@
qpc = Clip3(0, MAX_MAX_QP, qp + chromaQPOffset);
double crWeight = pow(2.0, (qp - g_chromaScale[chFmt][qpc]) / 3.0); // takes into account of the chroma qp mapping and chroma qp Offset
- m_rows[row].m_search.setQP(qp, crWeight, cbWeight);
+ m_rows[row].m_search.setQP(qp, cbWeight, crWeight);
}
void FrameEncoder::compressFrame()
@@ -450,7 +450,7 @@
for (int i = 0; i < m_numRows; i++)
{
m_rows[i].m_search.m_me.setSourcePlane(fenc->getLumaAddr(), fenc->getStride());
- m_rows[i].m_search.setQP(qp, crWeight, cbWeight);
+ m_rows[i].m_search.setQP(qp, cbWeight, crWeight);
}
m_frameFilter.m_sao.lumaLambda = lambda;
More information about the x265-devel
mailing list