[x265] TComTrQuant: lambda for each Cb and Cr
Steve Borho
steve at borho.org
Tue Jun 10 18:26:53 CEST 2014
On Sat, Jun 7, 2014 at 9:01 PM, Satoshi Nakagawa <nakagawa424 at oki.com> wrote:
> # HG changeset patch
> # User Satoshi Nakagawa <nakagawa424 at oki.com>
> # Date 1402192642 -32400
> # Sun Jun 08 10:57:22 2014 +0900
> # Node ID 188e115f07427c759f47154a864467be21b5b6a1
> # Parent e5656f1e190453efa84732269b259a6dee608ff9
> TComTrQuant: lambda for each Cb and Cr
Queued for testing. This commit message only describes about a third
of what this patch does, but it does look like it shouldn't change
outputs if the chroma weights are not changed from the default.
>
> diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibCommon/TComTrQuant.cpp
> --- a/source/Lib/TLibCommon/TComTrQuant.cpp Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/Lib/TLibCommon/TComTrQuant.cpp Sun Jun 08 10:57:22 2014 +0900
> @@ -262,49 +262,35 @@
> }
>
> uint32_t TComTrQuant::xQuant(TComDataCU* cu, int32_t* coef, coeff_t* qCoef, int trSize,
> - TextType ttype, uint32_t absPartIdx, int32_t *lastPos, bool curUseRDOQ)
> + TextType ttype, uint32_t absPartIdx, int32_t *lastPos)
> {
> - uint32_t acSum = 0;
> - int add = 0;
> - bool useRDOQ = m_useRDOQ && curUseRDOQ;
> + const uint32_t log2TrSize = g_convertToBit[trSize] + 2;
> + TUEntropyCodingParameters codingParameters;
> + getTUEntropyCodingParameters(cu, codingParameters, absPartIdx, log2TrSize, ttype);
> + int deltaU[32 * 32];
>
> - if (useRDOQ)
> + int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
> + X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
> + int32_t *quantCoeff = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
> +
> + int transformShift = MAX_TR_DYNAMIC_RANGE - X265_DEPTH - log2TrSize; // Represents scaling through forward transform
> +
> + int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift;
> + int add = (cu->getSlice()->getSliceType() == I_SLICE ? 171 : 85) << (qbits - 9);
> +
> + int numCoeff = 1 << log2TrSize * 2;
> + uint32_t acSum = primitives.quant(coef, quantCoeff, deltaU, qCoef, qbits, add, numCoeff, lastPos);
> +
> + if (acSum >= 2 && cu->getSlice()->getPPS()->getSignHideFlag())
> {
> - acSum = xRateDistOptQuant(cu, coef, qCoef, trSize, ttype, absPartIdx, lastPos);
> - }
> - else
> - {
> - const uint32_t log2TrSize = g_convertToBit[trSize] + 2;
> - TUEntropyCodingParameters codingParameters;
> - getTUEntropyCodingParameters(cu, codingParameters, absPartIdx, log2TrSize, ttype);
> - int deltaU[32 * 32];
> -
> - int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
> - X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
> - int32_t *quantCoeff = 0;
> - quantCoeff = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
> -
> - int transformShift = MAX_TR_DYNAMIC_RANGE - X265_DEPTH - log2TrSize; // Represents scaling through forward transform
> -
> - int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift;
> - add = (cu->getSlice()->getSliceType() == I_SLICE ? 171 : 85) << (qbits - 9);
> -
> - int numCoeff = 1 << log2TrSize * 2;
> - acSum += primitives.quant(coef, quantCoeff, deltaU, qCoef, qbits, add, numCoeff, lastPos);
> -
> - if (cu->getSlice()->getPPS()->getSignHideFlag() && acSum >= 2)
> - {
> - signBitHidingHDQ(qCoef, coef, deltaU, codingParameters);
> - }
> + signBitHidingHDQ(qCoef, coef, deltaU, codingParameters);
> }
> return acSum;
> }
>
> -void TComTrQuant::init(uint32_t maxTrSize, bool useRDOQ, bool useTransformSkipFast)
> +void TComTrQuant::init(bool useRDOQ)
> {
> - m_maxTrSize = maxTrSize;
> - m_useRDOQ = useRDOQ;
> - m_useTransformSkipFast = useTransformSkipFast;
> + m_useRDOQ = useRDOQ;
> }
>
> uint32_t TComTrQuant::transformNxN(TComDataCU* cu,
> @@ -363,7 +349,12 @@
> }
> }
> }
> - return xQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos, curUseRDOQ);
> +
> + if (m_useRDOQ && curUseRDOQ)
> + {
> + return xRateDistOptQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos);
> + }
> + return xQuant(cu, m_tmpCoeff, coeff, trSize, ttype, absPartIdx, lastPos);
> }
>
> void TComTrQuant::invtransformNxN(bool transQuantBypass, uint32_t mode, int16_t* residual, uint32_t stride, coeff_t* coeff, uint32_t trSize, int scalingListType, bool useTransformSkip, int lastPos)
> @@ -525,10 +516,9 @@
> X265_CHECK(scalingListType < 6, "scaling list type out of range\n");
>
> int qbits = QUANT_SHIFT + m_qpParam.m_per + transformShift; // Right shift of non-RDOQ quantizer; level = (coeff*Q + offset)>>q_bits
> - double *errScaleOrg = getErrScaleCoeff(scalingListType, log2TrSize - 2, m_qpParam.m_rem);
> - int32_t *qCoefOrg = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
> - int32_t *qCoef = qCoefOrg;
> - double *errScale = errScaleOrg;
> + int add = (1 << (qbits - 1));
> + double *errScale = getErrScaleCoeff(scalingListType, log2TrSize - 2, m_qpParam.m_rem);
> + int32_t *qCoef = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
>
> double costCoeff[32 * 32];
> double costSig[32 * 32];
> @@ -575,9 +565,9 @@
> int Q = qCoef[blkPos];
> double scaleFactor = errScale[blkPos];
> int levelDouble = srcCoeff[blkPos];
> - levelDouble = (int)std::min<int64_t>((int64_t)abs((int)levelDouble) * Q, MAX_INT - (1 << (qbits - 1)));
> + levelDouble = (int)std::min<int64_t>((int64_t)abs((int)levelDouble) * Q, MAX_INT - add);
>
> - uint32_t maxAbsLevel = (levelDouble + (1 << (qbits - 1))) >> qbits;
> + uint32_t maxAbsLevel = (levelDouble + add) >> qbits;
>
> costCoeff0[scanPos] = ((uint64_t)levelDouble * levelDouble) * scaleFactor;
> blockUncodedCost += costCoeff0[scanPos];
> @@ -599,11 +589,6 @@
> X265_CHECK(!!(c2Idx == 0) == ((-(int)c2Idx) >> (sizeof(int) * CHAR_BIT - 1)) + 1, "scan validation 3\n");
> X265_CHECK(baseLevel == ((c1Idx < C1FLAG_NUMBER) ? (2 + (c2Idx == 0)) : 1), "scan validation 4\n");
>
> - rateIncUp[blkPos] = 0;
> - rateIncDown[blkPos] = 0;
> - deltaU[blkPos] = 0;
> - sigRateDelta[blkPos] = 0;
> -
> //===== coefficient level estimation =====
> uint32_t level;
> const uint32_t oneCtx = 4 * ctxSet + c1;
> @@ -618,6 +603,7 @@
> level = xGetCodedLevel(costCoeff[scanPos], curCostSig, costSig[scanPos],
> levelDouble, maxAbsLevel, baseLevel, greaterOneBits, levelAbsBits, goRiceParam,
> c1c2Idx, qbits, scaleFactor, 1);
> + sigRateDelta[blkPos] = 0;
> }
> else
> {
> @@ -651,6 +637,7 @@
> else // level == 0
> {
> rateIncUp[blkPos] = greaterOneBits[0];
> + rateIncDown[blkPos] = 0;
> }
> dstCoeff[blkPos] = level;
> baseCost += costCoeff[scanPos];
> @@ -869,7 +856,7 @@
> {
> int64_t rdFactor = (int64_t)(
> g_invQuantScales[m_qpParam.rem()] * g_invQuantScales[m_qpParam.rem()] * (1 << (2 * m_qpParam.m_per))
> - / m_lambda / 16 / (1 << DISTORTION_PRECISION_ADJUSTMENT(2 * (X265_DEPTH - 8)))
> + / (m_lambda * (16 << DISTORTION_PRECISION_ADJUSTMENT(2 * (X265_DEPTH - 8))))
> + 0.5);
> int lastCG = 1;
>
> diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibCommon/TComTrQuant.h
> --- a/source/Lib/TLibCommon/TComTrQuant.h Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/Lib/TLibCommon/TComTrQuant.h Sun Jun 08 10:57:22 2014 +0900
> @@ -124,7 +124,7 @@
> ~TComTrQuant();
>
> // initialize class
> - void init(uint32_t maxTrSize, bool useRDOQ, bool useTransformSkipFast);
> + void init(bool useRDOQ);
>
> // transform & inverse transform functions
> uint32_t transformNxN(TComDataCU* cu, int16_t* residual, uint32_t stride, coeff_t* coeff, uint32_t trSize,
> @@ -134,9 +134,9 @@
>
> // Misc functions
> void setQPforQuant(int qpy, TextType ttype, int qpBdOffset, int chromaQPOffset, int chFmt);
> - void setLambda(double lambdaLuma, double lambdaChroma) { m_lumaLambda = lambdaLuma; m_chromaLambda = lambdaChroma; }
> + void setLambdas(double lambdaY, double lambdaCb, double lambdaCr) { m_lambdas[0] = lambdaY; m_lambdas[1] = lambdaCb; m_lambdas[2] = lambdaCr; }
>
> - void selectLambda(TextType ttype) { m_lambda = (ttype == TEXT_LUMA) ? m_lumaLambda : m_chromaLambda; }
> + void selectLambda(TextType ttype) { m_lambda = m_lambdas[ttype]; }
>
> void initScalingList();
> void destroyScalingList();
> @@ -206,12 +206,9 @@
> QpParam m_qpParam;
>
> double m_lambda;
> - double m_lumaLambda;
> - double m_chromaLambda;
> + double m_lambdas[3];
>
> - uint32_t m_maxTrSize;
> bool m_useRDOQ;
> - bool m_useTransformSkipFast;
> bool m_scalingListEnabledFlag;
>
> int32_t* m_tmpCoeff;
> @@ -224,7 +221,7 @@
>
> void xTransformSkip(int16_t* resiBlock, uint32_t stride, int32_t* coeff, int trSize);
> void signBitHidingHDQ(coeff_t* qcoeff, coeff_t* coeff, int32_t* deltaU, const TUEntropyCodingParameters &codingParameters);
> - uint32_t xQuant(TComDataCU* cu, int32_t* src, coeff_t* dst, int trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos, bool curUseRDOQ = true);
> + uint32_t xQuant(TComDataCU* cu, int32_t* src, coeff_t* dst, int trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos);
>
> // RDOQ functions
> uint32_t xRateDistOptQuant(TComDataCU* cu, int32_t* srcCoeff, coeff_t* dstCoeff, uint32_t trSize, TextType ttype, uint32_t absPartIdx, int32_t *lastPos);
> diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibEncoder/TEncSearch.cpp
> --- a/source/Lib/TLibEncoder/TEncSearch.cpp Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/Lib/TLibEncoder/TEncSearch.cpp Sun Jun 08 10:57:22 2014 +0900
> @@ -143,13 +143,12 @@
> return false;
> }
>
> -void TEncSearch::setQP(int qp, double crWeight, double cbWeight)
> +void TEncSearch::setQP(int qp, double cbWeight, double crWeight)
> {
> double lambda2 = x265_lambda2_tab[qp];
> - double chromaLambda = lambda2 / crWeight;
>
> m_me.setQP(qp);
> - m_trQuant->setLambda(lambda2, chromaLambda);
> + m_trQuant->setLambdas(lambda2, lambda2 / cbWeight, lambda2 / crWeight);
> m_rdCost->setLambda(lambda2, x265_lambda_tab[qp]);
> m_rdCost->setCbDistortionWeight(cbWeight);
> m_rdCost->setCrDistortionWeight(crWeight);
> @@ -543,7 +542,7 @@
> curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> }
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> - m_trQuant->selectLambda(TEXT_CHROMA);
> + m_trQuant->selectLambda(ttype);
>
> absSum = m_trQuant->transformNxN(cu, residual, stride, coeff, tuSize, ttype, absPartIdx, &lastPos, useTransformSkipChroma);
>
> @@ -1475,8 +1474,7 @@
> curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> }
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> -
> - m_trQuant->selectLambda(TEXT_CHROMA);
> + m_trQuant->selectLambda(ttype);
>
> absSum = m_trQuant->transformNxN(cu, residual, stride, coeff, tuSize, ttype, absPartIdxC, &lastPos, useTransformSkipChroma);
>
> @@ -2930,14 +2928,13 @@
>
> int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> -
> - m_trQuant->selectLambda(TEXT_CHROMA);
> -
> + m_trQuant->selectLambda(TEXT_CHROMA_U);
> absSumU = m_trQuant->transformNxN(cu, curResiU, strideResiC, coeffCurU + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPosU, false, curuseRDOQ);
>
> curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> + m_trQuant->selectLambda(TEXT_CHROMA_V);
> absSumV = m_trQuant->transformNxN(cu, curResiV, strideResiC, coeffCurV + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPosV, false, curuseRDOQ);
>
> @@ -3131,14 +3128,13 @@
> //Cb transform
> int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> -
> - m_trQuant->selectLambda(TEXT_CHROMA);
> -
> + m_trQuant->selectLambda(TEXT_CHROMA_U);
> absSum[TEXT_CHROMA_U][tuIterator.m_section] = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurU + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPos[TEXT_CHROMA_U][tuIterator.m_section], false, curuseRDOQ);
> //Cr transform
> curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> + m_trQuant->selectLambda(TEXT_CHROMA_V);
> absSum[TEXT_CHROMA_V][tuIterator.m_section] = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurV + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPos[TEXT_CHROMA_V][tuIterator.m_section], false, curuseRDOQ);
>
> @@ -3415,7 +3411,6 @@
> }
>
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, QP_BD_OFFSET, 0, chFmt);
> -
> m_trQuant->selectLambda(TEXT_LUMA);
> absSumTransformSkipY = m_trQuant->transformNxN(cu, resiYuv->getLumaAddr(absPartIdx), resiYuv->m_width, coeffCurY,
> trSize, TEXT_LUMA, absPartIdx, &lastPosTransformSkip[TEXT_LUMA][0], true, curuseRDOQ);
> @@ -3496,12 +3491,12 @@
>
> int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> - m_trQuant->selectLambda(TEXT_CHROMA);
> -
> + m_trQuant->selectLambda(TEXT_CHROMA_U);
> absSumTransformSkipU = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurU + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_U, absPartIdxC, &lastPosTransformSkip[TEXT_CHROMA_U][tuIterator.m_section], true, curuseRDOQ);
> curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> + m_trQuant->selectLambda(TEXT_CHROMA_V);
> absSumTransformSkipV = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absPartIdxC), resiYuv->m_cwidth, coeffCurV + subTUBufferOffset,
> trSizeC, TEXT_CHROMA_V, absPartIdxC, &lastPosTransformSkip[TEXT_CHROMA_V][tuIterator.m_section], true, curuseRDOQ);
>
> diff -r e5656f1e1904 -r 188e115f0742 source/Lib/TLibEncoder/TEncSearch.h
> --- a/source/Lib/TLibEncoder/TEncSearch.h Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/Lib/TLibEncoder/TEncSearch.h Sun Jun 08 10:57:22 2014 +0900
> @@ -142,7 +142,7 @@
>
> void setRDGoOnSbacCoder(TEncSbac* rdGoOnSbacCoder) { m_rdGoOnSbacCoder = rdGoOnSbacCoder; }
>
> - void setQP(int QP, double crWeight, double cbWeight);
> + void setQP(int QP, double cbWeight, double crWeight);
>
> TEncSearch();
> virtual ~TEncSearch();
> diff -r e5656f1e1904 -r 188e115f0742 source/encoder/cturow.cpp
> --- a/source/encoder/cturow.cpp Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/encoder/cturow.cpp Sun Jun 08 10:57:22 2014 +0900
> @@ -33,7 +33,7 @@
> {
> m_rdGoOnSbacCoder.init(&m_rdGoOnBinCodersCABAC);
> m_sbacCoder.init(&m_binCoderCABAC);
> - m_trQuant.init(1 << top->m_quadtreeTULog2MaxSize, top->m_bEnableRDOQ, !!top->m_param->bEnableTSkipFast);
> + m_trQuant.init(top->m_bEnableRDOQ);
> m_rdCost.setPsyRdScale(top->m_param->rdLevel >= 5 ? top->m_param->psyRd : 0);
> m_rdSbacCoders = new TEncSbac * *[g_maxCUDepth + 1];
> m_binCodersCABAC = new TEncBinCABAC * *[g_maxCUDepth + 1];
> diff -r e5656f1e1904 -r 188e115f0742 source/encoder/frameencoder.cpp
> --- a/source/encoder/frameencoder.cpp Thu Jun 05 22:45:25 2014 -0500
> +++ b/source/encoder/frameencoder.cpp Sun Jun 08 10:57:22 2014 +0900
> @@ -392,7 +392,7 @@
> qpc = Clip3(0, MAX_MAX_QP, qp + chromaQPOffset);
> double crWeight = pow(2.0, (qp - g_chromaScale[chFmt][qpc]) / 3.0); // takes into account of the chroma qp mapping and chroma qp Offset
>
> - m_rows[row].m_search.setQP(qp, crWeight, cbWeight);
> + m_rows[row].m_search.setQP(qp, cbWeight, crWeight);
> }
>
> void FrameEncoder::compressFrame()
> @@ -450,7 +450,7 @@
> for (int i = 0; i < m_numRows; i++)
> {
> m_rows[i].m_search.m_me.setSourcePlane(fenc->getLumaAddr(), fenc->getStride());
> - m_rows[i].m_search.setQP(qp, crWeight, cbWeight);
> + m_rows[i].m_search.setQP(qp, cbWeight, crWeight);
> }
>
> m_frameFilter.m_sao.lumaLambda = lambda;
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
--
Steve Borho
More information about the x265-devel
mailing list