[x265] fix: 4:2:2 rdLevel <= 1
Steve Borho
steve at borho.org
Fri Apr 25 16:45:07 CEST 2014
On Fri, Apr 25, 2014 at 2:05 AM, Satoshi Nakagawa <nakagawa424 at oki.com> wrote:
> # HG changeset patch
> # User Satoshi Nakagawa <nakagawa424 at oki.com>
> # Date 1398409064 -32400
> # Fri Apr 25 15:57:44 2014 +0900
> # Node ID 9178bef9cb2b89befc082b42865ad85befe7db2b
> # Parent c630b0b393eec3a2e2bedf137fba0d56764672ef
> fix: 4:2:2 rdLevel <= 1
Thanks Satoshi, this is being reviewed internally. Hope to have some
feedback on Monday
> diff -r c630b0b393ee -r 9178bef9cb2b source/Lib/TLibEncoder/TEncSearch.cpp
> --- a/source/Lib/TLibEncoder/TEncSearch.cpp Thu Apr 24 15:59:05 2014 -0500
> +++ b/source/Lib/TLibEncoder/TEncSearch.cpp Fri Apr 25 15:57:44 2014 +0900
> @@ -2812,6 +2812,8 @@
> assert(cu->getDepth(0) == cu->getDepth(absPartIdx));
> const uint32_t trMode = depth - cu->getDepth(0);
> const uint32_t trSizeLog2 = g_convertToBit[cu->getSlice()->getSPS()->getMaxCUSize() >> depth] + 2;
> + uint32_t trSizeCLog2 = trSizeLog2 - m_hChromaShift;
> + const uint32_t setCbf = 1 << trMode;
> int chFmt = cu->getChromaFormat();
>
> bool bSplitFlag = ((cu->getSlice()->getSPS()->getQuadtreeTUMaxDepthInter() == 1) && cu->getPredictionMode(absPartIdx) == MODE_INTER && (cu->getPartitionSize(absPartIdx) != SIZE_2Nx2N));
> @@ -2825,16 +2827,17 @@
>
> bool bCodeChroma = true;
> uint32_t trModeC = trMode;
> - uint32_t trSizeCLog2 = trSizeLog2 - 1;
> - if (trSizeLog2 == 2)
> + if ((trSizeLog2 == 2) && !(chFmt == CHROMA_444))
> {
> trSizeCLog2++;
> trModeC--;
> - uint32_t qpdiv = cu->getPic()->getNumPartInCU() >> ((cu->getDepth(0) + trModeC) << 1);
> + uint32_t qpdiv = cu->getPic()->getNumPartInCU() >> ((depth - 1) << 1);
> bCodeChroma = ((absPartIdx % qpdiv) == 0);
> }
>
> - const uint32_t setCbf = 1 << trMode;
> + const bool splitIntoSubTUs = (chFmt == CHROMA_422);
> + uint32_t absPartIdxStep = cu->getPic()->getNumPartInCU() >> ((cu->getDepth(0) + trModeC) << 1);
> +
> // code full block
> uint32_t absSumY = 0, absSumU = 0, absSumV = 0;
> int lastPosY = -1, lastPosU = -1, lastPosV = -1;
> @@ -2847,18 +2850,12 @@
> coeff_t *coeffCurV = cu->getCoeffCr() + (numCoeffPerAbsPartIdxIncrement * absPartIdx >> (m_hChromaShift + m_vChromaShift));
>
> int trWidth = 0, trHeight = 0, trWidthC = 0, trHeightC = 0;
> - uint32_t absTUPartIdxC = absPartIdx;
>
> trWidth = trHeight = 1 << trSizeLog2;
> trWidthC = trHeightC = 1 << trSizeCLog2;
> cu->setTrIdxSubParts(depth - cu->getDepth(0), absPartIdx, depth);
>
> cu->setTransformSkipSubParts(0, TEXT_LUMA, absPartIdx, depth);
> - if (bCodeChroma)
> - {
> - cu->setTransformSkipSubParts(0, TEXT_CHROMA_U, absPartIdx, cu->getDepth(0) + trModeC);
> - cu->setTransformSkipSubParts(0, TEXT_CHROMA_V, absPartIdx, cu->getDepth(0) + trModeC);
> - }
>
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, QP_BD_OFFSET, 0, chFmt);
> m_trQuant->selectLambda(TEXT_LUMA);
> @@ -2868,25 +2865,6 @@
>
> cu->setCbfSubParts(absSumY ? setCbf : 0, TEXT_LUMA, absPartIdx, depth);
>
> - if (bCodeChroma)
> - {
> - int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> - m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> -
> - m_trQuant->selectLambda(TEXT_CHROMA);
> -
> - absSumU = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absTUPartIdxC), resiYuv->m_cwidth, coeffCurU,
> - trWidthC, TEXT_CHROMA_U, absPartIdx, &lastPosU, false, curuseRDOQ);
> -
> - curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> - m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> - absSumV = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absTUPartIdxC), resiYuv->m_cwidth, coeffCurV,
> - trWidthC, TEXT_CHROMA_V, absPartIdx, &lastPosV, false, curuseRDOQ);
> -
> - cu->setCbfSubParts(absSumU ? setCbf : 0, TEXT_CHROMA_U, absPartIdx, cu->getDepth(0) + trModeC);
> - cu->setCbfSubParts(absSumV ? setCbf : 0, TEXT_CHROMA_V, absPartIdx, cu->getDepth(0) + trModeC);
> - }
> -
> if (absSumY)
> {
> int16_t *curResiY = resiYuv->getLumaAddr(absTUPartIdx);
> @@ -2903,49 +2881,84 @@
> assert(trWidth == trHeight);
> primitives.blockfill_s[(int)g_convertToBit[trWidth]](ptr, resiYuv->m_width, 0);
> }
> + cu->setCbfSubParts(absSumY ? setCbf : 0, TEXT_LUMA, absPartIdx, depth);
>
> if (bCodeChroma)
> {
> - if (absSumU)
> + TComTURecurse tuIterator;
> + initSection(&tuIterator, splitIntoSubTUs ? VERTICAL_SPLIT : DONT_SPLIT, absPartIdxStep, absPartIdx);
> +
> + uint32_t widthC = trWidthC;
> + uint32_t heightC = trWidthC;
> +
> + do
> {
> - int16_t *pcResiCurrU = resiYuv->getCbAddr(absTUPartIdxC);
> + uint32_t absTUPartIdxC = tuIterator.m_absPartIdxTURelCU;
> + uint32_t subTUBufferOffset = widthC * heightC * tuIterator.m_section;
> +
> + cu->setTransformSkipPartRange(0, TEXT_CHROMA_U, absTUPartIdxC, tuIterator.m_absPartIdxStep);
> + cu->setTransformSkipPartRange(0, TEXT_CHROMA_V, absTUPartIdxC, tuIterator.m_absPartIdxStep);
>
> int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
>
> - int scalingListType = 3 + TEXT_CHROMA_U;
> - assert(scalingListType < 6);
> - m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, pcResiCurrU, resiYuv->m_cwidth, coeffCurU, trWidthC, scalingListType, false, lastPosU);
> - }
> - else
> + m_trQuant->selectLambda(TEXT_CHROMA);
> +
> + absSumU = m_trQuant->transformNxN(cu, resiYuv->getCbAddr(absTUPartIdxC), resiYuv->m_cwidth, coeffCurU + subTUBufferOffset,
> + trWidthC, TEXT_CHROMA_U, absTUPartIdxC, &lastPosU, false, curuseRDOQ);
> +
> + curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> + m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> + absSumV = m_trQuant->transformNxN(cu, resiYuv->getCrAddr(absTUPartIdxC), resiYuv->m_cwidth, coeffCurV + subTUBufferOffset,
> + trWidthC, TEXT_CHROMA_V, absTUPartIdxC, &lastPosV, false, curuseRDOQ);
> +
> + cu->setCbfPartRange(absSumU ? setCbf : 0, TEXT_CHROMA_U, absTUPartIdxC, tuIterator.m_absPartIdxStep);
> + cu->setCbfPartRange(absSumV ? setCbf : 0, TEXT_CHROMA_V, absTUPartIdxC, tuIterator.m_absPartIdxStep);
> +
> + if (absSumU)
> + {
> + int16_t *pcResiCurrU = resiYuv->getCbAddr(absTUPartIdxC);
> +
> + curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
> + m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> +
> + int scalingListType = 3 + TEXT_CHROMA_U;
> + assert(scalingListType < 6);
> + m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absTUPartIdxC), REG_DCT, pcResiCurrU, resiYuv->m_cwidth, coeffCurU + subTUBufferOffset, trWidthC, scalingListType, false, lastPosU);
> + }
> + else
> + {
> + int16_t *ptr = resiYuv->getCbAddr(absTUPartIdxC);
> + assert(trWidthC == trHeightC);
> + primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, resiYuv->m_cwidth, 0);
> + }
> + if (absSumV)
> + {
> + int16_t *curResiV = resiYuv->getCrAddr(absTUPartIdxC);
> + curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> + m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> +
> + int scalingListType = 3 + TEXT_CHROMA_V;
> + assert(scalingListType < 6);
> + m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absTUPartIdxC), REG_DCT, curResiV, resiYuv->m_cwidth, coeffCurV + subTUBufferOffset, trWidthC, scalingListType, false, lastPosV);
> + }
> + else
> + {
> + int16_t *ptr = resiYuv->getCrAddr(absTUPartIdxC);
> + assert(trWidthC == trHeightC);
> + primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, resiYuv->m_cwidth, 0);
> + }
> + cu->setCbfPartRange(absSumU ? setCbf : 0, TEXT_CHROMA_U, absTUPartIdxC, tuIterator.m_absPartIdxStep);
> + cu->setCbfPartRange(absSumV ? setCbf : 0, TEXT_CHROMA_V, absTUPartIdxC, tuIterator.m_absPartIdxStep);
> + } while (isNextSection(&tuIterator));
> +
> + if (splitIntoSubTUs)
> {
> - int16_t *ptr = resiYuv->getCbAddr(absTUPartIdxC);
> - assert(trWidthC == trHeightC);
> - primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, resiYuv->m_cwidth, 0);
> - }
> - if (absSumV)
> - {
> - int16_t *curResiV = resiYuv->getCrAddr(absTUPartIdxC);
> - int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
> - m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset, chFmt);
> -
> - int scalingListType = 3 + TEXT_CHROMA_V;
> - assert(scalingListType < 6);
> - m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, curResiV, resiYuv->m_cwidth, coeffCurV, trWidthC, scalingListType, false, lastPosV);
> - }
> - else
> - {
> - int16_t *ptr = resiYuv->getCrAddr(absTUPartIdxC);
> - assert(trWidthC == trHeightC);
> - primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, resiYuv->m_cwidth, 0);
> + offsetSubTUCBFs(cu, TEXT_CHROMA_U, trMode, absPartIdx);
> + offsetSubTUCBFs(cu, TEXT_CHROMA_V, trMode, absPartIdx);
> }
> }
> - cu->setCbfSubParts(absSumY ? setCbf : 0, TEXT_LUMA, absPartIdx, depth);
> - if (bCodeChroma)
> - {
> - cu->setCbfSubParts(absSumU ? setCbf : 0, TEXT_CHROMA_U, absPartIdx, cu->getDepth(0) + trModeC);
> - cu->setCbfSubParts(absSumV ? setCbf : 0, TEXT_CHROMA_V, absPartIdx, cu->getDepth(0) + trModeC);
> - }
> + return;
> }
>
> // code sub-blocks
> @@ -2976,15 +2989,6 @@
> }
> return;
> }
> -
> - cu->setTrIdxSubParts(trMode, absPartIdx, depth);
> - cu->setCbfSubParts(absSumY ? setCbf : 0, TEXT_LUMA, absPartIdx, depth);
> -
> - if (bCodeChroma)
> - {
> - cu->setCbfSubParts(absSumU ? setCbf : 0, TEXT_CHROMA_U, absPartIdx, cu->getDepth(0) + trModeC);
> - cu->setCbfSubParts(absSumV ? setCbf : 0, TEXT_CHROMA_V, absPartIdx, cu->getDepth(0) + trModeC);
> - }
> }
>
> void TEncSearch::xEstimateResidualQT(TComDataCU* cu,
> @@ -3001,7 +3005,7 @@
> assert(cu->getDepth(0) == cu->getDepth(absPartIdx));
> const uint32_t trMode = depth - cu->getDepth(0);
> const uint32_t trSizeLog2 = g_convertToBit[cu->getSlice()->getSPS()->getMaxCUSize() >> depth] + 2;
> - uint32_t trSizeCLog2 = g_convertToBit[(cu->getSlice()->getSPS()->getMaxCUSize() >> m_hChromaShift) >> depth] + 2;
> + uint32_t trSizeCLog2 = trSizeLog2 - m_hChromaShift;
> const uint32_t subTUDepth = trMode + 1;
> const uint32_t setCbf = 1 << trMode;
> int chFmt = cu->getChromaFormat();
> @@ -3777,7 +3781,7 @@
> const uint32_t trMode = cu->getTransformIdx(absPartIdx);
> const bool bSubdiv = curTrMode != trMode;
> const uint32_t trSizeLog2 = g_convertToBit[cu->getSlice()->getSPS()->getMaxCUSize() >> depth] + 2;
> - uint32_t trSizeCLog2 = g_convertToBit[(cu->getSlice()->getSPS()->getMaxCUSize() >> m_hChromaShift) >> depth] + 2;
> + uint32_t trSizeCLog2 = trSizeLog2 - m_hChromaShift;
> int chFmt = cu->getChromaFormat();
>
> if (bSubdivAndCbf && trSizeLog2 <= cu->getSlice()->getSPS()->getQuadtreeTULog2MaxSize() && trSizeLog2 > cu->getQuadtreeTULog2MinSizeInCU(absPartIdx))
> @@ -3887,7 +3891,7 @@
> if (curTrMode == trMode)
> {
> const uint32_t trSizeLog2 = g_convertToBit[cu->getSlice()->getSPS()->getMaxCUSize() >> depth] + 2;
> - uint32_t trSizeCLog2 = g_convertToBit[(cu->getSlice()->getSPS()->getMaxCUSize() >> cu->getHorzChromaShift()) >> depth] + 2;
> + uint32_t trSizeCLog2 = trSizeLog2 - m_hChromaShift;
> const uint32_t qtlayer = cu->getSlice()->getSPS()->getQuadtreeTULog2MaxSize() - trSizeLog2;
>
> bool bCodeChroma = true;
> diff -r c630b0b393ee -r 9178bef9cb2b source/encoder/compress.cpp
> --- a/source/encoder/compress.cpp Thu Apr 24 15:59:05 2014 -0500
> +++ b/source/encoder/compress.cpp Fri Apr 25 15:57:44 2014 +0900
> @@ -498,7 +498,11 @@
> if (outBestCU->getSlice()->getSliceType() == P_SLICE)
> {
> /*compute intra cost */
> - bool bdoIntra = true;
> +
> + // TODO: residualQTIntrachroma is broken for 4:2:2
> + bool bdoIntra = (m_param->internalCsp != X265_CSP_I422);
> + // bool bdoIntra = true;
> +
> if (m_param->rdLevel > 2)
> {
> bdoIntra = (outBestCU->getCbf(0, TEXT_LUMA) || outBestCU->getCbf(0, TEXT_CHROMA_U) ||
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
--
Steve Borho
More information about the x265-devel
mailing list