[x265] [PATCH] rd level: use cu coeff array while computing coefficients instead of temp buffer
Deepthi Devaki Akkoorath
deepthidevaki at multicorewareinc.com
Mon Dec 16 11:21:03 CET 2013
On Mon, Dec 16, 2013 at 3:36 PM, Deepthi Nandakumar <
deepthi at multicorewareinc.com> wrote:
> Build fails. Restore qtlayer.
>
>
It builds fine here. What is the error?
>
> On Mon, Dec 16, 2013 at 3:07 PM, <deepthidevaki at multicorewareinc.com>wrote:
>
>> # HG changeset patch
>> # User Deepthi Devaki <deepthidevaki at multicorewareinc.com>
>> # Date 1387185624 -19800
>> # Node ID f20e5ab835cd7071d9ebeabff50f6b9fef4d3e39
>> # Parent 9bb16a023918c342b907d106b4a3d59ec2473bc1
>> rd level: use cu coeff array while computing coefficients instead of temp
>> buffer
>>
>> diff -r 9bb16a023918 -r f20e5ab835cd source/Lib/TLibEncoder/TEncSearch.cpp
>> --- a/source/Lib/TLibEncoder/TEncSearch.cpp Mon Dec 16 14:33:57 2013
>> +0530
>> +++ b/source/Lib/TLibEncoder/TEncSearch.cpp Mon Dec 16 14:50:24 2013
>> +0530
>> @@ -3266,9 +3266,7 @@
>> if (cu->getPredictionMode(0) == MODE_INTER)
>> {
>> residualTransformQuantInter(cu, 0, 0, resiYuv, cu->getDepth(0),
>> true);
>> - xSetResidualQTData(cu, 0, 0, NULL, cu->getDepth(0), false);
>> uint32_t width = cu->getWidth(0);
>> - xSetResidualQTData(cu, 0, 0, resiYuv, cu->getDepth(0), true);
>> reconYuv->addClip(predYuv, resiYuv, 0, width);
>>
>> if (cu->getMergeFlag(0) && cu->getPartitionSize(0) == SIZE_2Nx2N
>> && cu->getQtRootCbf(0) == 0)
>> @@ -3322,10 +3320,10 @@
>> if (bCheckFull)
>> {
>> const uint32_t numCoeffPerAbsPartIdxIncrement =
>> cu->getSlice()->getSPS()->getMaxCUWidth() *
>> cu->getSlice()->getSPS()->getMaxCUHeight() >>
>> (cu->getSlice()->getSPS()->getMaxCUDepth() << 1);
>> - const uint32_t qtlayer =
>> cu->getSlice()->getSPS()->getQuadtreeTULog2MaxSize() - trSizeLog2;
>> - TCoeff *coeffCurY = m_qtTempCoeffY[qtlayer] +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx);
>> - TCoeff *coeffCurU = m_qtTempCoeffCb[qtlayer] +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx >> 2);
>> - TCoeff *coeffCurV = m_qtTempCoeffCr[qtlayer] +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx >> 2);
>> +
>> + TCoeff *coeffCurY = cu->getCoeffY() +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx);
>> + TCoeff *coeffCurU = cu->getCoeffCb() +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx >> 2);
>> + TCoeff *coeffCurV = cu->getCoeffCr() +
>> (numCoeffPerAbsPartIdxIncrement * absPartIdx >> 2);
>>
>> int trWidth = 0, trHeight = 0, trWidthC = 0, trHeightC = 0;
>> uint32_t absTUPartIdxC = absPartIdx;
>> @@ -3370,64 +3368,55 @@
>>
>> if (absSumY)
>> {
>> - int16_t *curResiY =
>> m_qtTempTComYuv[qtlayer].getLumaAddr(absTUPartIdx);
>> + int16_t *curResiY = resiYuv->getLumaAddr(absTUPartIdx);
>>
>> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA,
>> cu->getSlice()->getSPS()->getQpBDOffsetY(), 0);
>>
>> int scalingListType = 3 + g_eTTable[(int)TEXT_LUMA];
>> assert(scalingListType < 6);
>> - assert(m_qtTempTComYuv[qtlayer].m_width == MAX_CU_SIZE);
>> -
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> curResiY, MAX_CU_SIZE, coeffCurY, trWidth, trHeight, scalingListType,
>> false, lastPosY); //this is for inter mode only
>> +
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> curResiY, resiYuv->m_width, coeffCurY, trWidth, trHeight, scalingListType,
>> false, lastPosY); //this is for inter mode only
>> }
>> else
>> {
>> - int16_t *ptr =
>> m_qtTempTComYuv[qtlayer].getLumaAddr(absTUPartIdx);
>> - assert(m_qtTempTComYuv[qtlayer].m_width == MAX_CU_SIZE);
>> -
>> + int16_t *ptr = resiYuv->getLumaAddr(absTUPartIdx);
>> assert(trWidth == trHeight);
>> - primitives.blockfill_s[(int)g_convertToBit[trWidth]](ptr,
>> MAX_CU_SIZE, 0);
>> + primitives.blockfill_s[(int)g_convertToBit[trWidth]](ptr,
>> resiYuv->m_width, 0);
>> }
>>
>> if (bCodeChroma)
>> {
>> if (absSumU)
>> {
>> - int16_t *pcResiCurrU =
>> m_qtTempTComYuv[qtlayer].getCbAddr(absTUPartIdxC);
>> + int16_t *pcResiCurrU = resiYuv->getCbAddr(absTUPartIdxC);
>>
>> int curChromaQpOffset =
>> cu->getSlice()->getPPS()->getChromaCbQpOffset() +
>> cu->getSlice()->getSliceQpDeltaCb();
>> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA,
>> cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
>>
>> int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_U];
>> assert(scalingListType < 6);
>> - assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE
>> / 2);
>> -
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> pcResiCurrU, MAX_CU_SIZE / 2, coeffCurU, trWidthC, trHeightC,
>> scalingListType, false, lastPosU);
>> +
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> pcResiCurrU, resiYuv->m_cwidth, coeffCurU, trWidthC, trHeightC,
>> scalingListType, false, lastPosU);
>> }
>> else
>> {
>> - int16_t *ptr =
>> m_qtTempTComYuv[qtlayer].getCbAddr(absTUPartIdxC);
>> - assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE
>> / 2);
>> -
>> + int16_t *ptr = resiYuv->getCbAddr(absTUPartIdxC);
>> assert(trWidthC == trHeightC);
>> -
>> primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, MAX_CU_SIZE /
>> 2, 0);
>> +
>> primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr,
>> resiYuv->m_cwidth, 0);
>> }
>> if (absSumV)
>> {
>> - int16_t *curResiV =
>> m_qtTempTComYuv[qtlayer].getCrAddr(absTUPartIdxC);
>> + int16_t *curResiV = resiYuv->getCrAddr(absTUPartIdxC);
>> int curChromaQpOffset =
>> cu->getSlice()->getPPS()->getChromaCrQpOffset() +
>> cu->getSlice()->getSliceQpDeltaCr();
>> m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA,
>> cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
>>
>> int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_V];
>> assert(scalingListType < 6);
>> - assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE
>> / 2);
>> -
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> curResiV, MAX_CU_SIZE / 2, coeffCurV, trWidthC, trHeightC, scalingListType,
>> false, lastPosV);
>> +
>> m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT,
>> curResiV, resiYuv->m_cwidth, coeffCurV, trWidthC, trHeightC,
>> scalingListType, false, lastPosV);
>> }
>> else
>> {
>> - int16_t *ptr =
>> m_qtTempTComYuv[qtlayer].getCrAddr(absTUPartIdxC);
>> - assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE
>> / 2);
>> -
>> + int16_t *ptr = resiYuv->getCrAddr(absTUPartIdxC);
>> assert(trWidthC == trHeightC);
>> -
>> primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr, MAX_CU_SIZE /
>> 2, 0);
>> +
>> primitives.blockfill_s[(int)g_convertToBit[trWidthC]](ptr,
>> resiYuv->m_cwidth, 0);
>> }
>> }
>> cu->setCbfSubParts(absSumY ? setCbf : 0, TEXT_LUMA, absPartIdx,
>> depth);
>> diff -r 9bb16a023918 -r f20e5ab835cd source/encoder/compress.cpp
>> --- a/source/encoder/compress.cpp Mon Dec 16 14:33:57 2013 +0530
>> +++ b/source/encoder/compress.cpp Mon Dec 16 14:50:24 2013 +0530
>> @@ -927,7 +927,6 @@
>>
>> //Residual encoding
>> m_search->residualTransformQuantInter(cu, 0, 0,
>> m_tmpResiYuv[depth], cu->getDepth(0), true);
>> - m_search->xSetResidualQTData(cu, 0, 0, NULL,
>> cu->getDepth(0), false);
>>
>> if (lcu->getMergeFlag(absPartIdx) && cu->getPartitionSize(0)
>> == SIZE_2Nx2N && !cu->getQtRootCbf(0))
>> {
>> @@ -937,7 +936,6 @@
>> else
>> {
>> cu->copyCodedToPic(depth);
>> - m_search->xSetResidualQTData(cu, 0, 0,
>> m_tmpResiYuv[depth], cu->getDepth(0), true);
>>
>> //Generate Recon
>> Pel* pred = m_bestPredYuv[0]->getLumaAddr(absPartIdx);
>> _______________________________________________
>> x265-devel mailing list
>> x265-devel at videolan.org
>> https://mailman.videolan.org/listinfo/x265-devel
>>
>
>
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20131216/ade70366/attachment.html>
More information about the x265-devel
mailing list