[x265] [PATCH] fix bug on TSKIP
Min Chen
chenm003 at 163.com
Wed Feb 12 11:58:56 CET 2014
# HG changeset patch
# User Min Chen <chenm003 at 163.com>
# Date 1392202729 -28800
# Node ID 0fb7b29600d93aed9ea8ab9f75c4c4ee06f842cd
# Parent 19d7752a4f9ca0863ea5efea80a040bae022be2b
fix bug on TSKIP
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TComDataCU.cpp
--- a/source/Lib/TLibCommon/TComDataCU.cpp Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TComDataCU.cpp Wed Feb 12 18:58:49 2014 +0800
@@ -1291,7 +1291,7 @@
{
uint32_t curPartNum = m_pic->getNumPartInCU() >> (depth << 1);
- memset(m_cbf[ttype] + absPartIdx, cbf, sizeof(UChar) * curPartNum);
+ memset(m_cbf[g_aucConvertTxtTypeToIdx[ttype]] + absPartIdx, cbf, sizeof(UChar) * curPartNum);
}
/** Sets a coded block flag for all sub-partitions of a partition
@@ -1304,7 +1304,7 @@
*/
void TComDataCU::setCbfSubParts(uint32_t uiCbf, TextType ttype, uint32_t absPartIdx, uint32_t partIdx, uint32_t depth)
{
- setSubPart<UChar>(uiCbf, m_cbf[ttype], absPartIdx, depth, partIdx);
+ setSubPart<UChar>(uiCbf, m_cbf[g_aucConvertTxtTypeToIdx[ttype]], absPartIdx, depth, partIdx);
}
void TComDataCU::setDepthSubParts(uint32_t depth, uint32_t absPartIdx)
@@ -1534,7 +1534,7 @@
{
uint32_t curPartNum = m_pic->getNumPartInCU() >> (depth << 1);
- memset(m_transformSkip[ttype] + absPartIdx, useTransformSkip, sizeof(UChar) * curPartNum);
+ memset(m_transformSkip[g_aucConvertTxtTypeToIdx[ttype]] + absPartIdx, useTransformSkip, sizeof(UChar) * curPartNum);
}
UChar TComDataCU::getNumPartInter()
@@ -2503,7 +2503,8 @@
void TComDataCU::clearCbf(uint32_t idx, TextType ttype, uint32_t numParts)
{
- ::memset(&m_cbf[ttype][idx], 0, sizeof(UChar) * numParts);
+ assert((int)ttype <= 2);
+ ::memset(&m_cbf[g_aucConvertTxtTypeToIdx[ttype]][idx], 0, sizeof(UChar) * numParts);
}
/** Set a I_PCM flag for all sub-partitions of a partition.
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TComDataCU.h
--- a/source/Lib/TLibCommon/TComDataCU.h Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TComDataCU.h Wed Feb 12 18:58:49 2014 +0800
@@ -280,9 +280,9 @@
void setTrIdxSubParts(uint32_t uiTrIdx, uint32_t absPartIdx, uint32_t depth);
- UChar* getTransformSkip(TextType ttype) { return m_transformSkip[ttype]; }
+ UChar* getTransformSkip(TextType ttype) { assert(ttype != TEXT_CHROMA); return m_transformSkip[g_aucConvertTxtTypeToIdx[ttype]]; }
- UChar getTransformSkip(uint32_t idx, TextType ttype) { return m_transformSkip[ttype][idx]; }
+ UChar getTransformSkip(uint32_t idx, TextType ttype) { assert(ttype != TEXT_CHROMA); return m_transformSkip[g_aucConvertTxtTypeToIdx[ttype]][idx]; }
void setTransformSkipSubParts(uint32_t useTransformSkip, TextType ttype, uint32_t absPartIdx, uint32_t depth);
void setTransformSkipSubParts(uint32_t useTransformSkipY, uint32_t useTransformSkipU, uint32_t useTransformSkipV, uint32_t absPartIdx, uint32_t depth);
@@ -303,13 +303,13 @@
Pel*& getPCMSampleCr() { return m_iPCMSampleCr; }
- UChar getCbf(uint32_t idx, TextType ttype) { return m_cbf[ttype][idx]; }
+ UChar getCbf(uint32_t idx, TextType ttype) { assert(ttype != TEXT_CHROMA); return m_cbf[g_aucConvertTxtTypeToIdx[ttype]][idx]; }
- UChar* getCbf(TextType ttype) { return m_cbf[ttype]; }
+ UChar* getCbf(TextType ttype) { assert(ttype != TEXT_CHROMA); return m_cbf[g_aucConvertTxtTypeToIdx[ttype]]; }
UChar getCbf(uint32_t idx, TextType ttype, uint32_t trDepth) { return (getCbf(idx, ttype) >> trDepth) & 0x1; }
- void setCbf(uint32_t idx, TextType ttype, UChar uh) { m_cbf[ttype][idx] = uh; }
+ void setCbf(uint32_t idx, TextType ttype, UChar uh) { assert(ttype != TEXT_CHROMA); m_cbf[g_aucConvertTxtTypeToIdx[ttype]][idx] = uh; }
void clearCbf(uint32_t idx, TextType ttype, uint32_t numParts);
UChar getQtRootCbf(uint32_t idx) { return getCbf(idx, TEXT_LUMA, 0) || getCbf(idx, TEXT_CHROMA_U, 0) || getCbf(idx, TEXT_CHROMA_V, 0); }
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TComRom.cpp
--- a/source/Lib/TLibCommon/TComRom.cpp Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TComRom.cpp Wed Feb 12 18:58:49 2014 +0800
@@ -621,5 +621,8 @@
106, 110, 114, 118, 122, 126, 130, 135, 139, 143, 147, 152, 156, 161, 165, 170,
175, 179, 184, 189, 194, 198, 203, 208, 214, 219, 224, 229, 234, 240, 245, 250
};
+
+const UChar g_aucConvertTxtTypeToIdx[4] = {0, 1, 1, 2};
+const UChar g_eTTable[4] = {0,3,1,2};
}
//! \}
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TComRom.h
--- a/source/Lib/TLibCommon/TComRom.h Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TComRom.h Wed Feb 12 18:58:49 2014 +0800
@@ -293,6 +293,10 @@
extern const UChar g_lpsTable[64][4];
extern const UChar g_renormTable[32];
extern const UChar x265_exp2_lut[64];
+
+// TSKIP
+extern const UChar g_aucConvertTxtTypeToIdx[4];
+extern const UChar g_eTTable[4];
}
#endif //ifndef X265_TCOMROM_H
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TComTrQuant.cpp
--- a/source/Lib/TLibCommon/TComTrQuant.cpp Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TComTrQuant.cpp Wed Feb 12 18:58:49 2014 +0800
@@ -277,7 +277,7 @@
int deltaU[32 * 32];
uint32_t log2TrSize = g_convertToBit[width] + 2;
- int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
+ int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + g_eTTable[(int)ttype];
assert(scalingListType < 6);
int32_t *quantCoeff = 0;
quantCoeff = getQuantCoeff(scalingListType, m_qpParam.m_rem, log2TrSize - 2);
@@ -511,7 +511,7 @@
uint32_t goRiceParam = 0;
double blockUncodedCost = 0;
const uint32_t log2BlkSize = g_convertToBit[width] + 2;
- int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + ttype;
+ int scalingListType = (cu->isIntra(absPartIdx) ? 0 : 3) + g_eTTable[(int)ttype];
assert(scalingListType < 6);
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibCommon/TypeDef.h
--- a/source/Lib/TLibCommon/TypeDef.h Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibCommon/TypeDef.h Wed Feb 12 18:58:49 2014 +0800
@@ -109,8 +109,8 @@
{
TEXT_LUMA = 0, ///< luma
TEXT_CHROMA = 1, ///< chroma (U+V)
- TEXT_CHROMA_U = 1, ///< chroma U
- TEXT_CHROMA_V = 2, ///< chroma V
+ TEXT_CHROMA_U = 2, ///< chroma U
+ TEXT_CHROMA_V = 3, ///< chroma V
};
/// index for SBAC based RD optimization
diff -r 19d7752a4f9c -r 0fb7b29600d9 source/Lib/TLibEncoder/TEncSearch.cpp
--- a/source/Lib/TLibEncoder/TEncSearch.cpp Wed Feb 12 00:35:27 2014 -0600
+++ b/source/Lib/TLibEncoder/TEncSearch.cpp Wed Feb 12 18:58:49 2014 +0800
@@ -475,7 +475,7 @@
int size = g_convertToBit[width];
if (absSum)
{
- int scalingListType = 0 + TEXT_LUMA;
+ int scalingListType = 0 + g_eTTable[(int)TEXT_LUMA];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), cu->getLumaIntraDir(absPartIdx), residual, stride, coeff, width, height, scalingListType, useTransformSkip, lastPos);
}
@@ -610,7 +610,7 @@
//--- inverse transform ---
if (absSum)
{
- int scalingListType = 0 + ttype;
+ int scalingListType = 0 + g_eTTable[(int)ttype];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, residual, stride, coeff, width, height, scalingListType, useTransformSkipChroma, lastPos);
}
@@ -1044,7 +1044,7 @@
int size = g_convertToBit[width];
if (absSum)
{
- int scalingListType = 0 + TEXT_LUMA;
+ int scalingListType = 0 + g_eTTable[(int)TEXT_LUMA];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), cu->getLumaIntraDir(absPartIdx), residual, stride, coeff, width, height, scalingListType, useTransformSkip, lastPos);
}
@@ -1391,7 +1391,7 @@
if (trMode == trDepth)
{
- bool checkTransformSkip = false; //cu->getSlice()->getPPS()->getUseTransformSkip();
+ bool checkTransformSkip = cu->getSlice()->getPPS()->getUseTransformSkip();
uint32_t trSizeLog2 = g_convertToBit[cu->getSlice()->getSPS()->getMaxCUWidth() >> fullDepth] + 2;
uint32_t actualTrDepth = trDepth;
@@ -1443,7 +1443,7 @@
for (int chromaModeId = firstCheckId; chromaModeId < 2; chromaModeId++)
{
- cu->setTransformSkipSubParts(chromaModeId, (TextType)(chromaId + 2), absPartIdx, cu->getDepth(0) + actualTrDepth);
+ cu->setTransformSkipSubParts(chromaModeId, (TextType)(chromaId + TEXT_CHROMA_U), absPartIdx, cu->getDepth(0) + actualTrDepth);
if (chromaModeId == firstCheckId)
{
default0Save1Load2 = 1;
@@ -1454,7 +1454,7 @@
}
singleDistCTmp = 0;
xIntraCodingChromaBlk(cu, trDepth, absPartIdx, fencYuv, predYuv, resiYuv, singleDistCTmp, chromaId, default0Save1Load2);
- singleCbfCTmp = cu->getCbf(absPartIdx, (TextType)(chromaId + 2), trDepth);
+ singleCbfCTmp = cu->getCbf(absPartIdx, (TextType)(chromaId + TEXT_CHROMA_U), trDepth);
if (chromaModeId == 1 && singleCbfCTmp == 0)
{
@@ -1463,7 +1463,7 @@
}
else
{
- uint32_t bitsTmp = xGetIntraBitsQTChroma(cu, trDepth, absPartIdx, chromaId + 2);
+ uint32_t bitsTmp = xGetIntraBitsQTChroma(cu, trDepth, absPartIdx, chromaId + TEXT_CHROMA_U);
singleCostTmp = m_rdCost->calcRdCost(singleDistCTmp, bitsTmp);
}
@@ -1489,10 +1489,14 @@
if (bestModeId == firstCheckId)
{
xLoadIntraResultChromaQT(cu, trDepth, absPartIdx, chromaId);
- cu->setCbfSubParts(singleCbfC << trDepth, (TextType)(chromaId + 2), absPartIdx, cu->getDepth(0) + actualTrDepth);
+ cu->setCbfSubParts(singleCbfC << trDepth, (TextType)(chromaId + TEXT_CHROMA_U), absPartIdx, cu->getDepth(0) + actualTrDepth);
m_rdGoOnSbacCoder->load(m_rdSbacCoders[fullDepth][CI_TEMP_BEST]);
}
- cu->setTransformSkipSubParts(bestModeId, (TextType)(chromaId + 2), absPartIdx, cu->getDepth(0) + actualTrDepth);
+ cu->setTransformSkipSubParts(bestModeId, (TextType)(chromaId + TEXT_CHROMA_U), absPartIdx, cu->getDepth(0) + actualTrDepth);
+ if (bestModeId)
+ {
+ bestModeId += 0;
+ }
outDist += singleDistC;
if (chromaId == 0)
@@ -1672,7 +1676,7 @@
//--- inverse transform ---
if (absSum)
{
- int scalingListType = 0 + ttype;
+ int scalingListType = 0 + g_eTTable[(int)ttype];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, residual, stride, coeff, width, height, scalingListType, useTransformSkipChroma, lastPos);
}
@@ -3276,7 +3280,7 @@
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, cu->getSlice()->getSPS()->getQpBDOffsetY(), 0);
- int scalingListType = 3 + TEXT_LUMA;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_LUMA];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, curResiY, resiYuv->m_width, coeffCurY, trWidth, trHeight, scalingListType, false, lastPosY); //this is for inter mode only
}
@@ -3296,7 +3300,7 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_U;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_U];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, pcResiCurrU, resiYuv->m_cwidth, coeffCurU, trWidthC, trHeightC, scalingListType, false, lastPosU);
}
@@ -3312,7 +3316,7 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_V;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_V];
assert(scalingListType < 6);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, curResiV, resiYuv->m_cwidth, coeffCurV, trWidthC, trHeightC, scalingListType, false, lastPosV);
}
@@ -3520,7 +3524,7 @@
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, cu->getSlice()->getSPS()->getQpBDOffsetY(), 0);
- int scalingListType = 3 + TEXT_LUMA;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_LUMA];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_width == MAX_CU_SIZE);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, curResiY, MAX_CU_SIZE, coeffCurY, trWidth, trHeight, scalingListType, false, lastPosY); //this is for inter mode only
@@ -3592,7 +3596,7 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_U;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_U];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE / 2);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, pcResiCurrU, MAX_CU_SIZE / 2, coeffCurU, trWidthC, trHeightC, scalingListType, false, lastPosU);
@@ -3659,7 +3663,7 @@
int curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_V;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_V];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE / 2);
m_trQuant->invtransformNxN(cu->getCUTransquantBypass(absPartIdx), REG_DCT, curResiV, MAX_CU_SIZE / 2, coeffCurV, trWidthC, trHeightC, scalingListType, false, lastPosV);
@@ -3765,7 +3769,7 @@
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_LUMA, cu->getSlice()->getSPS()->getQpBDOffsetY(), 0);
- int scalingListType = 3 + TEXT_LUMA;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_LUMA];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_width == MAX_CU_SIZE);
@@ -3855,7 +3859,7 @@
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCbQpOffset() + cu->getSlice()->getSliceQpDeltaCb();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_U;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_U];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE / 2);
@@ -3894,7 +3898,7 @@
curChromaQpOffset = cu->getSlice()->getPPS()->getChromaCrQpOffset() + cu->getSlice()->getSliceQpDeltaCr();
m_trQuant->setQPforQuant(cu->getQP(0), TEXT_CHROMA, cu->getSlice()->getSPS()->getQpBDOffsetC(), curChromaQpOffset);
- int scalingListType = 3 + TEXT_CHROMA_V;
+ int scalingListType = 3 + g_eTTable[(int)TEXT_CHROMA_V];
assert(scalingListType < 6);
assert(m_qtTempTComYuv[qtlayer].m_cwidth == MAX_CU_SIZE / 2);
More information about the x265-devel
mailing list