[x265] [PATCH 1 of 3] analysis: fix inter hash mistake with --cu-lossless

Steve Borho steve at borho.org
Tue Aug 26 00:17:33 CEST 2014


# HG changeset patch
# User Min Chen <chenm003 at 163.com>
# Date 1409002891 18000
#      Mon Aug 25 16:41:31 2014 -0500
# Node ID 0bf2756898bc78e5660a6b607b2f3cda97834264
# Parent  5acfb12ec5d17cc700e313fc99248e2408e5967b
analysis: fix inter hash mistake with --cu-lossless

diff -r 5acfb12ec5d1 -r 0bf2756898bc source/Lib/TLibEncoder/TEncSearch.cpp
--- a/source/Lib/TLibEncoder/TEncSearch.cpp	Mon Aug 25 17:53:12 2014 +0900
+++ b/source/Lib/TLibEncoder/TEncSearch.cpp	Mon Aug 25 16:41:31 2014 -0500
@@ -2293,7 +2293,7 @@
  * \returns void
  */
 void TEncSearch::encodeResAndCalcRdInterCU(TComDataCU* cu, TComYuv* fencYuv, TComYuv* predYuv, ShortYuv* outResiYuv,
-                                           ShortYuv* outBestResiYuv, TComYuv* outReconYuv)
+                                           ShortYuv* outBestResiYuv, TComYuv* outReconYuv, TComDataCU* tmpCu)
 {
     X265_CHECK(!cu->isIntra(0), "intra CU not expected\n");
 
@@ -2321,6 +2321,7 @@
     }
 
     uint64_t bestCost = MAX_INT64;
+    bool bestTransquantBypassFlag = bIsTQBypassEnable;
 
     for (uint32_t modeId = 0; modeId < numModes; modeId++)
     {
@@ -2388,15 +2389,29 @@
             if (cu->getQtRootCbf(0))
                 xSetResidualQTData(cu, 0, outBestResiYuv, depth, true);
 
+            bestTransquantBypassFlag = bIsLosslessMode;
             bestBits = bits;
             bestCost = cost;
             bestCoeffBits = cu->m_coeffBits;
             m_entropyCoder->store(m_rdEntropyCoders[depth][CI_TEMP_BEST]);
         }
+
+        // Save lossless mode coeff
+        if (bIsLosslessMode)
+        {
+            tmpCu->copyPartFrom(cu, 0, depth, false);
+        }
     }
 
     X265_CHECK(bestCost != MAX_INT64, "no best cost\n");
 
+    if (bestTransquantBypassFlag && !m_param->bLossless)
+    {
+        assert(log2CUSize > 2);
+        cu->setCUTransquantBypassSubParts(true, 0, depth);
+        cu->copyPartFrom(tmpCu, 0, depth, false);
+    }
+
     if (cu->getQtRootCbf(0))
         outReconYuv->addClip(predYuv, outBestResiYuv, log2CUSize);
     else
diff -r 5acfb12ec5d1 -r 0bf2756898bc source/Lib/TLibEncoder/TEncSearch.h
--- a/source/Lib/TLibEncoder/TEncSearch.h	Mon Aug 25 17:53:12 2014 +0900
+++ b/source/Lib/TLibEncoder/TEncSearch.h	Mon Aug 25 16:41:31 2014 -0500
@@ -147,7 +147,7 @@
 
     /// encode residual and compute rd-cost for inter mode
     void encodeResAndCalcRdInterCU(TComDataCU* cu, TComYuv* fencYuv, TComYuv* predYuv, ShortYuv* resiYuv, ShortYuv* bestResiYuv,
-                                   TComYuv* reconYuv);
+                                   TComYuv* reconYuv, TComDataCU* tmpCu);
     void encodeResAndCalcRdSkipCU(TComDataCU* cu, TComYuv* fencYuv, TComYuv* predYuv, TComYuv* reconYuv);
 
     void xRecurIntraCodingQT(TComDataCU* cu, uint32_t trDepth, uint32_t absPartIdx, TComYuv* fencYuv,
diff -r 5acfb12ec5d1 -r 0bf2756898bc source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Mon Aug 25 17:53:12 2014 +0900
+++ b/source/encoder/analysis.cpp	Mon Aug 25 16:41:31 2014 -0500
@@ -82,7 +82,7 @@
         uint32_t sizeL = cuSize * cuSize;
         uint32_t sizeC = sizeL >> (CHROMA_H_SHIFT(csp) + CHROMA_V_SHIFT(csp));
 
-        ok &= m_memPool[i].initialize(numPartitions, sizeL, sizeC, 8, tqBypass);
+        ok &= m_memPool[i].initialize(numPartitions, sizeL, sizeC, 9, tqBypass);
 
         m_interCU_2Nx2N[i]  = new TComDataCU;
         m_interCU_2Nx2N[i]->create(&m_memPool[i], numPartitions, cuSize, csp, 0, tqBypass);
@@ -108,6 +108,9 @@
         m_tempCU[i]         = new TComDataCU;
         m_tempCU[i]->create(&m_memPool[i], numPartitions, cuSize, csp, 7, tqBypass);
 
+        m_tempLosslessCU[i] = new TComDataCU;
+        m_tempLosslessCU[i]->create(&m_memPool[i], numPartitions, cuSize, csp, 8, tqBypass);
+
         m_bestPredYuv[i] = new TComYuv;
         ok &= m_bestPredYuv[i]->create(cuSize, cuSize, csp);
 
@@ -158,6 +161,7 @@
         delete m_bestMergeCU[i];
         delete m_bestCU[i];
         delete m_tempCU[i];
+        delete m_tempLosslessCU[i];
 
         if (m_bestPredYuv && m_bestPredYuv[i])
         {
@@ -240,6 +244,7 @@
     // initialize CU data
     m_bestCU[0]->initCU(cu->m_pic, cu->getAddr());
     m_tempCU[0]->initCU(cu->m_pic, cu->getAddr());
+    m_tempLosslessCU[0]->initCU(cu->m_pic, cu->getAddr());
 
     // analysis of CU
     uint32_t numPartition = cu->getTotalNumPart();
@@ -394,6 +399,7 @@
         uint32_t    nextDepth     = depth + 1;
         TComDataCU* subBestPartCU = m_bestCU[nextDepth];
         TComDataCU* subTempPartCU = m_tempCU[nextDepth];
+        TComDataCU* subTempLosslessPartCU = m_tempLosslessCU[nextDepth];
         for (uint32_t partUnitIdx = 0; partUnitIdx < 4; partUnitIdx++)
         {
             int qp = outTempCU->getQP(0);
@@ -404,6 +410,7 @@
                  (subBestPartCU->getCUPelY() < slice->m_sps->picHeightInLumaSamples)))
             {
                 subTempPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp); // clear sub partition datas or init.
+                subTempLosslessPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp); // clear sub partition datas or init.
                 if (0 == partUnitIdx) //initialize RD with previous depth buffer
                 {
                     m_rdEntropyCoders[nextDepth][CI_CURR_BEST].load(m_rdEntropyCoders[depth][CI_CURR_BEST]);
@@ -663,7 +670,7 @@
                     }
 
                     encodeResAndCalcRdInterCU(outBestCU, m_origYuv[depth], m_bestPredYuv[depth], m_tmpResiYuv[depth],
-                                              m_bestResiYuv[depth], m_bestRecoYuv[depth]);
+                                              m_bestResiYuv[depth], m_bestRecoYuv[depth], m_tempLosslessCU[depth]);
                     uint64_t bestMergeCost = m_rdCost.m_psyRd ? m_bestMergeCU[depth]->m_totalPsyCost : m_bestMergeCU[depth]->m_totalRDCost;
                     uint64_t bestCost = m_rdCost.m_psyRd ? outBestCU->m_totalPsyCost : outBestCU->m_totalRDCost;
                     if (bestMergeCost < bestCost)
@@ -733,7 +740,7 @@
                         }
 
                         encodeResAndCalcRdInterCU(outBestCU, m_origYuv[depth], m_bestPredYuv[depth], m_tmpResiYuv[depth],
-                                                  m_bestResiYuv[depth], m_bestRecoYuv[depth]);
+                                                  m_bestResiYuv[depth], m_bestRecoYuv[depth], m_tempLosslessCU[depth]);
                         m_rdEntropyCoders[depth][CI_TEMP_BEST].store(m_rdEntropyCoders[depth][CI_NEXT_BEST]);
                     }
                     else if (outBestCU->getPredictionMode(0) == MODE_INTRA)
@@ -880,10 +887,12 @@
         outTempCU->setQPSubParts(qp, 0, depth);
         uint32_t    nextDepth = depth + 1;
         TComDataCU* subTempPartCU = m_tempCU[nextDepth];
+        TComDataCU* subTempLosslessPartCU = m_tempLosslessCU[nextDepth];
         for (uint32_t partUnitIdx = 0; partUnitIdx < 4; partUnitIdx++)
         {
             TComDataCU* subBestPartCU = NULL;
             subTempPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp); // clear sub partition datas or init.
+            subTempLosslessPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp);
 
             if (bInsidePicture ||
                 ((subTempPartCU->getCUPelX() < slice->m_sps->picWidthInLumaSamples) &&
@@ -1258,10 +1267,12 @@
         uint32_t    nextDepth     = depth + 1;
         TComDataCU* subBestPartCU = m_bestCU[nextDepth];
         TComDataCU* subTempPartCU = m_tempCU[nextDepth];
+        TComDataCU* subTempLosslessPartCU = m_tempLosslessCU[nextDepth];
         for (uint32_t partUnitIdx = 0; partUnitIdx < 4; partUnitIdx++)
         {
             int qp = outTempCU->getQP(0);
             subBestPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp); // clear sub partition datas or init.
+            subTempLosslessPartCU->initSubCU(outTempCU, partUnitIdx, nextDepth, qp);
 
             if (bInsidePicture ||
                 ((subBestPartCU->getCUPelX() < slice->m_sps->picWidthInLumaSamples) &&
@@ -1433,7 +1444,7 @@
             }
 
             //Encode with residue
-            encodeResAndCalcRdInterCU(outTempCU, m_origYuv[depth], bestPredYuv, m_tmpResiYuv[depth], m_bestResiYuv[depth], m_tmpRecoYuv[depth]);
+            encodeResAndCalcRdInterCU(outTempCU, m_origYuv[depth], bestPredYuv, m_tmpResiYuv[depth], m_bestResiYuv[depth], m_tmpRecoYuv[depth], m_tempLosslessCU[depth]);
 
             uint64_t tempCost = m_rdCost.m_psyRd ? outTempCU->m_totalPsyCost : outTempCU->m_totalRDCost;
             uint64_t bestCost = m_rdCost.m_psyRd ? outBestCU->m_totalPsyCost : outBestCU->m_totalRDCost;
@@ -1506,7 +1517,8 @@
                                                   m_tmpPredYuv[depth],
                                                   m_tmpResiYuv[depth],
                                                   m_bestResiYuv[depth],
-                                                  m_tmpRecoYuv[depth]);
+                                                  m_tmpRecoYuv[depth],
+                                                  m_tempLosslessCU[depth]);
 
 
                     /* Todo: Fix the satd cost estimates. Why is merge being chosen in high motion areas: estimated distortion is too low? */
@@ -1590,7 +1602,7 @@
 
     if (predInterSearch(outTempCU, m_tmpPredYuv[depth], bUseMRG, true))
     {
-        encodeResAndCalcRdInterCU(outTempCU, m_origYuv[depth], m_tmpPredYuv[depth], m_tmpResiYuv[depth], m_bestResiYuv[depth], m_tmpRecoYuv[depth]);
+        encodeResAndCalcRdInterCU(outTempCU, m_origYuv[depth], m_tmpPredYuv[depth], m_tmpResiYuv[depth], m_bestResiYuv[depth], m_tmpRecoYuv[depth], m_tempLosslessCU[depth]);
         checkDQP(outTempCU);
         checkBestMode(outBestCU, outTempCU, depth);
     }
diff -r 5acfb12ec5d1 -r 0bf2756898bc source/encoder/analysis.h
--- a/source/encoder/analysis.h	Mon Aug 25 17:53:12 2014 +0900
+++ b/source/encoder/analysis.h	Mon Aug 25 16:41:31 2014 -0500
@@ -82,6 +82,7 @@
     TComDataCU*  m_bestMergeCU[NUM_CU_DEPTH];
     TComDataCU*  m_bestCU[NUM_CU_DEPTH]; // Best CUs at each depth
     TComDataCU*  m_tempCU[NUM_CU_DEPTH]; // Temporary CUs at each depth
+    TComDataCU*  m_tempLosslessCU[NUM_CU_DEPTH]; // Temporary CUs for lossless at each depth
 
     TComYuv**    m_bestPredYuv;          // Best Prediction Yuv for each depth
     ShortYuv**   m_bestResiYuv;          // Best Residual Yuv for each depth


More information about the x265-devel mailing list