[x265] [PATCH] [REVIEW PATCH]sa8dCost threshold to reduce number of modes in intra analysis

[ashok at multicorewareinc.com]

# HG changeset patch
# User Ashok Kumar Mishra<ashok at multicorewareinc.com>
# Date 1421415068 -19800
#      Fri Jan 16 19:01:08 2015 +0530
# Node ID 4c200eea0b2210fabb718d7ed8980c1af530ea3a
# Parent  f9a54da187139a7331f8048365673787cbf14711
[REVIEW PATCH]sa8dCost threshold to reduce number of modes in intra analysis

diff -r f9a54da18713 -r 4c200eea0b22 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Thu Jan 15 11:49:26 2015 +0530
+++ b/source/encoder/analysis.cpp	Fri Jan 16 19:01:08 2015 +0530
@@ -1187,13 +1187,13 @@
                 (!m_param->bEnableCbfFastMode || md.bestMode->cu.getQtRootCbf(0)))
             {
                 md.pred[PRED_INTRA].cu.initSubCU(parentCTU, cuGeom);
-                checkIntra(md.pred[PRED_INTRA], cuGeom, SIZE_2Nx2N, NULL);
+                checkIntra(md.pred[PRED_INTRA], cuGeom, SIZE_2Nx2N, NULL, md.bestMode[PRED_INTRA].sa8dCost * 3);
                 checkBestMode(md.pred[PRED_INTRA], depth);
 
                 if (depth == g_maxCUDepth && cuGeom.log2CUSize > m_slice->m_sps->quadtreeTULog2MinSize)
                 {
                     md.pred[PRED_INTRA_NxN].cu.initSubCU(parentCTU, cuGeom);
-                    checkIntra(md.pred[PRED_INTRA_NxN], cuGeom, SIZE_NxN, NULL);
+                    checkIntra(md.pred[PRED_INTRA_NxN], cuGeom, SIZE_NxN, NULL, md.bestMode->sa8dCost);
                     checkBestMode(md.pred[PRED_INTRA_NxN], depth);
                 }
             }
@@ -1549,6 +1549,12 @@
     }
     if (predInterSearch(interMode, cuGeom, bMergeOnly, true))
     {
+        const Yuv& fencYuv = *interMode.fencYuv;
+        Yuv& predYuv = interMode.predYuv;
+        int part = partitionFromLog2Size(cuGeom.log2CUSize);
+        interMode.distortion = primitives.cu[part].sa8d(fencYuv.m_buf[0], fencYuv.m_size, predYuv.m_buf[0], predYuv.m_size);
+        interMode.sa8dCost = m_rdCost.calcRdSADCost(interMode.distortion, interMode.sa8dBits);
+
         /* predInterSearch sets interMode.sa8dBits, but this is ignored */
         encodeResAndCalcRdInterCU(interMode, cuGeom);
 
diff -r f9a54da18713 -r 4c200eea0b22 source/encoder/search.cpp
--- a/source/encoder/search.cpp	Thu Jan 15 11:49:26 2015 +0530
+++ b/source/encoder/search.cpp	Fri Jan 16 19:01:08 2015 +0530
@@ -1137,7 +1137,7 @@
     }
 }
 
-void Search::checkIntra(Mode& intraMode, const CUGeom& cuGeom, PartSize partSize, uint8_t* sharedModes)
+void Search::checkIntra(Mode& intraMode, const CUGeom& cuGeom, PartSize partSize, uint8_t* sharedModes, uint64_t sa8dThreshold)
 {
     CUData& cu = intraMode.cu;
 
@@ -1148,7 +1148,13 @@
     cu.getIntraTUQtDepthRange(tuDepthRange, 0);
 
     intraMode.initCosts();
-    intraMode.distortion += estIntraPredQT(intraMode, cuGeom, tuDepthRange, sharedModes);
+    intraMode.distortion  = estIntraPredQT(intraMode, cuGeom, tuDepthRange, sharedModes, sa8dThreshold);
+
+    if (intraMode.distortion == MAX_INT)
+    {
+        intraMode.rdCost = MAX_INT64;
+        return;
+    }
     intraMode.distortion += estIntraPredChromaQT(intraMode, cuGeom);
 
     m_entropyCoder.resetBits();
@@ -1396,7 +1402,7 @@
     updateModeCost(intraMode);
 }
 
-uint32_t Search::estIntraPredQT(Mode &intraMode, const CUGeom& cuGeom, const uint32_t depthRange[2], uint8_t* sharedModes)
+uint32_t Search::estIntraPredQT(Mode &intraMode, const CUGeom& cuGeom, const uint32_t depthRange[2], uint8_t* sharedModes, uint64_t sa8dThreshold)
 {
     CUData& cu = intraMode.cu;
     Yuv* reconYuv = &intraMode.reconYuv;
@@ -1539,7 +1545,7 @@
 
             uint64_t paddedBcost = bcost + (bcost >> 3); // 1.12%
             for (int mode = 0; mode < 35; mode++)
-                if (modeCosts[mode] < paddedBcost || (mpms & ((uint64_t)1 << mode)))
+                if ((modeCosts[mode] < paddedBcost || (mpms & ((uint64_t)1 << mode))) && modeCosts[mode] < sa8dThreshold)
                     updateCandList(mode, modeCosts[mode], maxCandCount, rdModeList, candCostList);
 
             /* measure best candidates using simple RDO (no TU splits) */
@@ -1556,8 +1562,11 @@
                     codeIntraLumaTSkip(intraMode, cuGeom, initTuDepth, absPartIdx, icosts);
                 else
                     codeIntraLumaQT(intraMode, cuGeom, initTuDepth, absPartIdx, false, icosts, depthRange);
-                COPY2_IF_LT(bcost, icosts.rdcost, bmode, rdModeList[i]);
+                COPY3_IF_LT(bcost, icosts.rdcost, bmode, rdModeList[i], intraMode.sa8dCost, candCostList[i]);
             }
+
+            if (bcost == MAX_INT64)
+                return MAX_INT;
         }
 
         /* remeasure best mode, allowing TU splits */
diff -r f9a54da18713 -r 4c200eea0b22 source/encoder/search.h
--- a/source/encoder/search.h	Thu Jan 15 11:49:26 2015 +0530
+++ b/source/encoder/search.h	Fri Jan 16 19:01:08 2015 +0530
@@ -163,7 +163,7 @@
     void     invalidateContexts(int fromDepth);
 
     // full RD search of intra modes. if sharedModes is not NULL, it directly uses them
-    void     checkIntra(Mode& intraMode, const CUGeom& cuGeom, PartSize partSize, uint8_t* sharedModes);
+    void     checkIntra(Mode& intraMode, const CUGeom& cuGeom, PartSize partSize, uint8_t* sharedModes, uint64_t sa8dThreshold = MAX_INT);
 
     // select best intra mode using only sa8d costs, cannot measure NxN intra
     void     checkIntraInInter(Mode& intraMode, const CUGeom& cuGeom);
@@ -204,7 +204,7 @@
     void     saveResidualQTData(CUData& cu, ShortYuv& resiYuv, uint32_t absPartIdx, uint32_t depth);
 
     // RDO search of luma intra modes; result is fully encoded luma. luma distortion is returned
-    uint32_t estIntraPredQT(Mode &intraMode, const CUGeom& cuGeom, const uint32_t depthRange[2], uint8_t* sharedModes);
+    uint32_t estIntraPredQT(Mode &intraMode, const CUGeom& cuGeom, const uint32_t depthRange[2], uint8_t* sharedModes, uint64_t sa8dThreshold);
 
     // RDO select best chroma mode from luma; result is fully encode chroma. chroma distortion is returned
     uint32_t estIntraPredChromaQT(Mode &intraMode, const CUGeom& cuGeom);