[x265] [PATCH 3 of 4] limitTU : use neighbouring CUs' TU depth to limit 1st subTU's depth

[bhavna at multicorewareinc.com]

# HG changeset patch
# User Bhavna Hariharan <bhavna at multicorewareinc.com>
# Date 1479380469 -19800
#      Thu Nov 17 16:31:09 2016 +0530
# Node ID f5d3a5aedcdf20235ec76dbf9aba516da83e8dd1
# Parent  07a4e4d785a69f719922129ca5997b12552bb4ab
limitTU : use neighbouring CUs' TU depth to limit 1st subTU's depth

diff -r 07a4e4d785a6 -r f5d3a5aedcdf source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Thu Nov 17 12:19:38 2016 +0530
+++ b/source/encoder/analysis.cpp	Thu Nov 17 16:31:09 2016 +0530
@@ -377,7 +377,7 @@
             checkBestMode(md.pred[PRED_INTRA_NxN], depth);
         }
 
-        if (limitTU == X265_TU_LIMIT_NEIGH && cuGeom.log2CUSize >= 4)
+        if ((limitTU & X265_TU_LIMIT_NEIGH) && cuGeom.log2CUSize >= 4)
         {
             CUData* ctu = md.bestMode->cu.m_encData->getPicCTU(parentCTU.m_cuAddr);
             int8_t maxTUDepth = -1;
@@ -954,7 +954,7 @@
     bool skipRectAmp = false;
     bool chooseMerge = false;
 
-    if (limitTU == X265_TU_LIMIT_NEIGH && cuGeom.log2CUSize >= 4)
+    if ((limitTU & X265_TU_LIMIT_NEIGH) && cuGeom.log2CUSize >= 4)
         m_maxTUDepth = loadTUDepth(cuGeom, parentCTU);
 
     SplitData splitData[4];
@@ -1463,7 +1463,7 @@
     if (m_param->rdLevel)
         md.bestMode->reconYuv.copyToPicYuv(reconPic, cuAddr, cuGeom.absPartIdx);
 
-    if (limitTU == X265_TU_LIMIT_NEIGH && cuGeom.log2CUSize >= 4)
+    if ((limitTU & X265_TU_LIMIT_NEIGH) && cuGeom.log2CUSize >= 4)
     {
         if (mightNotSplit)
         {
@@ -1498,7 +1498,7 @@
         md.pred[PRED_2Nx2N].rdCost = 0;
     }
 
-    if (limitTU == X265_TU_LIMIT_NEIGH && cuGeom.log2CUSize >= 4)
+    if ((limitTU & X265_TU_LIMIT_NEIGH) && cuGeom.log2CUSize >= 4)
         m_maxTUDepth =  loadTUDepth(cuGeom, parentCTU);
 
     SplitData splitData[4];
@@ -1827,7 +1827,7 @@
         if (mightSplit)
             addSplitFlagCost(*md.bestMode, cuGeom.depth);
     }
-    if (limitTU == X265_TU_LIMIT_NEIGH && cuGeom.log2CUSize >= 4)
+    if ((limitTU & X265_TU_LIMIT_NEIGH) && cuGeom.log2CUSize >= 4)
     {
         if (mightNotSplit)
         {
diff -r 07a4e4d785a6 -r f5d3a5aedcdf source/encoder/search.cpp
--- a/source/encoder/search.cpp	Thu Nov 17 12:19:38 2016 +0530
+++ b/source/encoder/search.cpp	Thu Nov 17 16:31:09 2016 +0530
@@ -2625,7 +2625,7 @@
 
     uint32_t tuDepthRange[2];
     cu.getInterTUQtDepthRange(tuDepthRange, 0);
-    if (limitTU == X265_TU_LIMIT_NEIGH)
+    if (limitTU & X265_TU_LIMIT_NEIGH)
     {
         int maxLog2CUSize = (int)g_log2Size[m_param->maxCUSize];
         m_maxTUDepth = x265_clip3(maxLog2CUSize - (int32_t)tuDepthRange[1], maxLog2CUSize - (int32_t)tuDepthRange[0], m_maxTUDepth);
@@ -2639,7 +2639,12 @@
         memset(&m_cacheTU, 0, sizeof(TUInfoCache));
 
     Cost costs;
-    estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
+    if (limitTU == X265_TU_LIMIT_DFS_NEIGH)
+    {
+        int32_t tempDepth = m_maxTUDepth;
+        m_maxTUDepth = tempDepth;
+    }
+        estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
 
     uint32_t tqBypass = cu.m_tqBypass[0];
     if (!tqBypass)
@@ -2898,10 +2903,11 @@
     uint32_t ycbf = 0, ucbf = 0, vcbf = 0;
     for (uint32_t qIdx = 0, qPartIdx = absPartIdx; qIdx < 4; ++qIdx, qPartIdx += qNumParts)
     {
-        if (limitTU == X265_TU_LIMIT_DFS && tuDepth == 0 && qIdx == 1)
+        if (limitTU & X265_TU_LIMIT_DFS && tuDepth == 0 && qIdx == 1)
         {
+            m_maxTUDepth = cu.m_tuDepth[0];
             // Fetch maximum TU depth of first sub partition to limit recursion of others
-            for (uint32_t i = 0; i < cuGeom.numPartitions / 4; i++)
+            for (uint32_t i = 1; i < cuGeom.numPartitions / 4; i++)
                 m_maxTUDepth = X265_MAX(m_maxTUDepth, cu.m_tuDepth[i]);
         }
         estimateResidualQT(mode, cuGeom, qPartIdx, tuDepth + 1, resiYuv, splitCost, depthRange, splitMore);
@@ -2968,7 +2974,7 @@
             }
         }
     }
-    else if (limitTU == X265_TU_LIMIT_DFS || limitTU == X265_TU_LIMIT_NEIGH)
+    else if (limitTU & X265_TU_LIMIT_DFS_NEIGH)
     {
         if (bCheckSplit && m_maxTUDepth >= 0)
         {