[x265] [PATCH 3 of 4] limitTU: use max TU depth of first subTU to limit recursion of others in split

[kavitha at multicorewareinc.com]

# HG changeset patch
# User Kavitha Sampath <kavitha at multicorewareinc.com>
# Date 1475238341 -19800
#      Fri Sep 30 17:55:41 2016 +0530
# Node ID 3ae30a43ac939fe875eaec7f22d134711b00c449
# Parent  c018bc0ffc156902b1a9a13ecd6996d30d7403df
limitTU: use max TU depth of first subTU to limit recursion of others in split

diff -r c018bc0ffc15 -r 3ae30a43ac93 source/encoder/search.cpp
--- a/source/encoder/search.cpp	Fri Sep 23 14:22:41 2016 +0530
+++ b/source/encoder/search.cpp	Fri Sep 30 17:55:41 2016 +0530
@@ -67,6 +67,7 @@
     m_param = NULL;
     m_slice = NULL;
     m_frame = NULL;
+    m_maxTUDepth = 0;
 }
 
 bool Search::initSearch(const x265_param& param, ScalingList& scalingList)
@@ -2617,6 +2618,8 @@
 
     m_entropyCoder.load(m_rqt[depth].cur);
 
+    if (m_param->limitTU == X265_TU_LIMIT_DFS)
+        m_maxTUDepth = 0;
     Cost costs;
     estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
 
@@ -2876,6 +2879,11 @@
 
     bool bCheckSplit = log2TrSize > depthRange[0];
     bool bCheckFull = log2TrSize <= depthRange[1];
+    if (m_param->limitTU == X265_TU_LIMIT_DFS && m_maxTUDepth)
+    {
+        uint32_t log2MaxTrSize = cuGeom.log2CUSize - m_maxTUDepth;
+        bCheckSplit = log2TrSize > log2MaxTrSize;
+    }
     bool bSplitPresentFlag = bCheckSplit && bCheckFull;
 
     if (cu.m_partSize[0] != SIZE_2Nx2N && !tuDepth && bCheckSplit)
@@ -3372,6 +3380,11 @@
         uint32_t ycbf = 0, ucbf = 0, vcbf = 0;
         for (uint32_t qIdx = 0, qPartIdx = absPartIdx; qIdx < 4; ++qIdx, qPartIdx += qNumParts)
         {
+            if (m_param->limitTU == X265_TU_LIMIT_DFS && tuDepth == 0 && qIdx == 1)
+            {
+                for (uint32_t i = 0; i < cuGeom.numPartitions / 4; i++)
+                    m_maxTUDepth = X265_MAX(m_maxTUDepth, cu.m_tuDepth[i]);
+            }
             estimateResidualQT(mode, cuGeom, qPartIdx, tuDepth + 1, resiYuv, splitCost, depthRange);
             ycbf |= cu.getCbf(qPartIdx, TEXT_LUMA,     tuDepth + 1);
             if (m_csp != X265_CSP_I400 && m_frame->m_fencPic->m_picCsp != X265_CSP_I400)
diff -r c018bc0ffc15 -r 3ae30a43ac93 source/encoder/search.h
--- a/source/encoder/search.h	Fri Sep 23 14:22:41 2016 +0530
+++ b/source/encoder/search.h	Fri Sep 30 17:55:41 2016 +0530
@@ -274,6 +274,7 @@
     bool            m_bFrameParallel;
     uint32_t        m_numLayers;
     uint32_t        m_refLagPixels;
+    uint32_t        m_maxTUDepth;
 
     int16_t         m_sliceMaxY;
     int16_t         m_sliceMinY;