[x265] [PATCH 3 of 4] limitTU : use neighbouring CUs' TU depth to limit 1st subTU's depth

[bhavna at multicorewareinc.com]

# HG changeset patch
# User Bhavna Hariharan <bhavna at multicorewareinc.com>
# Date 1479454023 -19800
#      Fri Nov 18 12:57:03 2016 +0530
# Node ID df9a0c94631c9abac75e5a7cf4bd48ff55adced8
# Parent  40a0a322b26fc0516a72d4de9a941e18b5bb97b9
limitTU : use neighbouring CUs' TU depth to limit 1st subTU's depth

diff -r 40a0a322b26f -r df9a0c94631c doc/reST/cli.rst
--- a/doc/reST/cli.rst	Fri Nov 18 12:06:08 2016 +0530
+++ b/doc/reST/cli.rst	Fri Nov 18 12:57:03 2016 +0530
@@ -869,7 +869,7 @@
 	partitions, in which case a TU split is implied and thus the
 	residual quad-tree begins one layer below the CU quad-tree.
 
-.. option:: --limit-tu <0..3>
+.. option:: --limit-tu <0..4>
 
 	Enables early exit from TU depth recursion, for inter coded blocks.
 	Level 1 - decides to recurse to next higher depth based on cost 
@@ -878,6 +878,9 @@
 	other split subTUs.
 	Level 3 - based on the average depth of the co-located and the neighbor
 	CUs' TU depth, limits recursion of the current CU.
+	Level 4 - uses the depth of the neighbouring/ co-located CUs TU depth 
+	to limit the 1st subTU depth. The 1st subTU depth is taken as the 
+	limiting depth for the other subTUs.
 
 	Default: 0
 
diff -r 40a0a322b26f -r df9a0c94631c source/common/param.cpp
--- a/source/common/param.cpp	Fri Nov 18 12:06:08 2016 +0530
+++ b/source/common/param.cpp	Fri Nov 18 12:57:03 2016 +0530
@@ -1126,7 +1126,7 @@
           "QuadtreeTUMaxDepthInter must be less than or equal to the difference between log2(maxCUSize) and QuadtreeTULog2MinSize plus 1");
     CHECK((param->maxTUSize != 32 && param->maxTUSize != 16 && param->maxTUSize != 8 && param->maxTUSize != 4),
           "max TU size must be 4, 8, 16, or 32");
-    CHECK(param->limitTU > 3, "Invalid limit-tu option, limit-TU must be between 0 and 3");
+    CHECK(param->limitTU > 4, "Invalid limit-tu option, limit-TU must be between 0 and 4");
     CHECK(param->maxNumMergeCand < 1, "MaxNumMergeCand must be 1 or greater.");
     CHECK(param->maxNumMergeCand > 5, "MaxNumMergeCand must be 5 or smaller.");
 
diff -r 40a0a322b26f -r df9a0c94631c source/encoder/search.cpp
--- a/source/encoder/search.cpp	Fri Nov 18 12:06:08 2016 +0530
+++ b/source/encoder/search.cpp	Fri Nov 18 12:57:03 2016 +0530
@@ -103,6 +103,8 @@
             m_limitTU = X265_TU_LIMIT_DFS;
         else if (m_param->limitTU == 3)
             m_limitTU = X265_TU_LIMIT_NEIGH;
+        else if (m_param->limitTU == 4)
+            m_limitTU = X265_TU_LIMIT_DFS + X265_TU_LIMIT_NEIGH;
     }
 
     /* these are indexed by qtLayer (log2size - 2) so nominally 0=4x4, 1=8x8, 2=16x16, 3=32x32
@@ -2638,13 +2640,20 @@
 
     m_entropyCoder.load(m_rqt[depth].cur);
 
-    if (m_limitTU & X265_TU_LIMIT_DFS)
+    if ((m_limitTU & X265_TU_LIMIT_DFS) && !(m_limitTU & X265_TU_LIMIT_NEIGH))
         m_maxTUDepth = -1;
     else if (m_limitTU & X265_TU_LIMIT_BFS)
         memset(&m_cacheTU, 0, sizeof(TUInfoCache));
 
     Cost costs;
-    estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
+    if ((m_limitTU & X265_TU_LIMIT_DFS) && (m_limitTU & X265_TU_LIMIT_NEIGH))
+    {
+        int32_t tempDepth = m_maxTUDepth;
+        estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
+        m_maxTUDepth = tempDepth;
+    }
+    else
+        estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
 
     uint32_t tqBypass = cu.m_tqBypass[0];
     if (!tqBypass)
@@ -2905,8 +2914,9 @@
     {
         if ((m_limitTU & X265_TU_LIMIT_DFS) && tuDepth == 0 && qIdx == 1)
         {
+            m_maxTUDepth = cu.m_tuDepth[0];
             // Fetch maximum TU depth of first sub partition to limit recursion of others
-            for (uint32_t i = 0; i < cuGeom.numPartitions / 4; i++)
+            for (uint32_t i = 1; i < cuGeom.numPartitions / 4; i++)
                 m_maxTUDepth = X265_MAX(m_maxTUDepth, cu.m_tuDepth[i]);
         }
         estimateResidualQT(mode, cuGeom, qPartIdx, tuDepth + 1, resiYuv, splitCost, depthRange, splitMore);