[x265] [PATCH 2 of 4] limit TU : use cbf and quantization coefficients to limit recursion

[kavitha at multicorewareinc.com]

# HG changeset patch
# User Bhavna Hariharan <bhavna at multicorewareinc.com>
# Date 1474620761 -19800
#      Fri Sep 23 14:22:41 2016 +0530
# Node ID c018bc0ffc156902b1a9a13ecd6996d30d7403df
# Parent  c10ef341f4e65883243f78040f52ed06ace99535
limit TU : use cbf and quantization coefficients to limit recursion

diff -r c10ef341f4e6 -r c018bc0ffc15 source/encoder/search.cpp
--- a/source/encoder/search.cpp	Tue Oct 04 13:27:48 2016 +0530
+++ b/source/encoder/search.cpp	Fri Sep 23 14:22:41 2016 +0530
@@ -3194,6 +3194,8 @@
                 singlePsyEnergy[TEXT_LUMA][0] = nonZeroPsyEnergyY;
                 cbfFlag[TEXT_LUMA][0] = !!numSigTSkipY;
                 bestTransformMode[TEXT_LUMA][0] = 1;
+                if (m_param->limitTU)
+                    numSig[TEXT_LUMA][0] = numSigTSkipY;
                 uint32_t numCoeffY = 1 << (log2TrSize << 1);
                 memcpy(coeffCurY, m_tsCoeff, sizeof(coeff_t) * numCoeffY);
                 primitives.cu[partSize].copy_ss(curResiY, strideResiY, m_tsResidual, trSize);
@@ -3331,6 +3333,21 @@
             fullCost.rdcost = m_rdCost.calcPsyRdCost(fullCost.distortion, fullCost.bits, fullCost.energy);
         else
             fullCost.rdcost = m_rdCost.calcRdCost(fullCost.distortion, fullCost.bits);
+
+        if (m_param->limitTU && bCheckSplit)
+        {
+            // Stop recursion if the TU's energy level is minimal
+            if (cbfFlag[TEXT_LUMA][0] == 0)
+                bCheckSplit = false;
+            else if (numSig[TEXT_LUMA][0] < (cuGeom.numPartitions / 16))
+            {
+                uint32_t energy = 0;
+                for (uint32_t i = 0; i < cuGeom.numPartitions; i++)
+                    energy += abs(coeffCurY[i]);
+                if (energy < numSig[TEXT_LUMA][0])
+                    bCheckSplit = false;
+            }
+        }
     }
 
     // code sub-blocks