[x265] [PATCH 4 of 6 REV2] analysis: hoist all adaptive-quant work to recursive callers

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1429910062 18000
#      Fri Apr 24 16:14:22 2015 -0500
# Node ID 0d99d6436375fbe5adb6eafa40cbbd0c4fba1acc
# Parent  5644bbd23e71996651f4ed558e0260201a91f70d
analysis: hoist all adaptive-quant work to recursive callers

Note that the depth-first code will probably need to reset lambda after
returning from recursion (even before any of these patches)

diff -r 5644bbd23e71 -r 0d99d6436375 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Fri Apr 24 16:05:12 2015 -0500
+++ b/source/encoder/analysis.cpp	Fri Apr 24 16:14:22 2015 -0500
@@ -257,12 +257,6 @@
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
         uint8_t* reuseDepth  = &m_reuseIntraDataCTU->depth[parentCTU.m_cuAddr * parentCTU.m_numPartitions];
@@ -322,6 +316,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int32_t nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -331,8 +326,13 @@
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
 
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressIntraCU(parentCTU, childGeom, zOrder, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -534,12 +534,6 @@
 
     X265_CHECK(m_param->rdLevel >= 2, "compressInterCU_dist does not support RD 0 or 1\n");
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (mightNotSplit && depth >= minDepth)
     {
         int bTryAmp = m_slice->m_sps->maxAMPDepth > depth && (cuGeom.log2CUSize < 6 || m_param->rdLevel > 4);
@@ -731,6 +725,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -739,8 +734,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_dist(parentCTU, childGeom, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -791,12 +792,6 @@
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
     uint32_t minDepth = topSkipMinDepth(parentCTU, cuGeom);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (mightNotSplit && depth >= minDepth)
     {
         bool bTryIntra = m_slice->m_sliceType != B_SLICE || m_param->bIntraInBFrames;
@@ -1011,6 +1006,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -1019,8 +1015,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_rd0_4(parentCTU, childGeom, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -1081,12 +1083,6 @@
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
         uint8_t* reuseDepth  = &m_reuseInterDataCTU->depth[parentCTU.m_cuAddr * parentCTU.m_numPartitions];
@@ -1218,6 +1214,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -1226,8 +1223,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_rd5_6(parentCTU, childGeom, zOrder, nextQP);
 
                 // Save best CU and pred data for this sub CU