[x265] [PATCH 4 of 4] analysis: hoist all adaptive-quant work to recursive callers

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1429910062 18000
#      Fri Apr 24 16:14:22 2015 -0500
# Node ID c3e5a01c51c6cd7e96f48ea294c7ed3f174b0915
# Parent  e456ee49307963093a9e65d5bb3fc28418efdc57
analysis: hoist all adaptive-quant work to recursive callers

Note that the depth-first code will probably need to reset lambda after
returning from recursion (even before any of these patches)

diff -r e456ee493079 -r c3e5a01c51c6 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Fri Apr 24 16:05:12 2015 -0500
+++ b/source/encoder/analysis.cpp	Fri Apr 24 16:14:22 2015 -0500
@@ -259,12 +259,6 @@
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
         uint8_t* reuseDepth  = &m_reuseIntraDataCTU->depth[parentCTU.m_cuAddr * parentCTU.m_numPartitions];
@@ -324,6 +318,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int32_t nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -333,8 +328,13 @@
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
 
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressIntraCU(parentCTU, childGeom, zOrder, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -536,12 +536,6 @@
 
     X265_CHECK(m_param->rdLevel >= 2, "compressInterCU_dist does not support RD 0 or 1\n");
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (mightNotSplit && depth >= minDepth)
     {
         int bTryAmp = m_slice->m_sps->maxAMPDepth > depth && (cuGeom.log2CUSize < 6 || m_param->rdLevel > 4);
@@ -733,6 +727,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -741,8 +736,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_dist(parentCTU, childGeom, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -793,12 +794,6 @@
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
     uint32_t minDepth = topSkipMinDepth(parentCTU, cuGeom);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (mightNotSplit && depth >= minDepth)
     {
         bool bTryIntra = m_slice->m_sliceType != B_SLICE || m_param->bIntraInBFrames;
@@ -1013,6 +1008,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -1021,8 +1017,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_rd0_4(parentCTU, childGeom, nextQP);
 
                 // Save best CU and pred data for this sub CU
@@ -1083,12 +1085,6 @@
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
 
-    if (m_slice->m_pps->bUseDQP && depth && depth <= m_slice->m_pps->maxCuDQPDepth)
-    {
-        setLambdaFromQP(*m_slice, qp);
-        qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
-    }
-
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
         uint8_t* reuseDepth  = &m_reuseInterDataCTU->depth[parentCTU.m_cuAddr * parentCTU.m_numPartitions];
@@ -1220,6 +1216,7 @@
         ModeDepth& nd = m_modeDepth[nextDepth];
         invalidateContexts(nextDepth);
         Entropy* nextContext = &m_rqt[depth].cur;
+        int nextQP = qp;
 
         for (uint32_t subPartIdx = 0; subPartIdx < 4; subPartIdx++)
         {
@@ -1228,8 +1225,14 @@
             {
                 m_modeDepth[0].fencYuv.copyPartToYuv(nd.fencYuv, childGeom.absPartIdx);
                 m_rqt[nextDepth].cur.load(*nextContext);
-                int32_t nextQP = m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth ?
-                                 m_aqQP[childGeom.index] : qp;
+
+                if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
+                {
+                    nextQP = m_aqQP[childGeom.index];
+                    setLambdaFromQP(*m_slice, nextQP);
+                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
+                }
+
                 compressInterCU_rd5_6(parentCTU, childGeom, zOrder, nextQP);
 
                 // Save best CU and pred data for this sub CU