[x265] [PATCH 4 of 4] implement QP based RD refinement for Intra slices [CHANGES OUTPUT]

[kavitha at multicorewareinc.com]

# HG changeset patch
# User Kavitha Sampath <kavitha at multicorewareinc.com>
# Date 1444801800 -19800
#      Wed Oct 14 11:20:00 2015 +0530
# Node ID 0c97c1dc969033237c5eaa8079604dab6b590070
# Parent  cab9ae709e7bdd5451b0d8d3ddfbdd201e802b8b
implement QP based RD refinement for Intra slices [CHANGES OUTPUT]

After CU analysis, calculate R-D cost on the best partition mode
for a range of QP values to find the optimal rounding effect.

diff -r cab9ae709e7b -r 0c97c1dc9690 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Tue Oct 13 17:26:53 2015 +0530
+++ b/source/encoder/analysis.cpp	Wed Oct 14 11:20:00 2015 +0530
@@ -197,6 +197,9 @@
         }
     }
 
+    if (m_slice->m_sliceType == I_SLICE && m_param->bEnableRdRefine)
+        qpRdRefine(ctu, cuGeom);
+
     return *m_modeDepth[0].bestMode;
 }
 
@@ -225,7 +228,54 @@
     }
 }
 
-void Analysis::compressIntraCU(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp)
+void Analysis::qpRdRefine(const CUData& parentCTU, const CUGeom& cuGeom)
+{
+    uint64_t origCost, bestCost, cost, prevCost;
+    int failure, nQP, lambdaQP;
+    double bestQp, origQP;
+    bestQp = origQP = m_frame->m_encData->m_cuStat[parentCTU.m_cuAddr].baseQp;
+    bestCost = origCost = m_modeDepth[0].bestMode->rdCost;
+    lambdaQP = calculateQpforCuSize(parentCTU, cuGeom);
+
+    for (int16_t dir = 1; dir >= -1; dir -= 2)
+    {
+        int threshold = !!m_param->psyRd;
+
+        failure = 0;
+        prevCost = origCost;
+        double modQP = origQP + dir;
+
+        while (modQP >= QP_MIN && modQP <= QP_MAX_SPEC)
+        {
+             /* set modified QP for quant, maintain constant lambda for all QPs
+              * use lambda of QP used for CU analysis for cost calculation */
+             nQP = setLambdaFromQP(parentCTU, calculateQpforCuSize(parentCTU, cuGeom, modQP), lambdaQP);
+
+            compressIntraCU(parentCTU, cuGeom, nQP, modQP);
+            cost = m_modeDepth[0].bestMode->rdCost;
+            COPY2_IF_LT(bestCost, cost, bestQp, modQP);
+
+            if (cost < prevCost)
+                failure = 0;
+            else
+                failure++;
+
+            if (failure > threshold)
+                break;
+
+            prevCost = cost;
+            modQP += dir;
+       }
+    }
+
+    /* TODO: Try last CU's QP to decide the bestQP before re-encode */
+
+    /* Re-encode CU for best chosen QP */
+    nQP = setLambdaFromQP(parentCTU, calculateQpforCuSize(parentCTU, cuGeom, bestQp));
+    compressIntraCU(parentCTU, cuGeom, nQP, bestQp);
+}
+
+void Analysis::compressIntraCU(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp, double modQp)
 {
     uint32_t depth = cuGeom.depth;
     ModeDepth& md = m_modeDepth[depth];
@@ -237,9 +287,6 @@
     bool bAlreadyDecided = parentCTU.m_lumaIntraDir[cuGeom.absPartIdx] != (uint8_t)ALL_IDX;
     bool bDecidedDepth = parentCTU.m_cuDepth[cuGeom.absPartIdx] == depth;
 
-    // stop recursion if we reach the depth of previous analysis decision
-    mightSplit &= !(bAlreadyDecided && bDecidedDepth);
-
     if (bAlreadyDecided)
     {
         if (bDecidedDepth)
@@ -278,6 +325,9 @@
             addSplitFlagCost(*md.bestMode, cuGeom.depth);
     }
 
+    // stop recursion if we reach the depth of previous analysis decision
+    mightSplit &= !(bAlreadyDecided && bDecidedDepth);
+
     if (mightSplit)
     {
         Mode* splitPred = &md.pred[PRED_SPLIT];
@@ -300,9 +350,11 @@
                 m_rqt[nextDepth].cur.load(*nextContext);
 
                 if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
-                    nextQP = setLambdaFromQP(parentCTU, calculateQpforCuSize(parentCTU, childGeom));
-
-                compressIntraCU(parentCTU, childGeom, nextQP);
+                {
+                    int constLambdaQp = bAlreadyDecided ? calculateQpforCuSize(parentCTU, childGeom) : -1;
+                    nextQP = setLambdaFromQP(parentCTU, calculateQpforCuSize(parentCTU, childGeom, modQp), constLambdaQp);
+                }
+                compressIntraCU(parentCTU, childGeom, nextQP, modQp);
 
                 // Save best CU and pred data for this sub CU
                 splitCU->copyPartFrom(nd.bestMode->cu, childGeom, subPartIdx);
@@ -314,6 +366,10 @@
             {
                 /* record the depth of this non-present sub-CU */
                 splitCU->setEmptyPart(childGeom, subPartIdx);
+
+                /* Set depth of non-present CU to 0 to ensure that correct CU is fetched as reference to code deltaQP */
+                if (bAlreadyDecided)
+                    memset(parentCTU.m_cuDepth + childGeom.absPartIdx, 0, childGeom.numPartitions);
             }
         }
         nextContext->store(splitPred->contexts);
diff -r cab9ae709e7b -r 0c97c1dc9690 source/encoder/analysis.h
--- a/source/encoder/analysis.h	Tue Oct 13 17:26:53 2015 +0530
+++ b/source/encoder/analysis.h	Wed Oct 14 11:20:00 2015 +0530
@@ -110,8 +110,11 @@
 
     uint32_t m_splitRefIdx[4];
 
+    /* refine RD based on QP for rd-levels 5 and 6 */
+    void qpRdRefine(const CUData& ctu, const CUGeom& cuGeom);
+
     /* full analysis for an I-slice CU */
-    void compressIntraCU(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp);
+    void compressIntraCU(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp, double modQp = 0);
 
     /* full analysis for a P or B slice CU */
     uint32_t compressInterCU_dist(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp);