[x265] [PATCH 2 of 2] analysis: include CU mode signaling overhead in sa8d costs [CHANGES OUTPUTS]

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1418327875 21600
#      Thu Dec 11 13:57:55 2014 -0600
# Node ID ef4587c47e0f27a71c92f5b4e2595d16093c311f
# Parent  a0bf346f31760598e327c218ef2b81c5cfad3f55
analysis: include CU mode signaling overhead in sa8d costs [CHANGES OUTPUTS]

At RD levels 0..4, some mode decisions are made based on just the residual cost
using the formula: sa8d(prediction, source) + lambda * bits.  This avoids
the expensive residual coding operations required to generate the reconstructed
pixels necessary to measure the final distortion (for real RDO).

checkIntraInInter(), predInterSearch(), and checkMerge2Nx2N_rd0_4() were not
including the cost of the mode decision fields in that bit count (aka: signaling
not-skip, pred mode, part size, etc).  This resulted in less compression
efficiency (spending more bits than required for quality).

diff -r a0bf346f3176 -r ef4587c47e0f source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Thu Dec 11 13:52:34 2014 -0600
+++ b/source/encoder/analysis.cpp	Thu Dec 11 13:57:55 2014 -0600
@@ -1303,7 +1303,13 @@
         md.bestMode = tempPred->rdCost < bestPred->rdCost ? tempPred : bestPred;
     }
     else
+    {
+        /* at rdLevel 0, all mode decisions are made via sa8d residual cost, add merge mode signal cost */
+        bestPred->sa8dBits += m_entropyCoder.bitsInterMode(bestPred->cu, 0, depth);  /* inter 2Nx2N cost */
+        bestPred->sa8dBits += 1;  /* merge flag */
+        bestPred->sa8dCost = m_rdCost.calcRdSADCost(bestPred->distortion, bestPred->sa8dBits);
         md.bestMode = bestPred;
+    }
 
     /* broadcast sets of MV field data */
     bestPred->cu.setPUInterDir(interDirNeighbours[bestSadCand], 0, 0);
@@ -1458,6 +1464,7 @@
             interMode.distortion += primitives.sa8d_inter[cpart](fencYuv.m_buf[1], fencYuv.m_csize, predYuv.m_buf[1], predYuv.m_csize);
             interMode.distortion += primitives.sa8d_inter[cpart](fencYuv.m_buf[2], fencYuv.m_csize, predYuv.m_buf[2], predYuv.m_csize);
         }
+        interMode.sa8dBits += m_entropyCoder.bitsInterMode(interMode.cu, 0, cuGeom.depth);
         interMode.sa8dCost = m_rdCost.calcRdSADCost(interMode.distortion, interMode.sa8dBits);
 
         if (m_param->analysisMode == X265_ANALYSIS_SAVE && m_reuseInterDataCTU)
@@ -1669,6 +1676,10 @@
             cu.m_mv[1][0] = bestME[1].mv;
         }
     }
+
+    uint32_t modeBits = m_entropyCoder.bitsIntraMode(bidir2Nx2N.cu, 0);
+    bidir2Nx2N.sa8dBits += modeBits;
+    bidir2Nx2N.sa8dCost += m_rdCost.getCost(modeBits);
 }
 
 void Analysis::encodeResidue(const CUData& ctu, const CUGeom& cuGeom)
diff -r a0bf346f3176 -r ef4587c47e0f source/encoder/search.cpp
--- a/source/encoder/search.cpp	Thu Dec 11 13:52:34 2014 -0600
+++ b/source/encoder/search.cpp	Thu Dec 11 13:57:55 2014 -0600
@@ -1325,10 +1325,9 @@
 
     cu.setLumaIntraDirSubParts((uint8_t)bmode, absPartIdx, depth + initTuDepth);
     intraMode.initCosts();
-    intraMode.totalBits = bbits;
     intraMode.distortion = bsad;
-    intraMode.sa8dCost = bcost;
-    intraMode.sa8dBits = bbits;
+    intraMode.sa8dBits = bbits + m_entropyCoder.bitsIntraMode(intraMode.cu, absPartIdx);
+    intraMode.sa8dCost = m_rdCost.calcRdSADCost(intraMode.sa8dBits, intraMode.sa8dBits);
 }
 
 void Search::encodeIntraInInter(Mode& intraMode, const CUGeom& cuGeom)