[x265] [PATCH 2 of 3] complexAnalysis: increase analysis for areas capped by VBV

[bhavna at multicorewareinc.com]

# HG changeset patch
# User Bhavna Hariharan <bhavna at multicorewareinc.com>
# Date 1485246147 -19800
#      Tue Jan 24 13:52:27 2017 +0530
# Node ID e335733309ff7a324b78c79ed12ea59d0a0bf4b9
# Parent  09a7c884054aa74f7a3c09978fc8a766164d35dd
complexAnalysis: increase analysis for areas capped by VBV

diff -r 09a7c884054a -r e335733309ff source/common/cudata.cpp
--- a/source/common/cudata.cpp	Fri Jan 27 12:30:36 2017 +0530
+++ b/source/common/cudata.cpp	Tue Jan 24 13:52:27 2017 +0530
@@ -306,6 +306,8 @@
     for (int8_t i = 0; i < NUM_TU_DEPTH; i++)
         m_refTuDepth[i] = -1;
 
+    m_vbvAffected = false;
+
     uint32_t widthInCU = m_slice->m_sps->numCuInWidth;
     m_cuLeft = (m_cuAddr % widthInCU) ? m_encData->getPicCTU(m_cuAddr - 1) : NULL;
     m_cuAbove = (m_cuAddr >= widthInCU) && !m_bFirstRowInSlice ? m_encData->getPicCTU(m_cuAddr - widthInCU) : NULL;
diff -r 09a7c884054a -r e335733309ff source/common/cudata.h
--- a/source/common/cudata.h	Fri Jan 27 12:30:36 2017 +0530
+++ b/source/common/cudata.h	Tue Jan 24 13:52:27 2017 +0530
@@ -164,6 +164,8 @@
     static cubcast_t s_partSet[NUM_FULL_DEPTH]; // pointer to broadcast set functions per absolute depth
     static uint32_t  s_numPartInCUSize;
 
+    bool          m_vbvAffected;
+
     FrameData*    m_encData;
     const Slice*  m_slice;
 
diff -r 09a7c884054a -r e335733309ff source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Fri Jan 27 12:30:36 2017 +0530
+++ b/source/encoder/analysis.cpp	Tue Jan 24 13:52:27 2017 +0530
@@ -969,6 +969,9 @@
 
 SplitData Analysis::compressInterCU_rd0_4(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp)
 {
+    if (parentCTU.m_vbvAffected && calculateQpforCuSize(parentCTU, cuGeom, 1))
+        return compressInterCU_rd5_6(parentCTU, cuGeom, qp);
+
     uint32_t depth = cuGeom.depth;
     uint32_t cuAddr = parentCTU.m_cuAddr;
     ModeDepth& md = m_modeDepth[depth];
@@ -1537,6 +1540,9 @@
 
 SplitData Analysis::compressInterCU_rd5_6(const CUData& parentCTU, const CUGeom& cuGeom, int32_t qp)
 {
+    if (parentCTU.m_vbvAffected && !calculateQpforCuSize(parentCTU, cuGeom, 1))
+        return compressInterCU_rd0_4(parentCTU, cuGeom, qp);
+
     uint32_t depth = cuGeom.depth;
     ModeDepth& md = m_modeDepth[depth];
     md.bestMode = NULL;
@@ -2874,7 +2880,7 @@
     return false;
 }
 
-int Analysis::calculateQpforCuSize(const CUData& ctu, const CUGeom& cuGeom, double baseQp)
+int Analysis::calculateQpforCuSize(const CUData& ctu, const CUGeom& cuGeom, int32_t complexCheck, double baseQp)
 {
     FrameData& curEncData = *m_frame->m_encData;
     double qp = baseQp >= 0 ? baseQp : curEncData.m_cuStat[ctu.m_cuAddr].baseQp;
@@ -2885,7 +2891,11 @@
         loopIncr = 16;
     /* Use cuTree offsets if cuTree enabled and frame is referenced, else use AQ offsets */
     bool isReferenced = IS_REFERENCED(m_frame);
-    double *qpoffs = (isReferenced && m_param->rc.cuTree) ? m_frame->m_lowres.qpCuTreeOffset : m_frame->m_lowres.qpAqOffset;
+    double *qpoffs;
+    if (complexCheck)
+        qpoffs = m_frame->m_lowres.qpAqOffset;
+    else
+        qpoffs = (isReferenced && m_param->rc.cuTree) ? m_frame->m_lowres.qpCuTreeOffset : m_frame->m_lowres.qpAqOffset;
     if (qpoffs)
     {
         uint32_t width = m_frame->m_fencPic->m_picWidth;
@@ -2910,6 +2920,16 @@
 
         qp_offset /= cnt;
         qp += qp_offset;
+        if (complexCheck)
+        {
+            int32_t offset = (int32_t)(qp_offset * 100 + .5);
+            double threshold = (1 - ((X265_MAX_ANALYSIS_STRENGTH - m_param->complexAnalysis) * 0.5));
+            int32_t max_threshold = (int32_t)(threshold * 100 + .5);
+            if (offset < max_threshold)
+                return 1;
+            else
+                return 0;
+        }
     }
 
     return x265_clip3(m_param->rc.qpMin, m_param->rc.qpMax, (int)(qp + 0.5));
diff -r 09a7c884054a -r e335733309ff source/encoder/analysis.h
--- a/source/encoder/analysis.h	Fri Jan 27 12:30:36 2017 +0530
+++ b/source/encoder/analysis.h	Tue Jan 24 13:52:27 2017 +0530
@@ -174,7 +174,7 @@
     /* generate residual and recon pixels for an entire CTU recursively (RD0) */
     void encodeResidue(const CUData& parentCTU, const CUGeom& cuGeom);
 
-    int calculateQpforCuSize(const CUData& ctu, const CUGeom& cuGeom, double baseQP = -1);
+    int calculateQpforCuSize(const CUData& ctu, const CUGeom& cuGeom, int32_t complexCheck = 0, double baseQP = -1);
 
     void calculateNormFactor(CUData& ctu, int qp);
     void normFactor(const pixel* src, uint32_t blockSize, CUData& ctu, int qp, TextType ttype);
diff -r 09a7c884054a -r e335733309ff source/encoder/frameencoder.cpp
--- a/source/encoder/frameencoder.cpp	Fri Jan 27 12:30:36 2017 +0530
+++ b/source/encoder/frameencoder.cpp	Tue Jan 24 13:52:27 2017 +0530
@@ -1333,6 +1333,9 @@
             && analysisFrameData->highDistortionCtuCount && analysisFrameData->lowDistortionCtuCount)
             curEncData.m_cuStat[cuAddr].baseQp += analysisFrameData->offset[cuAddr];
 
+        if (m_param->complexAnalysis && (int32_t)(m_rce.qpaRc - m_rce.qpNoVbv) > 0)
+            ctu->m_vbvAffected = true;
+
         // Does all the CU analysis, returns best top level mode decision
         Mode& best = tld.analysis.compressCTU(*ctu, *m_frame, m_cuGeoms[m_ctuGeomMap[cuAddr]], rowCoder);