[x265] [PATCH 2 of 2] rc: adjust qp for scene transitions and fade ins to avoid quality loss with vbv

[aarthi at multicorewareinc.com]

# HG changeset patch
# User Aarthi Thirumalai
# Date 1439287054 -19800
#      Tue Aug 11 15:27:34 2015 +0530
# Node ID 5583033c0c4629a8d67e2db98da32b64d1f181bc
# Parent  be31c36c1f50cc46f47ebfc4b3929dee9079c9d8
rc: adjust qp for scene transitions and fade ins to avoid quality loss with vbv

diff -r be31c36c1f50 -r 5583033c0c46 source/encoder/ratecontrol.cpp
--- a/source/encoder/ratecontrol.cpp	Wed Jul 22 00:29:26 2015 +0530
+++ b/source/encoder/ratecontrol.cpp	Tue Aug 11 15:27:34 2015 +0530
@@ -181,6 +181,7 @@
     m_bTerminated = false;
     m_finalFrameCount = 0;
     m_numEntries = 0;
+    m_isSceneTransition = false;
     if (m_param->rc.rateControlMode == X265_RC_CRF)
     {
         m_param->rc.qp = (int)m_param->rc.rfConstant;
@@ -273,7 +274,7 @@
     if(m_param->rc.bStrictCbr)
         m_rateTolerance = 0.7;
 
-    m_leadingBframes = m_param->bframes;
+    m_numFramesInMiniGop = 0;
     m_bframeBits = 0;
     m_leadingNoBSatd = 0;
     m_ipOffset = 6.0 * X265_LOG2(m_param->rc.ipFactor);
@@ -282,6 +283,7 @@
     /* Adjust the first frame in order to stabilize the quality level compared to the rest */
 #define ABR_INIT_QP_MIN (24)
 #define ABR_INIT_QP_MAX (40)
+#define ABR_SCENECUT_INIT_QP_MIN (12)
 #define CRF_INIT_QP (int)m_param->rc.rfConstant
     for (int i = 0; i < 3; i++)
         m_lastQScaleFor[i] = x265_qp2qScale(m_param->rc.rateControlMode == X265_RC_CRF ? CRF_INIT_QP : ABR_INIT_QP_MIN);
@@ -960,10 +962,22 @@
         copyRceData(rce, &m_rce2Pass[rce->poc]);
     }
     rce->isActive = true;
-    if (m_sliceType == B_SLICE)
-        rce->bframes = m_leadingBframes;
-    else
-        m_leadingBframes = curFrame->m_lowres.leadingBframes;
+    bool isframeAfterKeyframe = m_sliceType != I_SLICE && m_curSlice->m_refPicList[0][0]->m_encData->m_slice->m_sliceType == I_SLICE;
+    if (curFrame->m_lowres.bScenecut && m_param->scenecutThreshold)
+    {
+        m_isSceneTransition = true;
+            /* Frame Predictors and Row predictors used in vbv */
+    for (int i = 0; i < 4; i++)
+    {
+        m_pred[i].coeff = 1.0;
+        m_pred[i].count = 1.0;
+        m_pred[i].decay = 0.5;
+        m_pred[i].offset = 0.0;
+    }
+    m_pred[0].coeff = m_pred[3].coeff = 0.75;
+    }
+    else if (m_sliceType != B_SLICE && !isframeAfterKeyframe)
+        m_isSceneTransition = false;
 
     rce->bLastMiniGopBFrame = curFrame->m_lowres.bLastMiniGopBFrame;
     rce->bufferRate = m_bufferRate;
@@ -1040,6 +1054,10 @@
                 }
             }
         }
+        /* For a scenecut that occurs within the mini-gop, enable scene transition
+         * switch until the next mini-gop to ensure a min qp for all the frames within 
+         * the scene-transition mini-gop */
+
         double q = x265_qScale2qp(rateEstimateQscale(curFrame, rce));
         q = x265_clip3((double)QP_MIN, (double)QP_MAX_MAX, q);
         m_qp = int(q + 0.5);
@@ -1382,6 +1400,13 @@
         else
             q += m_pbOffset;
 
+        /* Set a min qp at scenechanges and transitions */
+        if (m_isSceneTransition)
+        {
+            q = X265_MAX(ABR_SCENECUT_INIT_QP_MIN, q);
+            double minScenecutQscale =x265_qp2qScale(ABR_SCENECUT_INIT_QP_MIN); 
+            m_lastQScaleFor[P_SLICE] = X265_MAX(minScenecutQscale, m_lastQScaleFor[P_SLICE]);
+        }
         double qScale = x265_qp2qScale(q);
         rce->qpNoVbv = q;
         double lmin = 0, lmax = 0;
@@ -1544,6 +1569,13 @@
                 q = X265_MIN(lqmax, q);
             }
             q = x265_clip3(MIN_QPSCALE, MAX_MAX_QPSCALE, q);
+            /* Set a min qp at scenechanges and transitions */
+            if (m_isSceneTransition)
+            {
+               double minScenecutQscale =x265_qp2qScale(ABR_SCENECUT_INIT_QP_MIN); 
+               q = X265_MAX(minScenecutQscale, q);
+               m_lastQScaleFor[P_SLICE] = X265_MAX(minScenecutQscale, m_lastQScaleFor[P_SLICE]);
+            }
             rce->qpNoVbv = x265_qScale2qp(q);
             q = clipQscale(curFrame, rce, q);
             /*  clip qp to permissible range after vbv-lookahead estimation to avoid possible
@@ -1750,7 +1782,7 @@
                 }
                 /* Try to get the buffer not more than 80% filled, but don't set an impossible goal. */
                 targetFill = x265_clip3(m_bufferSize * (1 - 0.2 * finalDur), m_bufferSize, m_bufferFill - totalDuration * m_vbvMaxRate * 0.5);
-                if (m_isCbr && bufferFillCur > targetFill)
+                if (m_isCbr && bufferFillCur > targetFill && !m_isSceneTransition)
                 {
                     q /= 1.01;
                     loopTerminate |= 2;
diff -r be31c36c1f50 -r 5583033c0c46 source/encoder/ratecontrol.h
--- a/source/encoder/ratecontrol.h	Wed Jul 22 00:29:26 2015 +0530
+++ b/source/encoder/ratecontrol.h	Tue Aug 11 15:27:34 2015 +0530
@@ -147,7 +147,7 @@
     double  m_pbOffset;
     int64_t m_bframeBits;
     int64_t m_currentSatd;
-    int     m_leadingBframes;
+    int     m_numFramesInMiniGop;
     int     m_qpConstant[3];
     int     m_lastNonBPictType;
     int     m_framesDone;        /* # of frames passed through RateCotrol already */
@@ -173,6 +173,7 @@
     int64_t m_lastBsliceSatdCost;
     int     m_numBframesInPattern;
     bool    m_isPatternPresent;
+    bool    m_isSceneTransition;
 
     /* a common variable on which rateControlStart, rateControlEnd and rateControUpdateStats waits to
      * sync the calls to these functions. For example