[x265] [PATCH] rc: fix GOP reencode logic for cappedvbr

[Divya Manivannan]

# HG changeset patch
# User Divya Manivannan <divya at multicorewareinc.com>
# Date 1455185954 -19800
#      Thu Feb 11 15:49:14 2016 +0530
# Node ID 7c8671c8d68e6720182aba049ad64bdf28ec2034
# Parent  5de8de77a68efc03cc5c06f3d8ff226914399af6
rc: fix GOP reencode logic for cappedvbr

diff -r 5de8de77a68e -r 7c8671c8d68e source/encoder/ratecontrol.cpp
--- a/source/encoder/ratecontrol.cpp	Fri Feb 05 19:43:20 2016 +0530
+++ b/source/encoder/ratecontrol.cpp	Thu Feb 11 15:49:14 2016 +0530
@@ -225,6 +225,7 @@
     m_statFileOut = NULL;
     m_cutreeStatFileOut = m_cutreeStatFileIn = NULL;
     m_rce2Pass = NULL;
+    m_encOrder = NULL;
     m_lastBsliceSatdCost = 0;
     m_movingAvgSum = 0.0;
 
@@ -571,6 +572,8 @@
             X265_FREE(statsBuf);
             if (m_param->rc.rateControlMode != X265_RC_CQP)
             {
+                m_start = 0;
+                m_isGopReEncoded = true;
                 if (!initPass2())
                     return false;
             } /* else we're using constant quant, so no need to run the bitrate allocation */
@@ -831,13 +834,13 @@
     uint64_t allConstBits = 0, allCodedBits = 0;
     uint64_t allAvailableBits = uint64_t(m_param->rc.bitrate * 1000. * m_numEntries * m_frameDuration);
     int startIndex, framesCount, endIndex;
-    int fps = (int)(m_fps + 0.5);
+    int fps = X265_MIN(m_param->keyframeMax, (int)(m_fps + 0.5));
     startIndex = endIndex = framesCount = 0;
-    bool isQpModified = true;
+    bool isQpModified = m_isGopReEncoded;
     int diffQp = 0;
     double targetBits = 0;
     double expectedBits = 0;
-    for (startIndex = 0, endIndex = 0; endIndex < m_numEntries; endIndex++)
+    for (startIndex = m_start, endIndex = m_start; endIndex < m_numEntries; endIndex++)
     {
         allConstBits += m_rce2Pass[endIndex].miscBits;
         allCodedBits += m_rce2Pass[endIndex].coeffBits + m_rce2Pass[endIndex].mvBits;
@@ -855,7 +858,12 @@
                     {
                         double factor = 2;
                         double step = 0;
-                        for (int start = endIndex; start <= endIndex + fps - 1 && start < m_numEntries; start++)
+                        if (endIndex + fps >= m_numEntries)
+                        {
+                            m_start = endIndex - fps + 1;
+                            return true;
+                        }
+                        for (int start = endIndex + 1; start <= endIndex + fps && start < m_numEntries; start++)
                         {
                             RateControlEntry *rce = &m_rce2Pass[start];
                             targetBits += qScale2bits(rce, x265_qp2qScale(rce->qpNoVbv));
@@ -864,11 +872,12 @@
                         if (expectedBits < 0.95 * targetBits)
                         {
                             isQpModified = true;
+                            m_isGopReEncoded = true;
                             while (endIndex + fps < m_numEntries)
                             {
                                 step = pow(2, factor / 6.0);
                                 expectedBits = 0;
-                                for (int start = endIndex; start <= endIndex + fps - 1; start++)
+                                for (int start = endIndex + 1; start <= endIndex + fps; start++)
                                 {
                                     RateControlEntry *rce = &m_rce2Pass[start];
                                     rce->newQScale = rce->qScale / step;
@@ -883,13 +892,13 @@
                             }
 
                             if (m_isVbv && endIndex + fps < m_numEntries)
-                                if (!vbv2Pass((uint64_t)targetBits, endIndex + fps - 1, endIndex))
+                                if (!vbv2Pass((uint64_t)targetBits, endIndex + fps, endIndex + 1))
                                     return false;
 
                             targetBits = 0;
                             expectedBits = 0;
 
-                            for (int start = endIndex - fps; start <= endIndex - 1; start++)
+                            for (int start = endIndex - fps + 1; start <= endIndex; start++)
                             {
                                 RateControlEntry *rce = &m_rce2Pass[start];
                                 targetBits += qScale2bits(rce, x265_qp2qScale(rce->qpNoVbv));
@@ -898,7 +907,7 @@
                             {
                                 step = pow(2, factor / 6.0);
                                 expectedBits = 0;
-                                for (int start = endIndex - fps; start <= endIndex - 1; start++)
+                                for (int start = endIndex - fps + 1; start <= endIndex; start++)
                                 {
                                     RateControlEntry *rce = &m_rce2Pass[start];
                                     rce->newQScale = rce->qScale * step;
@@ -912,10 +921,13 @@
                                      break;
                             }
                             if (m_isVbv)
-                                if (!vbv2Pass((uint64_t)targetBits, endIndex - 1, endIndex - fps))
+                                if (!vbv2Pass((uint64_t)targetBits, endIndex, endIndex - fps + 1))
                                     return false;
                             diffQp = 0;
+                            m_reencode = endIndex - fps + 1;
+                            endIndex = endIndex + fps;
                             startIndex = endIndex + 1;
+                            m_start = startIndex;
                             targetBits = expectedBits = 0;
                         }
                         else
@@ -939,6 +951,9 @@
         if (!analyseABR2Pass(0, m_numEntries - 1, allAvailableBits))
             return false;
     }
+
+    m_start = X265_MAX(m_start, endIndex - fps);
+
     return true;
 }
 
@@ -2582,6 +2597,7 @@
         fclose(m_cutreeStatFileIn);
 
     X265_FREE(m_rce2Pass);
+    X265_FREE(m_encOrder);
     for (int i = 0; i < 2; i++)
         X265_FREE(m_cuTreeStats.qpBuffer[i]);
     
diff -r 5de8de77a68e -r 7c8671c8d68e source/encoder/ratecontrol.h
--- a/source/encoder/ratecontrol.h	Fri Feb 05 19:43:20 2016 +0530
+++ b/source/encoder/ratecontrol.h	Thu Feb 11 15:49:14 2016 +0530
@@ -202,7 +202,10 @@
 
     /* 2 pass */
     bool    m_2pass;
+    bool    m_isGopReEncoded;
     int     m_numEntries;
+    int     m_start;
+    int     m_reencode;
     FILE*   m_statFileOut;
     FILE*   m_cutreeStatFileOut;
     FILE*   m_cutreeStatFileIn;
@@ -235,6 +238,8 @@
     bool cuTreeReadFor2Pass(Frame* curFrame);
     void hrdFullness(SEIBufferingPeriod* sei);
     int writeRateControlFrameStats(Frame* curFrame, RateControlEntry* rce);
+    bool   initPass2();
+
 protected:
 
     static const int   s_slidingWindowFrames;
@@ -261,7 +266,6 @@
     double predictSize(Predictor *p, double q, double var);
     void   checkAndResetABR(RateControlEntry* rce, bool isFrameDone);
     double predictRowsSizeSum(Frame* pic, RateControlEntry* rce, double qpm, int32_t& encodedBits);
-    bool   initPass2();
     bool   analyseABR2Pass(int startPoc, int endPoc, uint64_t allAvailableBits);
     void   initFramePredictors();
     double getDiffLimitedQScale(RateControlEntry *rce, double q);