[x265] [PATCH] vbv:Avoid infinite loop when row re-encode is triggered in vbv- clip qp inside rowDiagonalVbvRateControl()

[santhoshini at multicorewareinc.com]

# HG changeset patch
# User Santhoshini Sekar <santhoshini at multicorewareinc.com>
# Date 1397822534 -19800
#      Fri Apr 18 17:32:14 2014 +0530
# Node ID 03aa222ff8eb5ffbe65bd5bf522cad6561210024
# Parent  78c1f43f12f57ba60e63cb79ce567065c700483e
vbv:Avoid infinite loop when row re-encode is triggered in vbv- clip qp inside rowDiagonalVbvRateControl()

diff -r 78c1f43f12f5 -r 03aa222ff8eb source/encoder/frameencoder.cpp
--- a/source/encoder/frameencoder.cpp	Thu Apr 17 16:32:16 2014 -0500
+++ b/source/encoder/frameencoder.cpp	Fri Apr 18 17:32:14 2014 +0530
@@ -1141,7 +1141,6 @@
             {
                 double qpBase = cu->m_baseQp;
                 int reEncode = m_top->m_rateControl->rowDiagonalVbvRateControl(m_pic, row, &m_rce, qpBase);
-                qpBase = Clip3((double)MIN_QP, (double)MAX_MAX_QP, qpBase);
                 m_pic->m_rowDiagQp[row] = qpBase;
                 m_pic->m_rowDiagQScale[row] =  x265_qp2qScale(qpBase);
 
diff -r 78c1f43f12f5 -r 03aa222ff8eb source/encoder/ratecontrol.cpp
--- a/source/encoder/ratecontrol.cpp	Thu Apr 17 16:32:16 2014 -0500
+++ b/source/encoder/ratecontrol.cpp	Fri Apr 18 17:32:14 2014 +0530
@@ -861,6 +861,7 @@
     double bufferLeftPlanned = rce->bufferFill - rce->frameSizePlanned;
 
     double maxFrameError = X265_MAX(0.05, 1.0 / pic->getFrameHeightInCU());
+    int retReencodeFlag = 0;
 
     if (row < pic->getPicSym()->getFrameHeightInCU() - 1)
     {
@@ -922,7 +923,7 @@
         {
             /* Bump QP to halfway in between... close enough. */
             qpVbv = Clip3(prevRowQp + 1.0f, qpMax, (prevRowQp + qpVbv) * 0.5);
-            return -1;
+            retReencodeFlag = -1;
         }
     }
     else
@@ -936,10 +937,11 @@
              qpVbv < qpMax && canReencodeRow))
         {
             qpVbv = qpMax;
-            return -1;
+            retReencodeFlag = -1;
         }
     }
-    return 0;
+    qpVbv = Clip3((double)MIN_QP, (double)MAX_MAX_QP, qpVbv);
+    return retReencodeFlag;
 }
 
 /* modify the bitrate curve from pass1 for one frame */