[x265-commits] [x265] rc: fix buffer underflow issue in vbv

[Aarthi at videolan.org]

details:   http://hg.videolan.org/x265/rev/638a3af86fa1
branches:  
changeset: 9784:638a3af86fa1
user:      Aarthi Thirumalai
date:      Tue Mar 17 22:50:17 2015 +0530
description:
rc: fix buffer underflow issue in vbv
Subject: [x265] dpb: fix decoder crash with --temporal-layers option

details:   http://hg.videolan.org/x265/rev/572b8f2dc414
branches:  
changeset: 9785:572b8f2dc414
user:      Aarthi Thirumalai
date:      Wed Mar 18 00:43:45 2015 +0530
description:
dpb: fix decoder crash with --temporal-layers option

diffstat:

 source/encoder/dpb.cpp         |   3 +-
 source/encoder/ratecontrol.cpp |  52 +++++++++++++++++++++---------------------
 2 files changed, 28 insertions(+), 27 deletions(-)

diffs (79 lines):

diff -r cc579ffe4b7e -r 572b8f2dc414 source/encoder/dpb.cpp
--- a/source/encoder/dpb.cpp	Tue Mar 17 11:11:32 2015 +0530
+++ b/source/encoder/dpb.cpp	Wed Mar 18 00:43:45 2015 +0530
@@ -107,7 +107,8 @@ void DPB::prepareEncode(Frame *newFrame)
         newFrame->m_encData->m_bHasReferences = false;
 
         // Adjust NAL type for unreferenced B frames
-        if (m_bTemporalSublayer)
+        if (m_bTemporalSublayer && !(slice->m_nalUnitType == NAL_UNIT_CODED_SLICE_RASL_R
+            || slice->m_nalUnitType == NAL_UNIT_CODED_SLICE_RADL_R))
             slice->m_nalUnitType = NAL_UNIT_CODED_SLICE_TSA_N;
         else
         {
diff -r cc579ffe4b7e -r 572b8f2dc414 source/encoder/ratecontrol.cpp
--- a/source/encoder/ratecontrol.cpp	Tue Mar 17 11:11:32 2015 +0530
+++ b/source/encoder/ratecontrol.cpp	Wed Mar 18 00:43:45 2015 +0530
@@ -1369,36 +1369,36 @@ double RateControl::rateEstimateQscale(F
 
         double qScale = x265_qp2qScale(q);
         double lmin = 0, lmax = 0;
-        if (m_isCbr)
+        if (m_isVbv)
         {
-            qScale = tuneAbrQScaleFromFeedback(qScale);
-            if (!m_isAbrReset)
+            lmin = m_lastQScaleFor[P_SLICE] / m_lstep;
+            lmax = m_lastQScaleFor[P_SLICE] * m_lstep;
+            if (m_isCbr)
             {
-                lmin = m_lastQScaleFor[P_SLICE] / m_lstep;
-                lmax = m_lastQScaleFor[P_SLICE] * m_lstep;
-                qScale = x265_clip3(lmin, lmax, qScale);
+                qScale = tuneAbrQScaleFromFeedback(qScale);
+                if (!m_isAbrReset)
+                    qScale = x265_clip3(lmin, lmax, qScale);
+                q = x265_qScale2qp(qScale);
             }
-            q = x265_qScale2qp(qScale);
+            rce->qpNoVbv = q;
+            if (!m_2pass)
+            {
+                qScale = clipQscale(curFrame, rce, qScale);
+                /*  clip qp to permissible range after vbv-lookahead estimation to avoid possible 
+                 * mispredictions by initial frame size predictors */
+       //         if (m_pred[m_sliceType].count == 1)
+        //            qScale = x265_clip3(lmin, lmax, qScale);
+                m_lastQScaleFor[m_sliceType] = qScale;
+                rce->frameSizePlanned = predictSize(&m_pred[m_sliceType], qScale, (double)m_currentSatd);
+            }
+            else
+            {
+                rce->frameSizePlanned = qScale2bits(rce, qScale);
+            }
+            /* Limit planned size by MinCR */
+            rce->frameSizePlanned = X265_MIN(rce->frameSizePlanned, rce->frameSizeMaximum);
+            rce->frameSizeEstimated = rce->frameSizePlanned;
         }
-        rce->qpNoVbv = q;
-        if (!m_2pass && m_isVbv)
-        {
-            qScale = clipQscale(curFrame, rce, qScale);
-            /*  clip qp to permissible range after vbv-lookahead estimation to avoid possible 
-             * mispredictions by initial frame size predictors */
-            if (m_pred[m_sliceType].count == 1)
-                qScale = x265_clip3(lmin, lmax, qScale);
-            m_lastQScaleFor[m_sliceType] = qScale;
-            rce->frameSizePlanned = predictSize(&m_pred[m_sliceType], qScale, (double)m_currentSatd);
-        }
-        else if (m_2pass && m_isVbv)
-        {
-            rce->frameSizePlanned = qScale2bits(rce, qScale);
-        }
-        /* Limit planned size by MinCR */
-        if (m_isVbv)
-            rce->frameSizePlanned = X265_MIN(rce->frameSizePlanned, rce->frameSizeMaximum);
-        rce->frameSizeEstimated = rce->frameSizePlanned;
         rce->newQScale = qScale;
         return qScale;
     }