[x265] [PATCH] rc: fix buffer underflow issue in vbv

[Steve Borho]

On 03/17, aarthi at multicorewareinc.com wrote:
> # HG changeset patch
> # User Aarthi Thirumalai
> # Date 1426612817 -19800
> #      Tue Mar 17 22:50:17 2015 +0530
> # Node ID 638a3af86fa12132aae47ed0d4b321ce6cc36ed2
> # Parent  cc579ffe4b7e0cd2b554a573ba70ff8e8c726d42
> rc: fix buffer underflow issue in vbv

queued for testing

> diff -r cc579ffe4b7e -r 638a3af86fa1 source/encoder/ratecontrol.cpp
> --- a/source/encoder/ratecontrol.cpp	Tue Mar 17 11:11:32 2015 +0530
> +++ b/source/encoder/ratecontrol.cpp	Tue Mar 17 22:50:17 2015 +0530
> @@ -1369,36 +1369,36 @@
>  
>          double qScale = x265_qp2qScale(q);
>          double lmin = 0, lmax = 0;
> -        if (m_isCbr)
> +        if (m_isVbv)
>          {
> -            qScale = tuneAbrQScaleFromFeedback(qScale);
> -            if (!m_isAbrReset)
> +            lmin = m_lastQScaleFor[P_SLICE] / m_lstep;
> +            lmax = m_lastQScaleFor[P_SLICE] * m_lstep;
> +            if (m_isCbr)
>              {
> -                lmin = m_lastQScaleFor[P_SLICE] / m_lstep;
> -                lmax = m_lastQScaleFor[P_SLICE] * m_lstep;
> -                qScale = x265_clip3(lmin, lmax, qScale);
> +                qScale = tuneAbrQScaleFromFeedback(qScale);
> +                if (!m_isAbrReset)
> +                    qScale = x265_clip3(lmin, lmax, qScale);
> +                q = x265_qScale2qp(qScale);
>              }
> -            q = x265_qScale2qp(qScale);
> +            rce->qpNoVbv = q;
> +            if (!m_2pass)
> +            {
> +                qScale = clipQscale(curFrame, rce, qScale);
> +                /*  clip qp to permissible range after vbv-lookahead estimation to avoid possible 
> +                 * mispredictions by initial frame size predictors */
> +       //         if (m_pred[m_sliceType].count == 1)
> +        //            qScale = x265_clip3(lmin, lmax, qScale);
> +                m_lastQScaleFor[m_sliceType] = qScale;
> +                rce->frameSizePlanned = predictSize(&m_pred[m_sliceType], qScale, (double)m_currentSatd);
> +            }
> +            else
> +            {
> +                rce->frameSizePlanned = qScale2bits(rce, qScale);
> +            }
> +            /* Limit planned size by MinCR */
> +            rce->frameSizePlanned = X265_MIN(rce->frameSizePlanned, rce->frameSizeMaximum);
> +            rce->frameSizeEstimated = rce->frameSizePlanned;
>          }
> -        rce->qpNoVbv = q;
> -        if (!m_2pass && m_isVbv)
> -        {
> -            qScale = clipQscale(curFrame, rce, qScale);
> -            /*  clip qp to permissible range after vbv-lookahead estimation to avoid possible 
> -             * mispredictions by initial frame size predictors */
> -            if (m_pred[m_sliceType].count == 1)
> -                qScale = x265_clip3(lmin, lmax, qScale);
> -            m_lastQScaleFor[m_sliceType] = qScale;
> -            rce->frameSizePlanned = predictSize(&m_pred[m_sliceType], qScale, (double)m_currentSatd);
> -        }
> -        else if (m_2pass && m_isVbv)
> -        {
> -            rce->frameSizePlanned = qScale2bits(rce, qScale);
> -        }
> -        /* Limit planned size by MinCR */
> -        if (m_isVbv)
> -            rce->frameSizePlanned = X265_MIN(rce->frameSizePlanned, rce->frameSizeMaximum);
> -        rce->frameSizeEstimated = rce->frameSizePlanned;
>          rce->newQScale = qScale;
>          return qScale;
>      }
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel

-- 
Steve Borho