[x265] [PATCH] rdcost: use a more gradual fall-off function for psy-rd at high QP

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1422736037 21600
#      Sat Jan 31 14:27:17 2015 -0600
# Branch stable
# Node ID 129e79064b03bc9913627c99f55c4e6e0b042078
# Parent  e2c958ff874e2bf8992ba22605e993530e8a2d8c
rdcost: use a more gradual fall-off function for psy-rd at high QP

This algorithm results in:

qp 39 psyRd 256
qp 40 psyRd 253
qp 41 psyRd 227
qp 42 psyRd 183
qp 43 psyRd 131
qp 44 psyRd 82
qp 45 psyRd 44
qp 46 psyRd 19
qp 47 psyRd 6
qp 48 psyRd 1
qp 49 psyRd 0
qp 50 psyRd 0
qp 51 psyRd 0

diff -r e2c958ff874e -r 129e79064b03 source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Sat Jan 31 13:48:34 2015 -0600
+++ b/source/encoder/rdcost.h	Sat Jan 31 14:27:17 2015 -0600
@@ -47,18 +47,19 @@
     void setQP(const Slice& slice, int qp)
     {
         m_qp = qp;
-        int qpCb, qpCr;
 
         /* Scale PSY RD factor by a slice type factor */
         static const uint32_t psyScaleFix8[3] = { 300, 256, 96 }; /* B, P, I */
         m_psyRd = (m_psyRdBase * psyScaleFix8[slice.m_sliceType]) >> 8;
 
         /* Scale PSY RD factor by QP, at high QP psy-rd can cause artifacts */
-        if (qp >= 50)
-            m_psyRd = 0;
-        else if (qp >= 42)
-            m_psyRd >>= 1;
+        if (qp >= 40)
+        {
+            int scale = qp >= QP_MAX_SPEC ? 0 : (QP_MAX_SPEC - qp) * 23;
+            m_psyRd = (m_psyRd * scale) >> 8;
+        }
 
+        int qpCb, qpCr;
         setLambda(x265_lambda2_tab[qp], x265_lambda_tab[qp]);
         if (slice.m_sps->chromaFormatIdc == X265_CSP_I420)
             qpCb = x265_clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice.m_pps->chromaQpOffset[0]]);