[x265] [PATCH V3] rdcost: consistent clipping for chroma QP, combine conditionals

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1430840579 18000
#      Tue May 05 10:42:59 2015 -0500
# Node ID f8335da1174cbf96f7046fd3480149c60b370010
# Parent  d984cc4216878e9f4ccf422678afea397102981f
rdcost: consistent clipping for chroma QP, combine conditionals

clip chroma QP to QP_MIN..QP_MAX_MAX prior to g_chromaScale lookup, clip non-
i420 chroma QP to QP_MIN..QP_MAX_SPEC directly. So in either case chroma QP ends
up in spec range but in the case if I420 the chroma QP is scaled into the spec
range via the lookup table.

diff -r d984cc421687 -r f8335da1174c source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Tue May 05 11:54:26 2015 +0530
+++ b/source/encoder/rdcost.h	Tue May 05 10:42:59 2015 -0500
@@ -48,6 +48,7 @@
     {
         x265_emms(); /* TODO: if the lambda tables were ints, this would not be necessary */
         m_qp = qp;
+        setLambda(x265_lambda2_tab[qp], x265_lambda_tab[qp]);
 
         /* Scale PSY RD factor by a slice type factor */
         static const uint32_t psyScaleFix8[3] = { 300, 256, 96 }; /* B, P, I */
@@ -61,19 +62,21 @@
         }
 
         int qpCb, qpCr;
-        setLambda(x265_lambda2_tab[qp], x265_lambda_tab[qp]);
         if (slice.m_sps->chromaFormatIdc == X265_CSP_I420)
-            qpCb = x265_clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice.m_pps->chromaQpOffset[0]]);
+        {
+            qpCb = (int)g_chromaScale[x265_clip3(QP_MIN, QP_MAX_MAX, qp + slice.m_pps->chromaQpOffset[0])];
+            qpCr = (int)g_chromaScale[x265_clip3(QP_MIN, QP_MAX_MAX, qp + slice.m_pps->chromaQpOffset[1])];
+        }
         else
-            qpCb = X265_MIN(qp + slice.m_pps->chromaQpOffset[0], QP_MAX_SPEC);
+        {
+            qpCb = x265_clip3(QP_MIN, QP_MAX_SPEC, qp + slice.m_pps->chromaQpOffset[0]);
+            qpCr = x265_clip3(QP_MIN, QP_MAX_SPEC, qp + slice.m_pps->chromaQpOffset[1]);
+        }
+
         int chroma_offset_idx = X265_MIN(qp - qpCb + 12, MAX_CHROMA_LAMBDA_OFFSET);
         uint16_t lambdaOffset = m_psyRd ? x265_chroma_lambda2_offset_tab[chroma_offset_idx] : 256;
         m_chromaDistWeight[0] = lambdaOffset;
 
-        if (slice.m_sps->chromaFormatIdc == X265_CSP_I420)
-            qpCr = x265_clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice.m_pps->chromaQpOffset[1]]);
-        else
-            qpCr = X265_MIN(qp + slice.m_pps->chromaQpOffset[1], QP_MAX_SPEC);
         chroma_offset_idx = X265_MIN(qp - qpCr + 12, MAX_CHROMA_LAMBDA_OFFSET);
         lambdaOffset = m_psyRd ? x265_chroma_lambda2_offset_tab[chroma_offset_idx] : 256;
         m_chromaDistWeight[1] = lambdaOffset;