[x265] [PATCH] rc: fix chroma qp and chroma lambda derivations

[aarthi at multicorewareinc.com]

# HG changeset patch
# User Aarthi Thirumalai
# Date 1417675781 -19800
#      Thu Dec 04 12:19:41 2014 +0530
# Node ID 97f0f0e5d7efd1f883283617f657875340f381a8
# Parent  511dde5ac1deee96a0105bb87e14670ef5ed72a6
rc: fix chroma qp and chroma lambda derivations.

fix the chroma qp values for Main10 profile, derive chroma qp from luma qp values
according to the HEVC spec.
improves quality at high qps.

diff -r 511dde5ac1de -r 97f0f0e5d7ef source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Thu Dec 04 12:43:06 2014 +0530
+++ b/source/encoder/rdcost.h	Thu Dec 04 12:19:41 2014 +0530
@@ -50,19 +50,24 @@
     void setQP(const Slice& slice, int qp)
     {
         m_qp = qp;
-
+        int qpCb, qpCr;
         /* Scale PSY RD factor by a slice type factor */
         static const uint32_t psyScaleFix8[3] = { 300, 256, 96 }; /* B, P, I */
         m_psyRd = (m_psyRdBase * psyScaleFix8[slice.m_sliceType]) >> 8;
 
         setLambda(x265_lambda2_tab[qp], x265_lambda_tab[qp]);
-
-        int qpCb = Clip3(QP_MIN, QP_MAX_MAX, qp + slice.m_pps->chromaQpOffset[0]);
+        if (slice.m_sps->chromaFormatIdc == X265_CSP_I420)
+            qpCb = Clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice.m_pps->chromaQpOffset[0]]);
+        else
+            qpCb = X265_MIN(qp + slice.m_pps->chromaQpOffset[0], QP_MAX_SPEC);
         int chroma_offset_idx = X265_MIN(qp - qpCb + 12, MAX_CHROMA_LAMBDA_OFFSET);
         uint16_t lambdaOffset = m_psyRd ? x265_chroma_lambda2_offset_tab[chroma_offset_idx] : 256;
         setCbDistortionWeight(lambdaOffset);
 
-        int qpCr = Clip3(QP_MIN, QP_MAX_MAX, qp + slice.m_pps->chromaQpOffset[1]);
+        if (slice.m_sps->chromaFormatIdc == X265_CSP_I420)
+            qpCr = Clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice.m_pps->chromaQpOffset[0]]);
+        else
+            qpCr = X265_MIN(qp + slice.m_pps->chromaQpOffset[0], QP_MAX_SPEC);
         chroma_offset_idx = X265_MIN(qp - qpCr + 12, MAX_CHROMA_LAMBDA_OFFSET);
         lambdaOffset = m_psyRd ? x265_chroma_lambda2_offset_tab[chroma_offset_idx] : 256;
         setCrDistortionWeight(lambdaOffset);
diff -r 511dde5ac1de -r 97f0f0e5d7ef source/encoder/sao.cpp
--- a/source/encoder/sao.cpp	Thu Dec 04 12:43:06 2014 +0530
+++ b/source/encoder/sao.cpp	Thu Dec 04 12:19:41 2014 +0530
@@ -176,8 +176,11 @@
 void SAO::startSlice(Frame* frame, Entropy& initState, int qp)
 {
     Slice* slice = frame->m_encData->m_slice;
-
-    int qpCb = Clip3(0, QP_MAX_MAX, qp + slice->m_pps->chromaQpOffset[0]);
+    int qpCb = qp;
+    if (m_param->internalCsp == X265_CSP_I420)
+        qpCb = Clip3(QP_MIN, QP_MAX_MAX, (int)g_chromaScale[qp + slice->m_pps->chromaQpOffset[0]]);
+    else
+        qpCb = X265_MIN(qp + slice->m_pps->chromaQpOffset[0], QP_MAX_SPEC);
     m_lumaLambda = x265_lambda2_tab[qp];
     m_chromaLambda = x265_lambda2_tab[qpCb]; // Use Cb QP for SAO chroma
     m_frame = frame;