[x265] [PATCH] quant: rdoq can lookup QP in the lambda table itself [CHANGES OUTPUTS]

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1407090564 18000
#      Sun Aug 03 13:29:24 2014 -0500
# Node ID 9cef3d48555a64f96d4bb7491b4842f24ffab159
# Parent  01116c890510099031faf23a9a8b02361cbc8118
quant: rdoq can lookup QP in the lambda table itself [CHANGES OUTPUTS]

This fixes a somewhat subtle bug in that it was previously possible to
configure the QP used in quant without setting the lambdas, and vice-versa.

This changes outputs; was the old behavior wrong or is the new behavior wrong?
SSIM appears to be higher with this patch applied

diff -r 01116c890510 -r 9cef3d48555a source/Lib/TLibEncoder/TEncSearch.cpp
--- a/source/Lib/TLibEncoder/TEncSearch.cpp	Sun Aug 03 12:56:29 2014 -0500
+++ b/source/Lib/TLibEncoder/TEncSearch.cpp	Sun Aug 03 13:29:24 2014 -0500
@@ -85,7 +85,7 @@
 bool TEncSearch::initSearch(Encoder& top)
 {
     m_param = top.m_param;
-    bool ok = m_quant.init(top.m_bEnableRDOQ, m_param->psyRdoq, top.m_scalingList);
+    bool ok = m_quant.init(top.m_bEnableRDOQ, m_param->psyRdoq, top.m_scalingList, m_csp);
 
     m_rdCost.setPsyRdScale(m_param->psyRd);
     m_bEnableRDOQ = top.m_bEnableRDOQ;
@@ -130,12 +130,12 @@
 void TEncSearch::setQP(int qp, int qpCb, int qpCr)
 {
     double lambda2 = x265_lambda2_tab[qp];
-    double lambdaCb = x265_lambda2_tab[qpCb];
-    double lambdaCr = x265_lambda2_tab[qpCr];
 
     m_me.setQP(qp);
-    m_quant.setLambdas(lambda2, lambdaCb, lambdaCr);
+    m_quant.setQPforQuant(qp, qpCb, qpCr);
+
     m_rdCost.setLambda(lambda2, x265_lambda_tab[qp]);
+
     int chroma_offset_idx = X265_MIN(qp - qpCb + 12, MAX_CHROMA_LAMBDA_OFFSET);
     uint16_t lambdaOffset = m_rdCost.m_psyRd ? x265_chroma_lambda2_offset_tab[chroma_offset_idx] : 256;
     m_rdCost.setCbDistortionWeight(lambdaOffset);
diff -r 01116c890510 -r 9cef3d48555a source/common/quant.cpp
--- a/source/common/quant.cpp	Sun Aug 03 12:56:29 2014 -0500
+++ b/source/common/quant.cpp	Sun Aug 03 13:29:24 2014 -0500
@@ -170,9 +170,10 @@
     m_fencShortBuf = NULL;
 }
 
-bool Quant::init(bool useRDOQ, double psyScale, const ScalingList& scalingList)
+bool Quant::init(bool useRDOQ, double psyScale, const ScalingList& scalingList, int chfmt)
 {
     m_useRDOQ = useRDOQ;
+    m_chFmt = chfmt;
     m_psyRdoqScale = (uint64_t)(psyScale * 256.0);
     m_scalingList = &scalingList;
     m_resiDctCoeff = X265_MALLOC(coeff_t, MAX_TR_SIZE * MAX_TR_SIZE * 2);
@@ -191,21 +192,25 @@
 void Quant::setQPforQuant(TComDataCU* cu)
 {
     int qpy = cu->getQP(0);
-    int chFmt = cu->getChromaFormat();
 
     m_qpParam[TEXT_LUMA].setQpParam(qpy + QP_BD_OFFSET);
-    setQPforQuant(qpy, TEXT_CHROMA_U, cu->m_slice->m_pps->chromaCbQpOffset, chFmt);
-    setQPforQuant(qpy, TEXT_CHROMA_V, cu->m_slice->m_pps->chromaCrQpOffset, chFmt);
+    setChromaQP(qpy + cu->m_slice->m_pps->chromaCbQpOffset, TEXT_CHROMA_U);
+    setChromaQP(qpy + cu->m_slice->m_pps->chromaCrQpOffset, TEXT_CHROMA_V);
 }
 
-void Quant::setQPforQuant(int qpy, TextType ttype, int chromaQPOffset, int chFmt)
+void Quant::setQPforQuant(int qpY, int qpCb, int qpCr)
 {
-    X265_CHECK(ttype == TEXT_CHROMA_U || ttype == TEXT_CHROMA_V, "invalid ttype\n");
+    m_qpParam[TEXT_LUMA].setQpParam(qpY + QP_BD_OFFSET);
+    setChromaQP(qpCb, TEXT_CHROMA_U);
+    setChromaQP(qpCr, TEXT_CHROMA_V);
+}
 
-    int qp = Clip3(-QP_BD_OFFSET, 57, qpy + chromaQPOffset);
+void Quant::setChromaQP(int qp, TextType ttype)
+{
+    qp = Clip3(-QP_BD_OFFSET, 57, qp);
     if (qp >= 30)
     {
-        if (chFmt == X265_CSP_I420)
+        if (m_chFmt == X265_CSP_I420)
             qp = g_chromaScale[qp];
         else
             qp = X265_MIN(qp, 51);
@@ -515,7 +520,7 @@
     int unquantRound = 1 << (unquantShift - 1);
     int scaleBits = SCALE_BITS - 2 * transformShift;
 
-    double lambda2 = m_lambdas[ttype];
+    double lambda2 = x265_lambda2_tab[m_qpParam[ttype].qp];
     double *errScale = m_scalingList->m_errScale[log2TrSize - 2][scalingListType][rem];
     bool bIsLuma = ttype == TEXT_LUMA;
     bool usePsy = m_psyRdoqScale && bIsLuma;
diff -r 01116c890510 -r 9cef3d48555a source/common/quant.h
--- a/source/common/quant.h	Sun Aug 03 12:56:29 2014 -0500
+++ b/source/common/quant.h	Sun Aug 03 13:29:24 2014 -0500
@@ -80,11 +80,11 @@
     ~Quant();
 
     /* one-time setup */
-    bool init(bool useRDOQ, double scale, const ScalingList& scalingList);
+    bool init(bool useRDOQ, double scale, const ScalingList& scalingList, int chfmt);
 
     /* CU setup */
     void setQPforQuant(TComDataCU* cu);
-    void setLambdas(double lambdaY, double lambdaCb, double lambdaCr) { m_lambdas[0] = lambdaY; m_lambdas[1] = lambdaCb; m_lambdas[2] = lambdaCr; }
+    void setQPforQuant(int qpY, int qpCb, int qpCr);
 
     uint32_t transformNxN(TComDataCU* cu, pixel *fenc, uint32_t fencstride, int16_t* residual, uint32_t stride, coeff_t* coeff,
                           uint32_t log2TrSize, TextType ttype, uint32_t absPartIdx, bool useTransformSkip, bool curUseRDOQ);
@@ -97,17 +97,17 @@
     const ScalingList* m_scalingList;
 
     QpParam  m_qpParam[3];
-    double   m_lambdas[3];
 
     bool     m_useRDOQ;
+    int      m_chFmt;
     uint64_t m_psyRdoqScale;
     coeff_t* m_resiDctCoeff;
     coeff_t* m_fencDctCoeff;
     int16_t* m_fencShortBuf;
 
-    void setQPforQuant(int qpy, TextType ttype, int chromaQPOffset, int chFmt);
+protected:
 
-protected:
+    void setChromaQP(int qp, TextType ttype);
 
     enum { IEP_RATE = 32768 }; // cost of an equal probable bit