[x265] [PATCH] aq: set qp, lambda for every CU in the row before processing the CU

[Aarthi Thirumalai]

# HG changeset patch
# User Aarthi Thirumalai
# Date 1383219394 -19800
#      Thu Oct 31 17:06:34 2013 +0530
# Node ID 0e321e5dd34869c0054a5c218cd14e5dfd30fd38
# Parent  9a0da4e6d9e363e383eae7243f0c64026a5f6d00
aq: set qp, lambda for every CU in the row before processing the CU

enabled bUseDQP flag when AQ is mode is ON.

diff -r 9a0da4e6d9e3 -r 0e321e5dd348 source/Lib/TLibCommon/TComDataCU.cpp
--- a/source/Lib/TLibCommon/TComDataCU.cpp	Thu Oct 31 15:10:34 2013 +0530
+++ b/source/Lib/TLibCommon/TComDataCU.cpp	Thu Oct 31 17:06:34 2013 +0530
@@ -246,7 +246,7 @@
     m_totalDistortion  = 0;
     m_totalBits        = 0;
     m_numPartitions    = pic->getNumPartInCU();
-
+    int qp             = pic->m_lowres.m_invQscaleFactor ? pic->getCU(getAddr())->getQP(0) : m_slice->getSliceQp();
     for (int i = 0; i < 4; i++)
     {
         m_avgCost[i] = 0;
@@ -304,7 +304,7 @@
         memset(m_height           + firstElement, g_maxCUHeight,            numElements * sizeof(*m_height));
         memset(m_mvpNum[0]        + firstElement, -1,                       numElements * sizeof(*m_mvpNum[0]));
         memset(m_mvpNum[1]        + firstElement, -1,                       numElements * sizeof(*m_mvpNum[1]));
-        memset(m_qp               + firstElement, getSlice()->getSliceQp(), numElements * sizeof(*m_qp));
+        memset(m_qp               + firstElement, qp,                        numElements * sizeof(*m_qp));
         memset(m_bMergeFlags      + firstElement, false,                    numElements * sizeof(*m_bMergeFlags));
         memset(m_mergeIndex       + firstElement, 0,                        numElements * sizeof(*m_mergeIndex));
         memset(m_lumaIntraDir     + firstElement, DC_IDX,                   numElements * sizeof(*m_lumaIntraDir));
diff -r 9a0da4e6d9e3 -r 0e321e5dd348 source/Lib/TLibEncoder/TEncCu.cpp
--- a/source/Lib/TLibEncoder/TEncCu.cpp	Thu Oct 31 15:10:34 2013 +0530
+++ b/source/Lib/TLibEncoder/TEncCu.cpp	Thu Oct 31 17:06:34 2013 +0530
@@ -1691,7 +1691,8 @@
 
     if (cu->getSlice()->getPPS()->getUseDQP() && (g_maxCUWidth >> depth) >= cu->getSlice()->getPPS()->getMinCuDQPSize())
     {
-        cu->setQPSubParts(cu->getRefQP(0), 0, depth); // set QP to default QP
+        if (!cu->getCbf(0, TEXT_LUMA, 0) && !cu->getCbf(0, TEXT_CHROMA_U, 0) && !cu->getCbf(0, TEXT_CHROMA_V, 0))
+            cu->setQPSubParts(cu->getRefQP(0), 0, depth); // set QP to default QP
     }
 }
 
diff -r 9a0da4e6d9e3 -r 0e321e5dd348 source/encoder/encoder.cpp
--- a/source/encoder/encoder.cpp	Thu Oct 31 15:10:34 2013 +0530
+++ b/source/encoder/encoder.cpp	Thu Oct 31 17:06:34 2013 +0530
@@ -784,7 +784,7 @@
 void Encoder::initPPS(TComPPS *pps)
 {
     pps->setConstrainedIntraPred(param.bEnableConstrainedIntra);
-    bool bUseDQP = (getMaxCuDQPDepth() > 0) ? true : false;
+    bool bUseDQP = ((getMaxCuDQPDepth() > 0) || param.rc.aqMode) ? true : false;
 
     int lowestQP = -(6 * (X265_DEPTH - 8)); //m_cSPS.getQpBDOffsetY();
 
diff -r 9a0da4e6d9e3 -r 0e321e5dd348 source/encoder/frameencoder.cpp
--- a/source/encoder/frameencoder.cpp	Thu Oct 31 15:10:34 2013 +0530
+++ b/source/encoder/frameencoder.cpp	Thu Oct 31 17:06:34 2013 +0530
@@ -306,6 +306,40 @@
     while (m_threadActive);
 }
 
+void FrameEncoder::setLambda(int qp , int row)
+{
+    TComSlice*   slice        = m_pic->getSlice();
+    double lambda = 0;
+    if (m_pic->getSlice()->getSliceType() == I_SLICE)
+    {
+        lambda = X265_MAX(1, x265_lambda2_tab_I[qp]);
+    }
+    else
+    {
+        lambda = X265_MAX(1, x265_lambda2_non_I[qp]);
+    }
+
+    // for RDO
+    // in RdCost there is only one lambda because the luma and chroma bits are not separated,
+    // instead we weight the distortion of chroma.
+    int qpc;
+    int chromaQPOffset = slice->getPPS()->getChromaCbQpOffset() + slice->getSliceQpDeltaCb();
+    qpc = Clip3(0, 57, qp + chromaQPOffset);
+    double cbWeight = pow(2.0, (qp - g_chromaScale[qpc])); // takes into account of the chroma qp mapping and chroma qp Offset
+    chromaQPOffset = slice->getPPS()->getChromaCrQpOffset() + slice->getSliceQpDeltaCr();
+    qpc = Clip3(0, 57, qp + chromaQPOffset);
+    double crWeight = pow(2.0, (qp - g_chromaScale[qpc])); // takes into account of the chroma qp mapping and chroma qp Offset
+    double chromaLambda = lambda / crWeight;
+    TComPicYuv *fenc = slice->getPic()->getPicYuvOrg();
+    m_rows[row].m_search.setQPLambda(qp, lambda, chromaLambda);
+    m_rows[row].m_search.m_me.setSourcePlane(fenc->getLumaAddr(), fenc->getStride());
+    m_rows[row].m_rdCost.setLambda(lambda);
+    m_rows[row].m_rdCost.setCbDistortionWeight(cbWeight);
+    m_rows[row].m_rdCost.setCrDistortionWeight(crWeight);
+    m_frameFilter.m_sao.lumaLambda = lambda;
+    m_frameFilter.m_sao.chromaLambda = chromaLambda;
+}
+
 void FrameEncoder::compressFrame()
 {
     PPAScopeEvent(FrameEncoder_compressFrame);
@@ -1042,8 +1076,12 @@
         codeRow.m_entropyCoder.resetEntropy();
 
         TEncSbac *bufSbac = (m_cfg->param.bEnableWavefront && col == 0 && row > 0) ? &m_rows[row - 1].m_bufferSbacCoder : NULL;
-        int qp = calcQpForCu(m_pic, cuAddr);
-        cu->setQP(0,(char)qp);
+        if (m_cfg->param.rc.aqMode)
+        {
+            int qp = calcQpForCu(m_pic, cuAddr);
+            cu->setQP(0, (char)qp);
+            setLambda(qp, row);
+        }
         codeRow.processCU(cu, m_pic->getSlice(), bufSbac, m_cfg->param.bEnableWavefront && col == 1);
 
         // TODO: Keep atomic running totals for rate control?
diff -r 9a0da4e6d9e3 -r 0e321e5dd348 source/encoder/frameencoder.h
--- a/source/encoder/frameencoder.h	Thu Oct 31 15:10:34 2013 +0530
+++ b/source/encoder/frameencoder.h	Thu Oct 31 17:06:34 2013 +0530
@@ -149,7 +149,7 @@
 
     /* blocks until worker thread is done, returns encoded picture and bitstream */
     TComPic *getEncodedPicture(NALUnitEBSP **nalunits);
-
+    void setLambda(int qp , int row);
     // worker thread
     void threadMain();