[x265] [PATCH] analysis: always configure quant QP directly after setting RD lambda

Steve Borho steve at borho.org
Sat Apr 25 08:50:53 CEST 2015


# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1429943995 18000
#      Sat Apr 25 01:39:55 2015 -0500
# Node ID 67b0c29c45e67da8045b720bc0d657a14f952c33
# Parent  318aa6a4eaf2d58cf7d223bf92448d94cc439a5b
analysis: always configure quant QP directly after setting RD lambda

Basically, everywhere we adjust or assign QP we set quant QP immediately. This
removes a great many ad-hoc calls to setQPforQuant() and hopefully makes it
impossible to miss quant being configured properly.

This patch fixes a layering violation where the frame encoder was setting the
RDO lambdas directly, but only when delta-QP was not enabled.

diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/common/quant.cpp
--- a/source/common/quant.cpp	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/common/quant.cpp	Sat Apr 25 01:39:55 2015 -0500
@@ -225,16 +225,15 @@
     X265_FREE(m_fencShortBuf);
 }
 
-void Quant::setQPforQuant(const CUData& cu)
+void Quant::setQPforQuant(const CUData& ctu, int qp)
 {
-    m_tqBypass = !!cu.m_tqBypass[0];
+    m_tqBypass = !!ctu.m_tqBypass[0];
     if (m_tqBypass)
         return;
-    m_nr = m_frameNr ? &m_frameNr[cu.m_encData->m_frameEncoderID] : NULL;
-    int qpy = cu.m_qp[0];
-    m_qpParam[TEXT_LUMA].setQpParam(qpy + QP_BD_OFFSET);
-    setChromaQP(qpy + cu.m_slice->m_pps->chromaQpOffset[0], TEXT_CHROMA_U, cu.m_chromaFormat);
-    setChromaQP(qpy + cu.m_slice->m_pps->chromaQpOffset[1], TEXT_CHROMA_V, cu.m_chromaFormat);
+    m_nr = m_frameNr ? &m_frameNr[ctu.m_encData->m_frameEncoderID] : NULL;
+    m_qpParam[TEXT_LUMA].setQpParam(qp + QP_BD_OFFSET);
+    setChromaQP(qp + ctu.m_slice->m_pps->chromaQpOffset[0], TEXT_CHROMA_U, ctu.m_chromaFormat);
+    setChromaQP(qp + ctu.m_slice->m_pps->chromaQpOffset[1], TEXT_CHROMA_V, ctu.m_chromaFormat);
 }
 
 void Quant::setChromaQP(int qpin, TextType ttype, int chFmt)
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/common/quant.h
--- a/source/common/quant.h	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/common/quant.h	Sat Apr 25 01:39:55 2015 -0500
@@ -103,7 +103,7 @@
     bool allocNoiseReduction(const x265_param& param);
 
     /* CU setup */
-    void setQPforQuant(const CUData& cu);
+    void setQPforQuant(const CUData& ctu, int qp);
 
     uint32_t transformNxN(const CUData& cu, const pixel* fenc, uint32_t fencStride, const int16_t* residual, uint32_t resiStride, coeff_t* coeff,
                           uint32_t log2TrSize, TextType ttype, uint32_t absPartIdx, bool useTransformSkip);
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/encoder/analysis.cpp	Sat Apr 25 01:39:55 2015 -0500
@@ -146,18 +146,16 @@
 
     if (m_slice->m_pps->bUseDQP)
     {
-        m_aqQP[0] = calculateQpforCuSize(ctu, cuGeom);
-        setLambdaFromQP(*m_slice, m_aqQP[0]);
-        m_aqQP[0] = x265_clip3(QP_MIN, QP_MAX_SPEC, m_aqQP[0]);
-        ctu.setQPSubParts((int8_t)m_aqQP[0], 0, 0);
+        m_aqQP[0] = setLambdaFromQP(ctu, calculateQpforCuSize(ctu, cuGeom));
 
         if (m_slice->m_pps->maxCuDQPDepth)
             initAqQPs(1, ctu, &cuGeom + 1);
     }
     else
-        m_aqQP[0] = m_slice->m_sliceQp;
+        /* adaptive quant disabled, CTU QP is always slice QP, and within spec range */
+        m_aqQP[0] = setLambdaFromQP(ctu, m_slice->m_sliceQp);
 
-    m_quant.setQPforQuant(ctu);
+    ctu.setQPSubParts((int8_t)m_aqQP[0], 0, 0);
     m_rqt[0].cur.load(initialContext);
     m_modeDepth[0].fencYuv.copyFromPicYuv(*m_frame->m_fencPic, ctu.m_cuAddr, 0);
 
@@ -269,7 +267,6 @@
             PartSize size = (PartSize)reusePartSizes[zOrder];
             Mode& mode = size == SIZE_2Nx2N ? md.pred[PRED_INTRA] : md.pred[PRED_INTRA_NxN];
             mode.cu.initSubCU(parentCTU, cuGeom, qp);
-            m_quant.setQPforQuant(mode.cu);
             checkIntra(mode, cuGeom, size, &reuseModes[zOrder], &reuseChromaModes[zOrder]);
             checkBestMode(mode, depth);
 
@@ -287,7 +284,6 @@
     else if (mightNotSplit)
     {
         md.pred[PRED_INTRA].cu.initSubCU(parentCTU, cuGeom, qp);
-        m_quant.setQPforQuant(md.pred[PRED_INTRA].cu);
         checkIntra(md.pred[PRED_INTRA], cuGeom, SIZE_2Nx2N, NULL, NULL);
         checkBestMode(md.pred[PRED_INTRA], depth);
 
@@ -327,11 +323,7 @@
                 m_rqt[nextDepth].cur.load(*nextContext);
 
                 if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
-                {
-                    nextQP = m_aqQP[childGeom.index];
-                    setLambdaFromQP(*m_slice, nextQP);
-                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
-                }
+                    nextQP = setLambdaFromQP(parentCTU, m_aqQP[childGeom.index]);
 
                 compressIntraCU(parentCTU, childGeom, zOrder, nextQP);
 
@@ -400,14 +392,9 @@
     {
         slave.m_slice = m_slice;
         slave.m_frame = m_frame;
-        slave.setLambdaFromQP(*m_slice, m_rdCost.m_qp);
+        slave.setLambdaFromQP(md.pred[PRED_2Nx2N].cu, m_rdCost.m_qp);
         slave.invalidateContexts(0);
-
-        if (m_param->rdLevel >= 5)
-        {
-            slave.m_rqt[pmode.cuGeom.depth].cur.load(m_rqt[pmode.cuGeom.depth].cur);
-            slave.m_quant.setQPforQuant(md.pred[PRED_2Nx2N].cu);
-        }
+        slave.m_rqt[pmode.cuGeom.depth].cur.load(m_rqt[pmode.cuGeom.depth].cur);
     }
 
     /* perform Mode task, repeat until no more work is available */
@@ -418,11 +405,6 @@
             switch (pmode.modes[task])
             {
             case PRED_INTRA:
-                if (&slave != this)
-                {
-                    slave.m_rqt[pmode.cuGeom.depth].cur.load(m_rqt[pmode.cuGeom.depth].cur);
-                    slave.m_quant.setQPforQuant(md.pred[PRED_INTRA].cu);
-                }
                 slave.checkIntraInInter(md.pred[PRED_INTRA], pmode.cuGeom);
                 if (m_param->rdLevel > 2)
                     slave.encodeIntraInInter(md.pred[PRED_INTRA], pmode.cuGeom);
@@ -738,11 +720,7 @@
                 m_rqt[nextDepth].cur.load(*nextContext);
 
                 if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
-                {
-                    nextQP = m_aqQP[childGeom.index];
-                    setLambdaFromQP(*m_slice, nextQP);
-                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
-                }
+                    nextQP = setLambdaFromQP(parentCTU, m_aqQP[childGeom.index]);
 
                 compressInterCU_dist(parentCTU, childGeom, nextQP);
 
@@ -943,7 +921,6 @@
                     {
                         /* generate recon pixels with no rate distortion considerations */
                         CUData& cu = md.bestMode->cu;
-                        m_quant.setQPforQuant(cu);
 
                         uint32_t tuDepthRange[2];
                         cu.getInterTUQtDepthRange(tuDepthRange, 0);
@@ -968,7 +945,6 @@
                     {
                         /* generate recon pixels with no rate distortion considerations */
                         CUData& cu = md.bestMode->cu;
-                        m_quant.setQPforQuant(cu);
 
                         uint32_t tuDepthRange[2];
                         cu.getIntraTUQtDepthRange(tuDepthRange, 0);
@@ -1019,11 +995,7 @@
                 m_rqt[nextDepth].cur.load(*nextContext);
 
                 if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
-                {
-                    nextQP = m_aqQP[childGeom.index];
-                    setLambdaFromQP(*m_slice, nextQP);
-                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
-                }
+                    nextQP = setLambdaFromQP(parentCTU, m_aqQP[childGeom.index]);
 
                 compressInterCU_rd0_4(parentCTU, childGeom, nextQP);
 
@@ -1225,11 +1197,7 @@
                 m_rqt[nextDepth].cur.load(*nextContext);
 
                 if (m_slice->m_pps->bUseDQP && nextDepth <= m_slice->m_pps->maxCuDQPDepth)
-                {
-                    nextQP = m_aqQP[childGeom.index];
-                    setLambdaFromQP(*m_slice, nextQP);
-                    nextQP = x265_clip3(QP_MIN, QP_MAX_SPEC, nextQP);
-                }
+                    nextQP = setLambdaFromQP(parentCTU, m_aqQP[childGeom.index]);
 
                 compressInterCU_rd5_6(parentCTU, childGeom, zOrder, nextQP);
 
@@ -1755,7 +1723,6 @@
     CUData& cu = bestMode->cu;
 
     cu.copyFromPic(ctu, cuGeom);
-    m_quant.setQPforQuant(cu);
 
     Yuv& fencYuv = m_modeDepth[cuGeom.depth].fencYuv;
     if (cuGeom.depth)
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/encoder/frameencoder.cpp
--- a/source/encoder/frameencoder.cpp	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/encoder/frameencoder.cpp	Sat Apr 25 01:39:55 2015 -0500
@@ -851,12 +851,11 @@
 
         if (m_param->rc.aqMode || bIsVbv)
         {
+            X265_CHECK(slice->m_pps->bUseDQP, "adaptive quant in use without DQP\n");
             int qp = calcQpForCu(cuAddr, curEncData.m_cuStat[cuAddr].baseQp);
             qp = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
             curEncData.m_rowStat[row].sumQpAq += qp;
         }
-        else
-            tld.analysis.setLambdaFromQP(*slice, slice->m_sliceQp);
 
         if (m_param->bEnableWavefront && !col && row)
         {
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/encoder/rdcost.h	Sat Apr 25 01:39:55 2015 -0500
@@ -40,12 +40,13 @@
     uint32_t  m_chromaDistWeight[2];
     uint32_t  m_psyRdBase;
     uint32_t  m_psyRd;
-    int       m_qp;
+    int       m_qp; /* QP used to configure lambda, may be higher than QP_MAX_SPEC but <= QP_MAX_MAX */
 
     void setPsyRdScale(double scale)                { m_psyRdBase = (uint32_t)floor(65536.0 * scale * 0.33); }
 
     void setQP(const Slice& slice, int qp)
     {
+        x265_emms(); /* TODO: if the lambda tables were ints, this would not be necessary */
         m_qp = qp;
 
         /* Scale PSY RD factor by a slice type factor */
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/encoder/search.cpp
--- a/source/encoder/search.cpp	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/encoder/search.cpp	Sat Apr 25 01:39:55 2015 -0500
@@ -163,11 +163,16 @@
     X265_FREE(m_tsRecon);
 }
 
-void Search::setLambdaFromQP(const Slice& slice, int qp)
+int Search::setLambdaFromQP(const CUData& ctu, int qp)
 {
-    x265_emms(); /* TODO: if the lambda tables were ints, this would not be necessary */
+    X265_CHECK(qp >= QP_MIN && qp <= QP_MAX_MAX, "QP used for lambda is out of range\n");
+
     m_me.setQP(qp);
-    m_rdCost.setQP(slice, qp);
+    m_rdCost.setQP(*m_slice, qp);
+
+    int quantQP = x265_clip3(QP_MIN, QP_MAX_SPEC, qp);
+    m_quant.setQPforQuant(ctu, quantQP);
+    return quantQP;
 }
 
 #if CHECKED_BUILD || _DEBUG
@@ -1364,8 +1369,6 @@
     X265_CHECK(cu.m_partSize[0] == SIZE_2Nx2N, "encodeIntraInInter does not expect NxN intra\n");
     X265_CHECK(!m_slice->isIntra(), "encodeIntraInInter does not expect to be used in I slices\n");
 
-    m_quant.setQPforQuant(cu);
-
     uint32_t tuDepthRange[2];
     cu.getIntraTUQtDepthRange(tuDepthRange, 0);
 
@@ -1888,7 +1891,7 @@
     /* Setup slave Search instance for ME for master's CU */
     if (&slave != this)
     {
-        slave.setLambdaFromQP(*m_slice, m_rdCost.m_qp);
+        slave.setLambdaFromQP(pme.mode.cu, m_rdCost.m_qp);
         slave.m_slice = m_slice;
         slave.m_frame = m_frame;
 
@@ -2502,9 +2505,6 @@
     uint32_t log2CUSize = cuGeom.log2CUSize;
     int sizeIdx = log2CUSize - 2;
 
-    uint32_t tqBypass = cu.m_tqBypass[0];
-    m_quant.setQPforQuant(interMode.cu);
-
     resiYuv->subtract(*fencYuv, *predYuv, log2CUSize);
 
     uint32_t tuDepthRange[2];
@@ -2515,6 +2515,7 @@
     Cost costs;
     estimateResidualQT(interMode, cuGeom, 0, 0, *resiYuv, costs, tuDepthRange);
 
+    uint32_t tqBypass = cu.m_tqBypass[0];
     if (!tqBypass)
     {
         uint32_t cbf0Dist = primitives.cu[sizeIdx].sse_pp(fencYuv->m_buf[0], fencYuv->m_size, predYuv->m_buf[0], predYuv->m_size);
diff -r 318aa6a4eaf2 -r 67b0c29c45e6 source/encoder/search.h
--- a/source/encoder/search.h	Sat Apr 25 00:41:25 2015 -0500
+++ b/source/encoder/search.h	Sat Apr 25 01:39:55 2015 -0500
@@ -287,7 +287,7 @@
     ~Search();
 
     bool     initSearch(const x265_param& param, ScalingList& scalingList);
-    void     setLambdaFromQP(const Slice& slice, int qp);
+    int      setLambdaFromQP(const CUData& ctu, int qp); /* returns real quant QP in valid spec range */
 
     // mark temp RD entropy contexts as uninitialized; useful for finding loads without stores
     void     invalidateContexts(int fromDepth);


More information about the x265-devel mailing list