[x265] [PATCH] rdcost: allow psy-rd factor to be adjusted by CU size [EXPERIMENTAL]

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1423085727 21600
#      Wed Feb 04 15:35:27 2015 -0600
# Node ID e5360f3e3172daa14a4753177203e74a7d44102f
# Parent  f8da0be4329f9e95a59d70d1c76628374b89bda4
rdcost: allow psy-rd factor to be adjusted by CU size [EXPERIMENTAL]

diff -r f8da0be4329f -r e5360f3e3172 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Wed Feb 04 15:43:39 2015 -0600
+++ b/source/encoder/analysis.cpp	Wed Feb 04 15:35:27 2015 -0600
@@ -221,6 +221,7 @@
 
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
+    m_rdCost.setCULog2Size(*parentCTU.m_slice, cuGeom.log2CUSize);
 
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
@@ -404,6 +405,7 @@
         slave->m_slice = m_slice;
         slave->m_frame = m_frame;
         slave->setQP(*m_slice, m_rdCost.m_qp);
+        slave->m_rdCost.setCULog2Size(*m_slice, m_curGeom->log2CUSize);
         slave->invalidateContexts(0);
     }
 
@@ -524,6 +526,7 @@
 
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
+    m_rdCost.setCULog2Size(*parentCTU.m_slice, cuGeom.log2CUSize);
     uint32_t minDepth = m_param->rdLevel <= 4 ? topSkipMinDepth(parentCTU, cuGeom) : 0;
 
     X265_CHECK(m_param->rdLevel >= 2, "compressInterCU_dist does not support RD 0 or 1\n");
@@ -797,6 +800,7 @@
 
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
+    m_rdCost.setCULog2Size(*parentCTU.m_slice, cuGeom.log2CUSize);
     uint32_t minDepth = topSkipMinDepth(parentCTU, cuGeom);
 
     if (mightNotSplit && depth >= minDepth)
@@ -1080,6 +1084,7 @@
 
     bool mightSplit = !(cuGeom.flags & CUGeom::LEAF);
     bool mightNotSplit = !(cuGeom.flags & CUGeom::SPLIT_MANDATORY);
+    m_rdCost.setCULog2Size(*parentCTU.m_slice, cuGeom.log2CUSize);
 
     if (m_param->analysisMode == X265_ANALYSIS_LOAD)
     {
diff -r f8da0be4329f -r e5360f3e3172 source/encoder/rdcost.h
--- a/source/encoder/rdcost.h	Wed Feb 04 15:43:39 2015 -0600
+++ b/source/encoder/rdcost.h	Wed Feb 04 15:35:27 2015 -0600
@@ -47,17 +47,7 @@
     void setQP(const Slice& slice, int qp)
     {
         m_qp = qp;
-
-        /* Scale PSY RD factor by a slice type factor */
-        static const uint32_t psyScaleFix8[3] = { 300, 256, 96 }; /* B, P, I */
-        m_psyRd = (m_psyRdBase * psyScaleFix8[slice.m_sliceType]) >> 8;
-
-        /* Scale PSY RD factor by QP, at high QP psy-rd can cause artifacts */
-        if (qp >= 40)
-        {
-            int scale = qp >= QP_MAX_SPEC ? 0 : (QP_MAX_SPEC - qp) * 23;
-            m_psyRd = (m_psyRd * scale) >> 8;
-        }
+        m_psyRd = 0;
 
         int qpCb, qpCr;
         setLambda(x265_lambda2_tab[qp], x265_lambda_tab[qp]);
@@ -78,6 +68,26 @@
         m_chromaDistWeight[1] = lambdaOffset;
     }
 
+    void setCULog2Size(const Slice& slice, uint32_t cuLog2Size)
+    {
+        /* Scale PSY RD factor by a slice type factor */
+        static const uint32_t psySliceScaleFix8[3] = { 300, 256, 96 }; /* B, P, I */
+        m_psyRd = (m_psyRdBase * psySliceScaleFix8[slice.m_sliceType]) >> 8;
+
+        /* Scale PSY RD factor by a CU size */
+        static const uint32_t psySizeScaleFix8[4] = { 96, 128, 256, 276 }; /* 64x64, 32x32, 16x16, 8x8 */
+        m_psyRd = (m_psyRd * psySizeScaleFix8[cuLog2Size - 2]) >> 8;
+
+        /* Scale PSY RD factor by QP, at high QP psy-rd can cause artifacts */
+        if (m_qp >= 40)
+        {
+            int scale = m_qp >= QP_MAX_SPEC ? 0 : (QP_MAX_SPEC - m_qp) * 23;
+            m_psyRd = (m_psyRd * scale) >> 8;
+        }
+
+        /* TODO: also scale lambda by depth? */
+    }
+
     void setLambda(double lambda2, double lambda)
     {
         m_lambda2 = (uint64_t)floor(256.0 * lambda2);
@@ -86,6 +96,7 @@
 
     inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits) const
     {
+        X265_CHECK(!m_psyRdBase, "setCULog2Size() was not called\n");
         X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambda2,
                    "calcRdCost wrap detected dist: %u, bits %u, lambda: "X265_LL"\n", distortion, bits, m_lambda2);
         return distortion + ((bits * m_lambda2 + 128) >> 8);