[x265] [PATCH] analysis: Fix blocking artifacts while sharing best merge candidates

[gopu at multicorewareinc.com]

# HG changeset patch
# User Gopu Govindaswamy <gopu at multicorewareinc.com>
# Date 1434098206 -19800
#      Fri Jun 12 14:06:46 2015 +0530
# Node ID e0327556bbcda705af8689ec0a77206599f5de61
# Parent  2cd9183df03edff0b148bab6e133dfe1ae4f69a1
analysis: Fix blocking artifacts while sharing best merge candidates

While ecoding with --analysis-mode=load mode noticed there will blocking
artifacts, this issue is while sharing best merge candidate,

when sharing the merge candidate before selecting the best mode (SKIP MODE),
first encode merge with residual (no skip) and if this mode hasCbf then try
merge without residual(skip) then select the best mode (SKIP MODE) from this two

diff -r 2cd9183df03e -r e0327556bbcd source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Thu Jun 11 17:06:46 2015 +0530
+++ b/source/encoder/analysis.cpp	Fri Jun 12 14:06:46 2015 +0530
@@ -1455,19 +1455,50 @@
     bool foundCbf0Merge = false;
     bool triedPZero = false, triedBZero = false;
     bestPred->rdCost = MAX_INT64;
+    uint8_t hasCbf = true;
+    bool swapped = false;
 
     if (isSkipMode)
     {
         uint32_t i = *m_reuseBestMergeCand;
-        bestPred->cu.m_mvpIdx[0][0] = (uint8_t)i;
-        bestPred->cu.m_interDir[0] = candDir[i];
-        bestPred->cu.m_mv[0][0] = candMvField[i][0].mv;
-        bestPred->cu.m_mv[1][0] = candMvField[i][1].mv;
-        bestPred->cu.m_refIdx[0][0] = (int8_t)candMvField[i][0].refIdx;
-        bestPred->cu.m_refIdx[1][0] = (int8_t)candMvField[i][1].refIdx;
+        tempPred->cu.m_mvpIdx[0][0] = (uint8_t)i;    /* merge candidate ID is stored in L0 MVP idx */
+        tempPred->cu.m_interDir[0] = candDir[i];
+        tempPred->cu.m_mv[0][0] = candMvField[i][0].mv;
+        tempPred->cu.m_mv[1][0] = candMvField[i][1].mv;
+        tempPred->cu.m_refIdx[0][0] = (int8_t)candMvField[i][0].refIdx;
+        tempPred->cu.m_refIdx[1][0] = (int8_t)candMvField[i][1].refIdx;
+        tempPred->cu.setPredModeSubParts(MODE_INTER); /* must be cleared between encode iterations */
 
-        motionCompensation(bestPred->cu, pu, bestPred->predYuv, true, true);
-        encodeResAndCalcRdSkipCU(*bestPred);
+        motionCompensation(tempPred->cu, pu, tempPred->predYuv, true, true);
+
+        /* if the best prediction has CBF (not a skip) then try merge with residual */
+        encodeResAndCalcRdInterCU(*tempPred, cuGeom);
+        if (tempPred->rdCost < bestPred->rdCost)
+        {
+            hasCbf = tempPred->cu.getQtRootCbf(0);
+            std::swap(tempPred, bestPred);
+            swapped = true;
+        }
+
+        if (!m_param->bLossless && hasCbf)
+        {
+            if (swapped)
+            {
+                tempPred->cu.m_mvpIdx[0][0] = (uint8_t)i;
+                tempPred->cu.m_interDir[0] = candDir[i];
+                tempPred->cu.m_mv[0][0] = candMvField[i][0].mv;
+                tempPred->cu.m_mv[1][0] = candMvField[i][1].mv;
+                tempPred->cu.m_refIdx[0][0] = (int8_t)candMvField[i][0].refIdx;
+                tempPred->cu.m_refIdx[1][0] = (int8_t)candMvField[i][1].refIdx;
+                tempPred->cu.setPredModeSubParts(MODE_INTER);
+                tempPred->predYuv.copyFromYuv(bestPred->predYuv);
+            }
+
+            /* try merge without residual (skip), if not lossless coding */
+            encodeResAndCalcRdSkipCU(*tempPred);
+            if (tempPred->rdCost < bestPred->rdCost)
+                std::swap(tempPred, bestPred);
+        }
     }
     else
     {
@@ -1504,8 +1535,6 @@
 
             motionCompensation(tempPred->cu, pu, tempPred->predYuv, true, true);
 
-            uint8_t hasCbf = true;
-            bool swapped = false;
             if (!foundCbf0Merge)
             {
                 /* if the best prediction has CBF (not a skip) then try merge with residual */