[x265] [PATCH] analysis mode: fix encoder crash

[kavitha at multicorewareinc.com]

# HG changeset patch
# User Kavitha Sampath <kavitha at multicorewareinc.com>
# Date 1452231216 -19800
#      Fri Jan 08 11:03:36 2016 +0530
# Node ID b7cb7195ba2f14dad3d0b5c2a834ce29caacaed7
# Parent  c7bcea62a5711d6e1071a76764f7e100f34aa00f
analysis mode: fix encoder crash

diff -r c7bcea62a571 -r b7cb7195ba2f source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Thu Jan 07 09:57:36 2016 +0530
+++ b/source/encoder/analysis.cpp	Fri Jan 08 11:03:36 2016 +0530
@@ -1397,12 +1397,6 @@
             md.pred[PRED_MERGE].cu.initSubCU(parentCTU, cuGeom, qp);
             checkMerge2Nx2N_rd5_6(md.pred[PRED_SKIP], md.pred[PRED_MERGE], cuGeom, true);
 
-            if (m_bTryLossless)
-                tryLossless(cuGeom);
-
-            if (mightSplit)
-                addSplitFlagCost(*md.bestMode, cuGeom.depth);
-
             // increment zOrder offset to point to next best depth in sharedDepth buffer
             zOrder += g_depthInc[g_maxCUDepth - 1][reuseDepth[zOrder]];
 
@@ -1417,7 +1411,7 @@
     splitData[3].initSplitCUData();
 
     /* Step 1. Evaluate Merge/Skip candidates for likely early-outs */
-    if (mightNotSplit)
+    if (mightNotSplit && !foundSkip)
     {
         md.pred[PRED_SKIP].cu.initSubCU(parentCTU, cuGeom, qp);
         md.pred[PRED_MERGE].cu.initSubCU(parentCTU, cuGeom, qp);
@@ -2138,10 +2132,11 @@
             MotionData* bestME = interMode.bestME[puIdx];
             for (int32_t i = 0; i < numPredDir; i++)
             {
+                if (bestME[i].ref >= 0)
+                    *m_reuseMv = getLowresMV(interMode.cu, pu, i, bestME[i].ref);
+
                 *m_reuseRef = bestME[i].ref;
                 m_reuseRef++;
-
-                *m_reuseMv = getLowresMV(interMode.cu, pu, i, bestME[i].ref);
                 m_reuseMv++;
             }
         }
@@ -2185,10 +2180,11 @@
             MotionData* bestME = interMode.bestME[puIdx];
             for (int32_t i = 0; i < numPredDir; i++)
             {
+                if (bestME[i].ref >= 0)
+                    *m_reuseMv = getLowresMV(interMode.cu, pu, i, bestME[i].ref);
+
                 *m_reuseRef = bestME[i].ref;
                 m_reuseRef++;
-
-                *m_reuseMv = getLowresMV(interMode.cu, pu, i, bestME[i].ref);
                 m_reuseMv++;
             }
         }
diff -r c7bcea62a571 -r b7cb7195ba2f source/encoder/search.cpp
--- a/source/encoder/search.cpp	Thu Jan 07 09:57:36 2016 +0530
+++ b/source/encoder/search.cpp	Fri Jan 08 11:03:36 2016 +0530
@@ -2086,11 +2086,14 @@
         cu.getNeighbourMV(puIdx, pu.puAbsPartIdx, interMode.interNeighbours);
 
         /* Uni-directional prediction */
-        if (m_param->analysisMode == X265_ANALYSIS_LOAD && bestME[0].ref >= 0)
+        if (m_param->analysisMode == X265_ANALYSIS_LOAD)
         {
             for (int list = 0; list < numPredDir; list++)
             {
                 int ref = bestME[list].ref;
+                if (ref < 0)
+                    continue;
+
                 uint32_t bits = m_listSelBits[list] + MVP_IDX_BITS;
                 bits += getTUBits(ref, numRefIdx[list]);
 
@@ -2108,7 +2111,8 @@
 
                 /* Get total cost of partition, but only include MV bit cost once */
                 bits += m_me.bitcost(outmv);
-                uint32_t cost = (satdCost - m_me.mvcost(outmv)) + m_rdCost.getCost(bits);
+                uint32_t mvCost = m_me.mvcost(outmv);
+                uint32_t cost = (satdCost - mvCost) + m_rdCost.getCost(bits);
 
                 /* Refine MVP selection, updates: mvpIdx, bits, cost */
                 mvp = checkBestMVP(amvp, outmv, mvpIdx, bits, cost);
@@ -2120,6 +2124,7 @@
                     bestME[list].mvpIdx = mvpIdx;
                     bestME[list].cost = cost;
                     bestME[list].bits = bits;
+                    bestME[list].mvCost  = mvCost;
                 }
             }
             bDoUnidir = false;
@@ -2170,6 +2175,7 @@
         }
         if (bDoUnidir)
         {
+            interMode.bestME[puIdx][0].ref = interMode.bestME[puIdx][1].ref = -1;
             uint32_t refMask = refMasks[puIdx] ? refMasks[puIdx] : (uint32_t)-1;
 
             for (int list = 0; list < numPredDir; list++)
diff -r c7bcea62a571 -r b7cb7195ba2f source/encoder/search.h
--- a/source/encoder/search.h	Thu Jan 07 09:57:36 2016 +0530
+++ b/source/encoder/search.h	Fri Jan 08 11:03:36 2016 +0530
@@ -88,6 +88,11 @@
     int      bits;
     uint32_t mvCost;
     uint32_t cost;
+
+    MotionData()
+    {
+        memset(this, 0, sizeof(MotionData));
+    }
 };
 
 struct Mode