[x265-commits] [x265] fix CHECKED_BUILD

[Satoshi Nakagawa]

details:   http://hg.videolan.org/x265/rev/845e82c5d607
branches:  
changeset: 7987:845e82c5d607
user:      Satoshi Nakagawa <nakagawa424 at oki.com>
date:      Sun Sep 07 12:35:21 2014 +0900
description:
fix CHECKED_BUILD
Subject: [x265] fix sao

details:   http://hg.videolan.org/x265/rev/8cbfec8d6b4d
branches:  
changeset: 7988:8cbfec8d6b4d
user:      Satoshi Nakagawa <nakagawa424 at oki.com>
date:      Sun Sep 07 12:54:27 2014 +0900
description:
fix sao

diffstat:

 source/common/dct.cpp       |   2 +-
 source/encoder/analysis.cpp |  15 ++++++++++--
 source/encoder/sao.cpp      |  51 +++++++++++++++++++++-----------------------
 3 files changed, 37 insertions(+), 31 deletions(-)

diffs (130 lines):

diff -r ed4c9acafc11 -r 8cbfec8d6b4d source/common/dct.cpp
--- a/source/common/dct.cpp	Fri Sep 05 17:36:18 2014 -0700
+++ b/source/common/dct.cpp	Sun Sep 07 12:54:27 2014 +0900
@@ -729,7 +729,7 @@ void dequant_normal_c(const int16_t* qua
     X265_CHECK(num <= 32 * 32, "dequant num %d too large\n", num);
     X265_CHECK((num % 8) == 0, "dequant num %d not multiple of 8\n", num);
     X265_CHECK(shift <= 10, "shift too large %d\n", shift);
-    X265_CHECK(((int)coef & 31) == 0, "dequant coef buffer not aligned\n");
+    X265_CHECK(((intptr_t)coef & 31) == 0, "dequant coef buffer not aligned\n");
 
     int add, coeffQ;
 
diff -r ed4c9acafc11 -r 8cbfec8d6b4d source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Fri Sep 05 17:36:18 2014 -0700
+++ b/source/encoder/analysis.cpp	Sun Sep 07 12:54:27 2014 +0900
@@ -1056,21 +1056,30 @@ void Analysis::compressInterCU_rd0_4(TCo
             copyYuv2Pic(pic, outBestCU->getAddr(), absPartIdx, depth);
     }
 
+#if CHECKED_BUILD || _DEBUG
     /* Assert if Best prediction mode is NONE
      * Selected mode's RD-cost must be not MAX_INT64 */
     if (bInsidePicture)
     {
         X265_CHECK(outBestCU->getPartitionSize(0) != SIZE_NONE, "no best prediction size\n");
         X265_CHECK(outBestCU->getPredictionMode(0) != MODE_NONE, "no best prediction mode\n");
-        if (m_rdCost.m_psyRd)
+        if (m_param->rdLevel > 1)
         {
-            X265_CHECK(outBestCU->m_totalPsyCost != MAX_INT64, "no best partition cost\n");
+            if (m_rdCost.m_psyRd)
+            {
+                X265_CHECK(outBestCU->m_totalPsyCost != MAX_INT64, "no best partition cost\n");
+            }
+            else
+            {
+                X265_CHECK(outBestCU->m_totalRDCost != MAX_INT64, "no best partition cost\n");
+            }
         }
         else
         {
-            X265_CHECK(outBestCU->m_totalRDCost != MAX_INT64, "no best partition cost\n");
+            X265_CHECK(outBestCU->m_sa8dCost != MAX_INT64, "no best partition cost\n");
         }
     }
+#endif
 
     x265_emms();
 }
diff -r ed4c9acafc11 -r 8cbfec8d6b4d source/encoder/sao.cpp
--- a/source/encoder/sao.cpp	Fri Sep 05 17:36:18 2014 -0700
+++ b/source/encoder/sao.cpp	Sun Sep 07 12:54:27 2014 +0900
@@ -1481,14 +1481,10 @@ void SAO::calcSaoStatsCu_BeforeDblk(Fram
     pixel* fenc;
     pixel* recon;
     int stride;
-    int lcuHeight;
-    int lcuWidth;
     uint32_t rPelX;
     uint32_t bPelY;
     int64_t* stats;
     int64_t* count;
-    uint32_t picWidthTmp = 0;
-    uint32_t picHeightTmp = 0;
     int classIdx;
     int startX;
     int startY;
@@ -1510,38 +1506,39 @@ void SAO::calcSaoStatsCu_BeforeDblk(Fram
     {
         // NOTE: Col
         {
-            lcuHeight = g_maxCUSize;
-            lcuWidth  = g_maxCUSize;
             addr    = idxX + frameWidthInCU * idxY;
             cu      = pic->getCU(addr);
+
+            uint32_t picWidthTmp  = m_param->sourceWidth;
+            uint32_t picHeightTmp = m_param->sourceHeight;
+            int lcuWidth  = g_maxCUSize;
+            int lcuHeight = g_maxCUSize;
             lPelX   = cu->getCUPelX();
             tPelY   = cu->getCUPelY();
-
-            memset(m_countPreDblk[addr], 0, 3 * MAX_NUM_SAO_TYPE * MAX_NUM_SAO_CLASS * sizeof(int64_t));
-            memset(m_offsetOrgPreDblk[addr], 0, 3 * MAX_NUM_SAO_TYPE * MAX_NUM_SAO_CLASS * sizeof(int64_t));
+            rPelX     = lPelX + lcuWidth;
+            bPelY     = tPelY + lcuHeight;
+            rPelX     = rPelX > picWidthTmp  ? picWidthTmp  : rPelX;
+            bPelY     = bPelY > picHeightTmp ? picHeightTmp : bPelY;
+            lcuWidth  = rPelX - lPelX;
+            lcuHeight = bPelY - tPelY;
+
+            memset(m_countPreDblk[addr], 0, sizeof(PerPlane));
+            memset(m_offsetOrgPreDblk[addr], 0, sizeof(PerPlane));
+
             for (int plane = 0; plane < 3; plane++)
             {
                 isChroma = !!plane;
-                if (plane == 0)
+                if (plane == 1)
                 {
-                    picWidthTmp  = m_param->sourceWidth;
-                    picHeightTmp = m_param->sourceHeight;
+                    picWidthTmp  >>= m_hChromaShift;
+                    picHeightTmp >>= m_vChromaShift;
+                    lcuWidth     >>= m_hChromaShift;
+                    lcuHeight    >>= m_vChromaShift;
+                    lPelX        >>= m_hChromaShift;
+                    tPelY        >>= m_vChromaShift;
+                    rPelX     = lPelX + lcuWidth;
+                    bPelY     = tPelY + lcuHeight;
                 }
-                else if (plane == 1)
-                {
-                    picWidthTmp  = m_param->sourceWidth  >> isChroma;
-                    picHeightTmp = m_param->sourceHeight >> isChroma;
-                    lcuWidth     = lcuWidth    >> isChroma;
-                    lcuHeight    = lcuHeight   >> isChroma;
-                    lPelX        = lPelX       >> isChroma;
-                    tPelY        = tPelY       >> isChroma;
-                }
-                rPelX     = lPelX + lcuWidth;
-                bPelY     = tPelY + lcuHeight;
-                rPelX     = rPelX > picWidthTmp  ? picWidthTmp  : rPelX;
-                bPelY     = bPelY > picHeightTmp ? picHeightTmp : bPelY;
-                lcuWidth  = rPelX - lPelX;
-                lcuHeight = bPelY - tPelY;
 
                 stride   = (plane == 0) ? pic->getStride() : pic->getCStride();