[x264-devel] Fix regression in r2118

[Jason Garrett-Glaser]

x264 | branch: master | Jason Garrett-Glaser <jason at x264.com> | Tue Dec  6 14:39:21 2011 -0800| [0c7dab9c2a106ce3ee5d6ad7282afb49e1cc3954] | committer: Jason Garrett-Glaser

Fix regression in r2118
Broke trellis with i16x16 macroblocks.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=0c7dab9c2a106ce3ee5d6ad7282afb49e1cc3954
---

 encoder/rdo.c |   53 +++++++++++++++++++++++++++++++++--------------------
 1 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/encoder/rdo.c b/encoder/rdo.c
index 28461d6..2cd6110 100644
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -461,30 +461,43 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct,
     } level_tree[64*8*2];
     int i_levels_used = 1;
 
-    if( i_coefs == 64 )
+    if( dc )
     {
-        h->mc.memcpy_aligned( coefs, dct, sizeof(dctcoef)*64 );
-        if( !h->quantf.quant_8x8( dct, quant_mf, quant_bias ) )
-            return 0;
-        h->zigzagf.scan_8x8( quant_coefs, dct );
-    }
-    else if( i_coefs == 16 )
-    {
-        memcpy( coefs, dct, sizeof(dctcoef)*16 );
-        if( !h->quantf.quant_4x4( dct, quant_mf, quant_bias ) )
-            return 0;
-        h->zigzagf.scan_4x4( quant_coefs, dct );
+        if( i_coefs == 16 )
+        {
+            memcpy( coefs, dct, sizeof(dctcoef)*16 );
+            if( !h->quantf.quant_4x4_dc( dct, quant_mf[0] >> 1, quant_bias[0] << 1 ) )
+                return 0;
+            h->zigzagf.scan_4x4( quant_coefs, dct );
+        }
+        else
+        {
+            memcpy( coefs, dct, sizeof(dctcoef)*i_coefs );
+            int nz = h->quantf.quant_2x2_dc( &dct[0], quant_mf[0] >> 1, quant_bias[0] << 1 );
+            if( i_coefs == 8 )
+                nz |= h->quantf.quant_2x2_dc( &dct[4], quant_mf[0] >> 1, quant_bias[0] << 1 );
+            if( !nz )
+                return 0;
+            for( int i = 0; i < i_coefs; i++ )
+                quant_coefs[i] = dct[zigzag[i]];
+        }
     }
     else
     {
-        memcpy( coefs, dct, sizeof(dctcoef)*i_coefs );
-        int nz = h->quantf.quant_2x2_dc( &dct[0], quant_mf[0] >> 1, quant_bias[0] << 1 );
-        if( i_coefs == 8 )
-            nz |= h->quantf.quant_2x2_dc( &dct[4], quant_mf[0] >> 1, quant_bias[0] << 1 );
-        if( !nz )
-            return 0;
-        for( int i = 0; i < i_coefs; i++ )
-            quant_coefs[i] = dct[zigzag[i]];
+        if( i_coefs == 64 )
+        {
+            h->mc.memcpy_aligned( coefs, dct, sizeof(dctcoef)*64 );
+            if( !h->quantf.quant_8x8( dct, quant_mf, quant_bias ) )
+                return 0;
+            h->zigzagf.scan_8x8( quant_coefs, dct );
+        }
+        else //if( i_coefs == 16 )
+        {
+            memcpy( coefs, dct, sizeof(dctcoef)*16 );
+            if( !h->quantf.quant_4x4( dct, quant_mf, quant_bias ) )
+                return 0;
+            h->zigzagf.scan_4x4( quant_coefs, dct );
+        }
     }
 
     int i_last_nnz = h->quantf.coeff_last[ctx_block_cat]( quant_coefs+b_ac )+b_ac;