[x264-devel] Fix trellis 2 + subme >= 8

[Loren Merritt]

x264 | branch: master | Loren Merritt <pengvado at akuvian.org> | Wed Jan 18 15:47:07 2012 -0800| [bcd41dbcaa4430b2118d9f6828c2b9635cf9d58d] | committer: Jason Garrett-Glaser

Fix trellis 2 + subme >= 8
Trellis didn't return a boolean value as it was supposed to.
Regression in r2143-5.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=bcd41dbcaa4430b2118d9f6828c2b9635cf9d58d
---

 common/x86/trellis-64.asm |    7 +++----
 encoder/rdo.c             |    3 ++-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/common/x86/trellis-64.asm b/common/x86/trellis-64.asm
index f0683bd..1543835 100644
--- a/common/x86/trellis-64.asm
+++ b/common/x86/trellis-64.asm
@@ -261,9 +261,7 @@ cglobal %1, 4,15,9
     inc    iiq
     jle .writeback_loop
 
-%if dc
     mov eax, 1
-%endif
 .return:
     ADD rsp, pad
     RET
@@ -724,6 +722,7 @@ TRELLIS_COEF0 1
     mov [nodes_curq + node_cabac_state(node_ctx) + (coeff_abs_level1_offs>>2)], r11b ; delayed from x264_cabac_size_decision2
 %endif
 %if %1 && node_ctx == 7
+    mov  r6d, levelgt1_ctxm
     mov [nodes_curq + node_cabac_state(node_ctx) + coeff_abs_levelgt1_offs-6], r10b
 %endif
     mov  r6d, [nodes_prevq + node_level_idx(j)]
@@ -786,8 +785,8 @@ TRELLIS_COEF0 1
     add  bitsd, r5d ; bs_size_ue_big from COEFN_SUFFIX
     ; n.cabac_state[levelgt1_ctx]
 %if j == 7 ; && compiling support for 4:2:2
-    mov    r5d, levelgt1_ctxm
-    %define coeff_abs_levelgt1_offs r5
+    mov    r6d, levelgt1_ctxm
+    %define coeff_abs_levelgt1_offs r6
 %endif
 %if j == 7
     movzx  r10, byte [nodes_prevq + node_cabac_state(j) + coeff_abs_levelgt1_offs-6] ; -6 because node only stores ctx 8 and 9
diff --git a/encoder/rdo.c b/encoder/rdo.c
index 77137c0..e307fba 100644
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -691,7 +691,8 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct,
     {
         int cost_sig = x264_cabac_size_decision_noup2( &cabac_state_sig[0], 1 )
                      + x264_cabac_size_decision_noup2( &cabac_state_last[0], 1 );
-        return dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig );
+        dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig );
+        return !!dct[0];
     }
 
 #if HAVE_MMX && ARCH_X86_64