[x264-devel] Try 8x8 transform analysis even when sub8x8 partitions are present

[Jason Garrett-Glaser]

x264 | branch: master | Jason Garrett-Glaser <jason at x264.com> | Mon Jun 25 18:01:29 2012 -0700| [5af86bedd71c89fc48b50bbb7e8a8bec3d360d3a] | committer: Jason Garrett-Glaser

Try 8x8 transform analysis even when sub8x8 partitions are present
Turn off the sub8x8 partitions, try it, and turn them back on if it didn't help.
Small compression improvement with p4x4 on (~0.1-0.5%).
Also update related comments.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=5af86bedd71c89fc48b50bbb7e8a8bec3d360d3a
---

 common/macroblock.h |   16 ++++++++--------
 encoder/analyse.c   |   12 +++++++++++-
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/common/macroblock.h b/common/macroblock.h
index 43b2a9c..ad4ff2f 100644
--- a/common/macroblock.h
+++ b/common/macroblock.h
@@ -420,23 +420,23 @@ static ALWAYS_INLINE int x264_mb_predict_non_zero_code( x264_t *h, int idx )
         i_ret = ( i_ret + 1 ) >> 1;
     return i_ret & 0x7f;
 }
+
+/* intra and skip are disallowed, p8x8 is conditional. */
+static const uint8_t x264_transform_allowed[X264_MBTYPE_MAX] =
+{
+    0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0
+};
+
 /* x264_mb_transform_8x8_allowed:
  *      check whether any partition is smaller than 8x8 (or at least
  *      might be, according to just partition type.)
  *      doesn't check for cbp */
 static ALWAYS_INLINE int x264_mb_transform_8x8_allowed( x264_t *h )
 {
-    // intra and skip are disallowed
-    // large partitions are allowed
-    // direct and 8x8 are conditional
-    static const uint8_t partition_tab[X264_MBTYPE_MAX] = {
-        0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0,
-    };
-
     if( !h->pps->b_transform_8x8_mode )
         return 0;
     if( h->mb.i_type != P_8x8 )
-        return partition_tab[h->mb.i_type];
+        return x264_transform_allowed[h->mb.i_type];
     return M32( h->mb.i_sub_partition ) == D_L0_8x8*0x01010101;
 }
 
diff --git a/encoder/analyse.c b/encoder/analyse.c
index f72225d..1c8a681 100644
--- a/encoder/analyse.c
+++ b/encoder/analyse.c
@@ -2851,8 +2851,15 @@ static inline void x264_mb_analyse_transform( x264_t *h )
 
 static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *a, int *i_satd, int *i_rd )
 {
-    if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
+    if( h->param.analyse.b_transform_8x8 && h->pps->b_transform_8x8_mode )
     {
+        uint32_t subpart_bak = M32( h->mb.i_sub_partition );
+        /* Try switching the subpartitions to 8x8 so that we can use 8x8 transform mode */
+        if( h->mb.i_type == P_8x8 )
+            M32( h->mb.i_sub_partition ) = D_L0_8x8*0x01010101;
+        else if( !x264_transform_allowed[h->mb.i_type] )
+            return;
+
         x264_analyse_update_cache( h, a );
         h->mb.b_transform_8x8 ^= 1;
         /* FIXME only luma is needed for 4:2:0, but the score for comparison already includes chroma */
@@ -2865,7 +2872,10 @@ static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *
             *i_rd = i_rd8;
         }
         else
+        {
             h->mb.b_transform_8x8 ^= 1;
+            M32( h->mb.i_sub_partition ) = subpart_bak;
+        }
     }
 }