[x264-devel] Eliminate the possibility of CAVLC level code overflow

[Jason Garrett-Glaser]

x264 | branch: master | Jason Garrett-Glaser <jason at x264.com> | Fri Mar 18 18:24:33 2011 -0700| [08d04a4d30b452faed3b763528611737d994b30b] | committer: Jason Garrett-Glaser

Eliminate the possibility of CAVLC level code overflow
Instead, if it happens, just re-encode the MB at higher QPs until it fits.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=08d04a4d30b452faed3b763528611737d994b30b
---

 common/common.h   |    1 +
 common/set.c      |    4 ++++
 encoder/cavlc.c   |    5 ++---
 encoder/encoder.c |   17 ++++++++++++++++-
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/common/common.h b/common/common.h
index 868f526..f2021b6 100644
--- a/common/common.h
+++ b/common/common.h
@@ -645,6 +645,7 @@ struct x264_t
         int b_reencode_mb;
         int ip_offset; /* Used by PIR to offset the quantizer of intra-refresh blocks. */
         int b_deblock_rdo;
+        int b_overflow; /* If CAVLC had a level code overflow during bitstream writing. */
 
         struct
         {
diff --git a/common/set.c b/common/set.c
index 3690ed6..f8f6a11 100644
--- a/common/set.c
+++ b/common/set.c
@@ -236,6 +236,10 @@ int x264_cqm_init( x264_t *h )
             h->param.rc.i_qp_max = min_qp_err-1;
         if( max_qp_err >= h->param.rc.i_qp_min )
             h->param.rc.i_qp_min = max_qp_err+1;
+        /* If long level-codes aren't allowed, we need to allow QP high enough to avoid them. */
+        if( !h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH )
+            while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_max)] <= 12 || h->param.rc.i_qp_max <= 12 )
+                h->param.rc.i_qp_max++;
         if( h->param.rc.i_qp_min > h->param.rc.i_qp_max )
         {
             x264_log( h, X264_LOG_ERROR, "Impossible QP constraints for CQM (min=%d, max=%d)\n", h->param.rc.i_qp_min, h->param.rc.i_qp_max );
diff --git a/encoder/cavlc.c b/encoder/cavlc.c
index 720cd18..278b9a1 100644
--- a/encoder/cavlc.c
+++ b/encoder/cavlc.c
@@ -100,9 +100,8 @@ static inline int block_residual_write_cavlc_escape( x264_t *h, int i_suffix_len
                 /* Weight highly against overflows. */
                 s->i_bits_encoded += 2000;
 #else
-                x264_log(h, X264_LOG_WARNING, "OVERFLOW levelcode=%d is only allowed in High Profile\n", i_level_code );
-                /* clip level, preserving sign */
-                i_level_code = (1<<12) - 2 + (i_level_code & 1);
+                /* We've had an overflow; note it down and re-encode the MB later. */
+                h->mb.b_overflow = 1;
 #endif
             }
         }
diff --git a/encoder/encoder.c b/encoder/encoder.c
index b4dde0b..d42ca31 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -1883,6 +1883,7 @@ static int x264_slice_write( x264_t *h )
      * other inaccuracies. */
     int overhead_guess = (NALU_OVERHEAD - (h->param.b_annexb && h->out.i_nal)) + 1 + h->param.b_cabac + 5;
     int slice_max_size = h->param.i_slice_max_size > 0 ? (h->param.i_slice_max_size-overhead_guess)*8 : 0;
+    int back_up_bitstream = slice_max_size || (!h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH);
     int starting_bits = bs_pos(&h->out.bs);
     int b_deblock = h->sh.i_disable_deblocking_filter_idc != 1;
     int b_hpel = h->fdec->b_kept_as_ref;
@@ -1933,7 +1934,7 @@ static int x264_slice_write( x264_t *h )
         if( x264_bitstream_check_buffer( h ) )
             return -1;
 
-        if( slice_max_size )
+        if( back_up_bitstream )
         {
             mv_bits_bak = h->stat.frame.i_mv_bits;
             tex_bits_bak = h->stat.frame.i_tex_bits;
@@ -1963,6 +1964,7 @@ static int x264_slice_write( x264_t *h )
         x264_macroblock_analyse( h );
 
         /* encode this macroblock -> be careful it can change the mb type to P_SKIP if needed */
+reencode:
         x264_macroblock_encode( h );
 
         if( h->param.b_cabac )
@@ -1991,6 +1993,19 @@ static int x264_slice_write( x264_t *h )
                     i_skip = 0;
                 }
                 x264_macroblock_write_cavlc( h );
+                /* If there was a CAVLC level code overflow, try again at a higher QP. */
+                if( h->mb.b_overflow )
+                {
+                    h->mb.i_chroma_qp = h->chroma_qp_table[++h->mb.i_qp];
+                    h->mb.i_skip_intra = 0;
+                    h->mb.b_skip_mc = 0;
+                    h->mb.b_overflow = 0;
+                    h->out.bs = bs_bak;
+                    i_skip = i_skip_bak;
+                    h->stat.frame.i_mv_bits = mv_bits_bak;
+                    h->stat.frame.i_tex_bits = tex_bits_bak;
+                    goto reencode;
+                }
             }
         }