[x264-devel] commit: Check minimum chroma QP in addition to luma QP during CQM init ( Jason Garrett-Glaser )

[git version control]

x264 | branch: master | Jason Garrett-Glaser <darkshikari at gmail.com> | Thu Aug 27 15:21:22 2009 -0700| [1b2a1c699a0d5b93d13d9bb5b118b02d261ba193] | committer: Jason Garrett-Glaser 

Check minimum chroma QP in addition to luma QP during CQM init
Correctly error out if the implied minimum chroma QP is too low.
Add missing emms to checkasm macroblock_tree_propagate test.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=1b2a1c699a0d5b93d13d9bb5b118b02d261ba193
---

 common/set.c      |   16 ++++++++++++----
 encoder/encoder.c |    4 ++--
 tools/checkasm.c  |    3 +++
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/common/set.c b/common/set.c
index 7d4d23f..1f53cf3 100644
--- a/common/set.c
+++ b/common/set.c
@@ -78,6 +78,7 @@ int x264_cqm_init( x264_t *h )
                         32 - h->param.analyse.i_luma_deadzone[0],
                         32 - 11, 32 - 21 };
     int max_qp_err = -1;
+    int max_chroma_qp_err = -1;
 
     for( i = 0; i < 6; i++ )
     {
@@ -148,8 +149,10 @@ int x264_cqm_init( x264_t *h )
                 h->  quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][0][i], q/6 - 1);
                 // round to nearest, unless that would cause the deadzone to be negative
                 h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
-                if( j > 0xffff && q > max_qp_err )
+                if( j > 0xffff && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) )
                     max_qp_err = q;
+                if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) )
+                    max_chroma_qp_err = q;
             }
         if( h->param.analyse.b_transform_8x8 )
         for( i_list = 0; i_list < 2; i_list++ )
@@ -165,9 +168,14 @@ int x264_cqm_init( x264_t *h )
 
     if( !h->mb.b_lossless && max_qp_err >= h->param.rc.i_qp_min )
     {
-        x264_log( h, X264_LOG_ERROR, "Quantization overflow.\n" );
-        x264_log( h, X264_LOG_ERROR, "Your CQM is incompatible with QP < %d, but min QP is set to %d\n",
-                  max_qp_err+1, h->param.rc.i_qp_min );
+        x264_log( h, X264_LOG_ERROR, "Quantization overflow.  Your CQM is incompatible with QP < %d,\n", max_qp_err+1 );
+        x264_log( h, X264_LOG_ERROR, "but min QP is set to %d.\n", h->param.rc.i_qp_min );
+        return -1;
+    }
+    if( !h->mb.b_lossless && max_chroma_qp_err >= h->chroma_qp_table[h->param.rc.i_qp_min] )
+    {
+        x264_log( h, X264_LOG_ERROR, "Quantization overflow.  Your CQM is incompatible with QP < %d,\n", max_chroma_qp_err+1 );
+        x264_log( h, X264_LOG_ERROR, "but min chroma QP is implied to be %d.\n", h->chroma_qp_table[h->param.rc.i_qp_min] );
         return -1;
     }
     return 0;
diff --git a/encoder/encoder.c b/encoder/encoder.c
index e679688..d84457d 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -762,6 +762,8 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
 
     x264_validate_levels( h, 1 );
 
+    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;
+
     if( x264_cqm_init( h ) < 0 )
         goto fail;
 
@@ -796,8 +798,6 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
     h->i_ref0 = 0;
     h->i_ref1 = 0;
 
-    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;
-
     x264_rdo_init( );
 
     /* init CPU functions */
diff --git a/tools/checkasm.c b/tools/checkasm.c
index 63452c6..5ef1c9a 100644
--- a/tools/checkasm.c
+++ b/tools/checkasm.c
@@ -515,6 +515,7 @@ static int check_dct( int cpu_ref, int cpu_new )
     memset( h, 0, sizeof(*h) );
     h->pps = h->pps_array;
     x264_param_default( &h->param );
+    h->chroma_qp_table = i_chroma_qp_table + 12;
     h->param.analyse.i_luma_deadzone[0] = 0;
     h->param.analyse.i_luma_deadzone[1] = 0;
     h->param.analyse.b_transform_8x8 = 1;
@@ -978,6 +979,7 @@ static int check_mc( int cpu_ref, int cpu_new )
         uint16_t *inter = intra+400;
         uint16_t *qscale = inter+400;
         uint16_t *rand = (uint16_t*)buf2;
+        x264_emms();
         for( i=0; i<400; i++ )
         {
             intra[i]  = *rand++ & 0x7fff;
@@ -1077,6 +1079,7 @@ static int check_quant( int cpu_ref, int cpu_new )
     memset( h, 0, sizeof(*h) );
     h->pps = h->pps_array;
     x264_param_default( &h->param );
+    h->chroma_qp_table = i_chroma_qp_table + 12;
     h->param.rc.i_qp_min = 26;
     h->param.analyse.b_transform_8x8 = 1;