[x264-devel] [PATCH] set: fix transpose custom scaling lists

[Nicolas Gaullier]

x264_sps_init() may be called twice, resulting in a second call to transpose() which cancels the first one. This patch isolates input parameters from current scaling lists stored in x264_sps_t.

Fix AVC-Intra cqm_8iy scaling list in 1080i
---
 common/set.h      |  1 +
 encoder/encoder.c | 14 ++++++++++++--
 encoder/set.c     | 47 +++++++++++++++++++++++++++++------------------
 encoder/set.h     |  2 +-
 4 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/common/set.h b/common/set.h
index 32ea96ce..7f1e539a 100644
--- a/common/set.h
+++ b/common/set.h
@@ -143,6 +143,7 @@ typedef struct
     int b_avcintra;
     int i_cqm_preset;
     const uint8_t *scaling_list[8]; /* could be 12, but we don't allow separate Cb/Cr lists */
+    int b_scaling_list_allocated;
 
 } x264_sps_t;
 
diff --git a/encoder/encoder.c b/encoder/encoder.c
index 7316a586..e2e3a4c8 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -1194,7 +1194,8 @@ static int validate_parameters( x264_t *h, int b_open )
             int maxrate_bak = h->param.rc.i_vbv_max_bitrate;
             if( h->param.rc.i_rc_method == X264_RC_ABR && h->param.rc.i_vbv_buffer_size <= 0 )
                 h->param.rc.i_vbv_max_bitrate = h->param.rc.i_bitrate * 2;
-            x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
+            if (x264_sps_init( h->sps, h->param.i_sps_id, &h->param ))
+                return -1;
             do h->param.i_level_idc = l->level_idc;
                 while( l[1].level_idc && x264_validate_levels( h, 0 ) && l++ );
             h->param.rc.i_vbv_max_bitrate = maxrate_bak;
@@ -1507,7 +1508,8 @@ x264_t *x264_encoder_open( x264_param_t *param )
 
     set_aspect_ratio( h, &h->param, 1 );
 
-    x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
+    if (x264_sps_init( h->sps, h->param.i_sps_id, &h->param ))
+        goto fail;
     x264_pps_init( h->pps, h->param.i_sps_id, &h->param, h->sps );
 
     x264_validate_levels( h, 1 );
@@ -4468,6 +4470,14 @@ void    x264_encoder_close  ( x264_t *h )
 #if HAVE_OPENCL
     x264_opencl_close_library( ocl );
 #endif
+
+    /* set */
+    if (h->sps->b_scaling_list_allocated)
+    {
+        for( int i = 0; i < 8; i++ )
+            x264_free((void *)h->sps->scaling_list[i]);
+        h->sps->b_scaling_list_allocated = 0;
+    }
 }
 
 int x264_encoder_delayed_frames( x264_t *h )
diff --git a/encoder/set.c b/encoder/set.c
index da4a8a10..8ed426a6 100644
--- a/encoder/set.c
+++ b/encoder/set.c
@@ -98,9 +98,20 @@ void x264_sei_write( bs_t *s, uint8_t *payload, int payload_size, int payload_ty
     bs_flush( s );
 }
 
-void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
+int x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
 {
     int csp = param->i_csp & X264_CSP_MASK;
+    uint8_t *param_cqms[8] = {
+        param->cqm_4iy,
+        param->cqm_4py,
+        param->cqm_4ic,
+        param->cqm_4pc,
+        param->cqm_8iy,
+        param->cqm_8py,
+        param->cqm_8ic,
+        param->cqm_8pc
+    };
+
 
     sps->i_id = i_id;
     sps->i_mb_width = ( param->i_width + 15 ) / 16;
@@ -255,29 +266,29 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
             sps->scaling_list[i] = x264_cqm_jvt[i];
         break;
     case X264_CQM_CUSTOM:
-        /* match the transposed DCT & zigzag */
-        transpose( param->cqm_4iy, 4 );
-        transpose( param->cqm_4py, 4 );
-        transpose( param->cqm_4ic, 4 );
-        transpose( param->cqm_4pc, 4 );
-        transpose( param->cqm_8iy, 8 );
-        transpose( param->cqm_8py, 8 );
-        transpose( param->cqm_8ic, 8 );
-        transpose( param->cqm_8pc, 8 );
-        sps->scaling_list[CQM_4IY] = param->cqm_4iy;
-        sps->scaling_list[CQM_4PY] = param->cqm_4py;
-        sps->scaling_list[CQM_4IC] = param->cqm_4ic;
-        sps->scaling_list[CQM_4PC] = param->cqm_4pc;
-        sps->scaling_list[CQM_8IY+4] = param->cqm_8iy;
-        sps->scaling_list[CQM_8PY+4] = param->cqm_8py;
-        sps->scaling_list[CQM_8IC+4] = param->cqm_8ic;
-        sps->scaling_list[CQM_8PC+4] = param->cqm_8pc;
+        if (sps->b_scaling_list_allocated)
+            for( int i = 0; i < 8; i++ )
+                x264_free((void*)sps->scaling_list[i]);
+        else
+            sps->b_scaling_list_allocated = 1;
+        for( int i = 0; i < 8; i++ )
+        {
+            uint8_t *new_scaling_list;
+            CHECKED_MALLOC( new_scaling_list, i < 4 ? 16 : 64 );
+            memcpy(new_scaling_list, param_cqms[i], i < 4 ? 16 : 64);
+            /* match the transposed DCT & zigzag */
+            transpose( new_scaling_list, i < 4 ? 4 : 8);
+            sps->scaling_list[i] = new_scaling_list;
+        }
         for( int i = 0; i < 8; i++ )
             for( int j = 0; j < (i < 4 ? 16 : 64); j++ )
                 if( sps->scaling_list[i][j] == 0 )
                     sps->scaling_list[i] = x264_cqm_jvt[i];
         break;
     }
+    return 0;
+fail:
+    return -1;
 }
 
 void x264_sps_init_reconfigurable( x264_sps_t *sps, x264_param_t *param )
diff --git a/encoder/set.h b/encoder/set.h
index bd0c4d7b..1217475b 100644
--- a/encoder/set.h
+++ b/encoder/set.h
@@ -28,7 +28,7 @@
 #define X264_ENCODER_SET_H
 
 #define x264_sps_init x264_template(sps_init)
-void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param );
+int x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param );
 #define x264_sps_init_reconfigurable x264_template(sps_init_reconfigurable)
 void x264_sps_init_reconfigurable( x264_sps_t *sps, x264_param_t *param );
 #define x264_sps_write x264_template(sps_write)
-- 
2.14.1.windows.1