[x264-devel] Fix possible double transpose of custom CQM if --level is not set

[Anton Mitrofanov]

x264 | branch: master | Anton Mitrofanov <BugMaster at narod.ru> | Sun Sep 23 20:47:44 2018 +0300| [545de2ffec6ae9a80738de1b2c8cf820249a2530] | committer: Henrik Gramner

Fix possible double transpose of custom CQM if --level is not set

Bug reported by Nicolas Gaullier

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=545de2ffec6ae9a80738de1b2c8cf820249a2530
---

 encoder/encoder.c |  1 +
 encoder/set.c     | 39 +++++++++++++++++++++------------------
 encoder/set.h     |  2 ++
 3 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/encoder/encoder.c b/encoder/encoder.c
index 7316a586..074c4a5c 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -1508,6 +1508,7 @@ x264_t *x264_encoder_open( x264_param_t *param )
     set_aspect_ratio( h, &h->param, 1 );
 
     x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
+    x264_sps_init_scaling_list( h->sps, &h->param );
     x264_pps_init( h->pps, h->param.i_sps_id, &h->param, h->sps );
 
     x264_validate_levels( h, 1 );
diff --git a/encoder/set.c b/encoder/set.c
index da4a8a10..07a1261e 100644
--- a/encoder/set.c
+++ b/encoder/set.c
@@ -243,7 +243,28 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
 
     sps->b_avcintra = !!param->i_avcintra_class;
     sps->i_cqm_preset = param->i_cqm_preset;
+}
+
+void x264_sps_init_reconfigurable( x264_sps_t *sps, x264_param_t *param )
+{
+    sps->crop.i_left   = param->crop_rect.i_left;
+    sps->crop.i_top    = param->crop_rect.i_top;
+    sps->crop.i_right  = param->crop_rect.i_right + sps->i_mb_width*16 - param->i_width;
+    sps->crop.i_bottom = (param->crop_rect.i_bottom + sps->i_mb_height*16 - param->i_height) >> !sps->b_frame_mbs_only;
+    sps->b_crop = sps->crop.i_left  || sps->crop.i_top ||
+                  sps->crop.i_right || sps->crop.i_bottom;
+
+    sps->vui.b_aspect_ratio_info_present = 0;
+    if( param->vui.i_sar_width > 0 && param->vui.i_sar_height > 0 )
+    {
+        sps->vui.b_aspect_ratio_info_present = 1;
+        sps->vui.i_sar_width = param->vui.i_sar_width;
+        sps->vui.i_sar_height= param->vui.i_sar_height;
+    }
+}
 
+void x264_sps_init_scaling_list( x264_sps_t *sps, x264_param_t *param )
+{
     switch( sps->i_cqm_preset )
     {
     case X264_CQM_FLAT:
@@ -280,24 +301,6 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
     }
 }
 
-void x264_sps_init_reconfigurable( x264_sps_t *sps, x264_param_t *param )
-{
-    sps->crop.i_left   = param->crop_rect.i_left;
-    sps->crop.i_top    = param->crop_rect.i_top;
-    sps->crop.i_right  = param->crop_rect.i_right + sps->i_mb_width*16 - param->i_width;
-    sps->crop.i_bottom = (param->crop_rect.i_bottom + sps->i_mb_height*16 - param->i_height) >> !sps->b_frame_mbs_only;
-    sps->b_crop = sps->crop.i_left  || sps->crop.i_top ||
-                  sps->crop.i_right || sps->crop.i_bottom;
-
-    sps->vui.b_aspect_ratio_info_present = 0;
-    if( param->vui.i_sar_width > 0 && param->vui.i_sar_height > 0 )
-    {
-        sps->vui.b_aspect_ratio_info_present = 1;
-        sps->vui.i_sar_width = param->vui.i_sar_width;
-        sps->vui.i_sar_height= param->vui.i_sar_height;
-    }
-}
-
 void x264_sps_write( bs_t *s, x264_sps_t *sps )
 {
     bs_realign( s );
diff --git a/encoder/set.h b/encoder/set.h
index bd0c4d7b..1b3ab79e 100644
--- a/encoder/set.h
+++ b/encoder/set.h
@@ -31,6 +31,8 @@
 void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param );
 #define x264_sps_init_reconfigurable x264_template(sps_init_reconfigurable)
 void x264_sps_init_reconfigurable( x264_sps_t *sps, x264_param_t *param );
+#define x264_sps_init_scaling_list x264_template(sps_init_scaling_list)
+void x264_sps_init_scaling_list( x264_sps_t *sps, x264_param_t *param );
 #define x264_sps_write x264_template(sps_write)
 void x264_sps_write( bs_t *s, x264_sps_t *sps );
 #define x264_pps_init x264_template(pps_init)