[x264-devel] Fix array overreads that caused miscompilation in gcc 4.8
Jason Garrett-Glaser
git at videolan.org
Tue Apr 23 23:03:03 CEST 2013
x264 | branch: master | Jason Garrett-Glaser <jason at x264.com> | Wed Feb 27 13:30:22 2013 -0800| [89aecb440e2939be7fb72d8362eb12504711b94f] | committer: Jason Garrett-Glaser
Fix array overreads that caused miscompilation in gcc 4.8
> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=89aecb440e2939be7fb72d8362eb12504711b94f
---
common/common.h | 1 +
common/set.c | 78 ++++++++++++++++++++++++++++++------------------------
tools/checkasm.c | 6 ++---
3 files changed, 47 insertions(+), 38 deletions(-)
diff --git a/common/common.h b/common/common.h
index 39ad5cb..f5c69ba 100644
--- a/common/common.h
+++ b/common/common.h
@@ -40,6 +40,7 @@
#define IS_DISPOSABLE(type) ( type == X264_TYPE_B )
#define FIX8(f) ((int)(f*(1<<8)+.5))
#define ALIGN(x,a) (((x)+((a)-1))&~((a)-1))
+#define ARRAY_ELEMS(a) ((sizeof(a))/(sizeof(a[0])))
#define CHECKED_MALLOC( var, size )\
do {\
diff --git a/common/set.c b/common/set.c
index 4c72125..fa8b158 100644
--- a/common/set.c
+++ b/common/set.c
@@ -85,44 +85,49 @@ int x264_cqm_init( x264_t *h )
int max_qp_err = -1;
int max_chroma_qp_err = -1;
int min_qp_err = QP_MAX+1;
- int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4 : 2; /* Checkasm may segfault if optimized out by --chroma-format */
+ int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4
+ : h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */
- for( int i = 0; i < 4 + num_8x8_lists; i++ )
- {
- int size = i<4 ? 16 : 64;
- int j;
- for( j = (i<4 ? 0 : 4); j < i; j++ )
- if( !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
- break;
- if( j < i )
- {
- h-> quant4_mf[i] = h-> quant4_mf[j];
- h->dequant4_mf[i] = h->dequant4_mf[j];
- h->unquant4_mf[i] = h->unquant4_mf[j];
- }
- else
- {
- CHECKED_MALLOC( h-> quant4_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );
- CHECKED_MALLOC( h->dequant4_mf[i], 6*size*sizeof(int) );
- CHECKED_MALLOC( h->unquant4_mf[i], (QP_MAX+1)*size*sizeof(int) );
- }
-
- for( j = (i<4 ? 0 : 4); j < i; j++ )
- if( deadzone[j&3] == deadzone[i&3] &&
- !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
- break;
- if( j < i )
- {
- h->quant4_bias[i] = h->quant4_bias[j];
- h->quant4_bias0[i] = h->quant4_bias0[j];
- }
- else
- {
- CHECKED_MALLOC( h->quant4_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );
- CHECKED_MALLOC( h->quant4_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );
- }
+#define CQM_ALLOC( w, count )\
+ for( int i = 0; i < count; i++ )\
+ {\
+ int size = w*w;\
+ int start = w == 8 ? 4 : 0;\
+ int j;\
+ for( j = 0; j < i; j++ )\
+ if( !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
+ break;\
+ if( j < i )\
+ {\
+ h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\
+ h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\
+ h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\
+ }\
+ else\
+ {\
+ CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
+ CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\
+ CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX+1)*size*sizeof(int) );\
+ }\
+ for( j = 0; j < i; j++ )\
+ if( deadzone[j] == deadzone[i] &&\
+ !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
+ break;\
+ if( j < i )\
+ {\
+ h->quant##w##_bias[i] = h->quant##w##_bias[j];\
+ h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\
+ }\
+ else\
+ {\
+ CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
+ CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
+ }\
}
+ CQM_ALLOC( 4, 4 )
+ CQM_ALLOC( 8, num_8x8_lists )
+
for( int q = 0; q < 6; q++ )
{
for( int i = 0; i < 16; i++ )
@@ -204,6 +209,9 @@ int x264_cqm_init( x264_t *h )
for( int cat = 0; cat < 3 + CHROMA444; cat++ )
{
int dct8x8 = cat&1;
+ if( !h->param.analyse.b_transform_8x8 && dct8x8 )
+ continue;
+
int size = dct8x8 ? 64 : 16;
udctcoef *nr_offset = h->nr_offset_emergency[q][cat];
/* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
diff --git a/tools/checkasm.c b/tools/checkasm.c
index 9135b70..bbda21a 100644
--- a/tools/checkasm.c
+++ b/tools/checkasm.c
@@ -309,7 +309,7 @@ static int check_pixel( int cpu_ref, int cpu_new )
#define TEST_PIXEL( name, align ) \
ok = 1, used_asm = 0; \
- for( int i = 0; i < 8; i++ ) \
+ for( int i = 0; i < ARRAY_ELEMS(pixel_c.name); i++ ) \
{ \
int res_c, res_asm; \
if( pixel_asm.name[i] != pixel_ref.name[i] ) \
@@ -1087,13 +1087,13 @@ static int check_dct( int cpu_ref, int cpu_new )
x264_zigzag_init( cpu_new, &zigzag_asm[0], &zigzag_asm[1] );
ok = 1; used_asm = 0;
- TEST_INTERLEAVE( interleave_8x8_cavlc, level1, level2, dct1[0], 64 );
+ TEST_INTERLEAVE( interleave_8x8_cavlc, level1, level2, dct8[0], 64 );
report( "zigzag_interleave :" );
for( interlace = 0; interlace <= 1; interlace++ )
{
ok = 1; used_asm = 0;
- TEST_ZIGZAG_SCAN( scan_8x8, level1, level2, dct1[0], 8 );
+ TEST_ZIGZAG_SCAN( scan_8x8, level1, level2, dct8[0], 8 );
TEST_ZIGZAG_SCAN( scan_4x4, level1, level2, dct1[0], 4 );
TEST_ZIGZAG_SUB( sub_4x4, level1, level2, 16 );
TEST_ZIGZAG_SUBAC( sub_4x4ac, level1, level2 );
More information about the x264-devel
mailing list