[x264-devel] Fix possible crashes in resize and crop filters with high bitdepth input

[Anton Mitrofanov]

x264 | branch: master | Anton Mitrofanov <BugMaster at narod.ru> | Tue Oct  8 23:32:37 2013 +0400| [8202764ed57f776907e09afc8aa9d1bfb31ceb98] | committer: Jason Garrett-Glaser

Fix possible crashes in resize and crop filters with high bitdepth input

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=8202764ed57f776907e09afc8aa9d1bfb31ceb98
---

 filters/video/crop.c   |    3 +--
 filters/video/resize.c |    2 +-
 input/input.c          |   24 +++++++++++++++++++-----
 input/input.h          |    1 +
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/filters/video/crop.c b/filters/video/crop.c
index a58813b..98faab7 100644
--- a/filters/video/crop.c
+++ b/filters/video/crop.c
@@ -105,8 +105,7 @@ static int get_frame( hnd_t handle, cli_pic_t *output, int frame )
     for( int i = 0; i < output->img.planes; i++ )
     {
         intptr_t offset = output->img.stride[i] * h->dims[1] * h->csp->height[i];
-        offset += h->dims[0] * h->csp->width[i];
-        offset *= x264_cli_csp_depth_factor( output->img.csp );
+        offset += h->dims[0] * h->csp->width[i] * x264_cli_csp_depth_factor( output->img.csp );
         output->img.plane[i] += offset;
     }
     return 0;
diff --git a/filters/video/resize.c b/filters/video/resize.c
index 197a0e3..1974710 100644
--- a/filters/video/resize.c
+++ b/filters/video/resize.c
@@ -392,7 +392,7 @@ static int check_resizer( resizer_hnd_t *h, cli_pic_t *in )
     h->scale = input_prop;
     if( !h->buffer_allocated )
     {
-        if( x264_cli_pic_alloc( &h->buffer, h->dst_csp, h->dst.width, h->dst.height ) )
+        if( x264_cli_pic_alloc_aligned( &h->buffer, h->dst_csp, h->dst.width, h->dst.height ) )
             return -1;
         h->buffer_allocated = 1;
     }
diff --git a/input/input.c b/input/input.c
index 9c6763d..5cb277c 100644
--- a/input/input.c
+++ b/input/input.c
@@ -74,7 +74,7 @@ uint64_t x264_cli_pic_size( int csp, int width, int height )
     return size;
 }
 
-int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
+static int x264_cli_pic_alloc_internal( cli_pic_t *pic, int csp, int width, int height, int align )
 {
     memset( pic, 0, sizeof(cli_pic_t) );
     int csp_mask = csp & X264_CSP_MASK;
@@ -87,15 +87,29 @@ int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
     pic->img.height = height;
     for( int i = 0; i < pic->img.planes; i++ )
     {
-         pic->img.plane[i] = x264_malloc( x264_cli_pic_plane_size( csp, width, height, i ) );
-         if( !pic->img.plane[i] )
-             return -1;
-         pic->img.stride[i] = width * x264_cli_csps[csp_mask].width[i] * x264_cli_csp_depth_factor( csp );
+        int stride = width * x264_cli_csps[csp_mask].width[i];
+        stride *= x264_cli_csp_depth_factor( csp );
+        stride = ALIGN( stride, align );
+        uint64_t size = (uint64_t)(height * x264_cli_csps[csp_mask].height[i]) * stride;
+        pic->img.plane[i] = x264_malloc( size );
+        if( !pic->img.plane[i] )
+            return -1;
+        pic->img.stride[i] = stride;
     }
 
     return 0;
 }
 
+int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
+{
+    return x264_cli_pic_alloc_internal( pic, csp, width, height, 1 );
+}
+
+int x264_cli_pic_alloc_aligned( cli_pic_t *pic, int csp, int width, int height )
+{
+    return x264_cli_pic_alloc_internal( pic, csp, width, height, NATIVE_ALIGN );
+}
+
 void x264_cli_pic_clean( cli_pic_t *pic )
 {
     for( int i = 0; i < pic->img.planes; i++ )
diff --git a/input/input.h b/input/input.h
index 5137be3..a33d22a 100644
--- a/input/input.h
+++ b/input/input.h
@@ -124,6 +124,7 @@ extern const x264_cli_csp_t x264_cli_csps[];
 int      x264_cli_csp_is_invalid( int csp );
 int      x264_cli_csp_depth_factor( int csp );
 int      x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height );
+int      x264_cli_pic_alloc_aligned( cli_pic_t *pic, int csp, int width, int height );
 void     x264_cli_pic_clean( cli_pic_t *pic );
 uint64_t x264_cli_pic_plane_size( int csp, int width, int height, int plane );
 uint64_t x264_cli_pic_size( int csp, int width, int height );