[vlc-commits] video_converter: i420_rgb: lazy alloc conversion buffer
Francois Cartegnie
git at videolan.org
Tue Sep 25 16:34:59 CEST 2018
vlc | branch: master | Francois Cartegnie <fcvlcdev at free.fr> | Mon Sep 24 14:28:19 2018 +0200| [b3c301b6155a85eda8eef9d8a8d596c0e1d54783] | committer: Francois Cartegnie
video_converter: i420_rgb: lazy alloc conversion buffer
Quote from A heap overflow world
"4K ought to be enough for anybody"
> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=b3c301b6155a85eda8eef9d8a8d596c0e1d54783
---
modules/video_chroma/i420_rgb.c | 20 +++++-------
modules/video_chroma/i420_rgb.h | 22 +++++++++++++
modules/video_chroma/i420_rgb16.c | 20 ++++++++++--
modules/video_chroma/i420_rgb16_x86.c | 58 +++++++++++++++++++++++++++++++----
4 files changed, 99 insertions(+), 21 deletions(-)
diff --git a/modules/video_chroma/i420_rgb.c b/modules/video_chroma/i420_rgb.c
index 1c66223749..9f70694fe5 100644
--- a/modules/video_chroma/i420_rgb.c
+++ b/modules/video_chroma/i420_rgb.c
@@ -211,30 +211,26 @@ static int Activate( vlc_object_t *p_this )
return VLC_EGENERIC;
p_filter->p_sys = p_sys;
+ p_sys->i_buffer_size = 0;
+ p_sys->p_buffer = NULL;
switch( p_filter->fmt_out.video.i_chroma )
{
#ifdef PLAIN
case VLC_CODEC_RGB8:
- p_sys->p_buffer = malloc( VOUT_MAX_WIDTH );
+ p_sys->i_bytespp = 1;
break;
#endif
case VLC_CODEC_RGB15:
case VLC_CODEC_RGB16:
- p_sys->p_buffer = malloc( VOUT_MAX_WIDTH * 2 );
+ p_sys->i_bytespp = 2;
break;
case VLC_CODEC_RGB24:
case VLC_CODEC_RGB32:
- p_sys->p_buffer = malloc( VOUT_MAX_WIDTH * 4 );
+ p_sys->i_bytespp = 4;
break;
default:
- p_sys->p_buffer = NULL;
- break;
- }
-
- if( p_sys->p_buffer == NULL )
- {
- free( p_sys );
- return VLC_EGENERIC;
+ free( p_sys );
+ return VLC_EGENERIC;
}
p_sys->p_offset = malloc( p_filter->fmt_out.video.i_width
@@ -243,7 +239,6 @@ static int Activate( vlc_object_t *p_this )
* sizeof( int ) );
if( p_sys->p_offset == NULL )
{
- free( p_sys->p_buffer );
free( p_sys );
return VLC_EGENERIC;
}
@@ -267,7 +262,6 @@ static int Activate( vlc_object_t *p_this )
if( p_sys->p_base == NULL )
{
free( p_sys->p_offset );
- free( p_sys->p_buffer );
free( p_sys );
return -1;
}
diff --git a/modules/video_chroma/i420_rgb.h b/modules/video_chroma/i420_rgb.h
index 9bd156149e..6d060397b3 100644
--- a/modules/video_chroma/i420_rgb.h
+++ b/modules/video_chroma/i420_rgb.h
@@ -20,6 +20,7 @@
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
*****************************************************************************/
+#include <limits.h>
#if !defined (SSE2) && !defined (MMX)
# define PLAIN
@@ -37,6 +38,8 @@
typedef struct
{
uint8_t *p_buffer;
+ size_t i_buffer_size;
+ uint8_t i_bytespp;
int *p_offset;
#ifdef PLAIN
@@ -57,6 +60,25 @@ typedef struct
} filter_sys_t;
/*****************************************************************************
+ * Conversion buffer helper
+ *****************************************************************************/
+static inline int AllocateOrGrow( uint8_t **pp_buffer, size_t *pi_buffer,
+ unsigned i_width, uint8_t bytespp )
+{
+ if(UINT_MAX / bytespp < i_width)
+ return -1;
+ const size_t i_realloc = i_width * bytespp;
+ if(*pi_buffer >= i_realloc)
+ return 0;
+ uint8_t *p_realloc = realloc(*pp_buffer, i_realloc);
+ if(!p_realloc)
+ return -1;
+ *pp_buffer = p_realloc;
+ *pi_buffer = i_realloc;
+ return 0;
+}
+
+/*****************************************************************************
* Prototypes
*****************************************************************************/
#ifdef PLAIN
diff --git a/modules/video_chroma/i420_rgb16.c b/modules/video_chroma/i420_rgb16.c
index be37d8d355..85c7048ccd 100644
--- a/modules/video_chroma/i420_rgb16.c
+++ b/modules/video_chroma/i420_rgb16.c
@@ -130,7 +130,7 @@ void I420_RGB16( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
uint16_t * p_ybase; /* Y dependant conversion table */
/* Conversion buffer pointer */
- uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer;
+ uint16_t * p_buffer_start;
uint16_t * p_buffer;
/* Offset array pointer */
@@ -156,6 +156,14 @@ void I420_RGB16( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint16_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
@@ -237,7 +245,7 @@ void I420_RGB32( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
uint32_t * p_ybase; /* Y dependant conversion table */
/* Conversion buffer pointer */
- uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer;
+ uint32_t * p_buffer_start;
uint32_t * p_buffer;
/* Offset array pointer */
@@ -263,6 +271,14 @@ void I420_RGB32( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint32_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
diff --git a/modules/video_chroma/i420_rgb16_x86.c b/modules/video_chroma/i420_rgb16_x86.c
index 37c425cf3d..ce39051443 100644
--- a/modules/video_chroma/i420_rgb16_x86.c
+++ b/modules/video_chroma/i420_rgb16_x86.c
@@ -122,7 +122,7 @@ void I420_R5G5B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
uint16_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer;
+ uint16_t * p_buffer_start;
uint16_t * p_buffer;
/* Offset array pointer */
@@ -147,6 +147,13 @@ void I420_R5G5B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint16_t*)p_sys->p_buffer;
/*
* Perform conversion
@@ -356,7 +363,7 @@ void I420_R5G6B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
uint16_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer;
+ uint16_t * p_buffer_start;
uint16_t * p_buffer;
/* Offset array pointer */
@@ -381,6 +388,13 @@ void I420_R5G6B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint16_t*)p_sys->p_buffer;
/*
* Perform conversion
@@ -590,7 +604,7 @@ void I420_A8R8G8B8( filter_t *p_filter, picture_t *p_src,
int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */
uint32_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer;
+ uint32_t * p_buffer_start;
uint32_t * p_buffer;
/* Offset array pointer */
@@ -615,6 +629,14 @@ void I420_A8R8G8B8( filter_t *p_filter, picture_t *p_src,
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint32_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
@@ -822,7 +844,7 @@ void I420_R8G8B8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */
uint32_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer;
+ uint32_t * p_buffer_start;
uint32_t * p_buffer;
/* Offset array pointer */
@@ -847,6 +869,14 @@ void I420_R8G8B8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint32_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
@@ -1054,7 +1084,7 @@ void I420_B8G8R8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */
uint32_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer;
+ uint32_t * p_buffer_start;
uint32_t * p_buffer;
/* Offset array pointer */
@@ -1079,6 +1109,14 @@ void I420_B8G8R8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint32_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
@@ -1283,7 +1321,7 @@ void I420_A8B8G8R8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */
uint32_t * p_pic_start; /* beginning of the current line for copy */
/* Conversion buffer pointer */
- uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer;
+ uint32_t * p_buffer_start;
uint32_t * p_buffer;
/* Offset array pointer */
@@ -1308,6 +1346,14 @@ void I420_A8B8G8R8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest )
(p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height),
&b_hscale, &i_vscale, p_offset_start );
+ if(b_hscale &&
+ AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size,
+ p_filter->fmt_in.video.i_x_offset +
+ p_filter->fmt_in.video.i_visible_width,
+ p_sys->i_bytespp))
+ return;
+ else p_buffer_start = (uint32_t*)p_sys->p_buffer;
+
/*
* Perform conversion
*/
More information about the vlc-commits
mailing list