[vlc-devel] [PATCH 1/3] picture: round visible pitch/line to the upper integer

[Steve Lhomme]

From: Steve Lhomme <robUx4 at videolabs.io>

Otherwise in cases where there's pixel sub-sampling we may miss some visible
sub pixels.

Add some asserts to make sure the visible values never go above the actual
buffer sizes.
---
 src/misc/picture.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/misc/picture.c b/src/misc/picture.c
index 0a2dc8f..c5b2737 100644
--- a/src/misc/picture.c
+++ b/src/misc/picture.c
@@ -185,12 +185,14 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt )
         plane_t *p = &p_picture->p[i];
 
         p->i_lines         = (i_height_aligned + i_height_extra ) * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
-        p->i_visible_lines = fmt->i_visible_height * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
+        p->i_visible_lines = (fmt->i_visible_height + p_dsc->p[i].h.den - 1) * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
         p->i_pitch         = i_width_aligned * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
-        p->i_visible_pitch = fmt->i_visible_width * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
+        p->i_visible_pitch = (fmt->i_visible_width + p_dsc->p[i].w.den - 1) * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
         p->i_pixel_pitch   = p_dsc->pixel_size;
 
         assert( (p->i_pitch % 16) == 0 );
+        assert( p->i_visible_lines * p_dsc->p[i].h.den / p_dsc->p[i].h.num <= fmt->i_height * p_dsc->pixel_size );
+        assert( p->i_visible_pitch * p_dsc->p[i].w.den / p_dsc->p[i].w.num <= fmt->i_width * p_dsc->pixel_size );
     }
     p_picture->i_planes  = p_dsc->plane_count;
 
-- 
2.10.1.windows.1