[vlc-devel] [PATCH 3/3] picture: round visible pitch/line to the upper integer

[Steve Lhomme]

Otherwise in cases where there's pixel sub-sampling we may miss some visible
sub pixels.

Add some asserts to make sure the visible values never go above the actual
buffer sizes.

--
replaces https://patches.videolan.org/patch/15991/
- use VLC_UPPER_DIV
---
 src/misc/picture.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/misc/picture.c b/src/misc/picture.c
index c69ea72b76..dd66339a34 100644
--- a/src/misc/picture.c
+++ b/src/misc/picture.c
@@ -185,12 +185,14 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt )
         plane_t *p = &p_picture->p[i];
 
         p->i_lines         = (i_height_aligned + i_height_extra ) * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
-        p->i_visible_lines = fmt->i_visible_height * p_dsc->p[i].h.num / p_dsc->p[i].h.den;
+        p->i_visible_lines = VLC_UPPER_DIV( fmt->i_visible_height, p_dsc->p[i].h.den ) * p_dsc->p[i].h.num;
         p->i_pitch         = i_width_aligned * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
-        p->i_visible_pitch = fmt->i_visible_width * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size;
+        p->i_visible_pitch = VLC_UPPER_DIV( fmt->i_visible_width, p_dsc->p[i].w.den ) * p_dsc->p[i].w.num * p_dsc->pixel_size;
         p->i_pixel_pitch   = p_dsc->pixel_size;
 
         assert( (p->i_pitch % 16) == 0 );
+        assert( p->i_visible_lines * p_dsc->p[i].h.den / p_dsc->p[i].h.num <= fmt->i_height );
+        assert( p->i_visible_pitch * p_dsc->p[i].w.den / p_dsc->p[i].w.num <= fmt->i_width * p_dsc->pixel_size );
     }
     p_picture->i_planes  = p_dsc->plane_count;
 
-- 
2.11.1