[vlc-commits] opengl: on manual unpacking use picture visible width for calculations, fixes some mov-samples

[Ilkka Ollakka]

vlc | branch: master | Ilkka Ollakka <ileoo at videolan.org> | Tue Feb 12 10:21:41 2013 +0200| [356c37db5dfdf9da3a10a2318bebc8783cfed019] | committer: Ilkka Ollakka

opengl: on manual unpacking use picture visible width for calculations, fixes some mov-samples

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=356c37db5dfdf9da3a10a2318bebc8783cfed019
---

 modules/video_output/opengl.c |   17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/modules/video_output/opengl.c b/modules/video_output/opengl.c
index e89cf63..36a514d 100644
--- a/modules/video_output/opengl.c
+++ b/modules/video_output/opengl.c
@@ -659,20 +659,23 @@ int vout_display_opengl_Prepare(vout_display_opengl_t *vgl,
         }
         glBindTexture(vgl->tex_target, vgl->texture[0][j]);
 
-        if ( (picture->p[j].i_pitch / picture->p[j].i_pixel_pitch) != (vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den) ) {
-            uint8_t *new_plane = malloc(picture->p[j].i_pitch*picture->p[j].i_pixel_pitch *vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den);
 #ifndef GL_UNPACK_ROW_LENGTH
+        if ( (picture->p[j].i_pitch / picture->p[j].i_pixel_pitch) !=
+             ( picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den ) )
+        {
+            uint8_t *new_plane = malloc( picture->format.i_visible_width * vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num / (vgl->chroma->p[j].h.den *  vgl->chroma->p[j].w.den ) );
             uint8_t *destination = new_plane;
             const uint8_t *source = picture->p[j].p_pixels;
-            for( unsigned height = 0; height < vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den; height++ )
+
+            for( unsigned height = 0; height < (vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den) ; height++ )
             {
-                memcpy( destination, source, vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den );
-                source +=  picture->p[j].i_pitch*picture->p[j].i_pixel_pitch;
-                destination += vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den;
+                memcpy( destination, source, picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den );
+                source += picture->p[j].i_pitch / picture->p[j].i_pixel_pitch;
+                destination += picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den;
             }
             glTexSubImage2D( vgl->tex_target, 0,
                              0, 0,
-                             vgl->fmt.i_width  * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den,
+                             picture->format.i_visible_width * vgl->chroma->p[j].w.num / vgl->chroma->p[j].w.den,
                              vgl->fmt.i_height * vgl->chroma->p[j].h.num / vgl->chroma->p[j].h.den,
                              vgl->tex_format, vgl->tex_type, new_plane );
             free( new_plane );