[vlc-devel] [RFC PATCH 3/3] vout: gl: handle picture orientation

[Thomas Guillem]

TODO:
 - fix glwin32 and macos gl outputs
 - handle subpictures scaling
---
 modules/video_output/opengl/display.c     | 22 +++-------------
 modules/video_output/opengl/vout_helper.c | 43 ++++++++++++++++++++++++++++++-
 modules/video_output/opengl/vout_helper.h |  6 +++++
 3 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/modules/video_output/opengl/display.c b/modules/video_output/opengl/display.c
index 2ecdd0caaa..b40bda4803 100644
--- a/modules/video_output/opengl/display.c
+++ b/modules/video_output/opengl/display.c
@@ -242,37 +242,23 @@ static int Control (vout_display_t *vd, int query, va_list ap)
       case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
       case VOUT_DISPLAY_CHANGE_ZOOM:
       {
-        vout_display_cfg_t c = *va_arg (ap, const vout_display_cfg_t *);
-        const video_format_t *src = &vd->source;
         vout_display_place_t place;
-
-        /* Reverse vertical alignment as the GL tex are Y inverted */
-        if (c.align.vertical == VOUT_DISPLAY_ALIGN_TOP)
-            c.align.vertical = VOUT_DISPLAY_ALIGN_BOTTOM;
-        else if (c.align.vertical == VOUT_DISPLAY_ALIGN_BOTTOM)
-            c.align.vertical = VOUT_DISPLAY_ALIGN_TOP;
-
-        vout_display_PlacePicture (&place, src, &c, false);
-        vlc_gl_Resize (sys->gl, place.width, place.height);
         if (vlc_gl_MakeCurrent (sys->gl) != VLC_SUCCESS)
             return VLC_EGENERIC;
-        vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
-        vout_display_opengl_Viewport(sys->vgl, place.x, place.y, place.width, place.height);
+        vout_display_opengl_PlacePicture(sys->vgl, &place, &vd->source,
+                                         va_arg (ap, const vout_display_cfg_t *));
         vlc_gl_ReleaseCurrent (sys->gl);
+        vlc_gl_Resize (sys->gl, place.width, place.height);
         return VLC_SUCCESS;
       }
 
       case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
       case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
       {
-        const vout_display_cfg_t *cfg = vd->cfg;
         vout_display_place_t place;
-
-        vout_display_PlacePicture (&place, &vd->source, cfg, false);
         if (vlc_gl_MakeCurrent (sys->gl) != VLC_SUCCESS)
             return VLC_EGENERIC;
-        vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
-        vout_display_opengl_Viewport(sys->vgl, place.x, place.y, place.width, place.height);
+        vout_display_opengl_PlacePicture(sys->vgl, &place, &vd->source, vd->cfg);
         vlc_gl_ReleaseCurrent (sys->gl);
         return VLC_SUCCESS;
       }
diff --git a/modules/video_output/opengl/vout_helper.c b/modules/video_output/opengl/vout_helper.c
index 87e53a3a0f..ad8b675d32 100644
--- a/modules/video_output/opengl/vout_helper.c
+++ b/modules/video_output/opengl/vout_helper.c
@@ -135,6 +135,9 @@ struct vout_display_opengl_t {
     int         region_count;
     gl_region_t *region;
 
+    video_orientation_t display_orientation;
+    video_format_t source;
+    vout_display_cfg_t cfg;
 
     picture_pool_t *pool;
 
@@ -904,7 +907,7 @@ vout_display_opengl_t *vout_display_opengl_New(video_format_t *fmt,
     /* Update the fmt to main program one */
     vgl->fmt = vgl->prgm->tc->fmt;
     /* The orientation is handled by the orientation matrix */
-    vgl->fmt.orientation = fmt->orientation;
+    vgl->display_orientation = vgl->fmt.orientation = fmt->orientation;
 
     /* Texture size */
     const opengl_tex_converter_t *tc = vgl->prgm->tc;
@@ -1087,6 +1090,34 @@ void vout_display_opengl_SetWindowAspectRatio(vout_display_opengl_t *vgl,
     getViewpointMatrixes(vgl, vgl->fmt.projection_mode, vgl->prgm);
 }
 
+static void UpdatePlace(vout_display_opengl_t *vgl,
+                        vout_display_place_t *place)
+{
+    vout_display_PlacePicture (place, &vgl->source, &vgl->cfg, false);
+
+    vout_display_opengl_SetWindowAspectRatio(vgl,
+                                             (float)place->width / place->height);
+    vout_display_opengl_Viewport(vgl, place->x, place->y,
+                                 place->width, place->height);
+}
+
+void vout_display_opengl_PlacePicture(vout_display_opengl_t *vgl,
+                                      vout_display_place_t *place,
+                                      const video_format_t *source,
+                                      const vout_display_cfg_t *cfg)
+{
+    /* Reverse vertical alignment as the GL tex are Y inverted */
+    vgl->cfg = *cfg;
+    if (vgl->cfg.align.vertical == VOUT_DISPLAY_ALIGN_TOP)
+        vgl->cfg.align.vertical = VOUT_DISPLAY_ALIGN_BOTTOM;
+    else if (vgl->cfg.align.vertical == VOUT_DISPLAY_ALIGN_BOTTOM)
+        vgl->cfg.align.vertical = VOUT_DISPLAY_ALIGN_TOP;
+    vgl->source = *source;
+    vgl->source.orientation = vgl->display_orientation;
+
+    UpdatePlace(vgl, place);
+}
+
 void vout_display_opengl_Viewport(vout_display_opengl_t *vgl, int x, int y,
                                   unsigned width, unsigned height)
 {
@@ -1147,6 +1178,16 @@ int vout_display_opengl_Prepare(vout_display_opengl_t *vgl,
 
     opengl_tex_converter_t *tc = vgl->prgm->tc;
 
+    if (vgl->display_orientation != picture->display_orientation)
+    {
+        vgl->display_orientation = picture->display_orientation;
+        vgl->source.orientation = vgl->display_orientation;
+        vout_display_place_t place;
+        UpdatePlace(vgl, &place);
+        getOrientationTransformMatrix(vgl->display_orientation,
+                                      vgl->prgm->var.OrientationMatrix);
+    }
+
     /* Update the texture */
     int ret = tc->pf_update(tc, vgl->texture, vgl->tex_width, vgl->tex_height,
                             picture, NULL);
diff --git a/modules/video_output/opengl/vout_helper.h b/modules/video_output/opengl/vout_helper.h
index 666dd79291..214255db88 100644
--- a/modules/video_output/opengl/vout_helper.h
+++ b/modules/video_output/opengl/vout_helper.h
@@ -31,6 +31,7 @@
 
 #include <vlc_common.h>
 #include <vlc_picture_pool.h>
+#include <vlc_vout_display.h>
 #include <vlc_opengl.h>
 
 /* if USE_OPENGL_ES2 is defined, OpenGL ES version 2 will be used, otherwise
@@ -96,6 +97,11 @@ int vout_display_opengl_SetViewpoint(vout_display_opengl_t *vgl, const vlc_viewp
 void vout_display_opengl_SetWindowAspectRatio(vout_display_opengl_t *vgl,
                                               float f_sar);
 
+void vout_display_opengl_PlacePicture(vout_display_opengl_t *vgl,
+                                      vout_display_place_t *place,
+                                      const video_format_t *source,
+                                      const vout_display_cfg_t *cfg);
+
 void vout_display_opengl_Viewport(vout_display_opengl_t *vgl, int x, int y,
                                   unsigned width, unsigned height);
 
-- 
2.11.0