[vlc-devel] [PATCH] vdpau: reorient video

[Felix Abecassis]

---
 modules/hw/vdpau/display.c | 55 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/modules/hw/vdpau/display.c b/modules/hw/vdpau/display.c
index 1206cdc..0bed9af 100644
--- a/modules/hw/vdpau/display.c
+++ b/modules/hw/vdpau/display.c
@@ -65,6 +65,7 @@ struct vout_display_sys_t
     VdpPresentationQueueTarget target; /**< VDPAU presentation queue target */
     VdpPresentationQueue queue; /**< VDPAU presentation queue */
     VdpRGBAFormat rgb_fmt; /**< Output surface format */
+    VdpOutputSurface render_surface; /**< displayed surface */
 
     picture_pool_t *pool; /**< pictures pool */
 };
@@ -241,6 +242,17 @@ out:/* Destroy GPU surface */
     vdp_bitmap_surface_destroy(sys->vdp, surface);
 }
 
+static uint32_t GetVDPAURotation(const video_format_t *fmt)
+{
+    switch (fmt->orientation)
+    {
+    case ORIENT_ROTATED_90: return VDP_OUTPUT_SURFACE_RENDER_ROTATE_90;
+    case ORIENT_ROTATED_180: return VDP_OUTPUT_SURFACE_RENDER_ROTATE_180;
+    case ORIENT_ROTATED_270: return VDP_OUTPUT_SURFACE_RENDER_ROTATE_270;
+    default: return VDP_OUTPUT_SURFACE_RENDER_ROTATE_0;
+    }
+}
+
 static void Queue(vout_display_t *vd, picture_t *pic, subpicture_t *subpic)
 {
     vout_display_sys_t *sys = vd->sys;
@@ -254,10 +266,22 @@ static void Queue(vout_display_t *vd, picture_t *pic, subpicture_t *subpic)
     if (err == VDP_STATUS_OK && status != VDP_PRESENTATION_QUEUE_STATUS_IDLE)
         msg_Dbg(vd, "surface status: %u", status);
 
+    /* Composite the picture into the entire rendering surface and
+     * apply the rotation if needed. */
+    int rotation_flag = GetVDPAURotation(&vd->fmt);
+    err = vdp_output_surface_render_output_surface(sys->vdp, sys->render_surface, NULL,
+                                                   surface, NULL, NULL, NULL, rotation_flag);
+    if (err != VDP_STATUS_OK)
+    {
+        msg_Err(vd, "composition to render surface failure: %s",
+                vdp_get_error_string(sys->vdp, err));
+        return;
+    }
+
     if (subpic != NULL)
         for (subpicture_region_t *r = subpic->p_region; r != NULL;
              r = r->p_next)
-            RenderRegion(vd, surface, subpic, r);
+            RenderRegion(vd, sys->render_surface, subpic, r);
 
     /* Compute picture presentation time */
     mtime_t now = mdate();
@@ -284,7 +308,7 @@ static void Queue(vout_display_t *vd, picture_t *pic, subpicture_t *subpic)
     pts += delay * 1000;
 
     /* Queue picture */
-    err = vdp_presentation_queue_display(sys->vdp, sys->queue, surface, 0, 0,
+    err = vdp_presentation_queue_display(sys->vdp, sys->queue, sys->render_surface, 0, 0,
                                          pts);
     if (err != VDP_STATUS_OK)
         msg_Err(vd, "presentation queue display failure: %s",
@@ -356,6 +380,23 @@ static int Control(vout_display_t *vd, int query, va_list ap)
                              XCB_CONFIG_WINDOW_X|XCB_CONFIG_WINDOW_Y|
                              XCB_CONFIG_WINDOW_WIDTH|XCB_CONFIG_WINDOW_HEIGHT,
                              values);
+
+        VdpOutputSurface new_surface;
+        VdpStatus err = vdp_output_surface_create(sys->vdp, sys->device, sys->rgb_fmt,
+                                                  fmt->i_visible_width, fmt->i_visible_height,
+                                                  &new_surface);
+        if (err != VDP_STATUS_OK)
+        {
+            /* If allocation of the new surface failed, keep the old one. */
+            msg_Err(vd, "%s creation failure: %s", "render surface",
+                    vdp_get_error_string(sys->vdp, err));
+        }
+        else
+        {
+            vdp_output_surface_destroy(sys->vdp, sys->render_surface);
+            sys->render_surface = new_surface;
+        }
+
         break;
     }
     case VOUT_DISPLAY_CHANGE_FULLSCREEN:
@@ -636,6 +677,16 @@ static int Open(vlc_object_t *obj)
             spu_chromas = subpicture_chromas;
     }
 
+    err = vdp_output_surface_create(sys->vdp, sys->device, sys->rgb_fmt,
+                                    vd->fmt.i_visible_width, vd->fmt.i_visible_height,
+                                    &sys->render_surface);
+    if (err != VDP_STATUS_OK)
+    {
+        msg_Err(vd, "%s creation failure: %s", "render surface",
+                vdp_get_error_string(sys->vdp, err));
+        goto error;
+    }
+
     /* Initialize VDPAU queue */
     err = vdp_presentation_queue_target_create_x11(sys->vdp, sys->device,
                                                    sys->window, &sys->target);
-- 
1.9.1