[vlc-commits] direct3d11: fix texture positioning when there is rotation

[Steve Lhomme]

vlc/vlc-3.0 | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Thu Feb  8 10:34:14 2018 +0100| [330ea0cce4bacaff825d9cf52e24dbd0c55b77b3] | committer: Jean-Baptiste Kempf

direct3d11: fix texture positioning when there is rotation

Tested with offset_test.ogv and forcing the rotation in the decoder.

Fixes #19078

(cherry picked from commit 916b67b476d6df26b44050311ef79e634ecd7882)
Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=330ea0cce4bacaff825d9cf52e24dbd0c55b77b3
---

 modules/video_output/win32/direct3d11.c | 79 ++++++++++++++++++++++++++++++---
 1 file changed, 73 insertions(+), 6 deletions(-)

diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index 137205eeca..a4a64b396c 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -2300,14 +2300,81 @@ static void SetupQuadFlat(d3d_vertex_t *dst_data, const RECT *output,
 {
     unsigned int src_width = quad->i_width;
     unsigned int src_height = quad->i_height;
-    float MidY = (output->top + output->bottom) / 2.f;
-    float MidX = (output->left + output->right) / 2.f;
+    float MidX,MidY;
 
     float top, bottom, left, right;
-    top    =  MidY / (MidY - output->top);
-    bottom = -(src_height - MidY) / (output->bottom - MidY);
-    left   = -MidX / (MidX - output->left);
-    right  =  (src_width  - MidX) / (output->right - MidX);
+    switch (orientation)
+    {
+    case ORIENT_ROTATED_90: /* 90° anti clockwise */
+        /* right/top aligned */
+        MidY = (output->left + output->right) / 2.f;
+        MidX = (output->top + output->bottom) / 2.f;
+        top    =  (src_width  - MidX) / (output->right - MidX);
+        bottom = -MidY / (MidY - output->top);
+        left   = -(src_height - MidY) / (output->bottom - MidY);
+        right  =   MidX / (MidX - output->left);
+        break;
+    case ORIENT_ROTATED_180: /* 180° */
+        /* right/top aligned */
+        MidY = (output->top + output->bottom) / 2.f;
+        MidX = (output->left + output->right) / 2.f;
+        top    =  (src_height - MidY) / (output->bottom - MidY);
+        bottom = -MidY / (MidY - output->top);
+        left   = -MidX / (MidX - output->left);
+        right  =  (src_width  - MidX) / (output->right - MidX);
+        break;
+    case ORIENT_ROTATED_270: /* 90° clockwise */
+        /* right/top aligned */
+        MidY = (output->left + output->right) / 2.f;
+        MidX = (output->top + output->bottom) / 2.f;
+        top    =  (src_width  - MidX) / (output->right - MidX);
+        bottom = -MidY / (MidY - output->top);
+        left   = -MidX / (MidX - output->left);
+        right  =  (src_height - MidY) / (output->bottom - MidY);
+        break;
+    case ORIENT_ANTI_TRANSPOSED:
+        MidY = (output->left + output->right) / 2.f;
+        MidX = (output->top + output->bottom) / 2.f;
+        top    =  (src_width  - MidX) / (output->right - MidX);
+        bottom = -MidY / (MidY - output->top);
+        left   = -(src_height - MidY) / (output->bottom - MidY);
+        right  =  MidX / (MidX - output->left);
+        break;
+    case ORIENT_TRANSPOSED:
+        MidY = (output->left + output->right) / 2.f;
+        MidX = (output->top + output->bottom) / 2.f;
+        top    =  (src_width  - MidX) / (output->right - MidX);
+        bottom = -MidY / (MidY - output->top);
+        left   = -MidX / (MidX - output->left);
+        right  =  (src_height - MidY) / (output->bottom - MidY);
+        break;
+    case ORIENT_VFLIPPED:
+        MidY = (output->top + output->bottom) / 2.f;
+        MidX = (output->left + output->right) / 2.f;
+        top    =  (src_height - MidY) / (output->bottom - MidY);
+        bottom = -MidY / (MidY - output->top);
+        left   = -MidX / (MidX - output->left);
+        right  =  (src_width  - MidX) / (output->right - MidX);
+        break;
+    case ORIENT_HFLIPPED:
+        MidY = (output->top + output->bottom) / 2.f;
+        MidX = (output->left + output->right) / 2.f;
+        top    =  MidY / (MidY - output->top);
+        bottom = -(src_height - MidY) / (output->bottom - MidY);
+        left   = -(src_width  - MidX) / (output->right - MidX);
+        right  =  MidX / (MidX - output->left);
+        break;
+    case ORIENT_NORMAL:
+    default:
+        /* left/top aligned */
+        MidY = (output->top + output->bottom) / 2.f;
+        MidX = (output->left + output->right) / 2.f;
+        top    =  MidY / (MidY - output->top);
+        bottom = -(src_height - MidY) / (output->bottom - MidY);
+        left   = -MidX / (MidX - output->left);
+        right  =  (src_width  - MidX) / (output->right - MidX);
+        break;
+    }
 
     const float vertices_coords[4][2] = {
         { left,  bottom },