[vlc-devel] [PATCH 2/3] demux/avformat: Apply flip operations from the display matrix

[Vittorio Giovara]

---
 modules/demux/avformat/demux.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/modules/demux/avformat/demux.c b/modules/demux/avformat/demux.c
index b57438afea..f7e2531254 100644
--- a/modules/demux/avformat/demux.c
+++ b/modules/demux/avformat/demux.c
@@ -118,20 +118,38 @@ static void get_rotation(es_format_t *fmt, AVStream *s)
 
     int32_t *matrix = (int32_t *)av_stream_get_side_data(s, AV_PKT_DATA_DISPLAYMATRIX, NULL);
     if( matrix ) {
+        int64_t det = (int64_t)matrix[0] * matrix[4] - (int64_t)matrix[1] * matrix[3];
+        if (det < 0) {
+            /* Flip the matrix to decouple flip and rotation operations.
+             * Always assume an horizontal flip for simplicity,
+             * it can be changed later if rotation is 180º. */
+            av_display_matrix_flip(matrix, 1, 0);
+        }
         angle = lround(av_display_rotation_get(matrix));
 
         if (angle > 45 && angle < 135)
             fmt->video.orientation = ORIENT_ROTATED_270;
 
-        else if (angle > 135 || angle < -135)
-            fmt->video.orientation = ORIENT_ROTATED_180;
-
+        else if (angle > 135 || angle < -135) {
+            if (det < 0)
+                fmt->video.orientation = ORIENT_VFLIPPED;
+            else
+                fmt->video.orientation = ORIENT_ROTATED_180;
+        }
         else if (angle < -45 && angle > -135)
             fmt->video.orientation = ORIENT_ROTATED_90;
 
         else
             fmt->video.orientation = ORIENT_NORMAL;
 
+        /* Flip is already applied to the 180º case. */
+        if (det < 0 && !(angle > 135 || angle < -135)) {
+            video_transform_t transform = (video_transform_t)fmt->video.orientation;
+            /* Flip first then rotate */
+            fmt->video.orientation = ORIENT_HFLIPPED;
+            video_format_TransformBy(&fmt->video, transform);
+        }
+
     } else if( rotation ) {
         angle = strtol(rotation->value, NULL, 10);
 
-- 
2.20.1