[vlc-devel] [PATCH] swscale: modify output aspect ratio if transformation is not homothetic

[Felix Abecassis]

Close #10745
---
 modules/video_chroma/swscale.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/modules/video_chroma/swscale.c b/modules/video_chroma/swscale.c
index 4b763dd..03cc414 100644
--- a/modules/video_chroma/swscale.c
+++ b/modules/video_chroma/swscale.c
@@ -430,6 +430,24 @@ static int Init( filter_t *p_filter )
         return VLC_EGENERIC;
     }
 
+    if (p_filter->b_allow_fmt_out_change)
+    {
+        /*
+         * If the transformation is not homothetic we must modify the
+         * aspect ratio of the output format in order to have the
+         * output picture displayed correctly and not stretched
+         * horizontally or vertically.
+         */
+        unsigned i_sar_num = p_fmti->i_sar_num * p_fmti->i_visible_width;
+        unsigned i_sar_den = p_fmti->i_sar_den * p_fmto->i_visible_width;
+        vlc_ureduce(&i_sar_num, &i_sar_den, i_sar_num, i_sar_den, 65536);
+        i_sar_num *= p_fmto->i_visible_height;
+        i_sar_den *= p_fmti->i_visible_height;
+        vlc_ureduce(&i_sar_num, &i_sar_den, i_sar_num, i_sar_den, 65536);
+        p_fmto->i_sar_num = i_sar_num;
+        p_fmto->i_sar_den = i_sar_den;
+    }
+
     p_sys->b_add_a = cfg.b_add_a;
     p_sys->b_copy = cfg.b_copy;
     p_sys->fmt_in  = *p_fmti;
-- 
1.8.3.2