>From 9668f775076d5c0dd46d4f6f31442538eb235d95 Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <steinar+vlc@gunderson.no>
Date: Sat, 8 Jun 2013 23:07:48 +0200
Subject: [PATCH 3/5] In video transcode, use the correct format for
 calculations.

The current code doesn't properly take into account that filters
can change the format (including the frame rate) from that of the
decoder's output, and thus uncritically use the decoder's output
format where it should use the output of the decoder -> filter
-> user_filter chain.
---
 modules/stream_out/transcode/video.c |   55 +++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 8f97568..250d873 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -270,24 +270,23 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-       filter_chain_AppendFilter( id->p_f_chain,
-                                  p_stream->p_sys->psz_deinterlace,
-                                  p_stream->p_sys->p_deinterlace_cfg,
-                                  &id->p_decoder->fmt_out,
-                                  &id->p_decoder->fmt_out );
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   p_stream->p_sys->psz_deinterlace,
+                                   p_stream->p_sys->p_deinterlace_cfg,
+                                   &id->p_decoder->fmt_out,
+                                   &id->p_decoder->fmt_out );
+
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }
 
     /* Take care of the scaling and chroma conversions */
-    if( ( id->p_decoder->fmt_out.video.i_chroma !=
-          id->p_encoder->fmt_in.video.i_chroma ) ||
-        ( id->p_decoder->fmt_out.video.i_width !=
-          id->p_encoder->fmt_in.video.i_width ) ||
-        ( id->p_decoder->fmt_out.video.i_height !=
-          id->p_encoder->fmt_in.video.i_height ) )
+    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
+        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
+        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
     {
        filter_chain_AppendFilter( id->p_f_chain,
                                   NULL, NULL,
-                                  &id->p_decoder->fmt_out,
+                                  p_fmt_out,
                                   &id->p_encoder->fmt_in );
     }
 
@@ -321,10 +320,18 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
+    if( id->p_f_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
+    }
+    if( id->p_uf_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
+    }
+
     /* Calculate scaling
      * width/height of source */
-    int i_src_width = id->p_decoder->fmt_out.video.i_width;
-    int i_src_height = id->p_decoder->fmt_out.video.i_height;
+    int i_src_width = p_fmt_out->video.i_width;
+    int i_src_height = p_fmt_out->video.i_height;
 
     /* with/height scaling */
     float f_scale_width = 1;
@@ -335,10 +342,10 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     int i_dst_height;
 
     /* aspect ratio */
-    float f_aspect = (double)id->p_decoder->fmt_out.video.i_sar_num *
-                     id->p_decoder->fmt_out.video.i_width /
-                     id->p_decoder->fmt_out.video.i_sar_den /
-                     id->p_decoder->fmt_out.video.i_height;
+    float f_aspect = (double)p_fmt_out->video.i_sar_num *
+                     p_fmt_out->video.i_width /
+                     p_fmt_out->video.i_sar_den /
+                     p_fmt_out->video.i_height;
 
     msg_Dbg( p_stream, "decoder aspect is %f:1", f_aspect );
 
@@ -437,13 +444,13 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( !id->p_encoder->fmt_out.video.i_frame_rate ||
         !id->p_encoder->fmt_out.video.i_frame_rate_base )
     {
-        if( id->p_decoder->fmt_out.video.i_frame_rate &&
-            id->p_decoder->fmt_out.video.i_frame_rate_base )
+        if( p_fmt_out->video.i_frame_rate &&
+            p_fmt_out->video.i_frame_rate_base )
         {
             id->p_encoder->fmt_out.video.i_frame_rate =
-                id->p_decoder->fmt_out.video.i_frame_rate;
+                p_fmt_out->video.i_frame_rate;
             id->p_encoder->fmt_out.video.i_frame_rate_base =
-                id->p_decoder->fmt_out.video.i_frame_rate_base;
+                p_fmt_out->video.i_frame_rate_base;
         }
         else
         {
@@ -468,8 +475,8 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     {
         vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                      &id->p_encoder->fmt_out.video.i_sar_den,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     (uint64_t)p_fmt_out->video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)p_fmt_out->video.i_sar_den * i_src_height * i_dst_width,
                      0 );
     }
     else
-- 
1.7.10.4

