>From e2b93c44a0eac51abea6d380fc267d1b0f979f7f Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <steinar+vlc@gunderson.no>
Date: Sat, 8 Jun 2013 23:11:16 +0200
Subject: [PATCH 5/6] If the transcode filter chain changes the format, notify
 the encoder.

We need to initialize the encoder before the filter chain, since
the encoder can change the output size (e.g. to make the resolution
divisible by 8). However, the filter chain can also change the
format (in particular by changing colorspace, or by deinterlacing
to twice the frame rate), so once the filter chain is set up,
reinitialize the encoder with the new format.
---
 modules/stream_out/transcode/video.c |   43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 669e87c..1a23e8d 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -261,6 +261,14 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_t *id )
 {
+    /* Clean up if we already had a previous chain set up */
+    if( id->p_f_chain )
+        filter_chain_Delete( id->p_f_chain );
+    if( id->p_uf_chain )
+        filter_chain_Delete( id->p_uf_chain );
+    id->p_f_chain = id->p_uf_chain = NULL;
+        
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
 
     id->p_f_chain = filter_chain_New( p_stream, "video filter2",
                                      false,
@@ -796,6 +804,29 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
             id->p_encoder->fmt_out.video.i_height = p_sys->i_height & ~1;
             id->p_encoder->fmt_out.video.i_sar_num = id->p_encoder->fmt_out.video.i_sar_den = 0;
 
+            /* Initialize first the filter chain so that the encoder knows the desired
+             * output format for filters (important for e.g. deinterlacing). Then initialize
+             * the encoder, and then create the filter chain anew since we might need to
+             * do something differently once we know the encoder's preferred width/height etc.
+             */
+            if( id->p_f_chain )
+                filter_chain_Delete( id->p_f_chain );
+            if( id->p_uf_chain )
+                filter_chain_Delete( id->p_uf_chain );
+            id->p_f_chain = id->p_uf_chain = NULL;
+
+            int i_saved_frame_rate = id->p_encoder->fmt_out.video.i_frame_rate;
+            int i_saved_frame_rate_base = id->p_encoder->fmt_out.video.i_frame_rate_base;
+            transcode_video_encoder_init( p_stream, id );
+            transcode_video_filter_init( p_stream, id );
+
+            /* Reset the stored frame rate before the second initialization.
+             * Otherwise, frame rate detected from the filter output
+             * will be taken as a user-given, hard-coded frame rate.
+             */
+            id->p_encoder->fmt_out.video.i_frame_rate = i_saved_frame_rate;
+            id->p_encoder->fmt_out.video.i_frame_rate_base = i_saved_frame_rate_base;
+
             transcode_video_encoder_init( p_stream, id );
             transcode_video_filter_init( p_stream, id );
             memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
@@ -804,8 +835,20 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
 
         if( unlikely( !id->p_encoder->p_module ) )
         {
+            if( id->p_f_chain )
+                filter_chain_Delete( id->p_f_chain );
+            if( id->p_uf_chain )
+                filter_chain_Delete( id->p_uf_chain );
+            id->p_f_chain = id->p_uf_chain = NULL;
+
+            int i_saved_frame_rate = id->p_encoder->fmt_out.video.i_frame_rate;
+            int i_saved_frame_rate_base = id->p_encoder->fmt_out.video.i_frame_rate_base;
             transcode_video_encoder_init( p_stream, id );
+            transcode_video_filter_init( p_stream, id );
 
+            id->p_encoder->fmt_out.video.i_frame_rate = i_saved_frame_rate;
+            id->p_encoder->fmt_out.video.i_frame_rate_base = i_saved_frame_rate_base;
+            transcode_video_encoder_init( p_stream, id );
             transcode_video_filter_init( p_stream, id );
             memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
 
-- 
1.7.10.4

