>From 2d965498a232c9c3df8267753bcf09815198ab4f Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <steinar+vlc@gunderson.no>
Date: Sat, 8 Jun 2013 23:11:16 +0200
Subject: [PATCH 5/6] Set up the encoder after the transcode filter chain.

The filter chain can change the format (in particular by changing colorspace,
or by deinterlacing to twice the frame rate), so wait with initializing the
encoder format until the filter chain has been set up.

It might be that the encoder itself demands format changes, e.g. due to
chroma demands or to make the resolution divisible by 16, but we can add
that extra filter to the end of the filter chain after the encoder parameters
have been set up.

Also fix a bug where we'd leak the filter chain when changing it during the
encode.
---
 modules/stream_out/transcode/video.c |   67 ++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index bea36a2..8be2a36 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -261,12 +261,15 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_t *id )
 {
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
 
     id->p_f_chain = filter_chain_New( p_stream, "video filter2",
-                                     false,
-                                     transcode_video_filter_allocation_init,
-                                     transcode_video_filter_allocation_clear,
-                                     p_stream->p_sys );
+                                      false,
+                                      transcode_video_filter_allocation_init,
+                                      transcode_video_filter_allocation_clear,
+                                      p_stream->p_sys );
+    filter_chain_Reset( id->p_f_chain, p_fmt_out, p_fmt_out );
+
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
@@ -279,23 +282,6 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }
 
-    /* Take care of the scaling and chroma conversions */
-    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
-        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
-        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
-    {
-        /* NB! Take care not to reset the frame rate. */
-        es_format_t p_new_fmt;
-        es_format_Copy( &p_new_fmt, &id->p_encoder->fmt_in );
-        p_new_fmt.video.i_frame_rate = p_fmt_out->video.i_frame_rate;
-        p_new_fmt.video.i_frame_rate_base = p_fmt_out->video.i_frame_rate_base;
-
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   NULL, NULL,
-                                   p_fmt_out,
-                                   &p_new_fmt );
-    }
-
     if( p_stream->p_sys->psz_vf2 )
     {
         const es_format_t *p_fmt_out;
@@ -321,6 +307,34 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
 
 }
 
+/* Take care of the scaling and chroma conversions.
+ *
+ * XXX: Shouldn't this really be after p_uf_chain, not p_f_chain,
+ * in case p_uf_chain changes the format?
+ */
+static void conversion_video_filter_append( sout_stream_id_t *id )
+{
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
+    if( id->p_f_chain )
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
+
+    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
+        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
+        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
+    {
+        /* Take care not to reset the frame rate */
+        es_format_t p_new_fmt;
+        es_format_Copy( &p_new_fmt, &id->p_encoder->fmt_in );
+        p_new_fmt.video.i_frame_rate = p_fmt_out->video.i_frame_rate;
+        p_new_fmt.video.i_frame_rate_base = p_fmt_out->video.i_frame_rate_base;
+
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   NULL, NULL,
+                                   p_fmt_out,
+                                   &p_new_fmt );
+    }
+}
+
 static void transcode_video_encoder_init( sout_stream_t *p_stream,
                                           sout_stream_id_t *id )
 {
@@ -794,17 +808,24 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
             id->p_encoder->fmt_out.video.i_height = p_sys->i_height & ~1;
             id->p_encoder->fmt_out.video.i_sar_num = id->p_encoder->fmt_out.video.i_sar_den = 0;
 
-            transcode_video_encoder_init( p_stream, id );
             transcode_video_filter_init( p_stream, id );
+            transcode_video_encoder_init( p_stream, id );
+            conversion_video_filter_append( id );
             memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
         }
 
 
         if( unlikely( !id->p_encoder->p_module ) )
         {
-            transcode_video_encoder_init( p_stream, id );
+            if( id->p_f_chain )
+                filter_chain_Delete( id->p_f_chain );
+            if( id->p_uf_chain )
+                filter_chain_Delete( id->p_uf_chain );
+            id->p_f_chain = id->p_uf_chain = NULL;
 
             transcode_video_filter_init( p_stream, id );
+            transcode_video_encoder_init( p_stream, id );
+            conversion_video_filter_append( id );
             memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
 
             if( transcode_video_encoder_open( p_stream, id ) != VLC_SUCCESS )
-- 
1.7.10.4

