[vlc-devel] [PATCH 2/7] transcode: video: recreate the filters once we know the encoder input format

[Steve Lhomme]

We may get better filters if the encoder can handle GPU (which is not known
when test creating the encoder with no decoder).

This makes the p_final_conv_static filter useless as we always have the proper
output of the filter chain.
---
 modules/stream_out/transcode/video.c | 37 +++++++++++-----------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index f5100184c35..0e5f589b417 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -572,6 +572,7 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
             transcode_video_framerate_apply( &p_pic->format, &id->decoder_out.video );
             transcode_video_sar_apply( &p_pic->format, &id->decoder_out.video );
 
+            bool need_filters = false;
             if( !transcode_video_filters_configured( id ) )
             {
                 if( transcode_video_filters_init( p_stream,
@@ -582,13 +583,9 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
                                                  transcode_encoder_format_in( id->encoder ),
                                                  id ) != VLC_SUCCESS )
                     goto error;
+                need_filters = true;
             }
 
-            /* Store the current encoder input chroma to detect whether we need
-             * a converter in p_final_conv_static. The encoder will override it
-             * if it needs any different format or chroma. */
-            es_format_t filter_fmt_out;
-            es_format_Copy( &filter_fmt_out, transcode_encoder_format_in( id->encoder ) );
             bool is_encoder_open = transcode_encoder_opened( id->encoder );
 
             /* Start missing encoder */
@@ -602,25 +599,21 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
                 goto error;
             }
 
-            /* The fmt_in may have been overriden by the encoder. */
-            const es_format_t *encoder_fmt_in = transcode_encoder_format_in( id->encoder );
-
-            /* In case the encoder wasn't open yet, check if we need to add
-             * a converter between last user filter and encoder. */
-            if( !is_encoder_open &&
-                filter_fmt_out.i_codec != encoder_fmt_in->i_codec )
+            if ( need_filters || !is_encoder_open )
             {
-                if ( !id->p_final_conv_static )
-                    id->p_final_conv_static =
-                        filter_chain_NewVideo( p_stream, false, NULL );
-                filter_chain_Reset( id->p_final_conv_static,
-                                    &filter_fmt_out,
-                                    //encoder_vctx_in,
-                                    NULL,
-                                    encoder_fmt_in );
-                filter_chain_AppendConverter( id->p_final_conv_static, NULL );
+                transcode_video_filters_clean( id );
+                if( transcode_video_filters_init( p_stream,
+                                                  id->p_filterscfg,
+                                                 (id->p_enccfg->video.fps.num > 0),
+                                                 &id->decoder_out,
+                                                 picture_GetVideoContext(p_pic),
+                                                 transcode_encoder_format_in( id->encoder ),
+                                                 id ) != VLC_SUCCESS )
+                {
+                    msg_Err( p_stream, "failed to recreate filters" );
+                    goto error;
+                }
             }
-            es_format_Clean(&filter_fmt_out);
 
             msg_Dbg( p_stream, "destination (after video filters) %ux%u",
                                transcode_encoder_format_in( id->encoder )->video.i_width,
-- 
2.26.2