[vlc-devel] [PATCH 7/8] transcode: do the reorientation in the last filter before the encoder

[Steve Lhomme]

This is also how the display modules work, the orientation is usually done
while rendering the picture.

The p_final_conv_static filter is also the one that produces pictures on which
we render subpictures, which is when the reorientation is forced.
---
 modules/stream_out/transcode/video.c | 44 +++++++++++++---------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index a3e4c619a50..0aa60d7b93c 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -271,8 +271,7 @@ static int transcode_video_set_conversions( sout_stream_t *p_stream,
                                             sout_stream_id_sys_t *id,
                                             const es_format_t **pp_src,
                                             vlc_video_context **pp_src_vctx,
-                                            const es_format_t *p_dst,
-                                            bool b_reorient )
+                                            const es_format_t *p_dst )
 {
     filter_owner_t owner = {
         .video = &transcode_filter_video_cbs,
@@ -282,39 +281,25 @@ static int transcode_video_set_conversions( sout_stream_t *p_stream,
     const bool b_do_scale = (*pp_src)->video.i_visible_width != p_dst->video.i_visible_width ||
                             (*pp_src)->video.i_visible_height != p_dst->video.i_visible_height;
     const bool b_do_chroma = (*pp_src)->video.i_chroma != p_dst->video.i_chroma;
-    const bool b_do_orient = ((*pp_src)->video.orientation != ORIENT_NORMAL) && b_reorient;
 
-    const es_format_t *p_tmpdst = p_dst;
-
-    if( ! (b_do_scale || b_do_chroma || b_do_orient) )
+    if( ! (b_do_scale || b_do_chroma) )
         return VLC_SUCCESS;
 
-    es_format_t tmpdst;
-    if( b_do_orient )
-    {
-        es_format_Init( &tmpdst, VIDEO_ES, p_dst->video.i_chroma );
-        video_format_ApplyRotation( &tmpdst.video, &p_dst->video );
-        p_tmpdst = &tmpdst;
-    }
-
-    msg_Dbg( p_stream, "adding (scale %d,chroma %d, orient %d) converters",
-                b_do_scale, b_do_chroma, b_do_orient );
+    msg_Dbg( p_stream, "adding (scale %d,chroma %d) converters",
+                b_do_scale, b_do_chroma);
 
     id->p_conv_nonstatic = filter_chain_NewVideo( p_stream, true, &owner );
     if( !id->p_conv_nonstatic )
         return VLC_EGENERIC;
-    filter_chain_Reset( id->p_conv_nonstatic, *pp_src, *pp_src_vctx, p_tmpdst );
+    filter_chain_Reset( id->p_conv_nonstatic, *pp_src, *pp_src_vctx, p_dst );
 
-    if( filter_chain_AppendConverter( id->p_conv_nonstatic, p_tmpdst ) != VLC_SUCCESS )
+    if( filter_chain_AppendConverter( id->p_conv_nonstatic, p_dst ) != VLC_SUCCESS )
         return VLC_EGENERIC;
 
     *pp_src = filter_chain_GetFmtOut( id->p_conv_nonstatic );
     *pp_src_vctx = filter_chain_GetVideoCtxOut( id->p_conv_nonstatic );
     debug_format( p_stream, *pp_src );
 
-    if( ((*pp_src)->video.orientation != ORIENT_NORMAL) && b_reorient )
-        return VLC_EGENERIC;
-
     return VLC_SUCCESS;
 }
 
@@ -360,8 +345,8 @@ static int transcode_video_filters_init( sout_stream_t *p_stream,
     }
 
     /* Chroma and other conversions */
-    if( transcode_video_set_conversions( p_stream, id, &p_src, &src_ctx, p_dst,
-                                         p_cfg->video.b_reorient ) != VLC_SUCCESS )
+    if( transcode_video_set_conversions( p_stream, id, &p_src, &src_ctx, p_dst
+                                        ) != VLC_SUCCESS )
         return VLC_EGENERIC;
 
     /* User filters */
@@ -593,8 +578,19 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
                 if ( !id->p_final_conv_static )
                     id->p_final_conv_static =
                         filter_chain_NewVideo( p_stream, false, NULL );
+
+                const es_format_t *p_fmt_filtered = &filter_fmt_out;
+                es_format_t tmpdst;
+                if ( id->p_filterscfg->video.b_reorient &&
+                    filter_fmt_out.video.orientation != ORIENT_NORMAL )
+                {
+                    es_format_Init( &tmpdst, VIDEO_ES, p_fmt_filtered->video.i_chroma );
+                    video_format_ApplyRotation( &tmpdst.video, &p_fmt_filtered->video );
+                    p_fmt_filtered = &tmpdst;
+                }
+
                 filter_chain_Reset( id->p_final_conv_static,
-                                    &filter_fmt_out,
+                                    p_fmt_filtered,
                                     //encoder_vctx_in,
                                     NULL,
                                     encoder_fmt_in );
-- 
2.26.2