>From d0e4bc50d9f64cf704ae2cc917d162e9181ebf84 Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <steinar+vlc@gunderson.no>
Date: Sat, 8 Jun 2013 23:14:15 +0200
Subject: [PATCH 5/5] Fix deinterlacing of packed YUV formats.

If adding the deinterlacer filter fails, try conversion filters
until we try something the deinterlacer wants, and then try again.
This fixes the problem where the input from the decklink module
(at least with --no-decklink-tenbits) is UYVY, while the deinterlacer only
wants planar formats.

Another possibility would be going straight for the format the
encoder wants, but often, this would be a 4:2:0 format, and 4:2:0
is pretty bad to deinterlace in, since the vertical chroma resolution
is gone already (the chroma interlacing in 4:2:0 is rather odd).
---
 modules/stream_out/transcode/video.c |   56 +++++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 5 deletions(-)

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 09fa328..70936c8 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -278,12 +278,58 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   p_stream->p_sys->psz_deinterlace,
-                                   p_stream->p_sys->p_deinterlace_cfg,
-                                   &id->p_decoder->fmt_out,
-                                   &id->p_decoder->fmt_out );
+        filter_t *p_filter =
+            filter_chain_AppendFilter( id->p_f_chain,
+                                       p_stream->p_sys->psz_deinterlace,
+                                       p_stream->p_sys->p_deinterlace_cfg,
+                                       p_fmt_out,
+                                       p_fmt_out );
+        if( !p_filter )
+        {
+            /* Most likely it failed due to wrong (e.g. non-planar) input format.
+             * Try converting to the color space the encoder wants and try again.
+             */
+            msg_Dbg( p_stream, "deinterlace init failed, trying colorspace conversion" );
+
+            bool b_ok = false;
+            for( const vlc_fourcc_t *fourcc = vlc_fourcc_GetYUVFallback( p_fmt_out->video.i_chroma );
+                 *fourcc != 0;
+                 ++fourcc )
+            {
+                msg_Dbg( p_stream, "trying deinterlace through colorspace conversion to %4.4s", (char *)fourcc );
+
+                es_format_t p_new_fmt;
+                es_format_Copy( &p_new_fmt, p_fmt_out );
+                p_new_fmt.video.i_chroma = *fourcc;  // XXX: is this enough?
+
+                filter_chain_Reset( id->p_f_chain, NULL, NULL ); 
+
+                p_filter = filter_chain_AppendFilter( id->p_f_chain,
+                                                      NULL, NULL,
+                                                      p_fmt_out,
+                                                      &p_new_fmt );
+                if( !p_filter )
+                    continue;
+                   
+                p_filter = filter_chain_AppendFilter( id->p_f_chain,
+                                                      p_stream->p_sys->psz_deinterlace,
+                                                      p_stream->p_sys->p_deinterlace_cfg,
+                                                      &p_new_fmt,
+                                                      &p_new_fmt );
+                if( p_filter )
+                {
+                    msg_Dbg( p_stream, "deinterlace through colorspace conversion to %4.4s succeeded", (char *)fourcc );
+                    b_ok = true;
+                    break;
+                }
+            }
 
+            if( !b_ok )
+            {
+                msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+                return;
+            } 
+        }
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }
 
-- 
1.7.10.4

