[vlc-devel] [PATCH 2/3] decoder: don't control the vout when not started

[Thomas Guillem]

Calling vout functions that need to Hold the vout_control will deadlock
if the vout is not yet started.
---
 src/input/decoder.c | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/input/decoder.c b/src/input/decoder.c
index b9cdc7d4a81..488998fc35e 100644
--- a/src/input/decoder.c
+++ b/src/input/decoder.c
@@ -128,6 +128,7 @@ struct vlc_input_decoder_t
     audio_output_t *p_aout;
 
     vout_thread_t   *p_vout;
+    bool             vout_started;
     enum vlc_vout_order vout_order;
 
     /* -- Theses variables need locking on read *and* write -- */
@@ -476,12 +477,20 @@ static int ModuleThread_UpdateVideoFormat( decoder_t *p_dec, vlc_video_context *
                                    &has_started);
     if (p_vout != NULL)
     {
+        vlc_mutex_lock( &p_owner->lock );
+        p_owner->vout_started = true;
+        vlc_mutex_unlock( &p_owner->lock );
+
         if (has_started)
             decoder_Notify(p_owner, on_vout_started, p_vout, p_owner->vout_order);
         return 0;
     }
 
 error:
+    vlc_mutex_lock( &p_owner->lock );
+    p_owner->vout_started = false;
+    vlc_mutex_unlock( &p_owner->lock );
+
     if (p_owner->vctx != NULL)
     {
         vlc_video_context_Release(p_owner->vctx);
@@ -1023,6 +1032,8 @@ static int ModuleThread_PlayVideo( vlc_input_decoder_t *p_owner, picture_t *p_pi
     }
 
     vlc_mutex_lock( &p_owner->lock );
+    assert( p_owner->vout_started );
+
     bool prerolled = p_owner->i_preroll_end != PREROLL_NONE;
     if( prerolled && p_owner->i_preroll_end > p_picture->date )
     {
@@ -1474,7 +1485,7 @@ static void DecoderThread_Flush( vlc_input_decoder_t *p_owner )
     }
     else if( p_dec->fmt_out.i_cat == VIDEO_ES )
     {
-        if( p_owner->p_vout )
+        if( p_owner->p_vout && p_owner->vout_started )
             vout_FlushAll( p_owner->p_vout );
 
         /* Reset the pool cancel state, previously set by
@@ -1504,7 +1515,7 @@ static void DecoderThread_ChangePause( vlc_input_decoder_t *p_owner, bool paused
     {
         case VIDEO_ES:
             vlc_mutex_lock( &p_owner->lock );
-            if( p_owner->p_vout != NULL )
+            if( p_owner->p_vout != NULL && p_owner->vout_started )
                 vout_ChangePause( p_owner->p_vout, paused, date );
             vlc_mutex_unlock( &p_owner->lock );
             break;
@@ -1530,7 +1541,7 @@ static void DecoderThread_ChangeRate( vlc_input_decoder_t *p_owner, float rate )
     switch( p_dec->fmt_out.i_cat )
     {
         case VIDEO_ES:
-            if( p_owner->p_vout != NULL )
+            if( p_owner->p_vout != NULL && p_owner->vout_started )
                 vout_ChangeRate( p_owner->p_vout, rate );
             break;
         case AUDIO_ES:
@@ -1562,7 +1573,7 @@ static void DecoderThread_ChangeDelay( vlc_input_decoder_t *p_owner, vlc_tick_t
     {
         case VIDEO_ES:
             vlc_mutex_lock( &p_owner->lock );
-            if( p_owner->p_vout != NULL )
+            if( p_owner->p_vout != NULL && p_owner->vout_started )
                 vout_ChangeDelay( p_owner->p_vout, delay );
             vlc_mutex_unlock( &p_owner->lock );
             break;
@@ -1792,6 +1803,7 @@ CreateDecoder( vlc_object_t *p_parent,
     p_owner->cbs_userdata = cbs_userdata;
     p_owner->p_aout = NULL;
     p_owner->p_vout = NULL;
+    p_owner->vout_started = false;
     p_owner->i_spu_channel = VOUT_SPU_CHANNEL_INVALID;
     p_owner->i_spu_order = 0;
     p_owner->p_sout = p_sout;
@@ -2151,7 +2163,8 @@ void vlc_input_decoder_Delete( vlc_input_decoder_t *p_owner )
      *
      * This unblocks the thread, allowing the decoder module to join all its
      * worker threads (if any) and the decoder thread to terminate. */
-    if( p_dec->fmt_in.i_cat == VIDEO_ES && p_owner->p_vout != NULL )
+    if( p_dec->fmt_in.i_cat == VIDEO_ES && p_owner->p_vout != NULL
+     && p_owner->vout_started )
     {
         if (p_owner->out_pool)
             picture_pool_Cancel( p_owner->out_pool, true );
@@ -2324,7 +2337,8 @@ void vlc_input_decoder_Flush( vlc_input_decoder_t *p_owner )
          * after being unstuck. */
 
         vlc_mutex_lock( &p_owner->lock );
-        if( p_owner->dec.fmt_out.i_cat == VIDEO_ES && p_owner->p_vout )
+        if( p_owner->dec.fmt_out.i_cat == VIDEO_ES && p_owner->p_vout
+         && p_owner->vout_started )
             vout_FlushAll( p_owner->p_vout );
         vlc_mutex_unlock( &p_owner->lock );
     }
-- 
2.29.2