[vlc-devel] [RFC PATCH 3/4] avocodec: fix deadlock when closing with mt

[Thomas Guillem]

lavc_GetFrame can be stuck in decoder_GetPicture when decoder is closing.
---
 modules/codec/avcodec/video.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 69d6f6e..1b98c5c 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -74,6 +74,7 @@ struct decoder_sys_t
 
     /* */
     bool b_flush;
+    bool b_flushing_buffers;
 
     /* VA API */
     vlc_va_t *p_va;
@@ -536,7 +537,11 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             /* NOTE: data is good only the timeline changed so do not flush decoder */
             post_mt( p_sys );
             if( p_block->i_flags & BLOCK_FLAG_DISCONTINUITY )
+            {
+                p_sys->b_flushing_buffers = true;
                 avcodec_flush_buffers( p_context );
+                p_sys->b_flushing_buffers = false;
+            }
             wait_mt( p_sys );
 #endif
             if( p_block->i_flags & BLOCK_FLAG_CORRUPTED )
@@ -858,7 +863,10 @@ void EndVideoDec( decoder_t *p_dec )
 
     /* do not flush buffers if codec hasn't been opened (theora/vorbis/VC1) */
     if( p_sys->p_context->codec )
+    {
+        p_sys->b_flushing_buffers = true;
         avcodec_flush_buffers( p_sys->p_context );
+    }
 
     wait_mt( p_sys );
 
@@ -1065,6 +1073,12 @@ static int lavc_GetFrame(struct AVCodecContext *ctx, AVFrame *frame, int flags)
     frame->opaque = NULL;
 
     wait_mt(sys);
+    if (sys->b_flushing_buffers)
+    {
+        /* If decoder is flushing buffers, abort current lavc_GetFrame call */
+        post_mt(sys);
+        return -1;
+    }
     if (sys->p_va == NULL)
     {
         if (!sys->b_direct_rendering)
-- 
2.1.4