[vlc-commits] commit: Workaround a potential segfault when using VAAPI/DXVA2 (close #3606 ). (Laurent Aimar )

[git at videolan.org]

vlc/vlc-1.1 | branch: master | Laurent Aimar <fenrir at videolan.org> | Sun Jun 13 01:41:11 2010 +0200| [f134a12a7469c003ba95177dc23e65c70a4a5a72] | committer: Jean-Baptiste Kempf 

Workaround a potential segfault when using VAAPI/DXVA2 (close #3606).

It seems that some avcodec decoders release frames even after being flushed.

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-1.1.git/?a=commit;h=f134a12a7469c003ba95177dc23e65c70a4a5a72
---

 modules/codec/avcodec/video.c |   20 +++++++++++---------
 1 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 230010d..bc37e12 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -725,7 +725,10 @@ void EndVideoDec( decoder_t *p_dec )
     if( p_sys->p_ff_pic ) av_free( p_sys->p_ff_pic );
 
     if( p_sys->p_va )
+    {
         vlc_va_Delete( p_sys->p_va );
+        p_sys->p_va = NULL;
+    }
 }
 
 /*****************************************************************************
@@ -1079,25 +1082,24 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
     if( p_sys->p_va )
     {
         vlc_va_Release( p_sys->p_va, p_ff_pic );
-
-        /* */
-        for( int i = 0; i < 4; i++ )
-            p_ff_pic->data[i] = NULL;
     }
     else if( !p_ff_pic->opaque )
     {
-        avcodec_default_release_buffer( p_context, p_ff_pic );
+        /* We can end up here without the AVFrame being allocated by
+         * avcodec_default_get_buffer() if VA is used and the frame is
+         * released when the decoder is closed
+         */
+        if( p_ff_pic->type == FF_BUFFER_TYPE_INTERNAL )
+            avcodec_default_release_buffer( p_context, p_ff_pic );
     }
     else
     {
         picture_t *p_pic = (picture_t*)p_ff_pic->opaque;
 
         decoder_UnlinkPicture( p_dec, p_pic );
-
-        /* */
-        for( int i = 0; i < 4; i++ )
-            p_ff_pic->data[i] = NULL;
     }
+    for( int i = 0; i < 4; i++ )
+        p_ff_pic->data[i] = NULL;
 }
 
 static void ffmpeg_NextPts( decoder_t *p_dec )