[vlc-commits] hw:d3d11: Fix crashes when mixing D3D11 filters with CPU filters

[Steve Lhomme]

vlc/vlc-3.0 | branch: master | Steve Lhomme <robUx4 at videolabs.io> | Mon Dec 11 09:37:56 2017 +0100| [df045e5b7b7e3174363be19597a067b95f6d6b90] | committer: Hugo Beauzée-Luyssen

hw:d3d11: Fix crashes when mixing D3D11 filters with CPU filters

The filter chain is not recomputed and we may use pictures not coming from the
right pool.

Fix crash in #19266

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>
(cherry picked from commit c6ca37272832afedc761a7dc28d5a821e6689df3)
Signed-off-by: Hugo Beauzée-Luyssen <hugo at beauzee.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=df045e5b7b7e3174363be19597a067b95f6d6b90
---

 modules/hw/d3d11/d3d11_deinterlace.c | 8 ++++----
 modules/hw/d3d11/d3d11_filters.c     | 7 +++++++
 modules/hw/d3d11/d3d11_surface.c     | 6 ++++++
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/modules/hw/d3d11/d3d11_deinterlace.c b/modules/hw/d3d11/d3d11_deinterlace.c
index c8aaf69c78..b380a61247 100644
--- a/modules/hw/d3d11/d3d11_deinterlace.c
+++ b/modules/hw/d3d11/d3d11_deinterlace.c
@@ -146,17 +146,17 @@ static int RenderPic( filter_t *p_filter, picture_t *p_outpic, picture_t *p_pic,
     if( p_cur && p_next )
     {
         picture_sys_t *picsys_next = ActivePictureSys(p_next);
-        if ( assert_ProcessorInput(p_filter, picsys_next) )
+        if ( unlikely(!picsys_next) || assert_ProcessorInput(p_filter, picsys_next) )
             return VLC_EGENERIC;
 
         picture_sys_t *picsys_cur = ActivePictureSys(p_cur);
-        if ( assert_ProcessorInput(p_filter, picsys_cur) )
+        if ( unlikely(!picsys_cur) || assert_ProcessorInput(p_filter, picsys_cur) )
             return VLC_EGENERIC;
 
         if ( p_prev )
         {
             picture_sys_t *picsys_prev = ActivePictureSys(p_prev);
-            if ( assert_ProcessorInput(p_filter, picsys_prev) )
+            if ( unlikely(!picsys_prev) || assert_ProcessorInput(p_filter, picsys_prev) )
                 return VLC_EGENERIC;
 
             stream.pInputSurface    = picsys_cur->processorInput;
@@ -177,7 +177,7 @@ static int RenderPic( filter_t *p_filter, picture_t *p_outpic, picture_t *p_pic,
     else
     {
         picture_sys_t *p_sys_src = ActivePictureSys(p_pic);
-        if ( assert_ProcessorInput(p_filter, p_sys_src) )
+        if ( unlikely(!p_sys_src) || assert_ProcessorInput(p_filter, p_sys_src) )
             return VLC_EGENERIC;
 
         /* first single frame */
diff --git a/modules/hw/d3d11/d3d11_filters.c b/modules/hw/d3d11/d3d11_filters.c
index 615aa5763e..df36a82a89 100644
--- a/modules/hw/d3d11/d3d11_filters.c
+++ b/modules/hw/d3d11/d3d11_filters.c
@@ -225,6 +225,13 @@ static picture_t *Filter(filter_t *p_filter, picture_t *p_pic)
         picture_Release( p_pic );
         return NULL;
     }
+    if (unlikely(!p_outpic->p_sys))
+    {
+        /* the output filter configuration may have changed since the filter
+         * was opened */
+        picture_Release( p_pic );
+        return NULL;
+    }
 
     picture_CopyProperties( p_outpic, p_pic );
 
diff --git a/modules/hw/d3d11/d3d11_surface.c b/modules/hw/d3d11/d3d11_surface.c
index 502ff8ecf7..e8e34b7fe6 100644
--- a/modules/hw/d3d11/d3d11_surface.c
+++ b/modules/hw/d3d11/d3d11_surface.c
@@ -580,6 +580,12 @@ static void NV12_D3D11(filter_t *p_filter, picture_t *src, picture_t *dst)
 {
     filter_sys_t *sys = (filter_sys_t*) p_filter->p_sys;
     picture_sys_t *p_sys = dst->p_sys;
+    if (unlikely(p_sys==NULL))
+    {
+        /* the output filter configuration may have changed since the filter
+         * was opened */
+        return;
+    }
 
     D3D11_TEXTURE2D_DESC texDesc;
     ID3D11Texture2D_GetDesc( sys->staging_pic->p_sys->texture[KNOWN_DXGI_INDEX], &texDesc);