[vlc-commits] dxva2: fix the OutOfMemory crash on the first BeginFrame() differently

[Steve Lhomme]

vlc | branch: master | Steve Lhomme <robUx4 at videolabs.io> | Tue Aug  2 13:28:53 2016 +0200| [320662df328fb95063896612b649999d9312a68c] | committer: Jean-Baptiste Kempf

dxva2: fix the OutOfMemory crash on the first BeginFrame() differently

We must call CreateSurface() with N-1 buffer.
But for the OOM issue we also need to check an extra buffer can be allocated.

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=320662df328fb95063896612b649999d9312a68c
---

 modules/codec/avcodec/dxva2.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/modules/codec/avcodec/dxva2.c b/modules/codec/avcodec/dxva2.c
index f2adb47..4357d29 100644
--- a/modules/codec/avcodec/dxva2.c
+++ b/modules/codec/avcodec/dxva2.c
@@ -703,7 +703,7 @@ static int DxCreateVideoDecoder(vlc_va_t *va, int codec_id, const video_format_t
     hr = IDirectXVideoDecoderService_CreateSurface((IDirectXVideoDecoderService*) sys->d3ddec,
                                                          sys->surface_width,
                                                          sys->surface_height,
-                                                         sys->surface_count,
+                                                         sys->surface_count - 1,
                                                          p_sys->render,
                                                          D3DPOOL_DEFAULT,
                                                          0,
@@ -711,13 +711,30 @@ static int DxCreateVideoDecoder(vlc_va_t *va, int codec_id, const video_format_t
                                                          (LPDIRECT3DSURFACE9*) sys->hw_surface,
                                                          NULL);
     if (FAILED(hr)) {
-        msg_Err(va, "IDirectXVideoAccelerationService_CreateSurface %d failed (hr=0x%0lx)", sys->surface_count, hr);
+        msg_Err(va, "IDirectXVideoAccelerationService_CreateSurface %d failed (hr=0x%0lx)", sys->surface_count - 1, hr);
         sys->surface_count = 0;
         return VLC_EGENERIC;
     }
     msg_Dbg(va, "IDirectXVideoAccelerationService_CreateSurface succeed with %d surfaces (%dx%d)",
             sys->surface_count, sys->surface_width, sys->surface_height);
 
+    IDirect3DSurface9 *tstCrash;
+    hr = IDirectXVideoDecoderService_CreateSurface((IDirectXVideoDecoderService*) sys->d3ddec,
+                                                         sys->surface_width,
+                                                         sys->surface_height,
+                                                         0,
+                                                         p_sys->render,
+                                                         D3DPOOL_DEFAULT,
+                                                         0,
+                                                         DXVA2_VideoDecoderRenderTarget,
+                                                         &tstCrash,
+                                                         NULL);
+    if (FAILED(hr)) {
+        msg_Err(va, "extra buffer impossible, avoid a crash (hr=0x%0lx)", hr);
+        return VLC_EGENERIC;
+    }
+    IDirect3DSurface9_Release(tstCrash);
+
     /* */
     DXVA2_VideoDesc dsc;
     ZeroMemory(&dsc, sizeof(dsc));