[vlc-commits] vdpau/chroma: fix corner case of corner case when changing device

[Rémi Denis-Courmont]

vlc | branch: master | Rémi Denis-Courmont <remi at remlab.net> | Mon Mar 17 21:15:48 2014 +0200| [2ea9a9818e1d6ce5c27e261b4d0d5760f4dfd586] | committer: Rémi Denis-Courmont

vdpau/chroma: fix corner case of corner case when changing device

To ensure the vdp_t instance pointers comparison is valid, the old and
new instances must be contemporary. Otherwise, the heap could put both
instances at the same memory address. So keep a reference to the old
device until after the comparison.

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=2ea9a9818e1d6ce5c27e261b4d0d5760f4dfd586
---

 modules/hw/vdpau/chroma.c |   21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/modules/hw/vdpau/chroma.c b/modules/hw/vdpau/chroma.c
index 99f9b84..b1d92a1 100644
--- a/modules/hw/vdpau/chroma.c
+++ b/modules/hw/vdpau/chroma.c
@@ -289,20 +289,19 @@ static picture_t *OutputAllocate(filter_t *filter)
 
     picture_sys_t *psys = pic->p_sys;
     assert(psys->vdp != NULL);
-    if (unlikely(sys->vdp != psys->vdp))
+
+    if (unlikely(sys->vdp != psys->vdp) && (sys->mixer != VDP_INVALID_HANDLE))
     {
-        if (sys->mixer != VDP_INVALID_HANDLE)
-        {
-            Flush(filter);
-            vdp_video_mixer_destroy(sys->vdp, sys->mixer);
-            sys->mixer = VDP_INVALID_HANDLE;
-        }
-        sys->vdp = psys->vdp;
-        sys->device = psys->device;
+        Flush(filter); /* release surfaces from the old device */
+        vdp_video_mixer_destroy(sys->vdp, sys->mixer);
+        vdp_release_x11(sys->vdp);
+        sys->mixer = VDP_INVALID_HANDLE;
     }
 
     if (unlikely(sys->mixer == VDP_INVALID_HANDLE))
     {
+        sys->vdp = vdp_hold_x11(psys->vdp, NULL);
+        sys->device = psys->device;
         sys->mixer = MixerCreate(filter);
         if (sys->mixer == VDP_INVALID_HANDLE)
             goto error;
@@ -659,8 +658,10 @@ static void OutputClose(vlc_object_t *obj)
 
     Flush(filter);
     if (sys->mixer != VDP_INVALID_HANDLE)
+    {
         vdp_video_mixer_destroy(sys->vdp, sys->mixer);
-
+        vdp_release_x11(sys->vdp);
+    }
     free(sys);
 }