[vlc-commits] vdpau: fix hardware decoding with non-multiple of 16 heights

[Rémi Denis-Courmont]

vlc/vlc-2.2 | branch: master | Rémi Denis-Courmont <remi at remlab.net> | Wed Apr 29 18:33:37 2015 +0300| [c9ac27670f33cc3533131ffe8e927a923ea9dcf6] | committer: Rémi Denis-Courmont

vdpau: fix hardware decoding with non-multiple of 16 heights

On the one hand, when decoding in hardware, the video surface
dimensions will match the coded dimensions, rather than the visible
ones. Otherwise the hardware decoder would not have enough space for
its output. In this the libavcodec plugin does not add padding (since
libavcodec SIMD optimizations are not involved).

On the other hand, when decoding in software, the video surface height
must not include the padding added by libavcodec SIMD optimizations. So
then the visible height needs to be used to avoid bleeding green lines.

(cherry picked from commit b670381ed43a5635d5ad5df9a9fef9454ba7e5c2)

Conflicts:
	modules/hw/vdpau/chroma.c

> http://git.videolan.org/gitweb.cgi/vlc/vlc-2.2.git/?a=commit;h=c9ac27670f33cc3533131ffe8e927a923ea9dcf6
---

 modules/hw/vdpau/chroma.c |    9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/modules/hw/vdpau/chroma.c b/modules/hw/vdpau/chroma.c
index 346d329..f2ec3ac 100644
--- a/modules/hw/vdpau/chroma.c
+++ b/modules/hw/vdpau/chroma.c
@@ -95,7 +95,7 @@ static VdpStatus MixerSetupColors(filter_t *filter, const VdpProcamp *procamp,
 }
 
 /** Create VDPAU video mixer */
-static VdpVideoMixer MixerCreate(filter_t *filter)
+static VdpVideoMixer MixerCreate(filter_t *filter, bool import)
 {
     filter_sys_t *sys = filter->p_sys;
     VdpVideoMixer mixer;
@@ -187,7 +187,8 @@ static VdpVideoMixer MixerCreate(filter_t *filter)
         VDP_VIDEO_MIXER_PARAMETER_CHROMA_TYPE,
     };
     uint32_t width = filter->fmt_in.video.i_width;
-    uint32_t height = filter->fmt_in.video.i_visible_height;
+    uint32_t height = import ? filter->fmt_in.video.i_visible_height
+                             : filter->fmt_in.video.i_height;
     const void *values[3] = { &width, &height, &sys->chroma, };
 
     err = vdp_video_mixer_create(sys->vdp, sys->device, featc, featv,
@@ -279,6 +280,8 @@ static void Flush(filter_t *filter)
         }
 }
 
+static picture_t *YCbCrRender(filter_t *filter, picture_t *src);
+
 /** Get a VLC picture for a VDPAU output surface */
 static picture_t *OutputAllocate(filter_t *filter)
 {
@@ -300,7 +303,7 @@ static picture_t *OutputAllocate(filter_t *filter)
     /* First picture: get the context and allocate the mixer */
     sys->vdp = vdp_hold_x11(psys->vdp, NULL);
     sys->device = psys->device;
-    sys->mixer = MixerCreate(filter);
+    sys->mixer = MixerCreate(filter, filter->pf_video_filter == YCbCrRender);
     if (sys->mixer != VDP_INVALID_HANDLE)
         return pic;