[vlc-commits] hw: vaapi: fix p010 <=> I420_10 conversions

[Thomas Guillem]

vlc/vlc-3.0 | branch: master | Thomas Guillem <thomas at gllm.fr> | Fri Mar 16 09:06:53 2018 +0100| [4158368c00c1e0541d475e732e38103196b36e8e] | committer: Thomas Guillem

hw: vaapi: fix p010 <=> I420_10 conversions

P010 is on MSB, and VLC_CODEC_I420_10B/VLC_CODEC_I420_10L are on LSB, so we
need to shift bits by 6 when doing the conversion. VLC_CODEC_I420_10L should be
used instead of VLC_CODEC_I420_10B since it's little endian.

It "worked" only because the conversion was tested while downloading and
uploading, so the download error was canceled by the upload error. This fixes
screenshot colors.

(cherry picked from commit 73e94f4893c4c92a2ee7e5abae96d4aa04ccba2e)
Signed-off-by: Thomas Guillem <thomas at gllm.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=4158368c00c1e0541d475e732e38103196b36e8e
---

 modules/hw/vaapi/chroma.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/hw/vaapi/chroma.c b/modules/hw/vaapi/chroma.c
index 5fc965283a..9009b7e2b9 100644
--- a/modules/hw/vaapi/chroma.c
+++ b/modules/hw/vaapi/chroma.c
@@ -108,9 +108,9 @@ FillPictureFromVAImage(picture_t *dest,
                 Copy420_SP_to_SP(dest, src_planes, src_pitches, src_img->height,
                                  cache);
                 break;
-            case VLC_CODEC_I420_10B:
+            case VLC_CODEC_I420_10L:
                 Copy420_16_SP_to_P(dest, src_planes, src_pitches,
-                                   src_img->height, 0, cache);
+                                   src_img->height, 6, cache);
                 break;
             default:
                 vlc_assert_unreachable();
@@ -216,10 +216,10 @@ FillVAImageFromPicture(VAImage *dest_img, uint8_t *dest_buf,
                         src->format.i_height, cache);
 
         break;
-    case VLC_CODEC_I420_10B:
+    case VLC_CODEC_I420_10L:
         assert(dest_pic->format.i_chroma == VLC_CODEC_VAAPI_420_10BPP);
         Copy420_16_P_to_SP(dest_pic, src_planes, src_pitches,
-                           src->format.i_height, 0, cache);
+                           src->format.i_height, -6, cache);
         break;
     case VLC_CODEC_P010:
     {
@@ -289,7 +289,7 @@ static int CheckFmt(const video_format_t *in, const video_format_t *out,
             break;
         case VLC_CODEC_VAAPI_420_10BPP:
             if (out->i_chroma == VLC_CODEC_P010
-             || out->i_chroma == VLC_CODEC_I420_10B)
+             || out->i_chroma == VLC_CODEC_I420_10L)
             {
                 *pixel_bytes = 2;
                 return VLC_SUCCESS;
@@ -306,7 +306,7 @@ static int CheckFmt(const video_format_t *in, const video_format_t *out,
             break;
         case VLC_CODEC_VAAPI_420_10BPP:
             if (in->i_chroma == VLC_CODEC_P010
-             || in->i_chroma == VLC_CODEC_I420_10B)
+             || in->i_chroma == VLC_CODEC_I420_10L)
             {
                 *pixel_bytes = 2;
                 return VLC_SUCCESS;