[vlc-devel] [PATCH] videotoolbox: rework >= 10bit output handling

[Thomas Guillem]

 - Force BGRA on macbooks not handling HEVC. This was done automatically
   before but it will likely change in the future like iOS.

 - Also force BGRA for 12 and 16bits since the openGL* implementation
   can't handle it.
---
 modules/codec/videotoolbox.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/modules/codec/videotoolbox.c b/modules/codec/videotoolbox.c
index 707ac04f9c8..cda49c6c020 100644
--- a/modules/codec/videotoolbox.c
+++ b/modules/codec/videotoolbox.c
@@ -185,12 +185,18 @@ static void HXXXGetBestChroma(decoder_t *p_dec)
 #if !TARGET_OS_IPHONE
             if (deviceSupportsHEVC()) /* 42010bit went with HEVC on macOS */
                 p_sys->i_cvpx_format = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange;
-#else
-            /* Force BGRA output (and let VT handle the tone mapping) on iOS
-             * since the apple openGLES implementation can't handle 16 bit
-             * textures. */
-            p_sys->i_cvpx_format = kCVPixelFormatType_32BGRA;
+            else
 #endif
+            /* Force BGRA output (and let VT handle the tone mapping) since the
+             * apple openGL* implementation can't handle 16 bit textures (this
+             * is the case for iOS and mac book not handled HEVC). */
+            p_sys->i_cvpx_format = kCVPixelFormatType_32BGRA;
+        }
+        else if (i_depth_luma > 10 && i_depth_chroma > 10)
+        {
+            /* XXX: The apple openGL implementation doesn't support 12 or 16
+             * bit rendering */
+            p_sys->i_cvpx_format = kCVPixelFormatType_32BGRA;
         }
     }
 }
-- 
2.28.0