[vlc-commits] videotoolbox: fallback to original chroma in case of PixelTransfer error

[Thomas Guillem]

vlc | branch: master | Thomas Guillem <thomas at gllm.fr> | Fri Mar  9 12:59:40 2018 +0100| [044f6aa41f778d1666419847df35795ed819cb2e] | committer: Thomas Guillem

videotoolbox: fallback to original chroma in case of PixelTransfer error

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=044f6aa41f778d1666419847df35795ed819cb2e
---

 modules/codec/videotoolbox.m | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/modules/codec/videotoolbox.m b/modules/codec/videotoolbox.m
index b00eb467ac..6465fd8131 100644
--- a/modules/codec/videotoolbox.m
+++ b/modules/codec/videotoolbox.m
@@ -101,6 +101,7 @@ enum vtsession_status
 {
     VTSESSION_STATUS_OK,
     VTSESSION_STATUS_RESTART,
+    VTSESSION_STATUS_RESTART_CHROMA,
     VTSESSION_STATUS_ABORT,
 };
 
@@ -208,7 +209,7 @@ static void HXXXGetBestChroma(decoder_t *p_dec)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
 
-    if (p_sys->i_cvpx_format != 0)
+    if (p_sys->i_cvpx_format != 0 || p_sys->b_cvpx_format_forced)
         return;
 
     uint8_t i_chroma_format, i_depth_luma, i_depth_chroma;
@@ -1766,6 +1767,10 @@ static int HandleVTStatus(decoder_t *p_dec, OSStatus status,
             case kCVReturnInvalidArgument:
                 *p_vtsession_status = VTSESSION_STATUS_ABORT;
                 break;
+            case kVTPixelTransferNotSupportedErr:
+            case kVTPixelTransferNotPermittedErr:
+                *p_vtsession_status = VTSESSION_STATUS_RESTART_CHROMA;
+                break;
             case -8960 /* codecErr */:
             case kVTVideoDecoderMalfunctionErr:
             case -8969 /* codecBadDataErr */:
@@ -1862,9 +1867,28 @@ static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
 #endif
     }
 
-    if (p_sys->vtsession_status == VTSESSION_STATUS_RESTART)
+    if (p_sys->vtsession_status == VTSESSION_STATUS_RESTART ||
+        p_sys->vtsession_status == VTSESSION_STATUS_RESTART_CHROMA)
     {
-        if (p_sys->i_restart_count <= VT_RESTART_MAX)
+        bool do_restart;
+        if (p_sys->vtsession_status == VTSESSION_STATUS_RESTART_CHROMA)
+        {
+            if (p_sys->i_cvpx_format == 0 && p_sys->b_cvpx_format_forced)
+            {
+                /* Already tried to fallback to the original chroma, aborting... */
+                do_restart = false;
+            }
+            else
+            {
+                p_sys->i_cvpx_format = 0;
+                p_sys->b_cvpx_format_forced = true;
+                do_restart = true;
+            }
+        }
+        else
+            do_restart = p_sys->i_restart_count <= VT_RESTART_MAX;
+
+        if (do_restart)
         {
             msg_Warn(p_dec, "restarting vt session (dec callback failed)");
             vlc_mutex_unlock(&p_sys->lock);