[vlc-commits] codec: videotoolbox: fix BadDataErr when decoding HEVC

[Thomas Guillem]

vlc/vlc-3.0 | branch: master | Thomas Guillem <thomas at gllm.fr> | Thu Mar  8 09:57:03 2018 +0100| [a0288ee5fc7491c4514c411973012123f25564af] | committer: Thomas Guillem

codec: videotoolbox: fix BadDataErr when decoding HEVC

Cf. comment.

This fixes playback on TS files that have only CRA (and no IDR) frames.
This also improves seek speed on all other files (since you could start with a
CRA frame when seeking).

(cherry picked from commit 4997acdf9fedbc0b4588c9f16cb003407223aae4)
Signed-off-by: Thomas Guillem <thomas at gllm.fr>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=a0288ee5fc7491c4514c411973012123f25564af
---

 modules/codec/videotoolbox.m | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/modules/codec/videotoolbox.m b/modules/codec/videotoolbox.m
index e6121da6dc..e9245c8d6e 100644
--- a/modules/codec/videotoolbox.m
+++ b/modules/codec/videotoolbox.m
@@ -181,6 +181,7 @@ struct decoder_sys_t
 
     h264_poc_context_t          h264_pocctx;
     hevc_poc_ctx_t              hevc_pocctx;
+    bool                        b_drop_blocks;
     date_t                      pts;
 
     struct pic_holder          *pic_holder;
@@ -647,6 +648,25 @@ static bool FillReorderInfoHEVC(decoder_t *p_dec, const block_t *p_block,
             if(!p_sli)
                 return false;
 
+            /* XXX: Work-around a VT bug on recent devices (iPhone X, MacBook
+             * Pro 2017). The VT session will report a BadDataErr if you send a
+             * RASL frame just after a CRA one. Indeed, RASL frames are
+             * corrupted if the decoding start at an IRAP frame (IDR/CRA), VT
+             * is likely failing to handle this case. */
+            if (!p_sys->b_vt_feed && (i_nal_type != HEVC_NAL_IDR_W_RADL &&
+                                      i_nal_type != HEVC_NAL_IDR_N_LP))
+                p_sys->b_drop_blocks = true;
+            else if (p_sys->b_drop_blocks)
+            {
+                if (i_nal_type == HEVC_NAL_RASL_N || i_nal_type == HEVC_NAL_RASL_R)
+                {
+                    hevc_rbsp_release_slice_header(p_sli);
+                    return false;
+                }
+                else
+                    p_sys->b_drop_blocks = false;
+            }
+
             p_info->b_keyframe = i_nal_type >= HEVC_NAL_BLA_W_LP;
 
             hevc_sequence_parameter_set_t *p_sps;
@@ -1298,6 +1318,7 @@ static void StopVideoToolbox(decoder_t *p_dec)
         p_sys->videoFormatDescription = nil;
     }
     p_sys->b_vt_feed = false;
+    p_sys->b_drop_blocks = false;
 }
 
 #pragma mark - module open and close
@@ -1784,6 +1805,7 @@ static void Drain(decoder_t *p_dec, bool flush)
     assert(RemoveOneFrameFromDPB(p_sys) == NULL);
     p_sys->b_vt_flush = false;
     p_sys->b_vt_feed = false;
+    p_sys->b_drop_blocks = false;
     vlc_mutex_unlock(&p_sys->lock);
 }