[vlc-devel] [PATCH 3/3] codec: avcodec: add hevc frame dropping

[Francois Cartegnie]

On too slow decoding, hevc frames are queued
indefinitvely and the decoder never recovers
(only outputs late, undisplayed frames) and
keeps decoding after stream has ended.

Ex: 60s decoding for a 16s 4K stream and only
first frames really displayed, then all black.

Because avcodec has no discardability for hevc
frames, frame drop is done selectively, and
sync is done on IFrame (with drop LP if any,
and can't do it lightly on pyramid/B with refs)
---
 modules/codec/avcodec/video.c | 75 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 65 insertions(+), 10 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 5e442bd..37bfeb2 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -43,6 +43,8 @@
 #include "avcodec.h"
 #include "va.h"
 
+#include "../packetizer/hevc_nal.h"
+
 /*****************************************************************************
  * decoder_sys_t : decoder descriptor
  *****************************************************************************/
@@ -573,19 +575,71 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
 
     const mtime_t i_late = ( p_sys->i_late_frames > 0 ) ? mdate() - p_sys->i_late_frames_start : 0;
 
-    if( p_dec->b_frame_drop_allowed && i_late > CLOCK_FREQ * 5 )
+    if( p_dec->b_frame_drop_allowed && p_sys->i_late_frames > 0 )
     {
-        p_sys->i_pts = VLC_TS_INVALID; /* To make sure we recover properly */
-        if( p_block )
+        /* Workaround for avcodec which doesn't provide AVDISCARD with hevc */
+        if ( p_dec->fmt_in.i_codec == VLC_CODEC_HEVC )
         {
-            block_Release( p_block );
-            *pp_block = p_block = NULL;
+            if( p_block && p_sys->i_late_frames > 4 && i_late > CLOCK_FREQ / 20 )
+            {
+                if( p_block->i_flags & BLOCK_FLAG_TYPE_B )
+                {
+                    uint8_t i_prefix;
+                    if( hevc_ishvcC(p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra) )
+                    {
+                        i_prefix = hevc_getNALLengthSize( p_dec->fmt_in.p_extra );
+                    }
+                    else if ( p_block->i_buffer > 4 )
+                    {
+                        i_prefix = (p_block->p_buffer[3] == 1) ? 4 : 3;
+                    }
+                    if ( i_prefix && p_block->i_buffer > (size_t) i_prefix + 1 )
+                    {
+                        uint8_t i_nal_type = hevc_getNALType( &p_block->p_buffer[i_prefix] );
+                        if( i_nal_type == HEVC_NAL_TRAIL_N )
+                        {
+                            block_Release( p_block );
+                            *pp_block = p_block = NULL; /* don't double free if we return a pic */
+                            goto decode_hurryup_bypass;
+                        }
+                    }
+                }
+                else if( (p_block->i_flags & BLOCK_FLAG_TYPE_P) && i_late > CLOCK_FREQ / 10 )
+                {
+                    block_Release( p_block );
+                    *pp_block = NULL; /* don't double free if we return a pic */
+                    p_block = NULL;
+                    goto decode_hurryup_bypass;
+                }
+                else if( p_block->i_flags & BLOCK_FLAG_TYPE_I )
+                {
+                    if( i_late > CLOCK_FREQ / 5 )
+                    {
+                        msg_Warn( p_dec, "flushing queue on new syncpoint and %d late frames", p_sys->i_late_frames );
+                        Flush( p_dec );
+                        p_sys->b_first_frame = true; /* Will also force display */
+                    }
+                    p_sys->i_late_frames = 0; /* Can't be late now */
+                    goto decode_hurryup_bypass;
+                }
+                /* We pass everything else */
+            }
+        }
+        /* All other codecs */
+        else if( i_late > CLOCK_FREQ * 5 )
+        {
+            p_sys->i_pts = VLC_TS_INVALID; /* To make sure we recover properly */
+            if( p_block )
+            {
+                block_Release( p_block );
+                *pp_block = p_block = NULL;
+            }
+            p_sys->i_late_frames--;
+            msg_Err( p_dec, "more than 5 seconds of late video -> "
+                     "dropping frame (computer too slow ?)" );
+            if( (p_sys->p_codec->capabilities & CODEC_CAP_DELAY) == 0 )
+                return NULL;
         }
-        p_sys->i_late_frames--;
-        msg_Err( p_dec, "more than 5 seconds of late video -> "
-                 "dropping frame (computer too slow ?)" );
-        if( (p_sys->p_codec->capabilities & CODEC_CAP_DELAY) == 0 )
-            return NULL;
     }
 
     b_drawpicture = ( !p_block || !(p_block->i_flags & BLOCK_FLAG_PREROLL) );
@@ -630,6 +684,7 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
     /*
      * Do the actual decoding now */
 
+decode_hurryup_bypass:
     /* Don't forget that libavcodec requires a little more bytes
      * that the real frame size */
     if( p_block && p_block->i_buffer > 0 )
-- 
2.5.0