[vlc-devel] [PATCH] rework the video frame skipping/dropping strategy in the avcodec decoder

[Steve Lhomme]

Fixes #10114

When we are late for decoding we do the following:
- skip non-ref frames outside of the decoder (decoded but not producing output)

If we're still getting too late
- skip non-key frames outside of the decoder (decoded but not producing output)

If we're still too late
- drop non-key frames packets before sending them to the decoder

If we're still 5s late, the old drop packet method will be for many coming packets

This strategy is more graceful than either skipping non-ref of just dropping any packet.
---
 modules/codec/avcodec/video.c | 78 +++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 44 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 5ba6e2e..2c268c8 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -464,7 +464,6 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
     AVCodecContext *p_context = p_sys->p_context;
-    int b_drawpicture;
     block_t *p_block;
 
     if( !pp_block )
@@ -529,53 +528,43 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         return NULL;
     }
 
-    /* A good idea could be to decode all I pictures and see for the other */
-    if( !p_dec->b_pace_control &&
-        p_sys->b_hurry_up &&
-        (p_sys->i_late_frames > 4) )
+    if ( p_sys->b_hurry_up )
     {
-        b_drawpicture = 0;
-        if( p_sys->i_late_frames < 12 )
+        /* A good idea could be to decode all I pictures and see for the other */
+        if( !p_dec->b_pace_control &&
+            (p_sys->i_late_frames > 4) )
         {
-            p_context->skip_frame =
-                    (p_sys->i_skip_frame <= AVDISCARD_NONREF) ?
-                    AVDISCARD_NONREF : p_sys->i_skip_frame;
+            if ( p_sys->i_late_frames < 12 )
+            {
+                msg_Warn( p_dec, "More than 4 late frames, skip coming non-ref frames" );
+                p_context->skip_frame = __MAX( p_sys->i_skip_frame, AVDISCARD_NONREF );
+            }
+            else if ( p_sys->i_late_frames < 48 )
+            {
+                msg_Warn( p_dec, "More than 12 late frames, skip coming non-key frames" );
+                p_context->skip_frame = __MAX( p_sys->i_skip_frame, AVDISCARD_NONKEY );
+            }
+            else if ( !p_block ||
+                      !( p_block->i_flags & ( BLOCK_FLAG_TYPE_I|BLOCK_FLAG_PREROLL ) ) )
+            {
+                /* picture too late, won't decode
+                 * but break picture until a new I, and for mpeg4 ...*/
+                //p_sys->i_late_frames--; /* needed else it will never be decrease */
+                msg_Warn( p_dec, "More than 48 late frames, dropping non-key frame" );
+                if( p_block )
+                    block_Release( p_block );
+                return NULL;
+            }
         }
         else
         {
-            /* picture too late, won't decode
-             * but break picture until a new I, and for mpeg4 ...*/
-            p_sys->i_late_frames--; /* needed else it will never be decrease */
-            if( p_block )
-                block_Release( p_block );
-            msg_Warn( p_dec, "More than 4 late frames, dropping frame" );
-            return NULL;
-        }
-    }
-    else
-    {
-        if( p_sys->b_hurry_up )
             p_context->skip_frame = p_sys->i_skip_frame;
-        if( !p_block || !(p_block->i_flags & BLOCK_FLAG_PREROLL) )
-            b_drawpicture = 1;
-        else
-            b_drawpicture = 0;
-    }
+        }
 
-    if( p_context->width <= 0 || p_context->height <= 0 )
-    {
-        if( p_sys->b_hurry_up )
+        if( p_context->width <= 0 || p_context->height <= 0 )
+        {
             p_context->skip_frame = p_sys->i_skip_frame;
-    }
-    else if( !b_drawpicture )
-    {
-        /* It creates broken picture
-         * FIXME either our parser or ffmpeg is broken */
-#if 0
-        if( p_sys->b_hurry_up )
-            p_context->skip_frame = __MAX( p_context->skip_frame,
-                                                  AVDISCARD_NONREF );
-#endif
+        }
     }
 
     /*
@@ -652,9 +641,8 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
 
             if( i_used < 0 )
             {
-                if( b_drawpicture )
-                    msg_Warn( p_dec, "cannot decode one frame (%zu bytes)",
-                            p_block->i_buffer );
+                msg_Warn( p_dec, "cannot decode one frame (%zu bytes)",
+                        p_block->i_buffer );
                 block_Release( p_block );
                 return NULL;
             }
@@ -732,9 +720,11 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         else
         {
             p_sys->i_late_frames = 0;
+            if ( p_sys->b_hurry_up )
+                p_context->skip_frame = p_sys->i_skip_frame;
         }
 
-        if( !b_drawpicture || ( !p_sys->p_va && !p_sys->p_ff_pic->linesize[0] ) )
+        if( !p_sys->p_va && !p_sys->p_ff_pic->linesize[0] )
             continue;
 
         if( p_sys->p_va != NULL || p_sys->p_ff_pic->opaque == NULL )
-- 
2.2.2