[vlc-devel] [PATCH] rework the video frame skipping/dropping strategy in the avcodec decoder

[Steve Lhomme]

One of the odd things about this bug, that you can see in the issue:
https://trac.videolan.org/vlc/ticket/10114
Is that `main warning: picture is too late to be displayed (missing 54 ms)`

The renderer has the picture already decoded but refuses to show it.
Just because it's late. I understand it might just show some
desynchronization with the sound (this file has no sound) but is it
better to not display anything when your CPU is overloaded ?

On Thu, Feb 26, 2015 at 5:28 PM, Steve Lhomme <robUx4 at videolabs.io> wrote:
> Fixes #10114
>
> When we are late for decoding we do the following:
> - skip non-ref frames outside of the decoder (decoded but not producing output)
>
> If we're still getting too late
> - skip non-key frames outside of the decoder (decoded but not producing output)
>
> If we're still too late
> - drop non-key frames packets before sending them to the decoder
>
> If we're still 5s late, the old drop packet method will be for many coming packets
>
> This strategy is more graceful than either skipping non-ref of just dropping any packet.
> ---
>  modules/codec/avcodec/video.c | 78 +++++++++++++++++++------------------------
>  1 file changed, 34 insertions(+), 44 deletions(-)
>
> diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
> index 5ba6e2e..2c268c8 100644
> --- a/modules/codec/avcodec/video.c
> +++ b/modules/codec/avcodec/video.c
> @@ -464,7 +464,6 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
>  {
>      decoder_sys_t *p_sys = p_dec->p_sys;
>      AVCodecContext *p_context = p_sys->p_context;
> -    int b_drawpicture;
>      block_t *p_block;
>
>      if( !pp_block )
> @@ -529,53 +528,43 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
>          return NULL;
>      }
>
> -    /* A good idea could be to decode all I pictures and see for the other */
> -    if( !p_dec->b_pace_control &&
> -        p_sys->b_hurry_up &&
> -        (p_sys->i_late_frames > 4) )
> +    if ( p_sys->b_hurry_up )
>      {
> -        b_drawpicture = 0;
> -        if( p_sys->i_late_frames < 12 )
> +        /* A good idea could be to decode all I pictures and see for the other */
> +        if( !p_dec->b_pace_control &&
> +            (p_sys->i_late_frames > 4) )
>          {
> -            p_context->skip_frame =
> -                    (p_sys->i_skip_frame <= AVDISCARD_NONREF) ?
> -                    AVDISCARD_NONREF : p_sys->i_skip_frame;
> +            if ( p_sys->i_late_frames < 12 )
> +            {
> +                msg_Warn( p_dec, "More than 4 late frames, skip coming non-ref frames" );
> +                p_context->skip_frame = __MAX( p_sys->i_skip_frame, AVDISCARD_NONREF );
> +            }
> +            else if ( p_sys->i_late_frames < 48 )
> +            {
> +                msg_Warn( p_dec, "More than 12 late frames, skip coming non-key frames" );
> +                p_context->skip_frame = __MAX( p_sys->i_skip_frame, AVDISCARD_NONKEY );
> +            }
> +            else if ( !p_block ||
> +                      !( p_block->i_flags & ( BLOCK_FLAG_TYPE_I|BLOCK_FLAG_PREROLL ) ) )
> +            {
> +                /* picture too late, won't decode
> +                 * but break picture until a new I, and for mpeg4 ...*/
> +                //p_sys->i_late_frames--; /* needed else it will never be decrease */
> +                msg_Warn( p_dec, "More than 48 late frames, dropping non-key frame" );
> +                if( p_block )
> +                    block_Release( p_block );
> +                return NULL;
> +            }
>          }
>          else
>          {
> -            /* picture too late, won't decode
> -             * but break picture until a new I, and for mpeg4 ...*/
> -            p_sys->i_late_frames--; /* needed else it will never be decrease */
> -            if( p_block )
> -                block_Release( p_block );
> -            msg_Warn( p_dec, "More than 4 late frames, dropping frame" );
> -            return NULL;
> -        }
> -    }
> -    else
> -    {
> -        if( p_sys->b_hurry_up )
>              p_context->skip_frame = p_sys->i_skip_frame;
> -        if( !p_block || !(p_block->i_flags & BLOCK_FLAG_PREROLL) )
> -            b_drawpicture = 1;
> -        else
> -            b_drawpicture = 0;
> -    }
> +        }
>
> -    if( p_context->width <= 0 || p_context->height <= 0 )
> -    {
> -        if( p_sys->b_hurry_up )
> +        if( p_context->width <= 0 || p_context->height <= 0 )
> +        {
>              p_context->skip_frame = p_sys->i_skip_frame;
> -    }
> -    else if( !b_drawpicture )
> -    {
> -        /* It creates broken picture
> -         * FIXME either our parser or ffmpeg is broken */
> -#if 0
> -        if( p_sys->b_hurry_up )
> -            p_context->skip_frame = __MAX( p_context->skip_frame,
> -                                                  AVDISCARD_NONREF );
> -#endif
> +        }
>      }
>
>      /*
> @@ -652,9 +641,8 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
>
>              if( i_used < 0 )
>              {
> -                if( b_drawpicture )
> -                    msg_Warn( p_dec, "cannot decode one frame (%zu bytes)",
> -                            p_block->i_buffer );
> +                msg_Warn( p_dec, "cannot decode one frame (%zu bytes)",
> +                        p_block->i_buffer );
>                  block_Release( p_block );
>                  return NULL;
>              }
> @@ -732,9 +720,11 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
>          else
>          {
>              p_sys->i_late_frames = 0;
> +            if ( p_sys->b_hurry_up )
> +                p_context->skip_frame = p_sys->i_skip_frame;
>          }
>
> -        if( !b_drawpicture || ( !p_sys->p_va && !p_sys->p_ff_pic->linesize[0] ) )
> +        if( !p_sys->p_va && !p_sys->p_ff_pic->linesize[0] )
>              continue;
>
>          if( p_sys->p_va != NULL || p_sys->p_ff_pic->opaque == NULL )
> --
> 2.2.2
>
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel