[vlc-devel] [PATCH] avcodec: don't detect texture size changes as needing codec changes

[Steve Lhomme]

lavc forces the coded size at each updaye, even though that's not the actual
texture size.

Since 9c5c6d212fc60e887579357f86d50537d512c2d3 we can set the output size of
the module to the texture size so it is handled properly downstream and not
hidden in the video context/picture_sys_t. It doesn't work well when lavc
request a format update when the texture dimensions didn't actually changed.

So we keep a copy of the video format dimensions that lavc thinks we are using
to check for actual changes in what it is feeding us.
---
 modules/codec/avcodec/video.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 22a5ccec65a..533bc3c550e 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -107,6 +107,10 @@ typedef struct
     int profile;
     int level;
 
+    // decoder output seen by lavc, regardless of texture padding
+    unsigned decoder_width;
+    unsigned decoder_height;
+
     /* Protect dec->fmt_out, decoder_Update*() and decoder_NewPicture()
      * functions */
     vlc_mutex_t lock;
@@ -367,6 +371,8 @@ static int lavc_UpdateVideoFormat(decoder_t *dec, AVCodecContext *ctx,
     if ( dec->fmt_in.video.mastering.max_luminance )
         dec->fmt_out.video.mastering = dec->fmt_in.video.mastering;
     dec->fmt_out.video.lighting = dec->fmt_in.video.lighting;
+    p_sys->decoder_width  = dec->fmt_out.video.i_width;
+    p_sys->decoder_height = dec->fmt_out.video.i_height;
 
     if (pp_dec_device)
     {
@@ -1665,11 +1671,11 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
          msg_Dbg(p_dec, "get format failed");
          goto no_reuse;
      }
-     if (fmt.i_width  != p_dec->fmt_out.video.i_width ||
-         fmt.i_height != p_dec->fmt_out.video.i_height)
+     if (fmt.i_width  != p_sys->decoder_width ||
+         fmt.i_height != p_sys->decoder_height)
      {
          msg_Dbg(p_dec, "mismatched dimensions %ux%u was %ux%u", fmt.i_width, fmt.i_height,
-                 p_dec->fmt_out.video.i_width, p_dec->fmt_out.video.i_height);
+                 p_sys->decoder_width, p_sys->decoder_height);
          goto no_reuse;
      }
      if (p_context->profile != p_sys->profile || p_context->level > p_sys->level)
-- 
2.17.1