[vlc-commits] avcodec: fix palette propagation

[Thomas Guillem]

vlc | branch: master | Thomas Guillem <thomas at gllm.fr> | Mon Sep 26 18:53:12 2016 +0200| [3e541f1ac5680828cc8dafba4b634f807600b666] | committer: Thomas Guillem

avcodec: fix palette propagation

closes #9940, #14975
see #17446

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=3e541f1ac5680828cc8dafba4b634f807600b666
---

 modules/codec/avcodec/video.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 6b18458..2708ca3 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -135,6 +135,13 @@ static int lavc_GetVideoFormat(decoder_t *dec, video_format_t *restrict fmt,
         if (GetVlcChroma(fmt, pix_fmt))
             return -1;
 
+        /* The libavcodec palette can only be fetched when the first output
+         * frame is decoded. Assume that the current chroma is RGB32 while we
+         * are waiting for a valid palette. Indeed, fmt_out.video.p_palette
+         * doesn't trigger a new vout request, but a new chroma yes. */
+        if (pix_fmt == AV_PIX_FMT_PAL8 && !dec->fmt_out.video.p_palette)
+            fmt->i_chroma = VLC_CODEC_RGB32;
+
         avcodec_align_dimensions2(ctx, &width, &height, aligns);
     }
     else /* hardware decoding */
@@ -902,6 +909,36 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             continue;
         }
 
+        if( p_context->pix_fmt == AV_PIX_FMT_PAL8
+         && !p_dec->fmt_out.video.p_palette )
+        {
+            /* See AV_PIX_FMT_PAL8 comment in avc_GetVideoFormat(): update the
+             * fmt_out palette and change the fmt_out chroma to request a new
+             * vout */
+            assert( p_dec->fmt_out.video.i_chroma != VLC_CODEC_RGBP );
+
+            video_palette_t *p_palette;
+            p_palette = p_dec->fmt_out.video.p_palette
+                      = malloc( sizeof(video_palette_t) );
+            if( !p_palette )
+            {
+                p_dec->b_error = true;
+                av_frame_free(&frame);
+                break;
+            }
+            static_assert( sizeof(p_palette->palette) == AVPALETTE_SIZE,
+                           "Palette size mismatch between vlc and libavutil" );
+            assert( frame->data[1] != NULL );
+            memcpy( p_palette->palette, frame->data[1], AVPALETTE_SIZE );
+            p_palette->i_entries = AVPALETTE_COUNT;
+            p_dec->fmt_out.video.i_chroma = VLC_CODEC_RGBP;
+            if( decoder_UpdateVideoFormat( p_dec ) )
+            {
+                av_frame_free(&frame);
+                continue;
+            }
+        }
+
         picture_t *p_pic = frame->opaque;
         if( p_pic == NULL )
         {   /* When direct rendering is not used, get_format() and get_buffer()