[vlc-devel] Fwd: [PATCH] avi: Tidy the palette handling bit.

[Frode Tennebø]

I'm resending this as it was never accepted.

It's a patch on my inital patch
(http://mailman.videolan.org/pipermail/vlc-commits/2012-April/014222.html)
earlier this year.

After 2.0.2 was released I became aware that paletted avis is still a
problem for Windows.  I'm not sure how to handle that as the patches
fixes this for Linux.  What differs the Windows code path and/or build
chain from Linux?

 -Frode


---------- Forwarded message ----------
From: Frode Tennebø <frode at tennebo.com>
Date: 18 June 2012 23:35
Subject: [PATCH] avi: Tidy the palette handling bit.
To: vlc-devel at videolan.org
Cc: Frode Tennebø <frode at tennebo.com>


---
 modules/demux/avi/avi.c |   18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/modules/demux/avi/avi.c b/modules/demux/avi/avi.c
index cd1a5bc..81ec46c 100644
--- a/modules/demux/avi/avi.c
+++ b/modules/demux/avi/avi.c
@@ -543,15 +543,6 @@ static int Open( vlc_object_t * p_this )
                     fmt.video.i_sar_num = ((i_frame_aspect_ratio >>
16) & 0xffff) * fmt.video.i_height;
                     fmt.video.i_sar_den = ((i_frame_aspect_ratio >>
0) & 0xffff) * fmt.video.i_width;
                 }
-                fmt.i_extra =
-                    __MAX( p_vids->p_bih->biSize - sizeof(
VLC_BITMAPINFOHEADER ),
-                           p_vids->i_chunk_size -
sizeof(VLC_BITMAPINFOHEADER) );
-                if( fmt.i_extra > 0 )
-                {
-                    fmt.p_extra = malloc( fmt.i_extra );
-                    if( !fmt.p_extra ) goto error;
-                    memcpy( fmt.p_extra, &p_vids->p_bih[1], fmt.i_extra );
-                }

                 msg_Dbg( p_demux, "stream[%d] video(%4.4s)
%"PRIu32"x%"PRIu32" %dbpp %ffps",
                          i, (char*)&p_vids->p_bih->biCompression,
@@ -572,10 +563,15 @@ static int Open( vlc_object_t * p_this )
                  *  to be true for all palettized codecs we support) */
                 if( fmt.video.i_bits_per_pixel > 0 &&
fmt.video.i_bits_per_pixel <= 8 )
                 {
-                    /* The palette is not always included in biSize */
+                    /* The palette should not be included in biSize, but come
+                     * directly after BITMAPINFORHEADER in the
BITMAPINFO structure */
                     fmt.i_extra = p_vids->i_chunk_size -
sizeof(VLC_BITMAPINFOHEADER);
-                    if( fmt.i_extra > 0 && fmt.p_extra )
+                    if( fmt.i_extra > 0 )
                     {
+                        fmt.p_extra = malloc( fmt.i_extra );
+                        if( !fmt.p_extra ) goto error;
+                        memcpy( fmt.p_extra, &p_vids->p_bih[1], fmt.i_extra );
+
                         const uint8_t *p_pal = fmt.p_extra;

                         fmt.video.p_palette = calloc( 1,
sizeof(video_palette_t) );
--
1.7.9.5