[vlc-commits] demux: avi: workaround broken RGB in Compression usage (fix #20475)

[Francois Cartegnie]

vlc/vlc-3.0 | branch: master | Francois Cartegnie <fcvlcdev at free.fr> | Tue Aug  7 11:09:58 2018 +0200| [cfa02ee978523a85b76fd4259a82dc4958920d62] | committer: Francois Cartegnie

demux: avi: workaround broken RGB in Compression usage (fix #20475)

refs broken_RV24_as_Compression.avi.bz2

(cherry picked from commit 484ac1b76aa209cb7ab10e33daf65936d7113903)

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=cfa02ee978523a85b76fd4259a82dc4958920d62
---

 modules/demux/avi/avi.c | 55 ++++++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/modules/demux/avi/avi.c b/modules/demux/avi/avi.c
index 6f74445c94..3492485a25 100644
--- a/modules/demux/avi/avi.c
+++ b/modules/demux/avi/avi.c
@@ -278,6 +278,30 @@ static void Close ( vlc_object_t * p_this )
     free( p_sys );
 }
 
+static void Set_BMP_RGB_Masks( es_format_t *fmt )
+{
+    switch( fmt->i_codec )
+    {
+        case VLC_CODEC_RGB32:
+            fmt->video.i_bmask = 0xff000000;
+            fmt->video.i_gmask = 0x00ff0000;
+            fmt->video.i_rmask = 0x0000ff00;
+            break;
+        case VLC_CODEC_RGB24: /* BGR (see biBitCount) */
+            fmt->video.i_bmask = 0x00ff0000;
+            fmt->video.i_gmask = 0x0000ff00;
+            fmt->video.i_rmask = 0x000000ff;
+            break;
+        case VLC_CODEC_RGB15:
+            fmt->video.i_rmask = 0x7c00;
+            fmt->video.i_gmask = 0x03e0;
+            fmt->video.i_bmask = 0x001f;
+            break;
+        default:
+            break;
+    }
+}
+
 /*****************************************************************************
  * Open: check file and initializes AVI structures
  *****************************************************************************/
@@ -591,13 +615,16 @@ static int Open( vlc_object_t * p_this )
                     {
                         case 32:
                             tk->fmt.i_codec = VLC_CODEC_RGB32;
+                            Set_BMP_RGB_Masks( &tk->fmt );
                             break;
                         case 24:
-                            tk->fmt.i_codec = VLC_CODEC_RGB24;
+                            tk->fmt.i_codec = VLC_CODEC_RGB24; /* BGR (see biBitCount) */
+                            Set_BMP_RGB_Masks( &tk->fmt );
                             break;
                         case 16: /* Yes it is RV15 */
                         case 15:
                             tk->fmt.i_codec = VLC_CODEC_RGB15;
+                            Set_BMP_RGB_Masks( &tk->fmt );
                             break;
                         case 9: /* <- TODO check that */
                             tk->fmt.i_codec = VLC_CODEC_I410;
@@ -610,24 +637,7 @@ static int Open( vlc_object_t * p_this )
                             break;
                     }
 
-                    switch( tk->fmt.i_codec )
-                    {
-                    case VLC_CODEC_RGB32:
-                        tk->fmt.video.i_bmask = 0xff000000;
-                        tk->fmt.video.i_gmask = 0x00ff0000;
-                        tk->fmt.video.i_rmask = 0x0000ff00;
-                        break;
-                    case VLC_CODEC_RGB24: /* BGR (see biBitCount) */
-                        tk->fmt.video.i_bmask = 0x00ff0000;
-                        tk->fmt.video.i_gmask = 0x0000ff00;
-                        tk->fmt.video.i_rmask = 0x000000ff;
-                        break;
-                    case VLC_CODEC_RGB15: /* RGB (B least 5 bits) */
-                        tk->fmt.video.i_rmask = 0x7c00;
-                        tk->fmt.video.i_gmask = 0x03e0;
-                        tk->fmt.video.i_bmask = 0x001f;
-                        break;
-                    case VLC_CODEC_RGBP:
+                    if( tk->fmt.i_codec == VLC_CODEC_RGBP )
                     {
                         const VLC_BITMAPINFO *p_bi = (const VLC_BITMAPINFO *) p_vids->p_bih;
                         tk->fmt.video.p_palette = malloc( sizeof(video_palette_t) );
@@ -645,10 +655,6 @@ static int Open( vlc_object_t * p_this )
                             tk->fmt.video.p_palette->i_entries = p_vids->p_bih->biClrUsed;
                         }
                     }
-                        break;
-                    default:
-                        break;
-                    }
 
                     tk->i_width_bytes = p_vids->p_bih->biWidth * (p_vids->p_bih->biBitCount >> 3);
                     /* RGB DIB are coded from bottom to top */
@@ -663,6 +669,9 @@ static int Open( vlc_object_t * p_this )
                         tk->fmt.i_codec           =
                         tk->fmt.i_original_fourcc = VLC_FOURCC( 'X', 'V', 'I', 'D' );
                     }
+
+                    /* Shitty files storing chroma in biCompression */
+                    Set_BMP_RGB_Masks( &tk->fmt );
                 }
                 tk->i_samplesize = 0;