[vlc-devel] [PATCH] omxil: use OMX chroma type instead of VLC type in GetVlcChromaSizes.

[Felix Abecassis]

Using VLC type for lookup in chroma_format_table is not enough since
there can be several corresponding OMX types with different
dimensions.
---
 modules/codec/omxil/android_mediacodec.c | 2 +-
 modules/codec/omxil/omxil.c              | 8 ++++----
 modules/codec/omxil/omxil_utils.h        | 2 +-
 modules/codec/omxil/utils.c              | 8 +++-----
 4 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/modules/codec/omxil/android_mediacodec.c b/modules/codec/omxil/android_mediacodec.c
index d8cf09f..9fe9cb0 100644
--- a/modules/codec/omxil/android_mediacodec.c
+++ b/modules/codec/omxil/android_mediacodec.c
@@ -419,7 +419,7 @@ static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic, int loo
                 // the cropping, so the top/left cropping params should just be ignored.
                 unsigned int chroma_div;
                 p_pic->date = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
-                GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
+                GetVlcChromaSizes(p_sys->pixel_format, p_dec->fmt_out.video.i_width,
                                   p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
                 CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
                                ptr, chroma_div);
diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index da86e23..91e2249 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -160,7 +160,7 @@ static OMX_ERRORTYPE ImplementationSpecificWorkarounds(decoder_t *p_dec,
             def->format.video.eColorFormat = OMX_COLOR_FormatCbYCrY;
             GetVlcChromaFormat( def->format.video.eColorFormat,
                                 &p_fmt->i_codec, 0 );
-            GetVlcChromaSizes( p_fmt->i_codec,
+            GetVlcChromaSizes( def->format.video.eColorFormat,
                                def->format.video.nFrameWidth,
                                def->format.video.nFrameHeight,
                                &p_port->i_frame_size, &p_port->i_frame_stride,
@@ -245,7 +245,7 @@ static OMX_ERRORTYPE SetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                     CHECK_ERROR(omx_error, "codec %4.4s doesn't match any OMX format",
                                 (char *)&p_fmt->i_codec );
                 }
-                GetVlcChromaSizes( p_fmt->i_codec,
+                GetVlcChromaSizes( def->format.video.eColorFormat,
                                    def->format.video.nFrameWidth,
                                    def->format.video.nFrameHeight,
                                    &p_port->i_frame_size, &p_port->i_frame_stride,
@@ -263,7 +263,7 @@ static OMX_ERRORTYPE SetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                 CHECK_ERROR(omx_error, "OMX color format %i not supported",
                             (int)def->format.video.eColorFormat );
             }
-            GetVlcChromaSizes( p_fmt->i_codec,
+            GetVlcChromaSizes( def->format.video.eColorFormat,
                                def->format.video.nFrameWidth,
                                def->format.video.nFrameHeight,
                                &p_port->i_frame_size, &p_port->i_frame_stride,
@@ -453,7 +453,7 @@ static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                 CHECK_ERROR(omx_error, "OMX color format %i not supported",
                             (int)def->format.video.eColorFormat );
             }
-            GetVlcChromaSizes( p_fmt->i_codec,
+            GetVlcChromaSizes( def->format.video.eColorFormat,
                                def->format.video.nFrameWidth,
                                def->format.video.nFrameHeight,
                                &p_port->i_frame_size, &p_port->i_frame_stride,
diff --git a/modules/codec/omxil/omxil_utils.h b/modules/codec/omxil/omxil_utils.h
index 1a99dfc..f6e8b3d 100644
--- a/modules/codec/omxil/omxil_utils.h
+++ b/modules/codec/omxil/omxil_utils.h
@@ -217,7 +217,7 @@ int GetOmxChromaFormat( vlc_fourcc_t i_fourcc,
                         const char **ppsz_name );
 int GetVlcChromaFormat( OMX_COLOR_FORMATTYPE i_omx_codec,
                         vlc_fourcc_t *pi_fourcc, const char **ppsz_name );
-int GetVlcChromaSizes( vlc_fourcc_t i_fourcc,
+int GetVlcChromaSizes( OMX_COLOR_FORMATTYPE i_omx_codec,
                        unsigned int width, unsigned int height,
                        unsigned int *size, unsigned int *pitch,
                        unsigned int *chroma_pitch_div );
diff --git a/modules/codec/omxil/utils.c b/modules/codec/omxil/utils.c
index b6cacbb..755d466 100644
--- a/modules/codec/omxil/utils.c
+++ b/modules/codec/omxil/utils.c
@@ -586,17 +586,15 @@ int GetVlcChromaFormat( OMX_COLOR_FORMATTYPE i_omx_codec,
     return !!chroma_format_table[i].i_fourcc;
 }
 
-int GetVlcChromaSizes( vlc_fourcc_t i_fourcc,
+int GetVlcChromaSizes( OMX_COLOR_FORMATTYPE i_omx_codec,
                        unsigned int width, unsigned int height,
                        unsigned int *size, unsigned int *pitch,
                        unsigned int *chroma_pitch_div )
 {
     unsigned int i;
 
-    i_fourcc = vlc_fourcc_GetCodec( VIDEO_ES, i_fourcc );
-
     for( i = 0; chroma_format_table[i].i_codec != 0; i++ )
-        if( chroma_format_table[i].i_fourcc == i_fourcc ) break;
+        if( chroma_format_table[i].i_codec == i_omx_codec ) break;
 
     /* Align on macroblock boundary */
     width = (width + 15) & ~0xF;
@@ -882,7 +880,7 @@ OMX_ERRORTYPE GetAudioParameters(OMX_HANDLETYPE handle,
 }
 
 /*****************************************************************************
- * PrintOmx: print component summary 
+ * PrintOmx: print component summary
  *****************************************************************************/
 void PrintOmx(decoder_t *p_dec, OMX_HANDLETYPE omx_handle, OMX_U32 i_port)
 {
-- 
1.8.3.2