[vlc-devel] [PATCH 2/2] omxil: Don't trust a qcom decoder claiming to output 420 planar

[Martin Storsjö]

Also move the color format definition to the header, making
it available to omxil.c.

In practice on Nexus One with stock firmware, it is NV21
even though the decoder says it's planar. (On CyanogenMod with
OMX decoder built from source, the decoder returns
OMX_QCOM_COLOR_FormatYVU420SemiPlanar as pixel format, though.)
---
If you want to, this hack/workaround could go within
#ifdef __ANDROID__ if that would make you more comfortable with
it.

 modules/codec/omxil/omxil.c       |    6 ++++++
 modules/codec/omxil/omxil_utils.h |    5 +++++
 modules/codec/omxil/utils.c       |    2 --
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index 4a4510e..aa34f5c 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -387,6 +387,7 @@ static OMX_ERRORTYPE SetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
 static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                                        es_format_t *p_fmt)
 {
+    decoder_sys_t *p_sys = p_dec->p_sys;
     OMX_PARAM_PORTDEFINITIONTYPE *def = &p_port->definition;
     OMX_ERRORTYPE omx_error;
 
@@ -405,6 +406,11 @@ static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
         p_fmt->video.i_frame_rate = p_dec->fmt_in.video.i_frame_rate;
         p_fmt->video.i_frame_rate_base = p_dec->fmt_in.video.i_frame_rate_base;
 
+        if(def->format.video.eColorFormat == OMX_COLOR_FormatYUV420Planar &&
+           !strncmp(p_sys->psz_component, "OMX.qcom.video.decoder",
+                    strlen("OMX.qcom.video.decoder")))
+            def->format.video.eColorFormat = OMX_QCOM_COLOR_FormatYVU420SemiPlanar;
+
         if(!GetVlcVideoFormat( def->format.video.eCompressionFormat,
                                &p_fmt->i_codec, 0 ) )
         {
diff --git a/modules/codec/omxil/omxil_utils.h b/modules/codec/omxil/omxil_utils.h
index 5dc1e7d..f71916c 100644
--- a/modules/codec/omxil/omxil_utils.h
+++ b/modules/codec/omxil/omxil_utils.h
@@ -180,3 +180,8 @@ OMX_ERRORTYPE GetAudioParameters(OMX_HANDLETYPE handle,
     uint8_t *pi_channels, unsigned int *pi_samplerate,
     unsigned int *pi_bitrate, unsigned int *pi_bps, unsigned int *pi_blocksize);
 unsigned int GetAudioParamSize(OMX_INDEXTYPE index);
+
+/*****************************************************************************
+ * Vendor specific color formats
+ *****************************************************************************/
+#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
diff --git a/modules/codec/omxil/utils.c b/modules/codec/omxil/utils.c
index 4e3141b..c2c72d9 100644
--- a/modules/codec/omxil/utils.c
+++ b/modules/codec/omxil/utils.c
@@ -36,8 +36,6 @@
 
 #include "omxil.h"
 
-#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
-
 /*****************************************************************************
  * Events utility functions
  *****************************************************************************/
-- 
1.7.2.5