[vlc-devel] [PATCH v2] omxil: Don't trust a qcom decoder claiming to output 420 planar

[Martin Storsjö]

Also move the color format definition to the header, making
it available to omxil.c.

In practice on Nexus One with stock firmware, it is NV21
even though the decoder says it's planar. (On CyanogenMod with
OMX decoder built from source, the decoder returns
OMX_QCOM_COLOR_FormatYVU420SemiPlanar as pixel format, though.)
---
Added a comment in the source file explaining the hack.

 modules/codec/omxil/omxil.c       |    9 +++++++++
 modules/codec/omxil/omxil_utils.h |    5 +++++
 modules/codec/omxil/utils.c       |    2 --
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index 4a4510e..dde7799 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -387,6 +387,7 @@ static OMX_ERRORTYPE SetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
 static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                                        es_format_t *p_fmt)
 {
+    decoder_sys_t *p_sys = p_dec->p_sys;
     OMX_PARAM_PORTDEFINITIONTYPE *def = &p_port->definition;
     OMX_ERRORTYPE omx_error;
 
@@ -405,6 +406,14 @@ static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
         p_fmt->video.i_frame_rate = p_dec->fmt_in.video.i_frame_rate;
         p_fmt->video.i_frame_rate_base = p_dec->fmt_in.video.i_frame_rate_base;
 
+        /* Hack: Nexus One (stock firmware with binary OMX driver blob)
+         * claims to output 420Planar even though it in in practice is
+         * NV21. */
+        if(def->format.video.eColorFormat == OMX_COLOR_FormatYUV420Planar &&
+           !strncmp(p_sys->psz_component, "OMX.qcom.video.decoder",
+                    strlen("OMX.qcom.video.decoder")))
+            def->format.video.eColorFormat = OMX_QCOM_COLOR_FormatYVU420SemiPlanar;
+
         if(!GetVlcVideoFormat( def->format.video.eCompressionFormat,
                                &p_fmt->i_codec, 0 ) )
         {
diff --git a/modules/codec/omxil/omxil_utils.h b/modules/codec/omxil/omxil_utils.h
index 5dc1e7d..f71916c 100644
--- a/modules/codec/omxil/omxil_utils.h
+++ b/modules/codec/omxil/omxil_utils.h
@@ -180,3 +180,8 @@ OMX_ERRORTYPE GetAudioParameters(OMX_HANDLETYPE handle,
     uint8_t *pi_channels, unsigned int *pi_samplerate,
     unsigned int *pi_bitrate, unsigned int *pi_bps, unsigned int *pi_blocksize);
 unsigned int GetAudioParamSize(OMX_INDEXTYPE index);
+
+/*****************************************************************************
+ * Vendor specific color formats
+ *****************************************************************************/
+#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
diff --git a/modules/codec/omxil/utils.c b/modules/codec/omxil/utils.c
index 4e3141b..c2c72d9 100644
--- a/modules/codec/omxil/utils.c
+++ b/modules/codec/omxil/utils.c
@@ -36,8 +36,6 @@
 
 #include "omxil.h"
 
-#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
-
 /*****************************************************************************
  * Events utility functions
  *****************************************************************************/
-- 
1.7.2.5