[vlc-devel] [PATCH 3/5] omxil: Handle planes with a height larger than the frame height

[Martin Storsjö]

The nSliceHeight parameter in OMX apparently can indicate plane
height. This shows up on Nexus One with recent CyanogenMod
with the OpenMAX driver built from source (as opposed to the
older ones with a binary-only OpenMAX driver).
---
 modules/codec/omxil/omxil.c       |    2 +-
 modules/codec/omxil/omxil_utils.h |    2 +-
 modules/codec/omxil/utils.c       |    9 ++++++++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index 2270022..dcd072e 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -1079,7 +1079,7 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
                 p_pic = decoder_NewPicture( p_dec );
                 if( !p_pic ) break; /* No picture available */
 
-                CopyOmxPicture(p_dec, p_pic, p_header);
+                CopyOmxPicture(p_dec, p_pic, p_header, p_sys->out.definition.format.video.nSliceHeight);
             }
 
             p_pic->date = p_header->nTimeStamp;
diff --git a/modules/codec/omxil/omxil_utils.h b/modules/codec/omxil/omxil_utils.h
index 959f934..99c4e6f 100644
--- a/modules/codec/omxil/omxil_utils.h
+++ b/modules/codec/omxil/omxil_utils.h
@@ -119,7 +119,7 @@ OMX_ERRORTYPE WaitForSpecificOmxEvent(decoder_t *p_dec,
 /*****************************************************************************
  * Picture utility functions
  *****************************************************************************/
-void CopyOmxPicture( decoder_t *, picture_t *, OMX_BUFFERHEADERTYPE * );
+void CopyOmxPicture( decoder_t *, picture_t *, OMX_BUFFERHEADERTYPE *, int );
 void CopyVlcPicture( decoder_t *, OMX_BUFFERHEADERTYPE *, picture_t * );
 
 /*****************************************************************************
diff --git a/modules/codec/omxil/utils.c b/modules/codec/omxil/utils.c
index 72d370e..468ba93 100644
--- a/modules/codec/omxil/utils.c
+++ b/modules/codec/omxil/utils.c
@@ -119,7 +119,7 @@ OMX_ERRORTYPE WaitForSpecificOmxEvent(decoder_t *p_dec,
  * Picture utility functions
  *****************************************************************************/
 void CopyOmxPicture( decoder_t *p_dec, picture_t *p_pic,
-                     OMX_BUFFERHEADERTYPE *p_header )
+                     OMX_BUFFERHEADERTYPE *p_header, int i_slice_height )
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
     int i_src_stride, i_dst_stride;
@@ -142,6 +142,13 @@ void CopyOmxPicture( decoder_t *p_dec, picture_t *p_pic,
             p_src += i_src_stride;
             p_dst += i_dst_stride;
         }
+        /* Handle plane height, which may be indicated via nSliceHeight in OMX.
+         * The handling for chroma planes currently assumes vertically
+         * subsampled chroma, e.g. 422 planar wouldn't work right. */
+        if( i_plane == 0 && i_slice_height > p_pic->p[i_plane].i_visible_lines )
+            p_src += i_src_stride * (i_slice_height - p_pic->p[i_plane].i_visible_lines);
+        else if ( i_plane > 0 && i_slice_height/2 > p_pic->p[i_plane].i_visible_lines )
+            p_src += i_src_stride * (i_slice_height/2 - p_pic->p[i_plane].i_visible_lines);
     }
 }
 
-- 
1.7.2.5