[vlc-devel] [PATCH] omxil: Feed H264 in annex b format

[Martin Storsjö]

The qcom OMX decoder doesn't handle NAL format (and stagefright
always feeds H264 in annex b format to the decoder, so the same
might apply for many other android devices, too).
---

This changes the behaviour for omxil on all platforms, but
IIRC the TI OMX decoder on OMAP3 handles annex b just fine,
so this shouldn't be an issue for that decoder.

Someone more knowledgeable on H.264 than me can hopefully
confirm that both 001 and 0001 are valid start codes for annex b -
NAL length sizes 3 and 4 can be converted to these start codes
inline without moving data.

 modules/codec/omxil/omxil.c |   47 ++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 46 insertions(+), 1 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index a1a9bb6..b339db1 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -898,7 +898,35 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode )
         OMX_FIFO_GET(&p_sys->in.fifo, p_header);
         p_header->nFilledLen = p_dec->fmt_in.i_extra;
 
-        if(p_sys->in.b_direct)
+        if( p_sys->i_nal_size_length )
+        {
+            int i_len = p_dec->fmt_in.i_extra - 6;
+            const uint8_t *p_in = (const uint8_t*)p_dec->fmt_in.p_extra + 6;
+            uint8_t *p_out = p_header->pBuffer;
+            int b_got_pps_count = false;
+            p_header->nFilledLen = 0;
+            while( i_len >= 2 )
+            {
+                uint16_t nal_len = (p_in[0] << 8) | p_in[1];
+                int i;
+                if( i_len < 2 + nal_len )
+                    nal_len = i_len - 2;
+                for( i = 0; i < p_sys->i_nal_size_length - 1; i++ )
+                    p_out[p_header->nFilledLen++] = 0;
+                p_out[p_header->nFilledLen++] = 1;
+                memcpy( p_out + p_header->nFilledLen, p_in + 2, nal_len );
+                p_header->nFilledLen += nal_len;
+                i_len -= nal_len + 2;
+                p_in  += nal_len + 2;
+                if( !b_got_pps_count && i_len > 0 )
+                {
+                    b_got_pps_count = true;
+                    i_len--;
+                    p_in++;
+                }
+            }
+        }
+        else if(p_sys->in.b_direct)
         {
             p_header->pOutputPortPrivate = p_header->pBuffer;
             p_header->pBuffer = p_dec->fmt_in.p_extra;
@@ -1149,6 +1177,23 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             block_Release(p_block);
         }
 
+        if( p_sys->i_nal_size_length )
+        {
+            /* This only works for nal sizes 3-4 */
+            int i_len = p_header->nFilledLen, i;
+            uint8_t* ptr = p_header->pBuffer;
+            while( i_len >= p_sys->i_nal_size_length )
+            {
+                int nal_len = 0;
+                for( i = 0; i < p_sys->i_nal_size_length; i++ ) {
+                    nal_len = (nal_len << 8) | ptr[i];
+                    ptr[i] = 0;
+                }
+                ptr[p_sys->i_nal_size_length - 1] = 1;
+                ptr   += nal_len + 4;
+                i_len -= nal_len + 4;
+            }
+        }
 #ifdef OMXIL_EXTRA_DEBUG
         msg_Dbg( p_dec, "EmptyThisBuffer %p, %p, %i", p_header, p_header->pBuffer,
                  (int)p_header->nFilledLen );
-- 
1.7.2.5