[vlc-devel] [PATCH v2] omxil: Feed H264 in annex b format

[Martin Storsjö]

The Qualcomm and Samsung OMX decoders (on Nexus One and Nexus S
at least) on android don't handle NAL format (and stagefright
always feeds H264 in annex b format to the decoder, so the same
might apply for many other android devices, too).
---

The OMAP decoders on N900 IIRC works just fine with annex b
format, so it shouldn't be an issue for that user either. But
if you'd feel better about it that way, it perhaps should
go within #ifdef __ANDROID__.

Only updated the commit message since the last version, to
reflect the need for this commit, since it seems to be needed on
many different devices, not only qualcomm.

 modules/codec/omxil/omxil.c |   47 ++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 46 insertions(+), 1 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index 979bf8d..761a1c8 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -904,7 +904,35 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode )
         OMX_FIFO_GET(&p_sys->in.fifo, p_header);
         p_header->nFilledLen = p_dec->fmt_in.i_extra;
 
-        if(p_sys->in.b_direct)
+        if( p_sys->i_nal_size_length )
+        {
+            int i_len = p_dec->fmt_in.i_extra - 6;
+            const uint8_t *p_in = (const uint8_t*)p_dec->fmt_in.p_extra + 6;
+            uint8_t *p_out = p_header->pBuffer;
+            int b_got_pps_count = false;
+            p_header->nFilledLen = 0;
+            while( i_len >= 2 )
+            {
+                uint16_t nal_len = (p_in[0] << 8) | p_in[1];
+                int i;
+                if( i_len < 2 + nal_len )
+                    nal_len = i_len - 2;
+                for( i = 0; i < p_sys->i_nal_size_length - 1; i++ )
+                    p_out[p_header->nFilledLen++] = 0;
+                p_out[p_header->nFilledLen++] = 1;
+                memcpy( p_out + p_header->nFilledLen, p_in + 2, nal_len );
+                p_header->nFilledLen += nal_len;
+                i_len -= nal_len + 2;
+                p_in  += nal_len + 2;
+                if( !b_got_pps_count && i_len > 0 )
+                {
+                    b_got_pps_count = true;
+                    i_len--;
+                    p_in++;
+                }
+            }
+        }
+        else if(p_sys->in.b_direct)
         {
             p_header->pOutputPortPrivate = p_header->pBuffer;
             p_header->pBuffer = p_dec->fmt_in.p_extra;
@@ -1155,6 +1183,23 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             block_Release(p_block);
         }
 
+        if( p_sys->i_nal_size_length )
+        {
+            /* This only works for nal sizes 3-4 */
+            int i_len = p_header->nFilledLen, i;
+            uint8_t* ptr = p_header->pBuffer;
+            while( i_len >= p_sys->i_nal_size_length )
+            {
+                int nal_len = 0;
+                for( i = 0; i < p_sys->i_nal_size_length; i++ ) {
+                    nal_len = (nal_len << 8) | ptr[i];
+                    ptr[i] = 0;
+                }
+                ptr[p_sys->i_nal_size_length - 1] = 1;
+                ptr   += nal_len + 4;
+                i_len -= nal_len + 4;
+            }
+        }
 #ifdef OMXIL_EXTRA_DEBUG
         msg_Dbg( p_dec, "EmptyThisBuffer %p, %p, %i", p_header, p_header->pBuffer,
                  (int)p_header->nFilledLen );
-- 
1.7.2.5