[vlc-devel] [PATCH v3] omxil: Feed H264 in annex b format

[Martin Storsjö]

The Qualcomm and Samsung OMX decoders (on Nexus One and Nexus S
at least) on android don't handle NAL format (and stagefright
always feeds H264 in annex b format to the decoder, so the same
might apply for many other android devices, too).
---
Added some comments, and added output buffer size checking in the
SPS/PPS conversion as pointed out by Laurent in his review on irc.

Is this acceptable now, or would it have to share code with the
same conversion in the crystalhd module? Only the first hunk
of this patch, adding 30 lines, is shared/similar to the
conversion in crystalhd, and here we already have part of the
work done elsewhere in omxil.c (parsing the first header fields
of the avc1 data).

 modules/codec/omxil/omxil.c |   52 ++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 51 insertions(+), 1 deletions(-)

diff --git a/modules/codec/omxil/omxil.c b/modules/codec/omxil/omxil.c
index 979bf8d..4a4510e 100644
--- a/modules/codec/omxil/omxil.c
+++ b/modules/codec/omxil/omxil.c
@@ -904,7 +904,39 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode )
         OMX_FIFO_GET(&p_sys->in.fifo, p_header);
         p_header->nFilledLen = p_dec->fmt_in.i_extra;
 
-        if(p_sys->in.b_direct)
+        /* Convert H.264 NAL format to annex b */
+        if( p_sys->i_nal_size_length && !p_sys->in.b_direct )
+        {
+            int i_len = p_dec->fmt_in.i_extra - 6;
+            const uint8_t *p_in = (const uint8_t*)p_dec->fmt_in.p_extra + 6;
+            uint8_t *p_out = p_header->pBuffer;
+            int b_got_pps_count = false;
+            p_header->nFilledLen = 0;
+            while( i_len >= 2 )
+            {
+                uint16_t nal_len = (p_in[0] << 8) | p_in[1];
+                int i;
+                if( i_len < 2 + nal_len )
+                    nal_len = i_len - 2;
+                if ( p_header->nFilledLen + p_sys->i_nal_size_length +
+                     nal_len > p_header->nAllocLen )
+                    break;
+                for( i = 0; i < p_sys->i_nal_size_length - 1; i++ )
+                    p_out[p_header->nFilledLen++] = 0;
+                p_out[p_header->nFilledLen++] = 1;
+                memcpy( p_out + p_header->nFilledLen, p_in + 2, nal_len );
+                p_header->nFilledLen += nal_len;
+                i_len -= nal_len + 2;
+                p_in  += nal_len + 2;
+                if( !b_got_pps_count && i_len > 0 )
+                {
+                    b_got_pps_count = true;
+                    i_len--;
+                    p_in++;
+                }
+            }
+        }
+        else if(p_sys->in.b_direct)
         {
             p_header->pOutputPortPrivate = p_header->pBuffer;
             p_header->pBuffer = p_dec->fmt_in.p_extra;
@@ -1155,6 +1187,24 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             block_Release(p_block);
         }
 
+        /* Convert H.264 NAL format to annex b */
+        if( p_sys->i_nal_size_length )
+        {
+            /* This only works for NAL sizes 3-4 */
+            int i_len = p_header->nFilledLen, i;
+            uint8_t* ptr = p_header->pBuffer;
+            while( i_len >= p_sys->i_nal_size_length )
+            {
+                int nal_len = 0;
+                for( i = 0; i < p_sys->i_nal_size_length; i++ ) {
+                    nal_len = (nal_len << 8) | ptr[i];
+                    ptr[i] = 0;
+                }
+                ptr[p_sys->i_nal_size_length - 1] = 1;
+                ptr   += nal_len + 4;
+                i_len -= nal_len + 4;
+            }
+        }
 #ifdef OMXIL_EXTRA_DEBUG
         msg_Dbg( p_dec, "EmptyThisBuffer %p, %p, %i", p_header, p_header->pBuffer,
                  (int)p_header->nFilledLen );
-- 
1.7.2.5