[x264-devel] [PATCH] Fix NAL size header for MP4/etc when producing AVC-Intra

[Sebastian Dröge]

From: Sebastian Dröge <sebastian at centricular.com>

AVC-Intra has custom padding that is applied in a few places after the NAL
size header is written. This causes the NAL size header to be too small,
causing a decoder to find just zero bytes after the previous NAL and then
failing as there's no good way to find the start of the next NAL.
---
 common/bitstream.c | 17 +++++++++++++++--
 encoder/encoder.c  | 24 ++++++++++++------------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/common/bitstream.c b/common/bitstream.c
index ec9836a..cc25a2c 100644
--- a/common/bitstream.c
+++ b/common/bitstream.c
@@ -82,6 +82,21 @@ void x264_nal_encode( x264_t *h, uint8_t *dst, x264_nal_t *nal )
     dst = h->bsf.nal_escape( dst, src, end );
     int size = (dst - orig_dst) - 4;
 
+    /* Apply additional padding for AVC Intra that was not part of i_payload */
+    if( h->param.i_avcintra_class )
+    {
+        nal->i_padding -= size + 4 - (nal->i_payload + NALU_OVERHEAD);
+        if( nal->i_padding > 0 )
+        {
+            memset( orig_dst + size + 4, 0, nal->i_padding );
+            size += nal->i_padding;
+        }
+        nal->i_padding = X264_MAX( nal->i_padding, 0 );
+    }
+
+    nal->i_payload = size+4;
+    nal->p_payload = orig_dst;
+
     /* Write the size header for mp4/etc */
     if( !h->param.b_annexb )
     {
@@ -92,8 +107,6 @@ void x264_nal_encode( x264_t *h, uint8_t *dst, x264_nal_t *nal )
         orig_dst[3] = size>> 0;
     }
 
-    nal->i_payload = size+4;
-    nal->p_payload = orig_dst;
     x264_emms();
 }
 
diff --git a/encoder/encoder.c b/encoder/encoder.c
index c87d79d..5f31128 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -1958,22 +1958,10 @@ static int x264_encoder_encapsulate_nals( x264_t *h, int start )
 
     for( int i = start; i < h->out.i_nal; i++ )
     {
-        int old_payload_len = h->out.nal[i].i_payload;
         h->out.nal[i].b_long_startcode = !i || h->out.nal[i].i_type == NAL_SPS || h->out.nal[i].i_type == NAL_PPS ||
                                          h->param.i_avcintra_class;
         x264_nal_encode( h, nal_buffer, &h->out.nal[i] );
         nal_buffer += h->out.nal[i].i_payload;
-        if( h->param.i_avcintra_class )
-        {
-            h->out.nal[i].i_padding -= h->out.nal[i].i_payload - (old_payload_len + NALU_OVERHEAD);
-            if( h->out.nal[i].i_padding > 0 )
-            {
-                memset( nal_buffer, 0, h->out.nal[i].i_padding );
-                nal_buffer += h->out.nal[i].i_padding;
-                h->out.nal[i].i_payload += h->out.nal[i].i_padding;
-            }
-            h->out.nal[i].i_padding = X264_MAX( h->out.nal[i].i_padding, 0 );
-        }
     }
 
     x264_emms();
@@ -3844,6 +3832,18 @@ static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
         memset( h->out.nal[0].p_payload + frame_size, 0, filler );
         h->out.nal[h->out.i_nal-1].i_payload += filler;
         h->out.nal[h->out.i_nal-1].i_padding = filler;
+
+        /* Fix up the size header for mp4/etc */
+        if( !h->param.b_annexb )
+        {
+            int size = h->out.nal[h->out.i_nal-1].i_payload - 4;
+
+            h->out.nal[h->out.i_nal-1].p_payload[0] = size>>24;
+            h->out.nal[h->out.i_nal-1].p_payload[1] = size>>16;
+            h->out.nal[h->out.i_nal-1].p_payload[2] = size>> 8;
+            h->out.nal[h->out.i_nal-1].p_payload[3] = size>> 0;
+        }
+
         frame_size += filler;
     }
     else
-- 
2.6.3