[vlc-devel] [PATCH] mux: mp4: avoid sending partial headers on error

[Alexandre Janniaux]

In non-quicktime mode, if the ftyp box is allocated but the mdat box
cannot be allocated because of memory errors, the ftyp box can be
written a second time by the FlushHeader function.

The muxer infrastructure should probably handle this ENOMEM case in a
smarter way, but reordering the code also prevent the doubling of the
box in every cases, by writing them all or none of them.

Issue randomly reproduced when muxing on iOS (~1 over 100 times).
---
 modules/mux/mp4/mp4.c | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/modules/mux/mp4/mp4.c b/modules/mux/mp4/mp4.c
index b58a999dbe..29e63bc8c0 100644
--- a/modules/mux/mp4/mp4.c
+++ b/modules/mux/mp4/mp4.c
@@ -227,33 +227,33 @@ static int MuxStream(sout_mux_t *p_mux, sout_input_t *p_input, mp4_stream_t *p_s
 static int WriteSlowStartHeader(sout_mux_t *p_mux)
 {
     sout_mux_sys_t *p_sys = p_mux->p_sys;
-    bo_t *box;
+
+    bo_t *mdat_box = box_new("mdat");
+    if (mdat_box == NULL)
+        return VLC_ENOMEM;
+    /* Now add mdat header */
+    bo_add_64be(mdat_box, 0); // enough to store an extended size
 
     if (!mp4mux_Is(p_sys->muxh, QUICKTIME))
     {
         /* Now add ftyp header */
-        box = mp4mux_GetFtyp(p_sys->muxh);
-        if(!box)
-            return VLC_ENOMEM;
+        bo_t *ftyp_box = mp4mux_GetFtyp(p_sys->muxh);
+        if(!ftyp_box)
+            goto ftyp_error;
 
-        p_sys->i_pos += bo_size(box);
+        p_sys->i_pos += bo_size(ftyp_box);
         p_sys->i_mdat_pos = p_sys->i_pos;
-        box_send(p_mux, box);
+        box_send(p_mux, ftyp_box);
     }
 
-    /* Now add mdat header */
-    box = box_new("mdat");
-    if(!box)
-        return VLC_ENOMEM;
-
-    bo_add_64be(box, 0); // enough to store an extended size
-
-    if(box->b)
-        p_sys->i_pos += bo_size(box);
-
-    box_send(p_mux, box);
+    if(mdat_box->b)
+        p_sys->i_pos += bo_size(mdat_box);
+    box_send(p_mux, mdat_box);
 
     return VLC_SUCCESS;
+ftyp_error:
+    bo_free(mdat_box);
+    return VLC_ENOMEM;
 }
 
 /*****************************************************************************
-- 
2.31.0