[vlc-devel] [PATCH 3/5] mediacodec: change the way to send codec specific data

Thomas Guillem thomas at gllm.fr
Tue May 12 18:13:06 CEST 2015


The CSD is now split into several ones, therefore there is one CSD per H264
Parameter Sets (SPS and PPS).

There is now only one way to send a CSD. It's sent via PutInput with the
BUFFER_FLAG_CODEC_CONFIG flag.
---
 modules/codec/omxil/android_mediacodec.c | 356 ++++++++++++++++++-------------
 1 file changed, 203 insertions(+), 153 deletions(-)

diff --git a/modules/codec/omxil/android_mediacodec.c b/modules/codec/omxil/android_mediacodec.c
index 27d3443..740c124 100644
--- a/modules/codec/omxil/android_mediacodec.c
+++ b/modules/codec/omxil/android_mediacodec.c
@@ -36,6 +36,7 @@
 #include <vlc_codec.h>
 #include <vlc_block_helper.h>
 #include <vlc_cpu.h>
+#include <vlc_memory.h>
 
 #include "../../packetizer/h264_nal.h"
 #include "../../packetizer/hevc_nal.h"
@@ -133,6 +134,13 @@ static int64_t timestamp_FifoGet(timestamp_fifo_t *fifo)
     return result;
 }
 
+/* Codec Specific Data */
+struct csd
+{
+    uint8_t *p_buf;
+    size_t i_size;
+};
+
 struct decoder_sys_t
 {
     uint32_t nal_size;
@@ -144,13 +152,11 @@ struct decoder_sys_t
     int stride, slice_height;
     char *name;
 
-    /* "csd-0" buffer */
-    void *p_csd0_buffer;
-    size_t i_csd0_buffer;
-    /* or buffer sent via BUFFER_FLAG_CODEC_CONFIG flag */
-    uint8_t *p_config_buffer;
-    size_t i_config_buffer;
-    bool b_config_resend;
+    /* Codec Specific Data buffer: sent in PutInput after a start or a flush
+     * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
+    struct csd *p_csd;
+    size_t i_csd_count;
+    size_t i_csd_send;
 
     bool b_update_format;
     int i_width;
@@ -408,30 +414,127 @@ end:
     return ret;
 }
 
-static int H264GetSPSPPS(uint8_t *p_buf, size_t i_buf,
-                         uint8_t **pp_sps_buf, size_t *p_sps_size,
-                         uint8_t **pp_pps_buf, size_t *p_pps_size,
-                         struct nal_sps *p_sps)
+static void CSDFree(decoder_t *p_dec)
+{
+    decoder_sys_t *p_sys = p_dec->p_sys;
+
+    if (p_sys->p_csd)
+    {
+        for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
+            free(p_sys->p_csd[i].p_buf);
+        free(p_sys->p_csd);
+        p_sys->p_csd = NULL;
+    }
+    p_sys->i_csd_count = 0;
+}
+
+/* Create the p_sys->p_csd that will be sent via PutInput */
+static int CSDDup(decoder_t *p_dec, const struct csd *p_csd, size_t i_count)
+{
+    decoder_sys_t *p_sys = p_dec->p_sys;
+    unsigned int i_last_csd_count = p_sys->i_csd_count;
+
+    p_sys->i_csd_count = i_count;
+    /* free previous p_buf if old count is bigger */
+    for (size_t i = p_sys->i_csd_count; i < i_last_csd_count; ++i)
+        free(p_sys->p_csd[i].p_buf);
+
+    p_sys->p_csd = realloc_or_free(p_sys->p_csd, p_sys->i_csd_count *
+                                   sizeof(struct csd));
+    if (!p_sys->p_csd)
+    {
+        CSDFree(p_dec);
+        return VLC_ENOMEM;
+    }
+
+    if (p_sys->i_csd_count > i_last_csd_count)
+        memset(&p_sys->p_csd[i_last_csd_count], 0,
+               (p_sys->i_csd_count - i_last_csd_count) * sizeof(struct csd));
+
+    for (size_t i = 0; i < p_sys->i_csd_count; ++i)
+    {
+        p_sys->p_csd[i].p_buf = realloc_or_free(p_sys->p_csd[i].p_buf,
+                                                p_csd[i].i_size);
+        if (!p_sys->p_csd[i].p_buf)
+        {
+            CSDFree(p_dec);
+            return VLC_ENOMEM;
+        }
+        memcpy(p_sys->p_csd[i].p_buf, p_csd[i].p_buf, p_csd[i].i_size);
+        p_sys->p_csd[i].i_size = p_csd[i].i_size;
+    }
+    return VLC_SUCCESS;
+}
+
+static bool CSDCmp(decoder_t *p_dec, struct csd *p_csd, size_t i_csd_count)
 {
-    uint8_t *p_sps_buf, *p_pps_buf;
-    size_t i_sps_size, i_pps_size;
+    decoder_sys_t *p_sys = p_dec->p_sys;
+
+    if (p_sys->i_csd_count != i_csd_count)
+        return false;
+    for (size_t i = 0; i < i_csd_count; ++i)
+    {
+        if (p_sys->p_csd[i].i_size != p_csd[i].i_size
+         || memcmp(p_sys->p_csd[i].p_buf, p_csd[i].p_buf, p_csd[i].i_size) != 0)
+            return false;
+    }
+    return true;
+}
 
-    if (h264_get_spspps(p_buf, i_buf, &p_sps_buf, &i_sps_size,
-                        &p_pps_buf, &i_pps_size) == 0)
+/* Fill the p_sys->p_csd struct with H264 Parameter Sets */
+static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
+                      bool *p_size_changed)
+{
+    decoder_sys_t *p_sys = p_dec->p_sys;
+    struct nal_sps sps;
+    uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
+    size_t i_sps_size = 0, i_pps_size = 0;
+
+    /* Check if p_buf contains a valid SPS PPS */
+    if (h264_get_spspps(p_buf, i_size, &p_sps_buf, &i_sps_size,
+                        &p_pps_buf, &i_pps_size) == 0
+     && h264_parse_sps(p_sps_buf, i_sps_size, &sps) == 0
+     && sps.i_width && sps.i_height)
     {
-        if (pp_sps_buf && p_sps_size )
+        struct csd csd[2];
+        int i_csd_count = 0;
+
+        if (i_sps_size)
         {
-            *pp_sps_buf = p_sps_buf;
-            *p_sps_size = i_sps_size;
+            csd[i_csd_count].p_buf = p_sps_buf;
+            csd[i_csd_count].i_size = i_sps_size;
+            i_csd_count++;
         }
-        if (pp_pps_buf && p_pps_size )
+        if (i_pps_size)
         {
-            *pp_pps_buf = p_pps_buf;
-            *p_pps_size = i_pps_size;
+            csd[i_csd_count].p_buf = p_pps_buf;
+            csd[i_csd_count].i_size = i_pps_size;
+            i_csd_count++;
         }
-        return h264_parse_sps(p_sps_buf, i_sps_size, p_sps);
-    } else
-        return -1;
+
+        /* Compare the SPS PPS with the old one */
+        if (!CSDCmp(p_dec, csd, i_csd_count))
+        {
+            msg_Warn(p_dec, "New SPS/PPS found, id: %d size: %dx%d sps: %d pps: %d",
+                     sps.i_id, sps.i_width, sps.i_height,
+                     i_sps_size, i_pps_size);
+
+            /* In most use cases, p_sys->p_csd[0] contains a SPS, and
+             * p_sys->p_csd[1] contains a PPS */
+            if (CSDDup(p_dec, csd, i_csd_count))
+                return VLC_ENOMEM;
+
+            if (p_size_changed)
+                *p_size_changed = (sps.i_width != p_sys->i_width
+                                || sps.i_height != p_sys->i_height);
+
+            p_sys->i_csd_send = 0;
+            p_sys->i_width = sps.i_width;
+            p_sys->i_height = sps.i_height;
+            return VLC_SUCCESS;
+        }
+    }
+    return VLC_EGENERIC;
 }
 
 static jstring GetMediaCodecName(decoder_t *p_dec, JNIEnv *env,
@@ -560,8 +663,6 @@ static int OpenMediaCodec(decoder_t *p_dec, JNIEnv *env)
     jstring jmime = NULL;
     jstring jcodec_name = NULL;
     jobject jcodec = NULL;
-    jobject jcsd0_buffer = NULL;
-    jstring jcsd0_string = NULL;
     jobject jformat = NULL;
     jstring jrotation_string = NULL;
     jobject jinput_buffers = NULL;
@@ -608,46 +709,55 @@ static int OpenMediaCodec(decoder_t *p_dec, JNIEnv *env)
      * initialisation via the MediaFormat class, or use a CODEC_CONFIG buffer
      * that can be provided during playback (and must be provided after a flush
      * and a start). */
-    if (p_dec->fmt_in.i_extra && !p_sys->p_config_buffer) {
-        uint32_t size = p_dec->fmt_in.i_extra;
-        int buf_size = p_dec->fmt_in.i_extra + 20;
-
-        /* Don't free p_csd0_buffer until Format use it, so until MediaCodec
-         * is closed */
-        p_sys->p_csd0_buffer = malloc(buf_size);
-        if (!p_sys->p_csd0_buffer)
+    if (p_dec->fmt_in.i_extra && !p_sys->p_csd)
+    {
+        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
+         || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
         {
-            msg_Warn(p_dec, "extra buffer allocation failed");
-            goto error;
-        }
-        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264 && ((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1) {
-            convert_sps_pps(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
-                            p_sys->p_csd0_buffer, buf_size,
-                            &size, &p_sys->nal_size);
-        } else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC) {
-            convert_hevc_nal_units(p_dec, p_dec->fmt_in.p_extra,
-                                   p_dec->fmt_in.i_extra,
-                                   p_sys->p_csd0_buffer, buf_size,
-                                   &size, &p_sys->nal_size);
-        } else {
-            memcpy(p_sys->p_csd0_buffer, p_dec->fmt_in.p_extra, size);
-        }
-        p_sys->i_csd0_buffer = size;
+            int buf_size = p_dec->fmt_in.i_extra + 20;
+            uint32_t size = p_dec->fmt_in.i_extra;
+            void *p_buf = malloc(buf_size);
 
-        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
-        {
-            struct nal_sps sps;
+            if (!p_buf)
+            {
+                msg_Warn(p_dec, "extra buffer allocation failed");
+                goto error;
+            }
 
-            if (H264GetSPSPPS(p_sys->p_csd0_buffer, p_sys->i_csd0_buffer,
-                              NULL, NULL, NULL, NULL, &sps) == 0)
+            if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
+            {
+                if (((uint8_t*)p_dec->fmt_in.p_extra)[0] == 1
+                 && convert_sps_pps(p_dec, p_dec->fmt_in.p_extra,
+                                    p_dec->fmt_in.i_extra,
+                                    p_buf, buf_size, &size,
+                                    &p_sys->nal_size) == VLC_SUCCESS)
+                    H264SetCSD(p_dec, p_buf, size, NULL);
+            } else
             {
-                msg_Warn(p_dec, "SPS found, id: %d size: %dx%d (vs %dx%d)",
-                         sps.i_id, sps.i_width, sps.i_height,
-                         p_sys->i_width, p_sys->i_height);
-                p_sys->i_width = sps.i_width;
-                p_sys->i_height = sps.i_height;
+                if (convert_hevc_nal_units(p_dec, p_dec->fmt_in.p_extra,
+                                           p_dec->fmt_in.i_extra,
+                                           p_buf, buf_size, &size,
+                                           &p_sys->nal_size) == VLC_SUCCESS)
+                {
+                    struct csd csd;
+
+                    csd.p_buf = p_buf;
+                    csd.i_size = size;
+                    CSDDup(p_dec, &csd, 1);
+                }
             }
+            free(p_buf);
         }
+        if (!p_sys->p_csd)
+        {
+            struct csd csd;
+
+            csd.p_buf = p_dec->fmt_in.p_extra;
+            csd.i_size = p_dec->fmt_in.i_extra;
+            CSDDup(p_dec, &csd, 1);
+        }
+
+        p_sys->i_csd_send = 0;
     }
 
     if (!p_sys->i_width && !p_sys->i_height)
@@ -659,23 +769,6 @@ static int OpenMediaCodec(decoder_t *p_dec, JNIEnv *env)
                                              jfields.create_video_format, jmime,
                                              p_sys->i_width, p_sys->i_height);
 
-    if (p_sys->p_csd0_buffer)
-    {
-        jcsd0_buffer = (*env)->NewDirectByteBuffer( env,
-                                                    p_sys->p_csd0_buffer,
-                                                    p_sys->i_csd0_buffer);
-        if (CHECK_EXCEPTION() || !jcsd0_buffer)
-        {
-            msg_Warn(p_dec, "java extra buffer allocation failed");
-            free(p_sys->p_csd0_buffer);
-            p_sys->p_csd0_buffer = NULL;
-            goto error;
-        }
-        jcsd0_string = (*env)->NewStringUTF(env, "csd-0");
-        (*env)->CallVoidMethod(env, jformat, jfields.set_bytebuffer,
-                               jcsd0_string, jcsd0_buffer);
-    }
-
     p_sys->direct_rendering = var_InheritBool(p_dec, CFG_PREFIX "dr");
 
     /* There is no way to rotate the video using direct rendering (and using a
@@ -775,10 +868,6 @@ error:
         (*env)->DeleteLocalRef(env, jcodec_name);
     if (jcodec)
         (*env)->DeleteLocalRef(env, jcodec);
-    if (jcsd0_buffer)
-        (*env)->DeleteLocalRef(env, jcsd0_buffer);
-    if (jcsd0_string)
-        (*env)->DeleteLocalRef(env, jcsd0_string);
     if (jformat)
         (*env)->DeleteLocalRef(env, jformat);
     if (jrotation_string)
@@ -946,8 +1035,7 @@ static void CloseDecoder(vlc_object_t *p_this)
     else
         msg_Warn(p_dec, "Can't get a JNIEnv, can't close mediacodec !");
 
-    free(p_sys->p_csd0_buffer);
-    free(p_sys->p_config_buffer);
+    CSDFree(p_dec);
     free(p_sys->name);
     ArchitectureSpecificCopyHooksDestroy(p_sys->pixel_format, &p_sys->architecture_specific_data);
     free(p_sys->pp_inflight_pictures);
@@ -1256,29 +1344,12 @@ static int GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t *p_pic, jlong time
     return 0;
 }
 
-static bool spsppscmp(decoder_t *p_dec, uint8_t *p_sps_buf, size_t i_sps_size,
-                      uint8_t *p_pps_buf, size_t i_pps_size)
-{
-    decoder_sys_t *p_sys = p_dec->p_sys;
-
-    if (p_sys->i_config_buffer != i_sps_size + i_pps_size)
-        return false;
-    if (p_sps_buf && memcmp(p_sys->p_config_buffer, p_sps_buf, i_sps_size) != 0)
-        return false;
-    if (p_pps_buf && memcmp(p_sys->p_config_buffer + i_sps_size,
-                            p_pps_buf, i_pps_size) != 0)
-        return false;
-    return true;
-}
-
 static void H264ProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t *p_block,
                              bool *p_delayed_open)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    uint8_t *p_sps_buf, *p_pps_buf;
-    size_t i_sps_size, i_pps_size;
-    struct nal_sps sps;
     struct H264ConvertState convert_state = { 0, 0 };
+    bool b_size_changed;
 
     assert(p_dec->fmt_in.i_codec == VLC_CODEC_H264 && p_block);
 
@@ -1286,49 +1357,18 @@ static void H264ProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t *p_block,
     {
         convert_h264_to_annexb(p_block->p_buffer, p_block->i_buffer,
                                p_sys->nal_size, &convert_state);
-    } else if (H264GetSPSPPS(p_block->p_buffer, p_block->i_buffer,
-                             &p_sps_buf, &i_sps_size,
-                             &p_pps_buf, &i_pps_size, &sps) == 0
-        && sps.i_width && sps.i_height
-        && !spsppscmp(p_dec, p_sps_buf, i_sps_size, p_pps_buf, i_pps_size))
+    } else if (H264SetCSD(p_dec, p_block->p_buffer, p_block->i_buffer,
+                          &b_size_changed) == VLC_SUCCESS)
     {
-        void *p_config_buffer;
-
-        msg_Warn(p_dec, "New SPS/PPS found, id: %d size: %dx%d (vs %dx%d) %d %d",
-                 sps.i_id, sps.i_width, sps.i_height,
-                 p_sys->i_width, p_sys->i_height,
-                 i_sps_size, i_pps_size);
-
-        if (p_sys->codec && (sps.i_width != p_sys->i_width ||
-                             sps.i_height != p_sys->i_height))
+        if (p_sys->codec && b_size_changed)
         {
-            msg_Err(p_dec, "SPS/PPS changed during the stream and "
+            msg_Err(p_dec, "SPS/PPS changed during playback and "
                     "MediaCodec configured with a different video size. "
                     "Restart it !");
             CloseMediaCodec(p_dec, env);
         }
         if (!p_sys->codec)
             *p_delayed_open = true;
-
-        p_config_buffer = realloc(p_sys->p_config_buffer,
-                                  i_sps_size + i_pps_size);
-        if (!p_config_buffer)
-            free(p_sys->p_config_buffer);
-        p_sys->p_config_buffer = p_config_buffer;
-
-        if (p_sys->p_config_buffer)
-        {
-            if (p_sps_buf && i_sps_size)
-                memcpy(p_sys->p_config_buffer, p_sps_buf, i_sps_size);
-            if (p_pps_buf && i_pps_size)
-                memcpy(p_sys->p_config_buffer + i_sps_size,
-                       p_pps_buf, i_pps_size);
-            p_sys->i_config_buffer = i_sps_size + i_pps_size;
-            p_sys->b_config_resend = true;
-        }
-
-        p_sys->i_width = sps.i_width;
-        p_sys->i_height = sps.i_height;
     }
 }
 
@@ -1383,26 +1423,28 @@ static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
     }
 
     if (p_block && p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
-        p_sys->i_preroll_end = 0;
-        timestamp_FifoEmpty(p_sys->timestamp_fifo);
-        if (p_sys->decoded) {
+        if (p_sys->decoded)
+        {
+            p_sys->i_preroll_end = 0;
+            timestamp_FifoEmpty(p_sys->timestamp_fifo);
             /* Invalidate all pictures that are currently in flight
              * since flushing make all previous indices returned by
              * MediaCodec invalid. */
             if (p_sys->direct_rendering)
                 InvalidateAllPictures(p_dec);
+        }
 
+        if (p_sys->decoded || p_sys->i_csd_send > 0)
+        {
             (*env)->CallVoidMethod(env, p_sys->codec, jfields.flush);
             if (CHECK_EXCEPTION()) {
                 msg_Warn(p_dec, "Exception occurred in MediaCodec.flush");
                 b_error = true;
             }
+            /* resend CODEC_CONFIG buffer after a flush */
+            p_sys->i_csd_send = 0;
         }
         p_sys->decoded = false;
-
-        /* resend CODEC_CONFIG buffer after a flush */
-        if (p_sys->p_config_buffer)
-            p_sys->b_config_resend = true;
         goto endclean;
     }
 
@@ -1449,22 +1491,23 @@ static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
 
     do
     {
-        if ((p_sys->b_config_resend || p_block) && i_input_ret == 0)
+        if ((p_sys->i_csd_send < p_sys->i_csd_count || p_block)
+         && i_input_ret == 0)
         {
             const void *p_buf;
             size_t i_size;
             jint jflags = 0;
             mtime_t i_ts = 0;
 
-            if (p_sys->b_config_resend)
+            if (p_sys->i_csd_send < p_sys->i_csd_count)
             {
-                p_buf = p_sys->p_config_buffer;
-                i_size = p_sys->i_config_buffer;
+                /* Try to send Codec Specific Data */
+                p_buf = p_sys->p_csd[p_sys->i_csd_send].p_buf;
+                i_size = p_sys->p_csd[p_sys->i_csd_send].i_size;
                 jflags = BUFFER_FLAG_CODEC_CONFIG;
-                msg_Dbg(p_dec, "sending codec specific data of size %d "
-                               "via BUFFER_FLAG_CODEC_CONFIG flag", i_size);
             } else
             {
+                /* Try to send p_block input buffer */
                 p_buf = p_block->p_buffer;
                 i_size = p_block->i_buffer;
                 i_ts = p_block->i_pts;
@@ -1477,11 +1520,18 @@ static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
 
             if (i_input_ret == 1)
             {
-                p_sys->decoded = true;
-                if (p_sys->b_config_resend)
-                    p_sys->b_config_resend = false;
+                if (p_sys->i_csd_send < p_sys->i_csd_count)
+                {
+                    msg_Dbg(p_dec, "sent codec specific data(%d) of size %d "
+                            "via BUFFER_FLAG_CODEC_CONFIG flag",
+                            p_sys->i_csd_send, i_size);
+                    p_sys->i_csd_send++;
+                    i_input_ret = 0;
+                    continue;
+                }
                 else
                 {
+                    p_sys->decoded = true;
                     if (p_block->i_flags & BLOCK_FLAG_PREROLL )
                         p_sys->i_preroll_end = i_ts;
                     timestamp_FifoPut(p_sys->timestamp_fifo,
@@ -1490,7 +1540,7 @@ static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
             }
         }
 
-        if (i_input_ret != -1 && i_output_ret == 0)
+        if (i_input_ret != -1 && p_sys->decoded && i_output_ret == 0)
         {
             /* FIXME: A new picture shouldn't be created each time.
              * If decoder_NewPicture fails because the decoder is
-- 
2.1.4




More information about the vlc-devel mailing list