[vlc-devel] [RFC PATCH] mediacodec: parse SPS/PPS, fix H264 via live streaming
Thomas Guillem
thomas at gllm.fr
Fri Apr 24 16:28:15 CEST 2015
On Fri, Apr 24, 2015, at 16:21, Thomas Guillem wrote:
> Don't abort if H264 fmt doesn't have any valid size or extra. The SPS/PPS
> will
> be parsed from DecodeVideo and MediaCodec will be (re)started if video
> size and
> SPS/PPS are valid.
> ---
> modules/codec/omxil/android_mediacodec.c | 158
> +++++++++++++++++++++++++++++--
> 1 file changed, 149 insertions(+), 9 deletions(-)
>
> diff --git a/modules/codec/omxil/android_mediacodec.c
> b/modules/codec/omxil/android_mediacodec.c
> index 66f82bc..27d7a4c 100644
> --- a/modules/codec/omxil/android_mediacodec.c
> +++ b/modules/codec/omxil/android_mediacodec.c
> @@ -142,6 +142,9 @@ struct decoder_sys_t
> int stride, slice_height;
> char *name;
>
> + int i_width;
> + int i_height;
> + int i_sps_id;
> void *p_extra_buffer;
> size_t i_extra_buffer;
>
> @@ -149,6 +152,7 @@ struct decoder_sys_t
> bool started;
> bool decoded;
> bool error_state;
> + bool b_new_block;
>
> ArchitectureSpecificCopyData architecture_specific_data;
>
> @@ -388,6 +392,32 @@ end:
> return ret;
> }
>
> +static int H264GetSPSPPS(uint8_t *p_buf, size_t i_buf,
> + uint8_t **pp_sps_buf, size_t *p_sps_size,
> + uint8_t **pp_pps_buf, size_t *p_pps_size,
> + struct nal_sps *p_sps)
> +{
> + uint8_t *p_sps_buf, *p_pps_buf;
> + size_t i_sps_size, i_pps_size;
> +
> + if (h264_get_spspps(p_buf, i_buf, &p_sps_buf, &i_sps_size,
> + &p_pps_buf, &i_pps_size) == 0)
> + {
> + if (pp_sps_buf && p_sps_size )
> + {
> + *pp_sps_buf = p_sps_buf;
> + *p_sps_size = i_sps_size;
> + }
> + if (pp_pps_buf && p_pps_size )
> + {
> + *pp_pps_buf = p_pps_buf;
> + *p_pps_size = i_pps_size;
> + }
> + return h264_parse_sps(p_sps_buf, i_sps_size, p_sps);
> + } else
> + return -1;
> +}
> +
> /*****************************************************************************
> * OpenMediaCodec: Create the mediacodec instance
> *****************************************************************************/
> @@ -526,10 +556,6 @@ loopclean:
> p_sys->allocated = true;
> p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
>
> - jobject format = (*env)->CallStaticObjectMethod(env,
> jfields.media_format_class,
> - jfields.create_video_format,
> (*env)->NewStringUTF(env, mime),
> - p_dec->fmt_in.video.i_width,
> p_dec->fmt_in.video.i_height);
> -
> if (p_dec->fmt_in.i_extra && !p_sys->p_extra_buffer) {
> uint32_t size = p_dec->fmt_in.i_extra;
> int buf_size = p_dec->fmt_in.i_extra + 20;
> @@ -555,7 +581,32 @@ loopclean:
> memcpy(p_sys->p_extra_buffer, p_dec->fmt_in.p_extra, size);
> }
> p_sys->i_extra_buffer = size;
> +
> + if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
> + {
> + struct nal_sps sps;
> +
> + if (H264GetSPSPPS(p_sys->p_extra_buffer,
> p_sys->i_extra_buffer,
> + NULL, NULL, NULL, NULL, &sps) == 0)
> + {
> + msg_Warn(p_dec, "SPS found, id: %d size: %dx%d (vs
> %dx%d)",
> + sps.i_id, sps.i_width, sps.i_height,
> + p_sys->i_width, p_sys->i_height);
> + p_sys->i_sps_id = sps.i_id;
> + p_sys->i_width = sps.i_width;
> + p_sys->i_height = sps.i_height;
> + }
> + }
> + }
> + if (!p_sys->i_width && !p_sys->i_height)
> + {
> + msg_Err(p_dec, "invalid size, abort MediaCodec");
> + goto error;
> }
> + jobject format = (*env)->CallStaticObjectMethod(env,
> jfields.media_format_class,
> + jfields.create_video_format,
> (*env)->NewStringUTF(env, mime),
> + p_sys->i_width, p_sys->i_height);
> +
> if (p_sys->p_extra_buffer)
> {
> jobject jextra_buffer;
> @@ -731,6 +782,8 @@ static int OpenDecoder(vlc_object_t *p_this)
>
> switch (p_dec->fmt_in.i_codec) {
> case VLC_CODEC_H264:
> + /* We can handle h264 without a valid video size */
> + break;
> case VLC_CODEC_HEVC:
> case VLC_CODEC_H263:
> case VLC_CODEC_MP4V:
> @@ -764,12 +817,25 @@ static int OpenDecoder(vlc_object_t *p_this)
> p_dec->fmt_out.audio = p_dec->fmt_in.audio;
> p_dec->b_need_packetized = true;
>
> + p_dec->p_sys->i_sps_id = -1;
> + p_dec->p_sys->i_width = p_dec->fmt_in.video.i_width;
> + p_dec->p_sys->i_height = p_dec->fmt_in.video.i_height;
> +
> p_dec->p_sys->timestamp_fifo = timestamp_FifoNew(32);
> if (!p_dec->p_sys->timestamp_fifo)
> goto error;
>
> + p_dec->p_sys->b_new_block = true;
> +
> switch (p_dec->fmt_in.i_codec)
> {
> + case VLC_CODEC_H264:
> + if (!p_dec->p_sys->i_width || !p_dec->p_sys->i_height)
> + {
> + msg_Warn(p_dec, "waiting for sps/pps for codec %4.4s",
> + (const char *)&p_dec->fmt_in.i_codec);
> + return VLC_SUCCESS;
> + }
> case VLC_CODEC_VC1:
> if (!p_dec->fmt_in.i_extra)
> {
> @@ -903,7 +969,6 @@ static int PutInput(decoder_t *p_dec, JNIEnv *env,
> block_t *p_block, jlong timeo
> jobject buf;
> jsize size;
> uint8_t *bufptr;
> - struct H264ConvertState convert_state = { 0, 0 };
>
> index = (*env)->CallIntMethod(env, p_sys->codec,
> jfields.dequeue_input_buffer,
> timeout);
> @@ -928,8 +993,6 @@ static int PutInput(decoder_t *p_dec, JNIEnv *env,
> block_t *p_block, jlong timeo
> size = p_block->i_buffer;
> memcpy(bufptr, p_block->p_buffer, size);
>
> - convert_h264_to_annexb(bufptr, size, p_sys->nal_size,
> &convert_state);
> -
> int64_t ts = p_block->i_pts;
> if (!ts && p_block->i_dts)
> ts = p_block->i_dts;
> @@ -1108,6 +1171,71 @@ static int GetOutput(decoder_t *p_dec, JNIEnv
> *env, picture_t *p_pic, jlong time
> return 0;
> }
>
> +static void H264ProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t
> *p_block,
> + bool *p_delayed_open)
> +{
> + decoder_sys_t *p_sys = p_dec->p_sys;
> + uint8_t *p_sps_buf, *p_pps_buf;
> + size_t i_sps_size, i_pps_size;
> + struct nal_sps sps;
> + struct H264ConvertState convert_state = { 0, 0 };
> +
> + assert(p_dec->fmt_in.i_codec == VLC_CODEC_H264 && p_block);
> +
> + convert_h264_to_annexb(p_block->p_buffer, p_block->i_buffer,
> + p_sys->nal_size, &convert_state);
> +
> + if (H264GetSPSPPS(p_block->p_buffer, p_block->i_buffer,
> + &p_sps_buf, &i_sps_size,
> + &p_pps_buf, &i_pps_size, &sps) == 0)
> + {
> + if (sps.i_id != p_sys->i_sps_id && sps.i_width && sps.i_height)
> + {
> + msg_Warn(p_dec, "SPS found, id: %d size: %dx%d (vs %dx%d)"
> + ", restart MediaCodec !",
> + sps.i_id, sps.i_width, sps.i_height,
> + p_sys->i_width, p_sys->i_height);
That's the part I'm not sure and still thinking about.
if sps.i_width != p_sys->i_width && sps.i_height != p_sys->i_height, we
should restart MediaCodec.
For the sps/pps, maybe I should send it each time I see with
BUFFER_FLAG_CODEC_CONFIG
> +
> + if (p_sys->codec)
> + CloseMediaCodec(p_dec, env);
> +
> + if (p_sys->p_extra_buffer)
> + free(p_sys->p_extra_buffer);
> + p_sys->p_extra_buffer = malloc(i_sps_size + i_pps_size);
> + if (p_sys->p_extra_buffer)
> + {
> + if (p_sps_buf && i_sps_size)
> + memcpy(p_sys->p_extra_buffer, p_sps_buf,
> i_sps_size);
> + if (p_pps_buf && i_pps_size)
> + memcpy((uint8_t *)p_sys->p_extra_buffer +
> i_sps_size,
> + p_pps_buf, i_pps_size);
> + p_sys->i_extra_buffer = i_sps_size + i_pps_size;
> + }
> +
> + p_sys->i_sps_id = sps.i_id;
> + p_sys->i_width = sps.i_width;
> + p_sys->i_height = sps.i_height;
> + *p_delayed_open = true;
> + }
> + }
> +}
> +
> +static void HEVCProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t
> *p_block,
> + bool *p_delayed_open)
> +{
> + decoder_sys_t *p_sys = p_dec->p_sys;
> + struct H264ConvertState convert_state = { 0, 0 };
> +
> + assert(p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && p_block);
> +
> + convert_h264_to_annexb(p_block->p_buffer, p_block->i_buffer,
> + p_sys->nal_size, &convert_state);
> +
> + /* TODO */
> + VLC_UNUSED(env);
> + VLC_UNUSED(p_delayed_open);
> +}
> +
> static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
> {
> decoder_sys_t *p_sys = p_dec->p_sys;
> @@ -1119,10 +1247,15 @@ static picture_t *DecodeVideo(decoder_t *p_dec,
> block_t **pp_block)
> int i_output_ret = 0;
> int i_input_ret = 0;
> bool b_error = false;
> + bool b_delayed_open = false;
> + bool b_new_block = p_block ? p_sys->b_new_block : false;
>
> if (p_sys->error_state)
> goto endclean;
>
> + if (b_new_block)
> + p_sys->b_new_block = false;
> +
> if (!(env = jni_get_env(THREAD_NAME)))
> {
> b_error = true;
> @@ -1149,11 +1282,17 @@ static picture_t *DecodeVideo(decoder_t *p_dec,
> block_t **pp_block)
> goto endclean;
> }
>
> + if (b_new_block)
> + {
> + if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
> + H264ProcessBlock(p_dec, env, p_block, &b_delayed_open);
> + else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
> + HEVCProcessBlock(p_dec, env, p_block, &b_delayed_open);
> + }
> +
> /* try delayed opening if there is a new extra data */
> if (!p_sys->codec)
> {
> - bool b_delayed_open = false;
> -
> switch (p_dec->fmt_in.i_codec)
> {
> case VLC_CODEC_VC1:
> @@ -1242,6 +1381,7 @@ endclean:
> {
> block_Release(p_block);
> *pp_block = NULL;
> + p_sys->b_new_block = true;
> }
> if (b_error && !p_sys->error_state) {
> /* Signal the error to the Java. */
> --
> 2.1.3
>
More information about the vlc-devel
mailing list