[vlc-devel] [RFC PATCH] mediacodec: parse SPS/PPS, fix H264 via live streaming
Thomas Guillem
thomas at gllm.fr
Fri Apr 24 17:44:30 CEST 2015
This patch doesn't work, I shouldn't compare the sps.id but the whole
SPS/PPS.
I'm now able to play 2 differents h264 streams concatenated (with
different resolution).
On Fri, Apr 24, 2015, at 16:28, Thomas Guillem wrote:
>
>
> On Fri, Apr 24, 2015, at 16:21, Thomas Guillem wrote:
> > Don't abort if H264 fmt doesn't have any valid size or extra. The SPS/PPS
> > will
> > be parsed from DecodeVideo and MediaCodec will be (re)started if video
> > size and
> > SPS/PPS are valid.
> > ---
> > modules/codec/omxil/android_mediacodec.c | 158
> > +++++++++++++++++++++++++++++--
> > 1 file changed, 149 insertions(+), 9 deletions(-)
> >
> > diff --git a/modules/codec/omxil/android_mediacodec.c
> > b/modules/codec/omxil/android_mediacodec.c
> > index 66f82bc..27d7a4c 100644
> > --- a/modules/codec/omxil/android_mediacodec.c
> > +++ b/modules/codec/omxil/android_mediacodec.c
> > @@ -142,6 +142,9 @@ struct decoder_sys_t
> > int stride, slice_height;
> > char *name;
> >
> > + int i_width;
> > + int i_height;
> > + int i_sps_id;
> > void *p_extra_buffer;
> > size_t i_extra_buffer;
> >
> > @@ -149,6 +152,7 @@ struct decoder_sys_t
> > bool started;
> > bool decoded;
> > bool error_state;
> > + bool b_new_block;
> >
> > ArchitectureSpecificCopyData architecture_specific_data;
> >
> > @@ -388,6 +392,32 @@ end:
> > return ret;
> > }
> >
> > +static int H264GetSPSPPS(uint8_t *p_buf, size_t i_buf,
> > + uint8_t **pp_sps_buf, size_t *p_sps_size,
> > + uint8_t **pp_pps_buf, size_t *p_pps_size,
> > + struct nal_sps *p_sps)
> > +{
> > + uint8_t *p_sps_buf, *p_pps_buf;
> > + size_t i_sps_size, i_pps_size;
> > +
> > + if (h264_get_spspps(p_buf, i_buf, &p_sps_buf, &i_sps_size,
> > + &p_pps_buf, &i_pps_size) == 0)
> > + {
> > + if (pp_sps_buf && p_sps_size )
> > + {
> > + *pp_sps_buf = p_sps_buf;
> > + *p_sps_size = i_sps_size;
> > + }
> > + if (pp_pps_buf && p_pps_size )
> > + {
> > + *pp_pps_buf = p_pps_buf;
> > + *p_pps_size = i_pps_size;
> > + }
> > + return h264_parse_sps(p_sps_buf, i_sps_size, p_sps);
> > + } else
> > + return -1;
> > +}
> > +
> > /*****************************************************************************
> > * OpenMediaCodec: Create the mediacodec instance
> > *****************************************************************************/
> > @@ -526,10 +556,6 @@ loopclean:
> > p_sys->allocated = true;
> > p_sys->codec = (*env)->NewGlobalRef(env, p_sys->codec);
> >
> > - jobject format = (*env)->CallStaticObjectMethod(env,
> > jfields.media_format_class,
> > - jfields.create_video_format,
> > (*env)->NewStringUTF(env, mime),
> > - p_dec->fmt_in.video.i_width,
> > p_dec->fmt_in.video.i_height);
> > -
> > if (p_dec->fmt_in.i_extra && !p_sys->p_extra_buffer) {
> > uint32_t size = p_dec->fmt_in.i_extra;
> > int buf_size = p_dec->fmt_in.i_extra + 20;
> > @@ -555,7 +581,32 @@ loopclean:
> > memcpy(p_sys->p_extra_buffer, p_dec->fmt_in.p_extra, size);
> > }
> > p_sys->i_extra_buffer = size;
> > +
> > + if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
> > + {
> > + struct nal_sps sps;
> > +
> > + if (H264GetSPSPPS(p_sys->p_extra_buffer,
> > p_sys->i_extra_buffer,
> > + NULL, NULL, NULL, NULL, &sps) == 0)
> > + {
> > + msg_Warn(p_dec, "SPS found, id: %d size: %dx%d (vs
> > %dx%d)",
> > + sps.i_id, sps.i_width, sps.i_height,
> > + p_sys->i_width, p_sys->i_height);
> > + p_sys->i_sps_id = sps.i_id;
> > + p_sys->i_width = sps.i_width;
> > + p_sys->i_height = sps.i_height;
> > + }
> > + }
> > + }
> > + if (!p_sys->i_width && !p_sys->i_height)
> > + {
> > + msg_Err(p_dec, "invalid size, abort MediaCodec");
> > + goto error;
> > }
> > + jobject format = (*env)->CallStaticObjectMethod(env,
> > jfields.media_format_class,
> > + jfields.create_video_format,
> > (*env)->NewStringUTF(env, mime),
> > + p_sys->i_width, p_sys->i_height);
> > +
> > if (p_sys->p_extra_buffer)
> > {
> > jobject jextra_buffer;
> > @@ -731,6 +782,8 @@ static int OpenDecoder(vlc_object_t *p_this)
> >
> > switch (p_dec->fmt_in.i_codec) {
> > case VLC_CODEC_H264:
> > + /* We can handle h264 without a valid video size */
> > + break;
> > case VLC_CODEC_HEVC:
> > case VLC_CODEC_H263:
> > case VLC_CODEC_MP4V:
> > @@ -764,12 +817,25 @@ static int OpenDecoder(vlc_object_t *p_this)
> > p_dec->fmt_out.audio = p_dec->fmt_in.audio;
> > p_dec->b_need_packetized = true;
> >
> > + p_dec->p_sys->i_sps_id = -1;
> > + p_dec->p_sys->i_width = p_dec->fmt_in.video.i_width;
> > + p_dec->p_sys->i_height = p_dec->fmt_in.video.i_height;
> > +
> > p_dec->p_sys->timestamp_fifo = timestamp_FifoNew(32);
> > if (!p_dec->p_sys->timestamp_fifo)
> > goto error;
> >
> > + p_dec->p_sys->b_new_block = true;
> > +
> > switch (p_dec->fmt_in.i_codec)
> > {
> > + case VLC_CODEC_H264:
> > + if (!p_dec->p_sys->i_width || !p_dec->p_sys->i_height)
> > + {
> > + msg_Warn(p_dec, "waiting for sps/pps for codec %4.4s",
> > + (const char *)&p_dec->fmt_in.i_codec);
> > + return VLC_SUCCESS;
> > + }
> > case VLC_CODEC_VC1:
> > if (!p_dec->fmt_in.i_extra)
> > {
> > @@ -903,7 +969,6 @@ static int PutInput(decoder_t *p_dec, JNIEnv *env,
> > block_t *p_block, jlong timeo
> > jobject buf;
> > jsize size;
> > uint8_t *bufptr;
> > - struct H264ConvertState convert_state = { 0, 0 };
> >
> > index = (*env)->CallIntMethod(env, p_sys->codec,
> > jfields.dequeue_input_buffer,
> > timeout);
> > @@ -928,8 +993,6 @@ static int PutInput(decoder_t *p_dec, JNIEnv *env,
> > block_t *p_block, jlong timeo
> > size = p_block->i_buffer;
> > memcpy(bufptr, p_block->p_buffer, size);
> >
> > - convert_h264_to_annexb(bufptr, size, p_sys->nal_size,
> > &convert_state);
> > -
> > int64_t ts = p_block->i_pts;
> > if (!ts && p_block->i_dts)
> > ts = p_block->i_dts;
> > @@ -1108,6 +1171,71 @@ static int GetOutput(decoder_t *p_dec, JNIEnv
> > *env, picture_t *p_pic, jlong time
> > return 0;
> > }
> >
> > +static void H264ProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t
> > *p_block,
> > + bool *p_delayed_open)
> > +{
> > + decoder_sys_t *p_sys = p_dec->p_sys;
> > + uint8_t *p_sps_buf, *p_pps_buf;
> > + size_t i_sps_size, i_pps_size;
> > + struct nal_sps sps;
> > + struct H264ConvertState convert_state = { 0, 0 };
> > +
> > + assert(p_dec->fmt_in.i_codec == VLC_CODEC_H264 && p_block);
> > +
> > + convert_h264_to_annexb(p_block->p_buffer, p_block->i_buffer,
> > + p_sys->nal_size, &convert_state);
> > +
> > + if (H264GetSPSPPS(p_block->p_buffer, p_block->i_buffer,
> > + &p_sps_buf, &i_sps_size,
> > + &p_pps_buf, &i_pps_size, &sps) == 0)
> > + {
> > + if (sps.i_id != p_sys->i_sps_id && sps.i_width && sps.i_height)
> > + {
> > + msg_Warn(p_dec, "SPS found, id: %d size: %dx%d (vs %dx%d)"
> > + ", restart MediaCodec !",
> > + sps.i_id, sps.i_width, sps.i_height,
> > + p_sys->i_width, p_sys->i_height);
>
> That's the part I'm not sure and still thinking about.
> if sps.i_width != p_sys->i_width && sps.i_height != p_sys->i_height, we
> should restart MediaCodec.
> For the sps/pps, maybe I should send it each time I see with
> BUFFER_FLAG_CODEC_CONFIG
>
> > +
> > + if (p_sys->codec)
> > + CloseMediaCodec(p_dec, env);
> > +
> > + if (p_sys->p_extra_buffer)
> > + free(p_sys->p_extra_buffer);
> > + p_sys->p_extra_buffer = malloc(i_sps_size + i_pps_size);
> > + if (p_sys->p_extra_buffer)
> > + {
> > + if (p_sps_buf && i_sps_size)
> > + memcpy(p_sys->p_extra_buffer, p_sps_buf,
> > i_sps_size);
> > + if (p_pps_buf && i_pps_size)
> > + memcpy((uint8_t *)p_sys->p_extra_buffer +
> > i_sps_size,
> > + p_pps_buf, i_pps_size);
> > + p_sys->i_extra_buffer = i_sps_size + i_pps_size;
> > + }
> > +
> > + p_sys->i_sps_id = sps.i_id;
> > + p_sys->i_width = sps.i_width;
> > + p_sys->i_height = sps.i_height;
> > + *p_delayed_open = true;
> > + }
> > + }
> > +}
> > +
> > +static void HEVCProcessBlock(decoder_t *p_dec, JNIEnv *env, block_t
> > *p_block,
> > + bool *p_delayed_open)
> > +{
> > + decoder_sys_t *p_sys = p_dec->p_sys;
> > + struct H264ConvertState convert_state = { 0, 0 };
> > +
> > + assert(p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && p_block);
> > +
> > + convert_h264_to_annexb(p_block->p_buffer, p_block->i_buffer,
> > + p_sys->nal_size, &convert_state);
> > +
> > + /* TODO */
> > + VLC_UNUSED(env);
> > + VLC_UNUSED(p_delayed_open);
> > +}
> > +
> > static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
> > {
> > decoder_sys_t *p_sys = p_dec->p_sys;
> > @@ -1119,10 +1247,15 @@ static picture_t *DecodeVideo(decoder_t *p_dec,
> > block_t **pp_block)
> > int i_output_ret = 0;
> > int i_input_ret = 0;
> > bool b_error = false;
> > + bool b_delayed_open = false;
> > + bool b_new_block = p_block ? p_sys->b_new_block : false;
> >
> > if (p_sys->error_state)
> > goto endclean;
> >
> > + if (b_new_block)
> > + p_sys->b_new_block = false;
> > +
> > if (!(env = jni_get_env(THREAD_NAME)))
> > {
> > b_error = true;
> > @@ -1149,11 +1282,17 @@ static picture_t *DecodeVideo(decoder_t *p_dec,
> > block_t **pp_block)
> > goto endclean;
> > }
> >
> > + if (b_new_block)
> > + {
> > + if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
> > + H264ProcessBlock(p_dec, env, p_block, &b_delayed_open);
> > + else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
> > + HEVCProcessBlock(p_dec, env, p_block, &b_delayed_open);
> > + }
> > +
> > /* try delayed opening if there is a new extra data */
> > if (!p_sys->codec)
> > {
> > - bool b_delayed_open = false;
> > -
> > switch (p_dec->fmt_in.i_codec)
> > {
> > case VLC_CODEC_VC1:
> > @@ -1242,6 +1381,7 @@ endclean:
> > {
> > block_Release(p_block);
> > *pp_block = NULL;
> > + p_sys->b_new_block = true;
> > }
> > if (b_error && !p_sys->error_state) {
> > /* Signal the error to the Java. */
> > --
> > 2.1.3
> >
More information about the vlc-devel
mailing list