[vlc-devel] [PATCH] [WIP] mediacodec: implementation of MediaCodec direct rendering based on the work by Martin Storsjö.
Rémi Denis-Courmont
remi at remlab.net
Wed Nov 6 22:14:39 CET 2013
Le mercredi 6 novembre 2013 11:59:00 Felix Abecassis a écrit :
> The decodes stores opaque buffers in the p_sys member of the picture and the
> vout uses a callback from the decoder to render these buffers.
>
> WIP:
> - The decoder and the vout are sharing a structure (stored in p_sys), do you
> think it's good practice?
The video output can do whatever it wants with the p_sys, so long as the p_sys
address is invariable for the picture lifetime. It is not a very good practice
but the decoder can use p_sys if it is sure what it contains, in other words
how the picture was allocated.
> - In the decoder we need to configure MediaCodec
> with a Surface object, currently we are waiting using msleep until the
> Surface object is created.
Can the video output not store a reference to the surface in every picture and
the decoder allocate a picture during its start-up?
> - The code was not thoroughly tested yet, Martin
> reported a deadlock I couldn't reproduce yet on my devices. - In the vout
> we need to allocate a picture pool of size > 30 in order to be connected
> directly to the decoder. This value is currently hardcoded. ---
> include/vlc_fourcc.h | 2 +
> modules/codec/omxil/android_mediacodec.c | 143 +++++++++++++++----
> modules/codec/omxil/android_mediacodec.h | 30 ++++
> modules/video_output/Modules.am | 1 +
> modules/video_output/androidsurface_opaque.c | 203
> +++++++++++++++++++++++++++ src/misc/fourcc.c |
> 3 +-
> 6 files changed, 353 insertions(+), 29 deletions(-)
> create mode 100644 modules/codec/omxil/android_mediacodec.h
> create mode 100644 modules/video_output/androidsurface_opaque.c
>
> diff --git a/include/vlc_fourcc.h b/include/vlc_fourcc.h
> index 70bc15e..c9830de 100644
> --- a/include/vlc_fourcc.h
> +++ b/include/vlc_fourcc.h
> @@ -245,6 +245,8 @@
> /* VDPAU output surface RGBA */
> #define VLC_CODEC_VDPAU_OUTPUT VLC_FOURCC('V','D','O','R')
>
> +#define VLC_CODEC_OPAQUE_VBUF VLC_FOURCC('O','P','A','Q')
That name is too generic in my opinion.
> +
> /* Image codec (video) */
> #define VLC_CODEC_PNG VLC_FOURCC('p','n','g',' ')
> #define VLC_CODEC_PPM VLC_FOURCC('p','p','m',' ')
> a/modules/codec/omxil/android_mediacodec.h
> b/modules/codec/omxil/android_mediacodec.h new file mode 100644
> index 0000000..54aad92
> --- /dev/null
> +++ b/modules/codec/omxil/android_mediacodec.h
> @@ -0,0 +1,30 @@
> +/**************************************************************************
> *** + * android_mediacodec.h: shared structures between MediaCodec decoder +
> * and MediaCodec video output
> +
> ***************************************************************************
> ** + * Copyright (C) 2013 Felix Abecassis
> + *
> + * Authors: Felix Abecassis <felix.abecassis at gmail.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU Lesser General Public License as published by
> + * the Free Software Foundation; either version 2.1 of the License, or + *
> (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public License
> + * along with this program; if not, write to the Free Software Foundation,
> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. +
> ***************************************************************************
> **/ +
> +struct picture_sys_t
> +{
> + void (*pf_callback)(picture_sys_t*);
> + decoder_t *p_dec;
This is not good. The decoder might be destroyed or even replaced while the
picture is still in flight.
> + uint32_t i_index;
> + int b_displayed;
> +};
> diff --git a/modules/video_output/Modules.am
> b/modules/video_output/Modules.am index 041dc29..e29c4ec 100644
> --- a/modules/video_output/Modules.am
> +++ b/modules/video_output/Modules.am
> @@ -13,6 +13,7 @@ SOURCES_vout_macosx = macosx.m opengl.h opengl.c
> SOURCES_vout_coregraphicslayer = coregraphicslayer.m
> SOURCES_vout_ios2 = ios2.m opengl.h opengl.c
> SOURCES_android_surface = androidsurface.c
> +SOURCES_android_surface_opaque = androidsurface_opaque.c
>
> if HAVE_DECKLINK
> libdecklinkoutput_plugin_la_SOURCES = decklink.cpp
> diff --git a/modules/video_output/androidsurface_opaque.c
> b/modules/video_output/androidsurface_opaque.c new file mode 100644
> index 0000000..a8639a4
> --- /dev/null
> +++ b/modules/video_output/androidsurface_opaque.c
> @@ -0,0 +1,203 @@
> +/**************************************************************************
> *** + * androidsurface_opaque.c: Video output module using direct rendering
> with + * opaque buffers
> +
> ***************************************************************************
> ** + * Copyright (C) 2013 Felix Abecassis
> + *
> + * Authors: Felix Abecassis <felix.abecassis at gmail.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU Lesser General Public License as published by
> + * the Free Software Foundation; either version 2.1 of the License, or + *
> (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public License
> + * along with this program; if not, write to the Free Software Foundation,
> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. +
> ***************************************************************************
> **/ +
> +#ifdef HAVE_CONFIG_H
> +# include "config.h"
> +#endif
> +
> +#include <vlc_common.h>
> +#include <vlc_plugin.h>
> +#include <vlc_vout_display.h>
> +#include <vlc_picture_pool.h>
> +#include "../../codec/omxil/android_mediacodec.h"
> +
> +static int Open (vlc_object_t *);
> +static void Close(vlc_object_t *);
> +
> +vlc_module_begin()
> + set_category(CAT_VIDEO)
> + set_subcategory(SUBCAT_VIDEO_VOUT)
> + set_shortname("vout_mediacodec")
> + set_description(N_("Android MediaCodec direct rendering video output"))
> + set_capability("vout display", 200)
> + set_callbacks(Open, Close)
> +vlc_module_end()
> +
> +/**************************************************************************
> *** + * Local prototypes
> +
> ***************************************************************************
> **/ +
> +static picture_pool_t *Pool (vout_display_t *, unsigned);
> +static void Display(vout_display_t *, picture_t *, subpicture_t
> *); +static int Control(vout_display_t *, int, va_list);
> +
> +struct vout_display_sys_t
> +{
> + picture_pool_t *pool;
> + video_format_t fmt;
Write-only?
> +};
> +
> +static int LockSurface(picture_t *);
> +static void UnlockSurface(picture_t *);
> +
> +#define POOL_SIZE 31
> +
> +static int Open(vlc_object_t *p_this)
> +{
> + vout_display_t *vd = (vout_display_t*)p_this;
> +
> + video_format_t fmt = vd->fmt;
> +
> + if (fmt.i_chroma != VLC_CODEC_OPAQUE_VBUF)
> + return VLC_EGENERIC;
> +
> + /* Allocate structure */
> + vout_display_sys_t *sys = (struct vout_display_sys_t*)calloc(1,
> sizeof(*sys));
> + if (!sys)
> + return VLC_ENOMEM;
> +
> + sys->fmt = fmt;
> +
> + /* We need to allocate a picture pool of more than 30 buffers in
> + * order to be connected directly to the decoder without any
> intermediate buffer pool. */
> + int i_pictures = POOL_SIZE;
> + picture_t** pictures = calloc(sizeof(*pictures), i_pictures);
> + if (!pictures)
> + goto error;
> + for (int i = 0; i < i_pictures; i++)
> + {
> + picture_sys_t *p_picsys = calloc(1, sizeof(*p_picsys));
> + if (unlikely(p_picsys == NULL))
> + goto error;
> +
> + picture_resource_t resource = { .p_sys = p_picsys };
> + picture_t *picture = picture_NewFromResource(&fmt, &resource);
> + if (!picture)
> + {
> + free(p_picsys);
> + goto error;
> + }
> + pictures[i] = picture;
> + }
> +
> + /* Wrap it into a picture pool */
> + picture_pool_configuration_t pool_cfg;
> + memset(&pool_cfg, 0, sizeof(pool_cfg));
> + pool_cfg.picture_count = i_pictures;
> + pool_cfg.picture = pictures;
> + pool_cfg.lock = LockSurface;
> + pool_cfg.unlock = UnlockSurface;
> +
> + sys->pool = picture_pool_NewExtended(&pool_cfg);
> + if (!sys->pool)
> + {
> + for (int i = 0; i < i_pictures; i++)
> + picture_Release(pictures[i]);
> + goto error;
> + }
> +
> + /* Setup vout_display */
> + vd->sys = sys;
> + vd->fmt = fmt;
> + vd->pool = Pool;
> + vd->display = Display;
> + vd->control = Control;
> + vd->prepare = NULL;
> + vd->manage = NULL;
> +
> + /* Fix initial state */
> + vout_display_SendEventFullscreen(vd, false);
> +
> + return VLC_SUCCESS;
> +
> +error:
> + free(pictures);
> + Close(p_this);
> + return VLC_ENOMEM;
> +}
> +
> +static void Close(vlc_object_t *p_this)
> +{
> + vout_display_t *vd = (vout_display_t *)p_this;
> + vout_display_sys_t *sys = vd->sys;
> +
> + picture_pool_Delete(sys->pool);
> + free(sys);
> +}
> +
> +static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
> +{
> + VLC_UNUSED(count);
> +
> + return vd->sys->pool;
> +}
> +
> +static int LockSurface(picture_t *picture)
> +{
> + VLC_UNUSED(picture);
> +
> + return VLC_SUCCESS;
> +}
> +
> +static void UnlockSurface(picture_t *picture)
> +{
> + picture_sys_t *p_picsys = picture->p_sys;
> + void (*display_callback)(picture_sys_t*) = p_picsys->pf_callback;
> + display_callback(p_picsys);
> +}
> +
> +static void Display(vout_display_t *vd, picture_t *picture, subpicture_t
> *subpicture) +{
> + VLC_UNUSED(vd);
> + VLC_UNUSED(subpicture);
> +
> + picture_sys_t *p_picsys = picture->p_sys;
> + void (*display_callback)(picture_sys_t*) = p_picsys->pf_callback;
> + display_callback(p_picsys);
> +
> + /* refcount lowers to 0, and pool_cfg.unlock is called */
> + picture_Release(picture);
> +}
> +
> +static int Control(vout_display_t *vd, int query, va_list args)
> +{
> + VLC_UNUSED(args);
> +
> + switch (query) {
> + case VOUT_DISPLAY_HIDE_MOUSE:
> + return VLC_SUCCESS;
> +
> + default:
> + msg_Err(vd, "Unknown request in vout mediacodec display");
> +
> + case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
> + case VOUT_DISPLAY_CHANGE_FULLSCREEN:
> + case VOUT_DISPLAY_CHANGE_WINDOW_STATE:
> + case VOUT_DISPLAY_CHANGE_DISPLAY_SIZE:
> + case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
> + case VOUT_DISPLAY_CHANGE_ZOOM:
> + case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
> + case VOUT_DISPLAY_GET_OPENGL:
> + return VLC_EGENERIC;
> + }
> +}
> diff --git a/src/misc/fourcc.c b/src/misc/fourcc.c
> index 78a77e1..d457973 100644
> --- a/src/misc/fourcc.c
> +++ b/src/misc/fourcc.c
> @@ -1993,7 +1993,8 @@ static const struct
>
> { { VLC_CODEC_VDPAU_VIDEO_420, VLC_CODEC_VDPAU_VIDEO_422,
> VLC_CODEC_VDPAU_VIDEO_444,
> - VLC_CODEC_VDPAU_OUTPUT, 0 }, FAKE_FMT() },
> + VLC_CODEC_VDPAU_OUTPUT,
> + VLC_CODEC_OPAQUE_VBUF, 0 }, FAKE_FMT() },
Probably missing es_format.c entry.
Also please split VLC_CODEC_* addition to a dedicated patch.
>
> { {0}, { 0, {}, 0, 0 } }
> };
--
Rémi Denis-Courmont
http://www.remlab.net/
More information about the vlc-devel
mailing list