[vlc-devel] [PATCH] [WIP] mediacodec: implementation of MediaCodec direct rendering based on the work by Martin Storsjö.
Felix Abecassis
felix.abecassis at gmail.com
Wed Nov 6 11:59:00 CET 2013
The decodes stores opaque buffers in the p_sys member of the picture and the vout uses a callback from the decoder to render these buffers.
WIP:
- The decoder and the vout are sharing a structure (stored in p_sys), do you think it's good practice?
- In the decoder we need to configure MediaCodec with a Surface object, currently we are waiting using msleep until the Surface object is created.
- The code was not thoroughly tested yet, Martin reported a deadlock I couldn't reproduce yet on my devices.
- In the vout we need to allocate a picture pool of size > 30 in order to be connected directly to the decoder. This value is currently hardcoded.
---
include/vlc_fourcc.h | 2 +
modules/codec/omxil/android_mediacodec.c | 143 +++++++++++++++----
modules/codec/omxil/android_mediacodec.h | 30 ++++
modules/video_output/Modules.am | 1 +
modules/video_output/androidsurface_opaque.c | 203 +++++++++++++++++++++++++++
src/misc/fourcc.c | 3 +-
6 files changed, 353 insertions(+), 29 deletions(-)
create mode 100644 modules/codec/omxil/android_mediacodec.h
create mode 100644 modules/video_output/androidsurface_opaque.c
diff --git a/include/vlc_fourcc.h b/include/vlc_fourcc.h
index 70bc15e..c9830de 100644
--- a/include/vlc_fourcc.h
+++ b/include/vlc_fourcc.h
@@ -245,6 +245,8 @@
/* VDPAU output surface RGBA */
#define VLC_CODEC_VDPAU_OUTPUT VLC_FOURCC('V','D','O','R')
+#define VLC_CODEC_OPAQUE_VBUF VLC_FOURCC('O','P','A','Q')
+
/* Image codec (video) */
#define VLC_CODEC_PNG VLC_FOURCC('p','n','g',' ')
#define VLC_CODEC_PPM VLC_FOURCC('p','p','m',' ')
diff --git a/modules/codec/omxil/android_mediacodec.c b/modules/codec/omxil/android_mediacodec.c
index ef1f90c..11b08a5 100644
--- a/modules/codec/omxil/android_mediacodec.c
+++ b/modules/codec/omxil/android_mediacodec.c
@@ -40,6 +40,7 @@
#include <OMX_Core.h>
#include <OMX_Component.h>
#include "omxil_utils.h"
+#include "android_mediacodec.h"
#define INFO_OUTPUT_BUFFERS_CHANGED -3
#define INFO_OUTPUT_FORMAT_CHANGED -2
@@ -47,6 +48,10 @@
extern JavaVM *myVm;
+extern jobject jni_LockAndGetAndroidJavaSurface();
+extern void jni_UnlockAndroidSurface();
+extern void jni_SetAndroidSurfaceSizeEnv(JNIEnv *p_env, int width, int height, int visible_width, int visible_height, int sar_num, int sar_den);
+
struct decoder_sys_t
{
jclass media_codec_list_class, media_codec_class, media_format_class;
@@ -76,6 +81,8 @@ struct decoder_sys_t
int started;
int decoded;
+ int closed;
+ int direct_rendering;
ArchitectureSpecificCopyData architecture_specific_data;
};
@@ -184,6 +191,8 @@ static int jstrcmp(JNIEnv* env, jobject str, const char* str2)
/*****************************************************************************
* OpenDecoder: Create the decoder instance
*****************************************************************************/
+#define SURFACE_RETRY_MAX 10
+#define SURFACE_RETRY_SLEEP 20000
static int OpenDecoder(vlc_object_t *p_this)
{
decoder_t *p_dec = (decoder_t*)p_this;
@@ -335,7 +344,27 @@ static int OpenDecoder(vlc_object_t *p_this)
(*env)->DeleteLocalRef(env, bytebuf);
}
- (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
+ // MediaCodec direct rendering is enabled by default.
+ p_sys->direct_rendering = 1;
+ if (p_sys->direct_rendering) {
+ // Wait until the Android surface object is created.
+ jobject surf = jni_LockAndGetAndroidJavaSurface();
+ for (int i = 0; i < SURFACE_RETRY_MAX; ++i) {
+ if (surf)
+ break;
+ jni_UnlockAndroidSurface();
+ msleep(SURFACE_RETRY_SLEEP);
+ surf = jni_LockAndGetAndroidJavaSurface();
+ }
+ if (!surf)
+ goto error;
+ // Configure MediaCodec with the Android surface.
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, surf, NULL, 0);
+ jni_UnlockAndroidSurface();
+ }
+ else
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->configure, format, NULL, NULL, 0);
+
if ((*env)->ExceptionOccurred(env)) {
msg_Warn(p_dec, "Exception occurred in MediaCodec.configure");
(*env)->ExceptionClear(env);
@@ -350,10 +379,12 @@ static int OpenDecoder(vlc_object_t *p_this)
p_sys->started = 1;
p_sys->input_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_input_buffers);
- p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
- p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
p_sys->input_buffers = (*env)->NewGlobalRef(env, p_sys->input_buffers);
- p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
+ if (!p_sys->direct_rendering) {
+ p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec, p_sys->get_output_buffers);
+ p_sys->output_buffers = (*env)->NewGlobalRef(env, p_sys->output_buffers);
+ }
+ p_sys->buffer_info = (*env)->NewObject(env, p_sys->buffer_info_class, p_sys->buffer_info_ctor);
p_sys->buffer_info = (*env)->NewGlobalRef(env, p_sys->buffer_info);
(*env)->DeleteLocalRef(env, format);
@@ -375,6 +406,7 @@ static void CloseDecoder(vlc_object_t *p_this)
if (!p_sys)
return;
+ p_sys->closed = 1;
(*myVm)->AttachCurrentThread(myVm, &env, NULL);
if (p_sys->input_buffers)
(*env)->DeleteGlobalRef(env, p_sys->input_buffers);
@@ -395,6 +427,34 @@ static void CloseDecoder(vlc_object_t *p_this)
free(p_sys);
}
+static void DisplayBuffer(picture_sys_t* p_picsys)
+{
+ decoder_t *p_dec = p_picsys->p_dec;
+ decoder_sys_t *p_sys = p_dec->p_sys;
+ if (p_sys->closed)
+ return;
+
+ int b_displayed = p_picsys->b_displayed;
+ if (b_displayed)
+ return;
+ uint32_t i_index = p_picsys->i_index;
+
+ JNIEnv *env = NULL;
+ (*myVm)->AttachCurrentThread(myVm, &env, NULL);
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, i_index, 1);
+ jthrowable exception = (*env)->ExceptionOccurred(env);
+ if(exception != NULL) {
+ jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
+ if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
+ msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
+ (*env)->ExceptionClear(env);
+ (*env)->DeleteLocalRef(env, illegalStateException);
+ }
+ }
+ (*myVm)->DetachCurrentThread(myVm);
+ p_picsys->b_displayed = 1;
+}
+
static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic)
{
decoder_sys_t *p_sys = p_dec->p_sys;
@@ -407,39 +467,60 @@ static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic)
(*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
continue;
}
- jobject buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
- jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
- uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
- if (!*pp_pic)
+
+ if (!*pp_pic) {
*pp_pic = decoder_NewPicture(p_dec);
+ } else if (p_sys->direct_rendering) {
+ picture_t *p_pic = *pp_pic;
+ picture_sys_t *p_picsys = p_pic->p_sys;
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, p_picsys->i_index, false);
+ }
if (*pp_pic) {
+
picture_t *p_pic = *pp_pic;
- int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
- int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
- ptr += offset; // Check the size parameter as well
// TODO: Use crop_top/crop_left as well? Or is that already taken into account?
// On OMX_TI_COLOR_FormatYUV420PackedSemiPlanar the offset already incldues
// the cropping, so the top/left cropping params should just be ignored.
- unsigned int chroma_div;
p_pic->date = (*env)->GetLongField(env, p_sys->buffer_info, p_sys->pts_field);
- GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
- p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
- CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
- ptr, chroma_div, &p_sys->architecture_specific_data);
- }
- (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
- jthrowable exception = (*env)->ExceptionOccurred(env);
- if(exception != NULL) {
- jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
- if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
- msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
- (*env)->ExceptionClear(env);
- (*env)->DeleteLocalRef(env, illegalStateException);
+ if (p_sys->direct_rendering) {
+ picture_sys_t *p_picsys = p_pic->p_sys;
+ p_picsys->pf_callback = DisplayBuffer;
+ p_picsys->p_dec = p_dec;
+ p_picsys->i_index = index;
+ p_picsys->b_displayed = 0;
+ } else {
+ jobject buf = (*env)->GetObjectArrayElement(env, p_sys->output_buffers, index);
+ jsize buf_size = (*env)->GetDirectBufferCapacity(env, buf);
+ uint8_t *ptr = (*env)->GetDirectBufferAddress(env, buf);
+
+ int size = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->size_field);
+ int offset = (*env)->GetIntField(env, p_sys->buffer_info, p_sys->offset_field);
+ ptr += offset; // Check the size parameter as well
+
+ unsigned int chroma_div;
+ GetVlcChromaSizes(p_dec->fmt_out.i_codec, p_dec->fmt_out.video.i_width,
+ p_dec->fmt_out.video.i_height, NULL, NULL, &chroma_div);
+ CopyOmxPicture(p_sys->pixel_format, p_pic, p_sys->slice_height, p_sys->stride,
+ ptr, chroma_div, &p_sys->architecture_specific_data);
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
+
+ jthrowable exception = (*env)->ExceptionOccurred(env);
+ if(exception != NULL) {
+ jclass illegalStateException = (*env)->FindClass(env, "java/lang/IllegalStateException");
+ if((*env)->IsInstanceOf(env, exception, illegalStateException)) {
+ msg_Err(p_dec, "Codec error (IllegalStateException) in MediaCodec.releaseOutputBuffer");
+ (*env)->ExceptionClear(env);
+ (*env)->DeleteLocalRef(env, illegalStateException);
+ }
+ }
+ (*env)->DeleteLocalRef(env, buf);
}
+ } else {
+ (*env)->CallVoidMethod(env, p_sys->codec, p_sys->release_output_buffer, index, false);
}
- (*env)->DeleteLocalRef(env, buf);
+
return;
- } else if (index == INFO_OUTPUT_BUFFERS_CHANGED) {
+ } else if (index == INFO_OUTPUT_BUFFERS_CHANGED && !p_sys->direct_rendering) {
msg_Dbg(p_dec, "output buffers changed");
(*env)->DeleteGlobalRef(env, p_sys->output_buffers);
p_sys->output_buffers = (*env)->CallObjectMethod(env, p_sys->codec,
@@ -468,7 +549,13 @@ static void GetOutput(decoder_t *p_dec, JNIEnv *env, picture_t **pp_pic)
int crop_bottom = GET_INTEGER(format, "crop-bottom");
const char *name = "unknown";
- GetVlcChromaFormat(p_sys->pixel_format, &p_dec->fmt_out.i_codec, &name);
+ if (p_sys->direct_rendering)
+ {
+ jni_SetAndroidSurfaceSizeEnv(env, width, height, width, height, 1, 1);
+ p_dec->fmt_out.i_codec = VLC_CODEC_OPAQUE_VBUF;
+ }
+ else
+ GetVlcChromaFormat(p_sys->pixel_format, &p_dec->fmt_out.i_codec, &name);
msg_Dbg(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
p_sys->pixel_format, name, width, height, p_sys->stride, p_sys->slice_height,
p_sys->crop_left, p_sys->crop_top, crop_right, crop_bottom);
diff --git a/modules/codec/omxil/android_mediacodec.h b/modules/codec/omxil/android_mediacodec.h
new file mode 100644
index 0000000..54aad92
--- /dev/null
+++ b/modules/codec/omxil/android_mediacodec.h
@@ -0,0 +1,30 @@
+/*****************************************************************************
+ * android_mediacodec.h: shared structures between MediaCodec decoder
+ * and MediaCodec video output
+ *****************************************************************************
+ * Copyright (C) 2013 Felix Abecassis
+ *
+ * Authors: Felix Abecassis <felix.abecassis at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+struct picture_sys_t
+{
+ void (*pf_callback)(picture_sys_t*);
+ decoder_t *p_dec;
+ uint32_t i_index;
+ int b_displayed;
+};
diff --git a/modules/video_output/Modules.am b/modules/video_output/Modules.am
index 041dc29..e29c4ec 100644
--- a/modules/video_output/Modules.am
+++ b/modules/video_output/Modules.am
@@ -13,6 +13,7 @@ SOURCES_vout_macosx = macosx.m opengl.h opengl.c
SOURCES_vout_coregraphicslayer = coregraphicslayer.m
SOURCES_vout_ios2 = ios2.m opengl.h opengl.c
SOURCES_android_surface = androidsurface.c
+SOURCES_android_surface_opaque = androidsurface_opaque.c
if HAVE_DECKLINK
libdecklinkoutput_plugin_la_SOURCES = decklink.cpp
diff --git a/modules/video_output/androidsurface_opaque.c b/modules/video_output/androidsurface_opaque.c
new file mode 100644
index 0000000..a8639a4
--- /dev/null
+++ b/modules/video_output/androidsurface_opaque.c
@@ -0,0 +1,203 @@
+/*****************************************************************************
+ * androidsurface_opaque.c: Video output module using direct rendering with
+ * opaque buffers
+ *****************************************************************************
+ * Copyright (C) 2013 Felix Abecassis
+ *
+ * Authors: Felix Abecassis <felix.abecassis at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_vout_display.h>
+#include <vlc_picture_pool.h>
+#include "../../codec/omxil/android_mediacodec.h"
+
+static int Open (vlc_object_t *);
+static void Close(vlc_object_t *);
+
+vlc_module_begin()
+ set_category(CAT_VIDEO)
+ set_subcategory(SUBCAT_VIDEO_VOUT)
+ set_shortname("vout_mediacodec")
+ set_description(N_("Android MediaCodec direct rendering video output"))
+ set_capability("vout display", 200)
+ set_callbacks(Open, Close)
+vlc_module_end()
+
+/*****************************************************************************
+ * Local prototypes
+ *****************************************************************************/
+
+static picture_pool_t *Pool (vout_display_t *, unsigned);
+static void Display(vout_display_t *, picture_t *, subpicture_t *);
+static int Control(vout_display_t *, int, va_list);
+
+struct vout_display_sys_t
+{
+ picture_pool_t *pool;
+ video_format_t fmt;
+};
+
+static int LockSurface(picture_t *);
+static void UnlockSurface(picture_t *);
+
+#define POOL_SIZE 31
+
+static int Open(vlc_object_t *p_this)
+{
+ vout_display_t *vd = (vout_display_t*)p_this;
+
+ video_format_t fmt = vd->fmt;
+
+ if (fmt.i_chroma != VLC_CODEC_OPAQUE_VBUF)
+ return VLC_EGENERIC;
+
+ /* Allocate structure */
+ vout_display_sys_t *sys = (struct vout_display_sys_t*)calloc(1, sizeof(*sys));
+ if (!sys)
+ return VLC_ENOMEM;
+
+ sys->fmt = fmt;
+
+ /* We need to allocate a picture pool of more than 30 buffers in
+ * order to be connected directly to the decoder without any intermediate buffer pool. */
+ int i_pictures = POOL_SIZE;
+ picture_t** pictures = calloc(sizeof(*pictures), i_pictures);
+ if (!pictures)
+ goto error;
+ for (int i = 0; i < i_pictures; i++)
+ {
+ picture_sys_t *p_picsys = calloc(1, sizeof(*p_picsys));
+ if (unlikely(p_picsys == NULL))
+ goto error;
+
+ picture_resource_t resource = { .p_sys = p_picsys };
+ picture_t *picture = picture_NewFromResource(&fmt, &resource);
+ if (!picture)
+ {
+ free(p_picsys);
+ goto error;
+ }
+ pictures[i] = picture;
+ }
+
+ /* Wrap it into a picture pool */
+ picture_pool_configuration_t pool_cfg;
+ memset(&pool_cfg, 0, sizeof(pool_cfg));
+ pool_cfg.picture_count = i_pictures;
+ pool_cfg.picture = pictures;
+ pool_cfg.lock = LockSurface;
+ pool_cfg.unlock = UnlockSurface;
+
+ sys->pool = picture_pool_NewExtended(&pool_cfg);
+ if (!sys->pool)
+ {
+ for (int i = 0; i < i_pictures; i++)
+ picture_Release(pictures[i]);
+ goto error;
+ }
+
+ /* Setup vout_display */
+ vd->sys = sys;
+ vd->fmt = fmt;
+ vd->pool = Pool;
+ vd->display = Display;
+ vd->control = Control;
+ vd->prepare = NULL;
+ vd->manage = NULL;
+
+ /* Fix initial state */
+ vout_display_SendEventFullscreen(vd, false);
+
+ return VLC_SUCCESS;
+
+error:
+ free(pictures);
+ Close(p_this);
+ return VLC_ENOMEM;
+}
+
+static void Close(vlc_object_t *p_this)
+{
+ vout_display_t *vd = (vout_display_t *)p_this;
+ vout_display_sys_t *sys = vd->sys;
+
+ picture_pool_Delete(sys->pool);
+ free(sys);
+}
+
+static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
+{
+ VLC_UNUSED(count);
+
+ return vd->sys->pool;
+}
+
+static int LockSurface(picture_t *picture)
+{
+ VLC_UNUSED(picture);
+
+ return VLC_SUCCESS;
+}
+
+static void UnlockSurface(picture_t *picture)
+{
+ picture_sys_t *p_picsys = picture->p_sys;
+ void (*display_callback)(picture_sys_t*) = p_picsys->pf_callback;
+ display_callback(p_picsys);
+}
+
+static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
+{
+ VLC_UNUSED(vd);
+ VLC_UNUSED(subpicture);
+
+ picture_sys_t *p_picsys = picture->p_sys;
+ void (*display_callback)(picture_sys_t*) = p_picsys->pf_callback;
+ display_callback(p_picsys);
+
+ /* refcount lowers to 0, and pool_cfg.unlock is called */
+ picture_Release(picture);
+}
+
+static int Control(vout_display_t *vd, int query, va_list args)
+{
+ VLC_UNUSED(args);
+
+ switch (query) {
+ case VOUT_DISPLAY_HIDE_MOUSE:
+ return VLC_SUCCESS;
+
+ default:
+ msg_Err(vd, "Unknown request in vout mediacodec display");
+
+ case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
+ case VOUT_DISPLAY_CHANGE_FULLSCREEN:
+ case VOUT_DISPLAY_CHANGE_WINDOW_STATE:
+ case VOUT_DISPLAY_CHANGE_DISPLAY_SIZE:
+ case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
+ case VOUT_DISPLAY_CHANGE_ZOOM:
+ case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
+ case VOUT_DISPLAY_GET_OPENGL:
+ return VLC_EGENERIC;
+ }
+}
diff --git a/src/misc/fourcc.c b/src/misc/fourcc.c
index 78a77e1..d457973 100644
--- a/src/misc/fourcc.c
+++ b/src/misc/fourcc.c
@@ -1993,7 +1993,8 @@ static const struct
{ { VLC_CODEC_VDPAU_VIDEO_420, VLC_CODEC_VDPAU_VIDEO_422,
VLC_CODEC_VDPAU_VIDEO_444,
- VLC_CODEC_VDPAU_OUTPUT, 0 }, FAKE_FMT() },
+ VLC_CODEC_VDPAU_OUTPUT,
+ VLC_CODEC_OPAQUE_VBUF, 0 }, FAKE_FMT() },
{ {0}, { 0, {}, 0, 0 } }
};
--
1.8.3.2
More information about the vlc-devel
mailing list