[vlc-devel] [PATCH 03/14] android: utils: refactor SurfaceTexture handling
Alexandre Janniaux
ajanni at videolabs.io
Mon Jun 15 10:09:37 CEST 2020
Hi,
> jsurface is always assigned from a jobject (in patch 7). Shouldn't it be
> declared as a jobject instead of void*?
That's a good question. I follow what has been done for AWindowHandler
which always exposes void* to avoid to avoid exposing jni.h outside of
android/utils, as forward-declaring jobject is tedious.
Regards,
--
Alexandre Janniaux
Videolabs
On Fri, Jun 12, 2020 at 04:38:21PM +0200, Romain Vimont wrote:
> On Fri, Jun 12, 2020 at 11:40:46AM +0200, Alexandre Janniaux wrote:
> > Refactor SurfaceTexture related API into a structure with vtable, so as
> > to implement both NDK and JNI API as separate vtable.
> >
> > It will be exposed to modules and will allow creating SurfaceTexture
> > directly in the module needing them, instead of relying on an external
> > SurfaceTexture provided by the binding.
> >
> > jsurface and ANativeWindow are also both exposed to the user since we
> > don't expose the ANativeWindow/JNI functions.
> >
> > The object is made so that SurfaceTexture are always created in the
> > detached state.
> > ---
> > modules/video_output/android/utils.c | 241 +++++++++++++++++++--------
> > modules/video_output/android/utils.h | 30 ++++
> > 2 files changed, 202 insertions(+), 69 deletions(-)
> >
> > diff --git a/modules/video_output/android/utils.c b/modules/video_output/android/utils.c
> > index 45a30af455e..e4d407f560f 100644
> > --- a/modules/video_output/android/utils.c
> > +++ b/modules/video_output/android/utils.c
> > @@ -20,6 +20,11 @@
> > * along with this program; if not, write to the Free Software Foundation,
> > * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
> > *****************************************************************************/
> > +
> > +#ifdef HAVE_CONFIG_H
> > +# include <config.h>
> > +#endif
> > +
> > #include "utils.h"
> > #include <dlfcn.h>
> > #include <jni.h>
> > @@ -30,20 +35,6 @@ typedef ANativeWindow* (*ptr_ANativeWindow_fromSurface)(JNIEnv*, jobject);
> > typedef ANativeWindow* (*ptr_ANativeWindow_fromSurfaceTexture)(JNIEnv*, jobject);
> > typedef void (*ptr_ANativeWindow_release)(ANativeWindow*);
> >
> > -typedef int (*ptr_SurfaceTexture_attachToGLContext)
> > - (AWindowHandler *p_awh, uint32_t tex_name);
> > -typedef int (*ptr_SurfaceTexture_updateTexImage)(AWindowHandler *p_awh,
> > - const float **pp_transform_mtx);
> > -typedef void (*ptr_SurfaceTexture_detachFromGLContext)
> > - (AWindowHandler *p_awh, JNIEnv *p_env);
> > -
> > -struct SurfaceTextureHandler
> > -{
> > - ptr_SurfaceTexture_attachToGLContext pf_attachToGL;
> > - ptr_SurfaceTexture_updateTexImage pf_updateTexImage;
> > - ptr_SurfaceTexture_detachFromGLContext pf_detachFromGL;
> > -};
> > -
> > typedef void (*ptr_ASurfaceTexture_getTransformMatrix)
> > (ASurfaceTexture *st, float mtx[16]);
> > typedef ASurfaceTexture* (*ptr_ASurfaceTexture_fromSurfaceTexture)
> > @@ -58,6 +49,20 @@ typedef int (*ptr_ASurfaceTexture_updateTexImage)(ASurfaceTexture* st);
> > typedef int (*ptr_ASurfaceTexture_detachFromGLContext)(ASurfaceTexture *st);
> > typedef void (*ptr_ASurfaceTexture_release)(ASurfaceTexture *st);
> >
> > +/*
> > + * Android ASurfaceTexture Android NDK
> > + */
> > +struct SurfaceTextureHandle {
> > + struct vlc_asurfacetexture surface;
> > +
> > + /* Underlying SurfaceTexture objects (JNI and NDK)*/
> > + jobject jtexture;
> > + ASurfaceTexture *texture;
> > +
> > + /* Android API are loaded into an AWindowHandler instance. */
> > + struct AWindowHandler *awh;
> > +};
> > +
> > struct ASurfaceTextureAPI
> > {
> > float transMat[16];
> > @@ -91,7 +96,9 @@ struct AWindowHandler
> > ptr_ANativeWindow_release pf_winRelease;
> > native_window_api_t anw_api;
> >
> > - struct SurfaceTextureHandler st;
> > + /* Store the surfacetexture that AWindowHandler will use. */
> > + struct vlc_asurfacetexture *st;
> > +
> > struct ASurfaceTextureAPI ndk_ast_api;
> > bool b_has_ndk_ast_api;
> >
> > @@ -384,89 +391,171 @@ LoadNativeSurfaceAPI(AWindowHandler *p_awh)
> > p_awh->anw_api.setBuffersGeometry = NULL;
> > }
> >
> > -/*
> > - * Android ASurfaceTexture Android NDK
> > - */
> > -
> > static int
> > -NDKSurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t texName)
> > +NDKSurfaceTexture_attachToGLContext(
> > + struct vlc_asurfacetexture *surface,
> > + uint32_t texName)
> > {
> > - return p_awh->ndk_ast_api.pf_attachToGL(p_awh->ndk_ast_api.p_ast, texName);
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > + return handle->awh->ndk_ast_api.pf_attachToGL(handle->texture, texName);
> > }
> >
> > static void
> > -NDKSurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> > +NDKSurfaceTexture_detachFromGLContext(
> > + struct vlc_asurfacetexture *surface)
> > {
> > - (void)p_env;
> > - p_awh->ndk_ast_api.pf_detachFromGL(p_awh->ndk_ast_api.p_ast);
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > + handle->awh->ndk_ast_api.pf_detachFromGL(handle->texture);
> > }
> >
> > static int
> > -NDKSurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> > +NDKSurfaceTexture_updateTexImage(
> > + struct vlc_asurfacetexture *surface,
> > + const float **pp_transform_mtx)
> > {
> > - if (p_awh->ndk_ast_api.pf_updateTexImage(p_awh->ndk_ast_api.p_ast))
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + /* ASurfaceTexture_updateTexImage can fail, for example if calling it
> > + * before having produced a new image. */
> > + if (handle->awh->ndk_ast_api.pf_updateTexImage(handle->texture))
> > return VLC_EGENERIC;
> >
> > - p_awh->ndk_ast_api.pf_getTransMatrix(p_awh->ndk_ast_api.p_ast,
> > - p_awh->ndk_ast_api.transMat);
> > - *pp_transform_mtx = p_awh->ndk_ast_api.transMat;
> > + handle->awh->ndk_ast_api.pf_getTransMatrix(handle->texture,
> > + handle->awh->ndk_ast_api.transMat);
> > + *pp_transform_mtx = handle->awh->ndk_ast_api.transMat;
> > return VLC_SUCCESS;
> > }
> >
> > +static void NDKSurfaceTexture_destroy(
> > + struct vlc_asurfacetexture *surface)
> > +{
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > + if (!p_env)
> > + return;
> > +
> > + handle->awh->pf_winRelease(handle->surface.window);
> > + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> > +
> > + handle->awh->ndk_ast_api.pf_releaseAst(handle->texture);
> > + (*p_env)->DeleteGlobalRef(p_env, handle->jtexture);
> > +
> > + free(handle);
> > +}
> > +
> > +static const struct vlc_asurfacetexture_operations NDKSurfaceAPI =
> > +{
> > + .attach_to_gl_context = NDKSurfaceTexture_attachToGLContext,
> > + .update_tex_image = NDKSurfaceTexture_updateTexImage,
> > + .detach_from_gl_context = NDKSurfaceTexture_detachFromGLContext,
> > + .destroy = NDKSurfaceTexture_destroy,
> > +};
> > +
> > static int
> > -JNISurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> > +JNISurfaceTexture_attachToGLContext(
> > + struct vlc_asurfacetexture *surface,
> > + uint32_t tex_name)
> > {
> > - JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > if (!p_env)
> > return VLC_EGENERIC;
> >
> > + AWindowHandler *p_awh = handle->awh;
> > +
> > return JNI_STEXCALL(CallBooleanMethod, attachToGLContext, tex_name) ?
> > VLC_SUCCESS : VLC_EGENERIC;
> > }
> >
> > static void
> > -JNISurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> > +JNISurfaceTexture_detachFromGLContext(
> > + struct vlc_asurfacetexture *surface)
> > {
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + AWindowHandler *p_awh = handle->awh;
> > + JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > + if (!p_env)
> > + return;
> > +
> > JNI_STEXCALL(CallVoidMethod, detachFromGLContext);
> >
> > - if (p_awh->stex.jtransform_mtx != NULL)
> > + if (handle->awh->stex.jtransform_mtx != NULL)
> > {
> > - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> > - p_awh->stex.jtransform_mtx,
> > + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> > + handle->awh->stex.jtransform_mtx,
> > JNI_ABORT);
> > - p_awh->stex.jtransform_mtx = NULL;
> > + handle->awh->stex.jtransform_mtx = NULL;
> > }
> > }
> >
> > static int
> > -JNISurfaceTexture_waitAndUpdateTexImage(AWindowHandler *p_awh,
> > - const float **pp_transform_mtx)
> > +JNISurfaceTexture_waitAndUpdateTexImage(
> > + struct vlc_asurfacetexture *surface,
> > + const float **pp_transform_mtx)
> > {
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + AWindowHandler *p_awh = handle->awh;
> > JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > if (!p_env)
> > return VLC_EGENERIC;
> >
> > - if (p_awh->stex.jtransform_mtx != NULL)
> > - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> > - p_awh->stex.jtransform_mtx,
> > +
> > + if (handle->awh->stex.jtransform_mtx != NULL)
> > + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> > + handle->awh->stex.jtransform_mtx,
> > JNI_ABORT);
> >
> > bool ret = JNI_STEXCALL(CallBooleanMethod, waitAndUpdateTexImage,
> > - p_awh->stex.jtransform_mtx_array);
> > + handle->awh->stex.jtransform_mtx_array);
> > if (ret)
> > {
> > - p_awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> > - p_awh->stex.jtransform_mtx_array, NULL);
> > - *pp_transform_mtx = p_awh->stex.jtransform_mtx;
> > + handle->awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> > + handle->awh->stex.jtransform_mtx_array, NULL);
> > + *pp_transform_mtx = handle->awh->stex.jtransform_mtx;
> > return VLC_SUCCESS;
> > }
> > else
> > {
> > - p_awh->stex.jtransform_mtx = NULL;
> > + handle->awh->stex.jtransform_mtx = NULL;
> > return VLC_EGENERIC;
> > }
> > }
> >
> > +static void JNISurfaceTexture_destroy(
> > + struct vlc_asurfacetexture *surface)
> > +{
> > + struct SurfaceTextureHandle *handle =
> > + container_of(surface, struct SurfaceTextureHandle, surface);
> > +
> > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > + if (!p_env)
> > + return;
> > +
> > + handle->awh->pf_winRelease(handle->surface.window);
> > + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> > +
> > + free(handle);
> > +}
> > +
> > +static const struct vlc_asurfacetexture_operations JNISurfaceAPI =
> > +{
> > + .attach_to_gl_context = JNISurfaceTexture_attachToGLContext,
> > + .update_tex_image = JNISurfaceTexture_waitAndUpdateTexImage,
> > + .detach_from_gl_context = JNISurfaceTexture_detachFromGLContext,
> > + .destroy = JNISurfaceTexture_destroy,
> > +};
> > +
> >
> > static int
> > LoadNDKSurfaceTextureAPI(AWindowHandler *p_awh, void *p_library, JNIEnv *p_env)
> > @@ -550,19 +639,6 @@ LoadNativeWindowAPI(AWindowHandler *p_awh, JNIEnv *p_env)
> > && p_awh->anw_api.setBuffersGeometry)
> > {
> > p_awh->b_has_ndk_ast_api = !LoadNDKSurfaceTextureAPI(p_awh, p_library, p_env);
> > - if (p_awh->b_has_ndk_ast_api)
> > - {
> > - p_awh->st.pf_attachToGL = NDKSurfaceTexture_attachToGLContext;
> > - p_awh->st.pf_updateTexImage = NDKSurfaceTexture_updateTexImage;
> > - p_awh->st.pf_detachFromGL = NDKSurfaceTexture_detachFromGLContext;
> > - }
> > - else
> > - {
> > - p_awh->st.pf_attachToGL = JNISurfaceTexture_attachToGLContext;
> > - p_awh->st.pf_updateTexImage = JNISurfaceTexture_waitAndUpdateTexImage;
> > - p_awh->st.pf_detachFromGL = JNISurfaceTexture_detachFromGLContext;
> > - }
> > -
> > p_awh->p_anw_dl = p_library;
> > }
> > else
> > @@ -774,12 +850,11 @@ AWindowHandler_destroy(AWindowHandler *p_awh)
> >
> > if (p_env)
> > {
> > - if (p_awh->b_has_ndk_ast_api)
> > - {
> > - p_awh->ndk_ast_api.pf_releaseAst(p_awh->ndk_ast_api.p_ast);
> > - (*p_env)->DeleteGlobalRef(p_env, p_awh->ndk_ast_api.surfacetexture);
> > + if (p_awh->st)
> > + p_awh->st->ops->destroy(p_awh->st);
> > +
> > + if (jfields.SurfaceTexture.clazz)
> > (*p_env)->DeleteGlobalRef(p_env, jfields.SurfaceTexture.clazz);
> > - }
> >
> > JNI_ANWCALL(CallVoidMethod, unregisterNative);
> > AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_Video);
> > @@ -857,11 +932,40 @@ WindowHandler_NewSurfaceEnv(AWindowHandler *p_awh, JNIEnv *p_env,
> > break;
> > case AWindow_SurfaceTexture:
> > {
> > + struct SurfaceTextureHandle *surfacetexture =
> > + malloc(sizeof *surfacetexture);
> > + if (surfacetexture == NULL)
> > + return VLC_EGENERIC;
> > +
> > if (p_awh->b_has_ndk_ast_api)
> > + {
> > jsurface = InitNDKSurfaceTexture(p_awh, p_env, id);
> > + surfacetexture->surface.ops = &NDKSurfaceAPI;
> > + surfacetexture->surface.window = p_awh->views[id].p_anw;
> > + }
> > else
> > + {
> > jsurface = JNI_STEXCALL(CallObjectMethod, getSurface);
> > - break;
> > + surfacetexture->surface.ops = &JNISurfaceAPI;
> > + surfacetexture->surface.window
> > + = p_awh->views[id].p_anw
> > + = p_awh->pf_winFromSurface(p_env, jsurface);
> > + }
> > +
> > + surfacetexture->awh = p_awh;
> > + surfacetexture->jtexture = p_awh->ndk_ast_api.surfacetexture;
> > + surfacetexture->texture = p_awh->ndk_ast_api.p_ast;
> > + surfacetexture->surface.jsurface
> > + = p_awh->views[id].jsurface
> > + = (*p_env)->NewGlobalRef(p_env, jsurface);
> > + (*p_env)->DeleteLocalRef(p_env, jsurface);
> > +
> > + assert(surfacetexture->surface.window);
> > + assert(surfacetexture->surface.jsurface);
> > +
> > + /* Store the vlc_asurfacetexture pointer for current AWH wrapper */
> > + p_awh->st = &surfacetexture->surface;
> > + return VLC_SUCCESS;
> > }
> > default:
> > vlc_assert_unreachable();
> > @@ -974,7 +1078,7 @@ AWindowHandler_setVideoLayout(AWindowHandler *p_awh,
> > int
> > SurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> > {
> > - return p_awh->st.pf_attachToGL(p_awh, tex_name);
> > + return p_awh->st->ops->attach_to_gl_context(p_awh->st, tex_name);
> > }
> >
> > void
> > @@ -985,12 +1089,11 @@ SurfaceTexture_detachFromGLContext(AWindowHandler *p_awh)
> > if (!p_env)
> > return;
> >
> > - p_awh->st.pf_detachFromGL(p_awh, p_env);
> > - AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_SurfaceTexture);
> > + p_awh->st->ops->detach_from_gl_context(p_awh->st);
> > }
> >
> > int
> > SurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> > {
> > - return p_awh->st.pf_updateTexImage(p_awh, pp_transform_mtx);
> > + return p_awh->st->ops->update_tex_image(p_awh->st, pp_transform_mtx);
> > }
> > diff --git a/modules/video_output/android/utils.h b/modules/video_output/android/utils.h
> > index 086819dc1ae..e8bec8b197c 100644
> > --- a/modules/video_output/android/utils.h
> > +++ b/modules/video_output/android/utils.h
> > @@ -79,6 +79,36 @@ struct android_video_context_t
> > bool (*render_ts)(struct picture_context_t *ctx, vlc_tick_t ts);
> > };
> >
> > +struct vlc_asurfacetexture
> > +{
> > + struct ANativeWindow *window;
> > + void *jsurface;
>
> jsurface is always assigned from a jobject (in patch 7). Shouldn't it be
> declared as a jobject instead of void*?
>
> > +
> > + const struct vlc_asurfacetexture_operations *ops;
> > +};
> > +
> > +/**
> > + * Wrapper structure for Android SurfaceTexture object.
> > + *
> > + * It can use either the NDK API or JNI API.
> > + */
> > +struct vlc_asurfacetexture_operations
> > +{
> > + int (*attach_to_gl_context)(
> > + struct vlc_asurfacetexture *surface,
> > + uint32_t tex_name);
> > +
> > + void (*detach_from_gl_context)(
> > + struct vlc_asurfacetexture *surface);
> > +
> > + int (*update_tex_image)(
> > + struct vlc_asurfacetexture *surface,
> > + const float **pp_transform_mtx);
> > +
> > + void (*destroy)(
> > + struct vlc_asurfacetexture *surface);
> > +};
> > +
> > /**
> > * Attach or get a JNIEnv*
> > *
> > --
> > 2.27.0
> > _______________________________________________
> > vlc-devel mailing list
> > To unsubscribe or modify your subscription options:
> > https://mailman.videolan.org/listinfo/vlc-devel
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel
More information about the vlc-devel
mailing list