[vlc-devel] [PATCH 03/14] android: utils: refactor SurfaceTexture handling
Romain Vimont
rom1v at videolabs.io
Mon Jun 15 10:44:45 CEST 2020
On Mon, Jun 15, 2020 at 10:09:37AM +0200, Alexandre Janniaux wrote:
> Hi,
>
> > jsurface is always assigned from a jobject (in patch 7). Shouldn't it be
> > declared as a jobject instead of void*?
>
> That's a good question. I follow what has been done for AWindowHandler
> which always exposes void* to avoid to avoid exposing jni.h outside of
> android/utils, as forward-declaring jobject is tedious.
It makes sense, but android/utils includes <jni.h> anyway (and
also declares AWindowHandler_getSurface() which returns a jobject), so
it will never need a forward-declaration IMO.
>
> Regards,
> --
> Alexandre Janniaux
> Videolabs
>
> On Fri, Jun 12, 2020 at 04:38:21PM +0200, Romain Vimont wrote:
> > On Fri, Jun 12, 2020 at 11:40:46AM +0200, Alexandre Janniaux wrote:
> > > Refactor SurfaceTexture related API into a structure with vtable, so as
> > > to implement both NDK and JNI API as separate vtable.
> > >
> > > It will be exposed to modules and will allow creating SurfaceTexture
> > > directly in the module needing them, instead of relying on an external
> > > SurfaceTexture provided by the binding.
> > >
> > > jsurface and ANativeWindow are also both exposed to the user since we
> > > don't expose the ANativeWindow/JNI functions.
> > >
> > > The object is made so that SurfaceTexture are always created in the
> > > detached state.
> > > ---
> > > modules/video_output/android/utils.c | 241 +++++++++++++++++++--------
> > > modules/video_output/android/utils.h | 30 ++++
> > > 2 files changed, 202 insertions(+), 69 deletions(-)
> > >
> > > diff --git a/modules/video_output/android/utils.c b/modules/video_output/android/utils.c
> > > index 45a30af455e..e4d407f560f 100644
> > > --- a/modules/video_output/android/utils.c
> > > +++ b/modules/video_output/android/utils.c
> > > @@ -20,6 +20,11 @@
> > > * along with this program; if not, write to the Free Software Foundation,
> > > * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
> > > *****************************************************************************/
> > > +
> > > +#ifdef HAVE_CONFIG_H
> > > +# include <config.h>
> > > +#endif
> > > +
> > > #include "utils.h"
> > > #include <dlfcn.h>
> > > #include <jni.h>
> > > @@ -30,20 +35,6 @@ typedef ANativeWindow* (*ptr_ANativeWindow_fromSurface)(JNIEnv*, jobject);
> > > typedef ANativeWindow* (*ptr_ANativeWindow_fromSurfaceTexture)(JNIEnv*, jobject);
> > > typedef void (*ptr_ANativeWindow_release)(ANativeWindow*);
> > >
> > > -typedef int (*ptr_SurfaceTexture_attachToGLContext)
> > > - (AWindowHandler *p_awh, uint32_t tex_name);
> > > -typedef int (*ptr_SurfaceTexture_updateTexImage)(AWindowHandler *p_awh,
> > > - const float **pp_transform_mtx);
> > > -typedef void (*ptr_SurfaceTexture_detachFromGLContext)
> > > - (AWindowHandler *p_awh, JNIEnv *p_env);
> > > -
> > > -struct SurfaceTextureHandler
> > > -{
> > > - ptr_SurfaceTexture_attachToGLContext pf_attachToGL;
> > > - ptr_SurfaceTexture_updateTexImage pf_updateTexImage;
> > > - ptr_SurfaceTexture_detachFromGLContext pf_detachFromGL;
> > > -};
> > > -
> > > typedef void (*ptr_ASurfaceTexture_getTransformMatrix)
> > > (ASurfaceTexture *st, float mtx[16]);
> > > typedef ASurfaceTexture* (*ptr_ASurfaceTexture_fromSurfaceTexture)
> > > @@ -58,6 +49,20 @@ typedef int (*ptr_ASurfaceTexture_updateTexImage)(ASurfaceTexture* st);
> > > typedef int (*ptr_ASurfaceTexture_detachFromGLContext)(ASurfaceTexture *st);
> > > typedef void (*ptr_ASurfaceTexture_release)(ASurfaceTexture *st);
> > >
> > > +/*
> > > + * Android ASurfaceTexture Android NDK
> > > + */
> > > +struct SurfaceTextureHandle {
> > > + struct vlc_asurfacetexture surface;
> > > +
> > > + /* Underlying SurfaceTexture objects (JNI and NDK)*/
> > > + jobject jtexture;
> > > + ASurfaceTexture *texture;
> > > +
> > > + /* Android API are loaded into an AWindowHandler instance. */
> > > + struct AWindowHandler *awh;
> > > +};
> > > +
> > > struct ASurfaceTextureAPI
> > > {
> > > float transMat[16];
> > > @@ -91,7 +96,9 @@ struct AWindowHandler
> > > ptr_ANativeWindow_release pf_winRelease;
> > > native_window_api_t anw_api;
> > >
> > > - struct SurfaceTextureHandler st;
> > > + /* Store the surfacetexture that AWindowHandler will use. */
> > > + struct vlc_asurfacetexture *st;
> > > +
> > > struct ASurfaceTextureAPI ndk_ast_api;
> > > bool b_has_ndk_ast_api;
> > >
> > > @@ -384,89 +391,171 @@ LoadNativeSurfaceAPI(AWindowHandler *p_awh)
> > > p_awh->anw_api.setBuffersGeometry = NULL;
> > > }
> > >
> > > -/*
> > > - * Android ASurfaceTexture Android NDK
> > > - */
> > > -
> > > static int
> > > -NDKSurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t texName)
> > > +NDKSurfaceTexture_attachToGLContext(
> > > + struct vlc_asurfacetexture *surface,
> > > + uint32_t texName)
> > > {
> > > - return p_awh->ndk_ast_api.pf_attachToGL(p_awh->ndk_ast_api.p_ast, texName);
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > + return handle->awh->ndk_ast_api.pf_attachToGL(handle->texture, texName);
> > > }
> > >
> > > static void
> > > -NDKSurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> > > +NDKSurfaceTexture_detachFromGLContext(
> > > + struct vlc_asurfacetexture *surface)
> > > {
> > > - (void)p_env;
> > > - p_awh->ndk_ast_api.pf_detachFromGL(p_awh->ndk_ast_api.p_ast);
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > + handle->awh->ndk_ast_api.pf_detachFromGL(handle->texture);
> > > }
> > >
> > > static int
> > > -NDKSurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> > > +NDKSurfaceTexture_updateTexImage(
> > > + struct vlc_asurfacetexture *surface,
> > > + const float **pp_transform_mtx)
> > > {
> > > - if (p_awh->ndk_ast_api.pf_updateTexImage(p_awh->ndk_ast_api.p_ast))
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + /* ASurfaceTexture_updateTexImage can fail, for example if calling it
> > > + * before having produced a new image. */
> > > + if (handle->awh->ndk_ast_api.pf_updateTexImage(handle->texture))
> > > return VLC_EGENERIC;
> > >
> > > - p_awh->ndk_ast_api.pf_getTransMatrix(p_awh->ndk_ast_api.p_ast,
> > > - p_awh->ndk_ast_api.transMat);
> > > - *pp_transform_mtx = p_awh->ndk_ast_api.transMat;
> > > + handle->awh->ndk_ast_api.pf_getTransMatrix(handle->texture,
> > > + handle->awh->ndk_ast_api.transMat);
> > > + *pp_transform_mtx = handle->awh->ndk_ast_api.transMat;
> > > return VLC_SUCCESS;
> > > }
> > >
> > > +static void NDKSurfaceTexture_destroy(
> > > + struct vlc_asurfacetexture *surface)
> > > +{
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > > + if (!p_env)
> > > + return;
> > > +
> > > + handle->awh->pf_winRelease(handle->surface.window);
> > > + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> > > +
> > > + handle->awh->ndk_ast_api.pf_releaseAst(handle->texture);
> > > + (*p_env)->DeleteGlobalRef(p_env, handle->jtexture);
> > > +
> > > + free(handle);
> > > +}
> > > +
> > > +static const struct vlc_asurfacetexture_operations NDKSurfaceAPI =
> > > +{
> > > + .attach_to_gl_context = NDKSurfaceTexture_attachToGLContext,
> > > + .update_tex_image = NDKSurfaceTexture_updateTexImage,
> > > + .detach_from_gl_context = NDKSurfaceTexture_detachFromGLContext,
> > > + .destroy = NDKSurfaceTexture_destroy,
> > > +};
> > > +
> > > static int
> > > -JNISurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> > > +JNISurfaceTexture_attachToGLContext(
> > > + struct vlc_asurfacetexture *surface,
> > > + uint32_t tex_name)
> > > {
> > > - JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > > if (!p_env)
> > > return VLC_EGENERIC;
> > >
> > > + AWindowHandler *p_awh = handle->awh;
> > > +
> > > return JNI_STEXCALL(CallBooleanMethod, attachToGLContext, tex_name) ?
> > > VLC_SUCCESS : VLC_EGENERIC;
> > > }
> > >
> > > static void
> > > -JNISurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> > > +JNISurfaceTexture_detachFromGLContext(
> > > + struct vlc_asurfacetexture *surface)
> > > {
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + AWindowHandler *p_awh = handle->awh;
> > > + JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > > + if (!p_env)
> > > + return;
> > > +
> > > JNI_STEXCALL(CallVoidMethod, detachFromGLContext);
> > >
> > > - if (p_awh->stex.jtransform_mtx != NULL)
> > > + if (handle->awh->stex.jtransform_mtx != NULL)
> > > {
> > > - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> > > - p_awh->stex.jtransform_mtx,
> > > + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> > > + handle->awh->stex.jtransform_mtx,
> > > JNI_ABORT);
> > > - p_awh->stex.jtransform_mtx = NULL;
> > > + handle->awh->stex.jtransform_mtx = NULL;
> > > }
> > > }
> > >
> > > static int
> > > -JNISurfaceTexture_waitAndUpdateTexImage(AWindowHandler *p_awh,
> > > - const float **pp_transform_mtx)
> > > +JNISurfaceTexture_waitAndUpdateTexImage(
> > > + struct vlc_asurfacetexture *surface,
> > > + const float **pp_transform_mtx)
> > > {
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + AWindowHandler *p_awh = handle->awh;
> > > JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> > > if (!p_env)
> > > return VLC_EGENERIC;
> > >
> > > - if (p_awh->stex.jtransform_mtx != NULL)
> > > - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> > > - p_awh->stex.jtransform_mtx,
> > > +
> > > + if (handle->awh->stex.jtransform_mtx != NULL)
> > > + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> > > + handle->awh->stex.jtransform_mtx,
> > > JNI_ABORT);
> > >
> > > bool ret = JNI_STEXCALL(CallBooleanMethod, waitAndUpdateTexImage,
> > > - p_awh->stex.jtransform_mtx_array);
> > > + handle->awh->stex.jtransform_mtx_array);
> > > if (ret)
> > > {
> > > - p_awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> > > - p_awh->stex.jtransform_mtx_array, NULL);
> > > - *pp_transform_mtx = p_awh->stex.jtransform_mtx;
> > > + handle->awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> > > + handle->awh->stex.jtransform_mtx_array, NULL);
> > > + *pp_transform_mtx = handle->awh->stex.jtransform_mtx;
> > > return VLC_SUCCESS;
> > > }
> > > else
> > > {
> > > - p_awh->stex.jtransform_mtx = NULL;
> > > + handle->awh->stex.jtransform_mtx = NULL;
> > > return VLC_EGENERIC;
> > > }
> > > }
> > >
> > > +static void JNISurfaceTexture_destroy(
> > > + struct vlc_asurfacetexture *surface)
> > > +{
> > > + struct SurfaceTextureHandle *handle =
> > > + container_of(surface, struct SurfaceTextureHandle, surface);
> > > +
> > > + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> > > + if (!p_env)
> > > + return;
> > > +
> > > + handle->awh->pf_winRelease(handle->surface.window);
> > > + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> > > +
> > > + free(handle);
> > > +}
> > > +
> > > +static const struct vlc_asurfacetexture_operations JNISurfaceAPI =
> > > +{
> > > + .attach_to_gl_context = JNISurfaceTexture_attachToGLContext,
> > > + .update_tex_image = JNISurfaceTexture_waitAndUpdateTexImage,
> > > + .detach_from_gl_context = JNISurfaceTexture_detachFromGLContext,
> > > + .destroy = JNISurfaceTexture_destroy,
> > > +};
> > > +
> > >
> > > static int
> > > LoadNDKSurfaceTextureAPI(AWindowHandler *p_awh, void *p_library, JNIEnv *p_env)
> > > @@ -550,19 +639,6 @@ LoadNativeWindowAPI(AWindowHandler *p_awh, JNIEnv *p_env)
> > > && p_awh->anw_api.setBuffersGeometry)
> > > {
> > > p_awh->b_has_ndk_ast_api = !LoadNDKSurfaceTextureAPI(p_awh, p_library, p_env);
> > > - if (p_awh->b_has_ndk_ast_api)
> > > - {
> > > - p_awh->st.pf_attachToGL = NDKSurfaceTexture_attachToGLContext;
> > > - p_awh->st.pf_updateTexImage = NDKSurfaceTexture_updateTexImage;
> > > - p_awh->st.pf_detachFromGL = NDKSurfaceTexture_detachFromGLContext;
> > > - }
> > > - else
> > > - {
> > > - p_awh->st.pf_attachToGL = JNISurfaceTexture_attachToGLContext;
> > > - p_awh->st.pf_updateTexImage = JNISurfaceTexture_waitAndUpdateTexImage;
> > > - p_awh->st.pf_detachFromGL = JNISurfaceTexture_detachFromGLContext;
> > > - }
> > > -
> > > p_awh->p_anw_dl = p_library;
> > > }
> > > else
> > > @@ -774,12 +850,11 @@ AWindowHandler_destroy(AWindowHandler *p_awh)
> > >
> > > if (p_env)
> > > {
> > > - if (p_awh->b_has_ndk_ast_api)
> > > - {
> > > - p_awh->ndk_ast_api.pf_releaseAst(p_awh->ndk_ast_api.p_ast);
> > > - (*p_env)->DeleteGlobalRef(p_env, p_awh->ndk_ast_api.surfacetexture);
> > > + if (p_awh->st)
> > > + p_awh->st->ops->destroy(p_awh->st);
> > > +
> > > + if (jfields.SurfaceTexture.clazz)
> > > (*p_env)->DeleteGlobalRef(p_env, jfields.SurfaceTexture.clazz);
> > > - }
> > >
> > > JNI_ANWCALL(CallVoidMethod, unregisterNative);
> > > AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_Video);
> > > @@ -857,11 +932,40 @@ WindowHandler_NewSurfaceEnv(AWindowHandler *p_awh, JNIEnv *p_env,
> > > break;
> > > case AWindow_SurfaceTexture:
> > > {
> > > + struct SurfaceTextureHandle *surfacetexture =
> > > + malloc(sizeof *surfacetexture);
> > > + if (surfacetexture == NULL)
> > > + return VLC_EGENERIC;
> > > +
> > > if (p_awh->b_has_ndk_ast_api)
> > > + {
> > > jsurface = InitNDKSurfaceTexture(p_awh, p_env, id);
> > > + surfacetexture->surface.ops = &NDKSurfaceAPI;
> > > + surfacetexture->surface.window = p_awh->views[id].p_anw;
> > > + }
> > > else
> > > + {
> > > jsurface = JNI_STEXCALL(CallObjectMethod, getSurface);
> > > - break;
> > > + surfacetexture->surface.ops = &JNISurfaceAPI;
> > > + surfacetexture->surface.window
> > > + = p_awh->views[id].p_anw
> > > + = p_awh->pf_winFromSurface(p_env, jsurface);
> > > + }
> > > +
> > > + surfacetexture->awh = p_awh;
> > > + surfacetexture->jtexture = p_awh->ndk_ast_api.surfacetexture;
> > > + surfacetexture->texture = p_awh->ndk_ast_api.p_ast;
> > > + surfacetexture->surface.jsurface
> > > + = p_awh->views[id].jsurface
> > > + = (*p_env)->NewGlobalRef(p_env, jsurface);
> > > + (*p_env)->DeleteLocalRef(p_env, jsurface);
> > > +
> > > + assert(surfacetexture->surface.window);
> > > + assert(surfacetexture->surface.jsurface);
> > > +
> > > + /* Store the vlc_asurfacetexture pointer for current AWH wrapper */
> > > + p_awh->st = &surfacetexture->surface;
> > > + return VLC_SUCCESS;
> > > }
> > > default:
> > > vlc_assert_unreachable();
> > > @@ -974,7 +1078,7 @@ AWindowHandler_setVideoLayout(AWindowHandler *p_awh,
> > > int
> > > SurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> > > {
> > > - return p_awh->st.pf_attachToGL(p_awh, tex_name);
> > > + return p_awh->st->ops->attach_to_gl_context(p_awh->st, tex_name);
> > > }
> > >
> > > void
> > > @@ -985,12 +1089,11 @@ SurfaceTexture_detachFromGLContext(AWindowHandler *p_awh)
> > > if (!p_env)
> > > return;
> > >
> > > - p_awh->st.pf_detachFromGL(p_awh, p_env);
> > > - AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_SurfaceTexture);
> > > + p_awh->st->ops->detach_from_gl_context(p_awh->st);
> > > }
> > >
> > > int
> > > SurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> > > {
> > > - return p_awh->st.pf_updateTexImage(p_awh, pp_transform_mtx);
> > > + return p_awh->st->ops->update_tex_image(p_awh->st, pp_transform_mtx);
> > > }
> > > diff --git a/modules/video_output/android/utils.h b/modules/video_output/android/utils.h
> > > index 086819dc1ae..e8bec8b197c 100644
> > > --- a/modules/video_output/android/utils.h
> > > +++ b/modules/video_output/android/utils.h
> > > @@ -79,6 +79,36 @@ struct android_video_context_t
> > > bool (*render_ts)(struct picture_context_t *ctx, vlc_tick_t ts);
> > > };
> > >
> > > +struct vlc_asurfacetexture
> > > +{
> > > + struct ANativeWindow *window;
> > > + void *jsurface;
> >
> > jsurface is always assigned from a jobject (in patch 7). Shouldn't it be
> > declared as a jobject instead of void*?
> >
> > > +
> > > + const struct vlc_asurfacetexture_operations *ops;
> > > +};
> > > +
> > > +/**
> > > + * Wrapper structure for Android SurfaceTexture object.
> > > + *
> > > + * It can use either the NDK API or JNI API.
> > > + */
> > > +struct vlc_asurfacetexture_operations
> > > +{
> > > + int (*attach_to_gl_context)(
> > > + struct vlc_asurfacetexture *surface,
> > > + uint32_t tex_name);
> > > +
> > > + void (*detach_from_gl_context)(
> > > + struct vlc_asurfacetexture *surface);
> > > +
> > > + int (*update_tex_image)(
> > > + struct vlc_asurfacetexture *surface,
> > > + const float **pp_transform_mtx);
> > > +
> > > + void (*destroy)(
> > > + struct vlc_asurfacetexture *surface);
> > > +};
> > > +
> > > /**
> > > * Attach or get a JNIEnv*
> > > *
> > > --
> > > 2.27.0
> > > _______________________________________________
> > > vlc-devel mailing list
> > > To unsubscribe or modify your subscription options:
> > > https://mailman.videolan.org/listinfo/vlc-devel
> > _______________________________________________
> > vlc-devel mailing list
> > To unsubscribe or modify your subscription options:
> > https://mailman.videolan.org/listinfo/vlc-devel
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel
More information about the vlc-devel
mailing list