[vlc-devel] [PATCH 03/14] android: utils: refactor SurfaceTexture handling
Romain Vimont
rom1v at videolabs.io
Fri Jun 12 14:41:26 CEST 2020
On Fri, Jun 12, 2020 at 11:40:46AM +0200, Alexandre Janniaux wrote:
> Refactor SurfaceTexture related API into a structure with vtable, so as
> to implement both NDK and JNI API as separate vtable.
>
> It will be exposed to modules and will allow creating SurfaceTexture
> directly in the module needing them, instead of relying on an external
> SurfaceTexture provided by the binding.
>
> jsurface and ANativeWindow are also both exposed to the user since we
> don't expose the ANativeWindow/JNI functions.
>
> The object is made so that SurfaceTexture are always created in the
> detached state.
> ---
> modules/video_output/android/utils.c | 241 +++++++++++++++++++--------
> modules/video_output/android/utils.h | 30 ++++
> 2 files changed, 202 insertions(+), 69 deletions(-)
>
> diff --git a/modules/video_output/android/utils.c b/modules/video_output/android/utils.c
> index 45a30af455e..e4d407f560f 100644
> --- a/modules/video_output/android/utils.c
> +++ b/modules/video_output/android/utils.c
> @@ -20,6 +20,11 @@
> * along with this program; if not, write to the Free Software Foundation,
> * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
> *****************************************************************************/
> +
> +#ifdef HAVE_CONFIG_H
> +# include <config.h>
> +#endif
> +
> #include "utils.h"
> #include <dlfcn.h>
> #include <jni.h>
> @@ -30,20 +35,6 @@ typedef ANativeWindow* (*ptr_ANativeWindow_fromSurface)(JNIEnv*, jobject);
> typedef ANativeWindow* (*ptr_ANativeWindow_fromSurfaceTexture)(JNIEnv*, jobject);
> typedef void (*ptr_ANativeWindow_release)(ANativeWindow*);
>
> -typedef int (*ptr_SurfaceTexture_attachToGLContext)
> - (AWindowHandler *p_awh, uint32_t tex_name);
> -typedef int (*ptr_SurfaceTexture_updateTexImage)(AWindowHandler *p_awh,
> - const float **pp_transform_mtx);
> -typedef void (*ptr_SurfaceTexture_detachFromGLContext)
> - (AWindowHandler *p_awh, JNIEnv *p_env);
> -
> -struct SurfaceTextureHandler
> -{
> - ptr_SurfaceTexture_attachToGLContext pf_attachToGL;
> - ptr_SurfaceTexture_updateTexImage pf_updateTexImage;
> - ptr_SurfaceTexture_detachFromGLContext pf_detachFromGL;
> -};
> -
> typedef void (*ptr_ASurfaceTexture_getTransformMatrix)
> (ASurfaceTexture *st, float mtx[16]);
> typedef ASurfaceTexture* (*ptr_ASurfaceTexture_fromSurfaceTexture)
> @@ -58,6 +49,20 @@ typedef int (*ptr_ASurfaceTexture_updateTexImage)(ASurfaceTexture* st);
> typedef int (*ptr_ASurfaceTexture_detachFromGLContext)(ASurfaceTexture *st);
> typedef void (*ptr_ASurfaceTexture_release)(ASurfaceTexture *st);
>
> +/*
> + * Android ASurfaceTexture Android NDK
> + */
> +struct SurfaceTextureHandle {
Is it a vlc_asurfacetexture_priv? ;-)
> + struct vlc_asurfacetexture surface;
> +
> + /* Underlying SurfaceTexture objects (JNI and NDK)*/
> + jobject jtexture;
> + ASurfaceTexture *texture;
> +
> + /* Android API are loaded into an AWindowHandler instance. */
> + struct AWindowHandler *awh;
> +};
> +
> struct ASurfaceTextureAPI
> {
> float transMat[16];
> @@ -91,7 +96,9 @@ struct AWindowHandler
> ptr_ANativeWindow_release pf_winRelease;
> native_window_api_t anw_api;
>
> - struct SurfaceTextureHandler st;
> + /* Store the surfacetexture that AWindowHandler will use. */
> + struct vlc_asurfacetexture *st;
> +
> struct ASurfaceTextureAPI ndk_ast_api;
> bool b_has_ndk_ast_api;
>
> @@ -384,89 +391,171 @@ LoadNativeSurfaceAPI(AWindowHandler *p_awh)
> p_awh->anw_api.setBuffersGeometry = NULL;
> }
>
> -/*
> - * Android ASurfaceTexture Android NDK
> - */
> -
> static int
> -NDKSurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t texName)
> +NDKSurfaceTexture_attachToGLContext(
> + struct vlc_asurfacetexture *surface,
> + uint32_t texName)
> {
> - return p_awh->ndk_ast_api.pf_attachToGL(p_awh->ndk_ast_api.p_ast, texName);
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> + return handle->awh->ndk_ast_api.pf_attachToGL(handle->texture, texName);
> }
>
> static void
> -NDKSurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> +NDKSurfaceTexture_detachFromGLContext(
> + struct vlc_asurfacetexture *surface)
> {
> - (void)p_env;
> - p_awh->ndk_ast_api.pf_detachFromGL(p_awh->ndk_ast_api.p_ast);
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> + handle->awh->ndk_ast_api.pf_detachFromGL(handle->texture);
> }
>
> static int
> -NDKSurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> +NDKSurfaceTexture_updateTexImage(
> + struct vlc_asurfacetexture *surface,
> + const float **pp_transform_mtx)
> {
> - if (p_awh->ndk_ast_api.pf_updateTexImage(p_awh->ndk_ast_api.p_ast))
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + /* ASurfaceTexture_updateTexImage can fail, for example if calling it
> + * before having produced a new image. */
> + if (handle->awh->ndk_ast_api.pf_updateTexImage(handle->texture))
> return VLC_EGENERIC;
>
> - p_awh->ndk_ast_api.pf_getTransMatrix(p_awh->ndk_ast_api.p_ast,
> - p_awh->ndk_ast_api.transMat);
> - *pp_transform_mtx = p_awh->ndk_ast_api.transMat;
> + handle->awh->ndk_ast_api.pf_getTransMatrix(handle->texture,
> + handle->awh->ndk_ast_api.transMat);
> + *pp_transform_mtx = handle->awh->ndk_ast_api.transMat;
> return VLC_SUCCESS;
> }
>
> +static void NDKSurfaceTexture_destroy(
> + struct vlc_asurfacetexture *surface)
> +{
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> + if (!p_env)
> + return;
> +
> + handle->awh->pf_winRelease(handle->surface.window);
> + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> +
> + handle->awh->ndk_ast_api.pf_releaseAst(handle->texture);
> + (*p_env)->DeleteGlobalRef(p_env, handle->jtexture);
> +
> + free(handle);
> +}
> +
> +static const struct vlc_asurfacetexture_operations NDKSurfaceAPI =
> +{
> + .attach_to_gl_context = NDKSurfaceTexture_attachToGLContext,
> + .update_tex_image = NDKSurfaceTexture_updateTexImage,
> + .detach_from_gl_context = NDKSurfaceTexture_detachFromGLContext,
> + .destroy = NDKSurfaceTexture_destroy,
> +};
> +
> static int
> -JNISurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> +JNISurfaceTexture_attachToGLContext(
> + struct vlc_asurfacetexture *surface,
> + uint32_t tex_name)
> {
> - JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> if (!p_env)
> return VLC_EGENERIC;
>
> + AWindowHandler *p_awh = handle->awh;
> +
> return JNI_STEXCALL(CallBooleanMethod, attachToGLContext, tex_name) ?
> VLC_SUCCESS : VLC_EGENERIC;
> }
>
> static void
> -JNISurfaceTexture_detachFromGLContext(AWindowHandler *p_awh, JNIEnv *p_env)
> +JNISurfaceTexture_detachFromGLContext(
> + struct vlc_asurfacetexture *surface)
> {
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + AWindowHandler *p_awh = handle->awh;
> + JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> + if (!p_env)
> + return;
> +
> JNI_STEXCALL(CallVoidMethod, detachFromGLContext);
>
> - if (p_awh->stex.jtransform_mtx != NULL)
> + if (handle->awh->stex.jtransform_mtx != NULL)
> {
> - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> - p_awh->stex.jtransform_mtx,
> + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> + handle->awh->stex.jtransform_mtx,
> JNI_ABORT);
> - p_awh->stex.jtransform_mtx = NULL;
> + handle->awh->stex.jtransform_mtx = NULL;
> }
> }
>
> static int
> -JNISurfaceTexture_waitAndUpdateTexImage(AWindowHandler *p_awh,
> - const float **pp_transform_mtx)
> +JNISurfaceTexture_waitAndUpdateTexImage(
> + struct vlc_asurfacetexture *surface,
> + const float **pp_transform_mtx)
> {
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + AWindowHandler *p_awh = handle->awh;
> JNIEnv *p_env = android_getEnvCommon(NULL, p_awh->p_jvm, "SurfaceTexture");
> if (!p_env)
> return VLC_EGENERIC;
>
> - if (p_awh->stex.jtransform_mtx != NULL)
> - (*p_env)->ReleaseFloatArrayElements(p_env, p_awh->stex.jtransform_mtx_array,
> - p_awh->stex.jtransform_mtx,
> +
> + if (handle->awh->stex.jtransform_mtx != NULL)
> + (*p_env)->ReleaseFloatArrayElements(p_env, handle->awh->stex.jtransform_mtx_array,
> + handle->awh->stex.jtransform_mtx,
> JNI_ABORT);
>
> bool ret = JNI_STEXCALL(CallBooleanMethod, waitAndUpdateTexImage,
> - p_awh->stex.jtransform_mtx_array);
> + handle->awh->stex.jtransform_mtx_array);
> if (ret)
> {
> - p_awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> - p_awh->stex.jtransform_mtx_array, NULL);
> - *pp_transform_mtx = p_awh->stex.jtransform_mtx;
> + handle->awh->stex.jtransform_mtx = (*p_env)->GetFloatArrayElements(p_env,
> + handle->awh->stex.jtransform_mtx_array, NULL);
> + *pp_transform_mtx = handle->awh->stex.jtransform_mtx;
> return VLC_SUCCESS;
> }
> else
> {
> - p_awh->stex.jtransform_mtx = NULL;
> + handle->awh->stex.jtransform_mtx = NULL;
> return VLC_EGENERIC;
> }
> }
>
> +static void JNISurfaceTexture_destroy(
> + struct vlc_asurfacetexture *surface)
> +{
> + struct SurfaceTextureHandle *handle =
> + container_of(surface, struct SurfaceTextureHandle, surface);
> +
> + JNIEnv *p_env = android_getEnvCommon(NULL, handle->awh->p_jvm, "SurfaceTexture");
> + if (!p_env)
> + return;
> +
> + handle->awh->pf_winRelease(handle->surface.window);
> + (*p_env)->DeleteGlobalRef(p_env, handle->surface.jsurface);
> +
> + free(handle);
> +}
> +
> +static const struct vlc_asurfacetexture_operations JNISurfaceAPI =
> +{
> + .attach_to_gl_context = JNISurfaceTexture_attachToGLContext,
> + .update_tex_image = JNISurfaceTexture_waitAndUpdateTexImage,
> + .detach_from_gl_context = JNISurfaceTexture_detachFromGLContext,
> + .destroy = JNISurfaceTexture_destroy,
> +};
> +
>
> static int
> LoadNDKSurfaceTextureAPI(AWindowHandler *p_awh, void *p_library, JNIEnv *p_env)
> @@ -550,19 +639,6 @@ LoadNativeWindowAPI(AWindowHandler *p_awh, JNIEnv *p_env)
> && p_awh->anw_api.setBuffersGeometry)
> {
> p_awh->b_has_ndk_ast_api = !LoadNDKSurfaceTextureAPI(p_awh, p_library, p_env);
> - if (p_awh->b_has_ndk_ast_api)
> - {
> - p_awh->st.pf_attachToGL = NDKSurfaceTexture_attachToGLContext;
> - p_awh->st.pf_updateTexImage = NDKSurfaceTexture_updateTexImage;
> - p_awh->st.pf_detachFromGL = NDKSurfaceTexture_detachFromGLContext;
> - }
> - else
> - {
> - p_awh->st.pf_attachToGL = JNISurfaceTexture_attachToGLContext;
> - p_awh->st.pf_updateTexImage = JNISurfaceTexture_waitAndUpdateTexImage;
> - p_awh->st.pf_detachFromGL = JNISurfaceTexture_detachFromGLContext;
> - }
> -
> p_awh->p_anw_dl = p_library;
> }
> else
> @@ -774,12 +850,11 @@ AWindowHandler_destroy(AWindowHandler *p_awh)
>
> if (p_env)
> {
> - if (p_awh->b_has_ndk_ast_api)
> - {
> - p_awh->ndk_ast_api.pf_releaseAst(p_awh->ndk_ast_api.p_ast);
> - (*p_env)->DeleteGlobalRef(p_env, p_awh->ndk_ast_api.surfacetexture);
> + if (p_awh->st)
> + p_awh->st->ops->destroy(p_awh->st);
> +
> + if (jfields.SurfaceTexture.clazz)
> (*p_env)->DeleteGlobalRef(p_env, jfields.SurfaceTexture.clazz);
> - }
>
> JNI_ANWCALL(CallVoidMethod, unregisterNative);
> AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_Video);
> @@ -857,11 +932,40 @@ WindowHandler_NewSurfaceEnv(AWindowHandler *p_awh, JNIEnv *p_env,
> break;
> case AWindow_SurfaceTexture:
> {
> + struct SurfaceTextureHandle *surfacetexture =
> + malloc(sizeof *surfacetexture);
> + if (surfacetexture == NULL)
> + return VLC_EGENERIC;
> +
> if (p_awh->b_has_ndk_ast_api)
> + {
> jsurface = InitNDKSurfaceTexture(p_awh, p_env, id);
> + surfacetexture->surface.ops = &NDKSurfaceAPI;
> + surfacetexture->surface.window = p_awh->views[id].p_anw;
> + }
> else
> + {
> jsurface = JNI_STEXCALL(CallObjectMethod, getSurface);
> - break;
> + surfacetexture->surface.ops = &JNISurfaceAPI;
> + surfacetexture->surface.window
> + = p_awh->views[id].p_anw
> + = p_awh->pf_winFromSurface(p_env, jsurface);
> + }
> +
> + surfacetexture->awh = p_awh;
> + surfacetexture->jtexture = p_awh->ndk_ast_api.surfacetexture;
> + surfacetexture->texture = p_awh->ndk_ast_api.p_ast;
Not only when p_awh->b_has_ndk_ast_api is true (according to the field
names)?
> + surfacetexture->surface.jsurface
> + = p_awh->views[id].jsurface
> + = (*p_env)->NewGlobalRef(p_env, jsurface);
> + (*p_env)->DeleteLocalRef(p_env, jsurface);
> +
> + assert(surfacetexture->surface.window);
> + assert(surfacetexture->surface.jsurface);
> +
> + /* Store the vlc_asurfacetexture pointer for current AWH wrapper */
> + p_awh->st = &surfacetexture->surface;
> + return VLC_SUCCESS;
> }
> default:
> vlc_assert_unreachable();
> @@ -974,7 +1078,7 @@ AWindowHandler_setVideoLayout(AWindowHandler *p_awh,
> int
> SurfaceTexture_attachToGLContext(AWindowHandler *p_awh, uint32_t tex_name)
> {
> - return p_awh->st.pf_attachToGL(p_awh, tex_name);
> + return p_awh->st->ops->attach_to_gl_context(p_awh->st, tex_name);
> }
>
> void
> @@ -985,12 +1089,11 @@ SurfaceTexture_detachFromGLContext(AWindowHandler *p_awh)
> if (!p_env)
> return;
>
> - p_awh->st.pf_detachFromGL(p_awh, p_env);
> - AWindowHandler_releaseANativeWindowEnv(p_awh, p_env, AWindow_SurfaceTexture);
> + p_awh->st->ops->detach_from_gl_context(p_awh->st);
> }
>
> int
> SurfaceTexture_updateTexImage(AWindowHandler *p_awh, const float **pp_transform_mtx)
> {
> - return p_awh->st.pf_updateTexImage(p_awh, pp_transform_mtx);
> + return p_awh->st->ops->update_tex_image(p_awh->st, pp_transform_mtx);
> }
> diff --git a/modules/video_output/android/utils.h b/modules/video_output/android/utils.h
> index 086819dc1ae..e8bec8b197c 100644
> --- a/modules/video_output/android/utils.h
> +++ b/modules/video_output/android/utils.h
> @@ -79,6 +79,36 @@ struct android_video_context_t
> bool (*render_ts)(struct picture_context_t *ctx, vlc_tick_t ts);
> };
>
> +struct vlc_asurfacetexture
> +{
> + struct ANativeWindow *window;
> + void *jsurface;
> +
> + const struct vlc_asurfacetexture_operations *ops;
> +};
> +
> +/**
> + * Wrapper structure for Android SurfaceTexture object.
> + *
> + * It can use either the NDK API or JNI API.
> + */
> +struct vlc_asurfacetexture_operations
> +{
> + int (*attach_to_gl_context)(
> + struct vlc_asurfacetexture *surface,
> + uint32_t tex_name);
> +
> + void (*detach_from_gl_context)(
> + struct vlc_asurfacetexture *surface);
> +
> + int (*update_tex_image)(
> + struct vlc_asurfacetexture *surface,
> + const float **pp_transform_mtx);
> +
> + void (*destroy)(
> + struct vlc_asurfacetexture *surface);
> +};
> +
> /**
> * Attach or get a JNIEnv*
> *
> --
> 2.27.0
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel
More information about the vlc-devel
mailing list