[vlc-devel] [PATCH 2/2] video_filter: add egl_surfacetexture filter plugin
Romain Vimont
rom1v at videolabs.io
Tue Apr 20 14:17:43 UTC 2021
From: Alexandre Janniaux <ajanni at videolabs.io>
On Android, this filter allows to render OpenGL filters to an Android
SurfaceTexture.
Contrary to egl_pbuffer, the resulting picture stays on the GPU (it is
not downloaded to main memory), providing much better performances.
Co-authored-by: Romain Vimont <rom1v at videolabs.io>
---
modules/video_filter/egl_surfacetexture.c | 473 ++++++++++++++++++
modules/video_output/opengl/Makefile.am | 10 +
modules/video_output/opengl/interop_android.c | 2 +-
3 files changed, 484 insertions(+), 1 deletion(-)
create mode 100644 modules/video_filter/egl_surfacetexture.c
diff --git a/modules/video_filter/egl_surfacetexture.c b/modules/video_filter/egl_surfacetexture.c
new file mode 100644
index 0000000000..eae1f6d329
--- /dev/null
+++ b/modules/video_filter/egl_surfacetexture.c
@@ -0,0 +1,473 @@
+/*****************************************************************************
+ * egl_surfacetexture.c: OpenGL offscreen provider with SurfaceTexture
+ *****************************************************************************
+ * Copyright (C) 2021 Videolabs
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_modules.h>
+#include <vlc_picture.h>
+#include <vlc_filter.h>
+#include <vlc_opengl.h>
+#include <vlc_vout_display.h>
+#include <vlc_atomic.h>
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+
+#include "../video_output/android/utils.h"
+#include "../video_output/opengl/gl_api.h"
+
+#define BUFFER_COUNT 3
+
+struct video_ctx
+{
+ android_video_context_t android;
+ picture_pool_t *pool;
+ picture_t *pictures[BUFFER_COUNT];
+
+ EGLDisplay display;
+ EGLContext context;
+};
+
+struct picture_ctx
+{
+ struct picture_context_t context;
+ EGLSurface surface;
+
+ struct vlc_asurfacetexture *texture;
+};
+
+#define PRIV(pic_ctx) container_of(pic_ctx, struct picture_ctx, context)
+
+struct surfacetexture_sys
+{
+ android_video_context_t *avctx;
+
+ video_format_t fmt_out;
+
+ size_t current_flip;
+
+ struct vlc_gl_api api;
+
+ EGLConfig cfgv;
+
+ PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR;
+ PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR;
+
+ picture_t *current_picture;
+};
+
+static inline struct video_ctx *GetVCtx(vlc_gl_t *gl)
+{
+ return vlc_video_context_GetPrivate(gl->offscreen_vctx_out,
+ VLC_VIDEO_CONTEXT_AWINDOW);
+}
+
+static int MakeCurrent(vlc_gl_t *gl)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ /* We must always have a surface mapped. */
+ assert(sys->current_picture);
+ assert(sys->current_picture->context);
+
+ struct picture_ctx *ctx = PRIV(sys->current_picture->context);
+
+ if (eglMakeCurrent (vctx->display, ctx->surface, ctx->surface,
+ vctx->context) != EGL_TRUE)
+ return VLC_EGENERIC;
+
+ return VLC_SUCCESS;
+}
+
+static void ReleaseCurrent(vlc_gl_t *gl)
+{
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ eglMakeCurrent(vctx->display, EGL_NO_SURFACE, EGL_NO_SURFACE,
+ EGL_NO_CONTEXT);
+}
+
+static void *GetSymbol(vlc_gl_t *gl, const char *procname)
+{
+ (void) gl;
+ return (void *)eglGetProcAddress(procname);
+}
+
+static const char *QueryString(vlc_gl_t *gl, int32_t name)
+{
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ return eglQueryString(vctx->display, name);
+}
+
+static void *CreateImageKHR(vlc_gl_t *gl, unsigned target, void *buffer,
+ const int32_t *attrib_list)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ return sys->eglCreateImageKHR(vctx->display, NULL, target, buffer,
+ attrib_list);
+}
+
+static bool DestroyImageKHR(vlc_gl_t *gl, void *image)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ return sys->eglDestroyImageKHR(vctx->display, image);
+}
+
+static picture_context_t *CopyPictureContext(picture_context_t *input)
+{
+ vlc_video_context_Hold(input->vctx);
+ return input;
+}
+
+static void DestroyPictureContext(picture_context_t *input)
+{
+ (void) input;
+ /* video context is already released by picture_Release. */
+}
+
+static picture_context_t *CreatePictureContext(vlc_gl_t *gl)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct picture_ctx *ctx = malloc(sizeof(*ctx));
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ ctx->texture = vlc_asurfacetexture_New(gl->device->opaque, false);
+
+ struct ANativeWindow *window = ctx->texture->window;
+ native_window_api_t *api =
+ AWindowHandler_getANativeWindowAPI(gl->device->opaque);
+ api->setBuffersGeometry(window, sys->fmt_out.i_width, sys->fmt_out.i_height,
+ AHARDWAREBUFFER_FORMAT_BLOB);
+
+ /* Create a drawing surface */
+ ctx->surface = eglCreateWindowSurface(vctx->display, sys->cfgv, window,
+ NULL);
+ if (ctx->surface == EGL_NO_SURFACE)
+ {
+ msg_Err(gl, "cannot create EGL window surface");
+ goto error;
+ }
+
+ ctx->context.vctx = gl->offscreen_vctx_out;
+ ctx->context.copy = CopyPictureContext;
+ ctx->context.destroy = DestroyPictureContext;
+
+ return &ctx->context;
+
+error:
+ free(ctx);
+ return NULL;
+}
+
+static int InitEGL(vlc_gl_t *gl)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ vctx->display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+ if (vctx->display == EGL_NO_DISPLAY)
+ return VLC_EGENERIC;
+
+ /* Initialize EGL display */
+ EGLint major, minor;
+ if (eglInitialize(vctx->display, &major, &minor) != EGL_TRUE)
+ goto error;
+ msg_Dbg(gl, "EGL version %s by %s, API %s",
+ eglQueryString(vctx->display, EGL_VERSION),
+ eglQueryString(vctx->display, EGL_VENDOR),
+ "OpenGL ES2"
+ );
+
+ const EGLint conf_attr[] = {
+ EGL_RED_SIZE, 8,
+ EGL_GREEN_SIZE, 8,
+ EGL_BLUE_SIZE, 8,
+ EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+ EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
+ EGL_NONE,
+ };
+
+ EGLConfig *cfgv = &sys->cfgv;
+
+ EGLint cfgc;
+ if (eglChooseConfig(vctx->display, conf_attr, cfgv, 1, &cfgc) != EGL_TRUE
+ || cfgc == 0)
+ {
+ msg_Err(gl, "cannot choose EGL configuration");
+ goto error;
+ }
+
+ if (eglBindAPI(EGL_OPENGL_ES_API) != EGL_TRUE)
+ {
+ msg_Err(gl, "cannot bind EGL OPENGL ES API");
+ goto error;
+ }
+
+ const EGLint ctx_attr[] = {
+ EGL_CONTEXT_CLIENT_VERSION, 2,
+ EGL_NONE
+ };
+
+ EGLContext ctx
+ = eglCreateContext(vctx->display, sys->cfgv, EGL_NO_CONTEXT, ctx_attr);
+ if (ctx == EGL_NO_CONTEXT)
+ {
+ msg_Err(gl, "cannot create EGL context");
+ goto error;
+ }
+
+ vctx->context = ctx;
+
+ return VLC_SUCCESS;
+
+error:
+ eglTerminate(vctx->display);
+ return VLC_EGENERIC;
+}
+
+static picture_t *SwapOffscreen(vlc_gl_t *gl)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ picture_t *current_picture = sys->current_picture;
+ assert(current_picture);
+ assert(current_picture->context);
+
+ /* Swap currently active surface */
+ struct picture_ctx *current_ctx = PRIV(current_picture->context);
+
+ eglSwapBuffers(vctx->display, current_ctx->surface);
+
+ /* Change currently active surface to the next available one */
+ picture_t *picture = picture_pool_Wait(vctx->pool);
+ assert(picture->context);
+ struct picture_ctx *ctx = PRIV(current_picture->context);
+
+ eglMakeCurrent(vctx->display, ctx->surface, ctx->surface, vctx->context);
+ sys->current_picture = picture;
+
+ return current_picture;
+}
+
+static void Close( vlc_gl_t *gl )
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ picture_Release(sys->current_picture);
+ vlc_video_context_Release(gl->offscreen_vctx_out);
+ free(sys);
+}
+
+static void DestroyVideoContext(void *priv)
+{
+ struct video_ctx *vctx = priv;
+ for (size_t i = 0; i < ARRAY_SIZE(vctx->pictures); ++i)
+ {
+ picture_t *picture = vctx->pictures[i];
+ struct picture_ctx *ctx = PRIV(picture->context);
+
+ eglDestroySurface(vctx->display, ctx->surface);
+ vlc_asurfacetexture_Delete(ctx->texture);
+
+ /* Delete context without calling the picture destructors */
+ free(ctx);
+ picture->context = NULL;
+ }
+
+ /* Picture pool will release the pictures too */
+ picture_pool_Release(vctx->pool);
+
+ /* Must be not be called from Close(), otherwise it might cause deadlocks
+ * on eglDestroySurface() */
+ eglDestroyContext(vctx->display, vctx->context);
+
+ eglTerminate(vctx->display);
+}
+
+static struct vlc_asurfacetexture *
+PictureContextGetTexture(picture_context_t *context)
+{
+ assert(context);
+
+ struct picture_ctx *ctx = PRIV(context);
+ return ctx->texture;
+}
+
+static int InitPicturePool(vlc_gl_t *gl)
+{
+ struct surfacetexture_sys *sys = gl->sys;
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ size_t i;
+ for (i = 0; i < ARRAY_SIZE(vctx->pictures); ++i)
+ {
+ picture_t *pic = picture_NewFromFormat(&sys->fmt_out);
+ if (!pic)
+ goto error;
+
+ pic->context = CreatePictureContext(gl);
+ if (!pic->context)
+ {
+ picture_Release(pic);
+ goto error;
+ }
+
+ vctx->pictures[i] = pic;
+ }
+
+ vctx->pool = picture_pool_New(ARRAY_SIZE(vctx->pictures), vctx->pictures);
+ if (!vctx->pool)
+ goto error;
+
+ return VLC_SUCCESS;
+
+error:
+ while (i--)
+ picture_Release(vctx->pictures[i]);
+ return VLC_EGENERIC;
+}
+
+static int Open(vlc_gl_t *gl, unsigned width, unsigned height)
+{
+ if (gl->device == NULL || gl->device->type != VLC_DECODER_DEVICE_AWINDOW)
+ {
+ msg_Err(gl, "Wrong decoder device");
+ return VLC_EGENERIC;
+ }
+
+ struct surfacetexture_sys *sys = malloc(sizeof *sys);
+ if (sys == NULL)
+ return VLC_ENOMEM;
+
+ sys->fmt_out.i_visible_width
+ = sys->fmt_out.i_width
+ = width;
+ sys->fmt_out.i_visible_height
+ = sys->fmt_out.i_height
+ = height;
+
+ sys->fmt_out.i_chroma
+ = gl->offscreen_chroma_out
+ = VLC_CODEC_ANDROID_OPAQUE;
+ gl->sys = sys;
+
+ static const struct vlc_video_context_operations ops = {
+ .destroy = DestroyVideoContext,
+ };
+
+ gl->offscreen_vctx_out =
+ vlc_video_context_Create(gl->device, VLC_VIDEO_CONTEXT_AWINDOW,
+ sizeof(struct video_ctx), &ops);
+ if (gl->offscreen_vctx_out == NULL)
+ goto error1;
+
+ if (InitEGL(gl) != VLC_SUCCESS)
+ {
+ msg_Err(gl, "Failed to create opengl context\n");
+ goto error2;
+ }
+
+ gl->ext = VLC_GL_EXT_EGL;
+ gl->make_current = MakeCurrent;
+ gl->release_current = ReleaseCurrent;
+ gl->resize = NULL;
+ gl->swap_offscreen = SwapOffscreen;
+ gl->get_proc_address = GetSymbol;
+ gl->destroy = Close;
+ gl->egl.queryString = QueryString;
+
+ struct video_ctx *vctx = GetVCtx(gl);
+
+ sys->eglCreateImageKHR =
+ (PFNEGLCREATEIMAGEKHRPROC) eglGetProcAddress("eglCreateImageKHR");
+ sys->eglDestroyImageKHR =
+ (PFNEGLDESTROYIMAGEKHRPROC) eglGetProcAddress("eglDestroyImageKHR");
+ if (sys->eglCreateImageKHR != NULL && sys->eglDestroyImageKHR != NULL)
+ {
+ gl->egl.createImageKHR = CreateImageKHR;
+ gl->egl.destroyImageKHR = DestroyImageKHR;
+ }
+
+ if (InitPicturePool(gl) != VLC_SUCCESS)
+ goto error3;
+
+ sys->current_picture = picture_pool_Get(vctx->pool);
+ assert(sys->current_picture);
+ assert(sys->current_picture->context);
+ assert(sys->current_picture->context->vctx);
+
+ vctx->android.texture = NULL;
+ vctx->android.render = NULL;
+ vctx->android.render_ts = NULL;
+ vctx->android.get_texture = PictureContextGetTexture;
+
+ int ret = vlc_gl_MakeCurrent(gl);
+ if (ret != VLC_SUCCESS)
+ goto error4;
+
+ ret = vlc_gl_api_Init(&sys->api, gl);
+ if (ret != VLC_SUCCESS)
+ {
+ msg_Err(gl, "Failed to initialize gl_api");
+ vlc_gl_ReleaseCurrent(gl);
+ goto error4;
+ }
+ const char *extensions =
+ (const char *) sys->api.vt.GetString(GL_EXTENSIONS);
+ vlc_gl_ReleaseCurrent(gl);
+
+ if (!vlc_gl_StrHasToken(extensions, "GL_OES_EGL_image_external"))
+ {
+ msg_Warn(gl, "GL_OES_EGL_image_external is not available,"
+ " disabling egl_surfacetexture.");
+ goto error4;
+ }
+
+ return VLC_SUCCESS;
+
+error4:
+ picture_Release(sys->current_picture);
+error3:
+ eglTerminate(vctx->display);
+error2:
+ vlc_video_context_Release(gl->offscreen_vctx_out);
+error1:
+ free(sys);
+ return VLC_EGENERIC;
+}
+
+vlc_module_begin()
+ set_shortname( N_("egl_surfacetexture") )
+ set_description( N_("EGL Android SurfaceTexture offscreen opengl provider") )
+ set_capability( "opengl es2 offscreen", 100)
+
+ add_shortcut( "egl_surfacetexture" )
+ set_callback( Open )
+vlc_module_end()
diff --git a/modules/video_output/opengl/Makefile.am b/modules/video_output/opengl/Makefile.am
index 2273a737e9..b8a8ca319d 100644
--- a/modules/video_output/opengl/Makefile.am
+++ b/modules/video_output/opengl/Makefile.am
@@ -121,3 +121,13 @@ libegl_pbuffer_filter_plugin_la_CPPFLAGS += -DUSE_OPENGL_ES2=1
vout_LTLIBRARIES += libegl_pbuffer_filter_plugin.la
endif
endif
+
+libegl_surfacetexture_plugin_la_SOURCES = video_filter/egl_surfacetexture.c
+libegl_surfacetexture_plugin_la_CFLAGS = $(AM_CFLAGS) $(EGL_CFLAGS) -DUSE_OPENGL_ES2
+libegl_surfacetexture_plugin_la_LIBADD = $(EGL_LIBS)
+
+if HAVE_ANDROID
+if HAVE_EGL
+vout_LTLIBRARIES += libegl_surfacetexture_plugin.la
+endif
+endif
diff --git a/modules/video_output/opengl/interop_android.c b/modules/video_output/opengl/interop_android.c
index f94a7c26e9..44d174ab81 100644
--- a/modules/video_output/opengl/interop_android.c
+++ b/modules/video_output/opengl/interop_android.c
@@ -96,7 +96,7 @@ tc_anop_update(struct vlc_gl_interop *interop, GLuint *textures,
priv->previous_texture = texture;
}
- if (!avctx->render(pic->context))
+ if (avctx->render && !avctx->render(pic->context))
goto success; /* already rendered */
/* Release previous image */
--
2.31.0
More information about the vlc-devel
mailing list