[vlc-devel] [PATCH v2 3/6] video_filter: add egl_pbuffer filter plugin
Romain Vimont
rom1v at videolabs.io
Fri Feb 26 17:10:33 UTC 2021
From: Alexandre Janniaux <ajanni at videolabs.io>
The video filter egl_pbuffer allows running OpenGL filters inside an EGL
pixel buffer context, using multiple framebuffers.
It is enabled only on Android, where the display EGL_DEFAULT_DISPLAY is
refcounted.
Co-authored-by: Romain Vimont <rom1v at videolabs.io>
---
modules/video_filter/egl_pbuffer.c | 467 ++++++++++++++++++++++++
modules/video_output/opengl/Makefile.am | 10 +
modules/video_output/opengl/gl_api.c | 3 +
modules/video_output/opengl/gl_common.h | 17 +
4 files changed, 497 insertions(+)
create mode 100644 modules/video_filter/egl_pbuffer.c
diff --git a/modules/video_filter/egl_pbuffer.c b/modules/video_filter/egl_pbuffer.c
new file mode 100644
index 0000000000..0ecc2562d0
--- /dev/null
+++ b/modules/video_filter/egl_pbuffer.c
@@ -0,0 +1,467 @@
+/*****************************************************************************
+ * egl_pbuffer.c: OpenGL filter in EGL offscreen framebuffer
+ *****************************************************************************
+ * Copyright (C) 2020 Videolabs
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_modules.h>
+#include <vlc_picture.h>
+#include <vlc_filter.h>
+#include <vlc_opengl.h>
+#include <vlc_vout_display.h>
+#include <vlc_atomic.h>
+#include "../video_output/opengl/vout_helper.h"
+#include "../video_output/opengl/filters.h"
+#include "../video_output/opengl/gl_api.h"
+#include "../video_output/opengl/gl_common.h"
+#include "../video_output/opengl/interop.h"
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+
+#define BUFFER_COUNT 4
+
+struct pbo_picture_context
+{
+ struct picture_context_t context;
+ void *buffer_mapping;
+ int rc;
+ vlc_mutex_t *lock;
+ vlc_cond_t *cond;
+};
+
+struct vlc_gl_pbuffer
+{
+ vlc_gl_t *gl;
+ vlc_mutex_t lock;
+ vlc_cond_t cond;
+
+ video_format_t fmt_out;
+
+ struct vlc_gl_api api;
+
+ size_t current_flip;
+ GLuint pixelbuffers[BUFFER_COUNT];
+ GLuint framebuffers[BUFFER_COUNT];
+ GLuint textures[BUFFER_COUNT];
+ struct pbo_picture_context picture_contexts[BUFFER_COUNT];
+
+ EGLDisplay display;
+ EGLSurface surface;
+ EGLContext context;
+
+ PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR;
+ PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR;
+
+ bool current;
+};
+
+static int MakeCurrent (vlc_gl_t *gl)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ assert(!sys->current);
+ if (eglMakeCurrent (sys->display, sys->surface, sys->surface,
+ sys->context) != EGL_TRUE)
+ return VLC_EGENERIC;
+
+ sys->current = true;
+ return VLC_SUCCESS;
+}
+
+static void ReleaseCurrent (vlc_gl_t *gl)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ assert(sys->current);
+ eglMakeCurrent (sys->display, EGL_NO_SURFACE, EGL_NO_SURFACE,
+ EGL_NO_CONTEXT);
+
+ sys->current = false;
+}
+
+static void *GetSymbol(vlc_gl_t *gl, const char *procname)
+{
+ (void) gl;
+ return (void *)eglGetProcAddress (procname);
+}
+
+static const char *QueryString(vlc_gl_t *gl, int32_t name)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ return eglQueryString(sys->display, name);
+}
+
+static void *CreateImageKHR(vlc_gl_t *gl, unsigned target, void *buffer,
+ const int32_t *attrib_list)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ return sys->eglCreateImageKHR(sys->display, NULL, target, buffer,
+ attrib_list);
+}
+
+static bool DestroyImageKHR(vlc_gl_t *gl, void *image)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ return sys->eglDestroyImageKHR(sys->display, image);
+}
+
+static int InitEGL(vlc_gl_t *gl, unsigned width, unsigned height)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ sys->display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+ if (sys->display == EGL_NO_DISPLAY)
+ return VLC_EGENERIC;
+
+ /* Initialize EGL display */
+ EGLint major, minor;
+ if (eglInitialize(sys->display, &major, &minor) != EGL_TRUE)
+ goto error;
+ msg_Dbg(gl, "EGL version %s by %s, API %s",
+ eglQueryString(sys->display, EGL_VERSION),
+ eglQueryString(sys->display, EGL_VENDOR),
+#ifdef USE_OPENGL_ES2
+ "OpenGL ES2"
+#else
+ "OpenGL"
+#endif
+ );
+
+ const EGLint conf_attr[] = {
+ EGL_RED_SIZE, 8,
+ EGL_GREEN_SIZE, 8,
+ EGL_BLUE_SIZE, 8,
+#ifdef USE_OPENGL_ES2
+ EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+#else
+ EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT,
+#endif
+ EGL_SURFACE_TYPE, EGL_PBUFFER_BIT,
+ EGL_NONE,
+ };
+ EGLConfig cfgv[1];
+ EGLint cfgc;
+
+ msg_Info(gl, "WIDTH=%u HEIGHT=%u", width, height);
+ const EGLint surface_attr[] = {
+ EGL_WIDTH, width,
+ EGL_HEIGHT, height,
+ EGL_NONE,
+ };
+
+ if (eglChooseConfig(sys->display, conf_attr, cfgv, 1, &cfgc) != EGL_TRUE
+ || cfgc == 0)
+ {
+ msg_Err (gl, "cannot choose EGL configuration");
+ goto error;
+ }
+
+ /* Create a drawing surface */
+ sys->surface = eglCreatePbufferSurface(sys->display, cfgv[0], surface_attr);
+ if (sys->surface == EGL_NO_SURFACE)
+ {
+ msg_Err (gl, "cannot create EGL window surface");
+ assert(false);
+ goto error;
+ }
+
+#ifdef USE_OPENGL_ES2
+ if (eglBindAPI (EGL_OPENGL_ES_API) != EGL_TRUE)
+ {
+ msg_Err (gl, "cannot bind EGL OPENGL ES API");
+ goto error;
+ }
+
+ const GLint ctx_attr[] = {
+ EGL_CONTEXT_CLIENT_VERSION, 2,
+ EGL_NONE
+ };
+#else
+ if (eglBindAPI (EGL_OPENGL_API) != EGL_TRUE)
+ {
+ msg_Err (gl, "cannot bind EGL OPENGL API");
+ goto error;
+ }
+
+ const GLint ctx_attr[] = {
+ EGL_CONTEXT_CLIENT_VERSION, 3,
+ EGL_NONE
+ };
+#endif
+
+ EGLContext ctx
+ = sys->context
+ = eglCreateContext(sys->display, cfgv[0], EGL_NO_CONTEXT, ctx_attr);
+
+ if (ctx == EGL_NO_CONTEXT)
+ {
+ msg_Err (gl, "cannot create EGL context");
+ goto error;
+ }
+
+ return VLC_SUCCESS;
+error:
+ return VLC_EGENERIC;
+}
+
+static picture_context_t *picture_context_copy(picture_context_t *input)
+{
+ struct pbo_picture_context *context =
+ (struct pbo_picture_context *)input;
+
+ vlc_mutex_lock(context->lock);
+ context->rc++;
+ vlc_mutex_unlock(context->lock);
+ return input;
+}
+
+static void picture_context_destroy(picture_context_t *input)
+{
+ struct pbo_picture_context *context = (struct pbo_picture_context *) input;
+
+ vlc_mutex_lock(context->lock);
+ context->rc--;
+ vlc_cond_signal(context->cond);
+ vlc_mutex_unlock(context->lock);
+}
+
+static inline void BindDrawFramebuffer(struct vlc_gl_pbuffer *sys)
+{
+ const opengl_vtable_t *vt = &sys->api.vt;
+ vt->BindFramebuffer(GL_DRAW_FRAMEBUFFER,
+ sys->framebuffers[sys->current_flip]);
+}
+
+static void UpdateBuffer(vlc_gl_t *gl)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+
+ vlc_mutex_lock(&sys->lock);
+ size_t index;
+
+ do {
+ for (index = 0; index < BUFFER_COUNT; ++index)
+ {
+ assert(sys->picture_contexts[index].rc >= 0);
+ if (sys->picture_contexts[index].rc == 0)
+ goto out_loop;
+ }
+ vlc_cond_wait(&sys->cond, &sys->lock);
+ } while (index == BUFFER_COUNT);
+out_loop:
+ vlc_mutex_unlock(&sys->lock);
+
+ sys->current_flip = index;
+ BindDrawFramebuffer(sys);
+}
+
+static picture_t *Swap(vlc_gl_t *gl)
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+ const opengl_vtable_t *vt = &sys->api.vt;
+
+ if (!sys->current)
+ eglMakeCurrent(sys->display, sys->surface, sys->surface, sys->context);
+
+ /* Read current framebuffer */
+ struct pbo_picture_context *context =
+ &sys->picture_contexts[sys->current_flip];
+
+ vt->BindBuffer(GL_PIXEL_PACK_BUFFER, sys->pixelbuffers[sys->current_flip]);
+ vt->BindFramebuffer(GL_FRAMEBUFFER, sys->framebuffers[sys->current_flip]);
+ if (context->buffer_mapping != NULL)
+ vt->UnmapBuffer(GL_PIXEL_PACK_BUFFER);
+
+ GLsizei width = sys->fmt_out.i_visible_width;
+ GLsizei height = sys->fmt_out.i_visible_height;
+ GLenum format = GL_RGBA;
+
+ vt->ReadPixels(0, 0, width, height, format, GL_UNSIGNED_BYTE, 0);
+
+ void *pixels = vt->MapBufferRange(
+ GL_PIXEL_PACK_BUFFER, 0, width*height*4, GL_MAP_READ_BIT);
+
+ GLsizei stride;
+ vt->GetIntegerv(GL_PACK_ROW_LENGTH, &stride);
+ stride = width;
+
+ context->buffer_mapping = pixels;
+ context->rc ++;
+
+ /* Swap framebuffer */
+ UpdateBuffer(gl);
+
+ if (!sys->current)
+ eglMakeCurrent(sys->display, EGL_NO_SURFACE, EGL_NO_SURFACE,
+ EGL_NO_CONTEXT);
+
+ /* Output as picture */
+ picture_resource_t pict_resource = {
+ .pf_destroy = NULL,
+ };
+
+ pict_resource.p[0].p_pixels = pixels;
+ pict_resource.p[0].i_lines = height;
+ pict_resource.p[0].i_pitch = stride * 4;
+
+
+ picture_t *output = picture_NewFromResource(&sys->fmt_out, &pict_resource);
+ assert(output);
+ if (output == NULL)
+ goto error;
+
+ output->context = (picture_context_t *)context;
+ output->context->vctx = NULL;
+
+ return output;
+
+error:
+ context->rc--;
+ return NULL;
+}
+
+static void Close( vlc_gl_t *gl )
+{
+ struct vlc_gl_pbuffer *sys = gl->sys;
+ const opengl_vtable_t *vt = &sys->api.vt;
+
+ vlc_gl_MakeCurrent(sys->gl);
+ vt->DeleteBuffers(BUFFER_COUNT, sys->pixelbuffers);
+ vt->DeleteFramebuffers(BUFFER_COUNT, sys->framebuffers);
+ vt->DeleteTextures(BUFFER_COUNT, sys->textures);
+ vlc_gl_ReleaseCurrent(sys->gl);
+}
+
+static int Open(vlc_gl_t *gl, unsigned width, unsigned height)
+{
+ struct vlc_gl_pbuffer *sys = vlc_obj_malloc(&gl->obj, sizeof *sys);
+ if (sys == NULL)
+ return VLC_ENOMEM;
+
+ sys->gl = gl;
+ sys->current = false;
+
+ video_format_Init(&sys->fmt_out, VLC_CODEC_RGBA);
+ sys->fmt_out.i_visible_width
+ = sys->fmt_out.i_width
+ = width;
+ sys->fmt_out.i_visible_height
+ = sys->fmt_out.i_height
+ = height;
+
+ gl->offscreen_chroma_out = VLC_CODEC_RGBA;
+ gl->offscreen_vctx_out = NULL;
+
+ vlc_mutex_init(&sys->lock);
+ vlc_cond_init(&sys->cond);
+
+ gl->sys = sys;
+
+ if (InitEGL(gl, width, height) != VLC_SUCCESS)
+ {
+ msg_Err(gl, "Failed to create opengl context\n");
+ goto error1;
+ }
+
+ gl->ext = VLC_GL_EXT_EGL;
+ gl->make_current = MakeCurrent;
+ gl->release_current = ReleaseCurrent;
+ gl->resize = NULL;
+ gl->swap_offscreen = Swap;
+ gl->get_proc_address = GetSymbol;
+ gl->destroy = Close;
+ gl->egl.queryString = QueryString;
+ gl->offscreen_vflip = true;
+
+ sys->eglCreateImageKHR = (void *)eglGetProcAddress("eglCreateImageKHR");
+ sys->eglDestroyImageKHR = (void *)eglGetProcAddress("eglDestroyImageKHR");
+ if (sys->eglCreateImageKHR != NULL && sys->eglDestroyImageKHR != NULL)
+ {
+ gl->egl.createImageKHR = CreateImageKHR;
+ gl->egl.destroyImageKHR = DestroyImageKHR;
+ }
+
+ vlc_gl_MakeCurrent(gl);
+ int ret = vlc_gl_api_Init(&sys->api, gl);
+ if (ret != VLC_SUCCESS)
+ {
+ msg_Err(gl, "Failed to initialize gl_api");
+ vlc_gl_ReleaseCurrent(gl);
+ goto error2;
+ }
+
+ const opengl_vtable_t *vt = &sys->api.vt;
+ vt->GenBuffers(BUFFER_COUNT, sys->pixelbuffers);
+ vt->GenFramebuffers(BUFFER_COUNT, sys->framebuffers);
+ vt->GenTextures(BUFFER_COUNT, sys->textures);
+
+ for (size_t i=0; i<BUFFER_COUNT; ++i)
+ {
+ vt->BindBuffer(GL_PIXEL_PACK_BUFFER, sys->pixelbuffers[i]);
+ vt->BufferData(GL_PIXEL_PACK_BUFFER, width*height*4, NULL,
+ GL_STREAM_READ);
+ vt->BindFramebuffer(GL_FRAMEBUFFER, sys->framebuffers[i]);
+ vt->BindTexture(GL_TEXTURE_2D, sys->textures[i]);
+ vt->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0,
+ GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+ vt->FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+ GL_TEXTURE_2D, sys->textures[i], 0);
+
+ struct pbo_picture_context *context = &sys->picture_contexts[i];
+ context->buffer_mapping = NULL;
+ context->lock = &sys->lock;
+ context->cond = &sys->cond;
+ context->context.destroy = picture_context_destroy;
+ context->context.copy = picture_context_copy;
+ context->rc = 0;
+ }
+
+ sys->current_flip = BUFFER_COUNT - 1;
+ BindDrawFramebuffer(sys);
+
+ vlc_gl_ReleaseCurrent(gl);
+
+ return VLC_SUCCESS;
+
+error2:
+ vlc_object_delete(sys->gl);
+error1:
+ vlc_obj_free(&gl->obj, sys);
+
+ return VLC_EGENERIC;
+}
+
+vlc_module_begin()
+ set_shortname( N_("egl_pbuffer") )
+ set_description( N_("EGL PBuffer offscreen opengl provider") )
+#ifdef USE_OPENGL_ES2
+ set_capability( "opengl es2 offscreen", 1)
+#else
+ set_capability( "opengl offscreen", 1 )
+#endif
+
+ add_shortcut( "egl_pbuffer" )
+ set_callback( Open )
+vlc_module_end()
diff --git a/modules/video_output/opengl/Makefile.am b/modules/video_output/opengl/Makefile.am
index 76f6df47a0..58b050b3f7 100644
--- a/modules/video_output/opengl/Makefile.am
+++ b/modules/video_output/opengl/Makefile.am
@@ -68,3 +68,13 @@ libgl_plugin_la_LIBADD = libvlc_opengl.la
if HAVE_GL
vout_LTLIBRARIES += libgl_plugin.la
endif # HAVE_GL
+
+libegl_pbuffer_filter_plugin_la_SOURCES = video_filter/egl_pbuffer.c
+libegl_pbuffer_filter_plugin_la_CPPFLAGS = $(AM_CPPFLAGS) $(EGL_FLAGS)
+libegl_pbuffer_filter_plugin_la_LIBADD = $(LIBM) $(EGL_LIBS)
+
+if HAVE_ANDROID
+libegl_pbuffer_filter_plugin_la_LIBADD += libvlc_opengles.la $(GLES2_LIBS)
+libegl_pbuffer_filter_plugin_la_CPPFLAGS += -DUSE_OPENGL_ES2=1
+vout_LTLIBRARIES += libegl_pbuffer_filter_plugin.la
+endif
diff --git a/modules/video_output/opengl/gl_api.c b/modules/video_output/opengl/gl_api.c
index 8cbce7a41d..92db2206f1 100644
--- a/modules/video_output/opengl/gl_api.c
+++ b/modules/video_output/opengl/gl_api.c
@@ -138,10 +138,13 @@ vlc_gl_api_Init(struct vlc_gl_api *api, vlc_gl_t *gl)
GET_PROC_ADDR_OPTIONAL(FramebufferRenderbuffer);
GET_PROC_ADDR_OPTIONAL(BlitFramebuffer);
+ GET_PROC_ADDR_OPTIONAL(ReadPixels);
+
GET_PROC_ADDR_OPTIONAL(BufferSubData);
GET_PROC_ADDR_OPTIONAL(BufferStorage);
GET_PROC_ADDR_OPTIONAL(MapBufferRange);
GET_PROC_ADDR_OPTIONAL(FlushMappedBufferRange);
+ GET_PROC_ADDR_OPTIONAL(MapBuffer);
GET_PROC_ADDR_OPTIONAL(UnmapBuffer);
GET_PROC_ADDR_OPTIONAL(FenceSync);
GET_PROC_ADDR_OPTIONAL(DeleteSync);
diff --git a/modules/video_output/opengl/gl_common.h b/modules/video_output/opengl/gl_common.h
index 90bfc0fb13..5a06336a90 100644
--- a/modules/video_output/opengl/gl_common.h
+++ b/modules/video_output/opengl/gl_common.h
@@ -156,6 +156,18 @@
# define GL_COLOR_ATTACHMENT7 0x8CE7
#endif
+#if !defined(GL_PIXEL_PACK_BUFFER)
+# define GL_PIXEL_PACK_BUFFER 0x88EB
+#endif
+
+#if !defined(GL_PACK_ROW_LENGTH)
+# define GL_PACK_ROW_LENGTH 0x0D02
+#endif
+
+#if !defined(GL_STREAM_READ)
+# define GL_STREAM_READ 0x88E1
+#endif
+
#ifndef APIENTRY
# define APIENTRY
#endif
@@ -220,6 +232,7 @@ typedef void (APIENTRY *PFNGLFRAMEBUFFERRENDERBUFFERPROC) (GLenum target, GLenum
typedef void (APIENTRY *PFNGLBLITFRAMEBUFFERPROC) (GLint srcX0, GLint srcY0, GLint srcX1, GLint srcY1,
GLint dstX0, GLint dstY0, GLint dstX1, GLint dstY1,
GLbitfield mask, GLenum filter);
+typedef void (APIENTRY *PFNGLREADPIXELSPROC) (GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, void *);
/* The following are defined in glext.h but not for GLES2 or on Apple systems */
#if defined(USE_OPENGL_ES2) || defined(__APPLE__)
@@ -275,6 +288,7 @@ typedef GLboolean (APIENTRY *PFNGLUNMAPBUFFERPROC) (GLenum target);
typedef GLsync (APIENTRY *PFNGLFENCESYNCPROC) (GLenum condition, GLbitfield flags);
typedef void (APIENTRY *PFNGLDELETESYNCPROC) (GLsync sync);
typedef GLenum (APIENTRY *PFNGLCLIENTWAITSYNCPROC) (GLsync sync, GLbitfield flags, GLuint64 timeout);
+typedef void *(APIENTRY *PFNGLMAPBUFFERPROC)(GLenum, GLbitfield);
#endif
/**
@@ -372,11 +386,14 @@ typedef struct {
PFNGLFRAMEBUFFERRENDERBUFFERPROC FramebufferRenderbuffer;
PFNGLBLITFRAMEBUFFERPROC BlitFramebuffer;
+ PFNGLREADPIXELSPROC ReadPixels;
+
/* Commands used for PBO and/or Persistent mapping */
PFNGLBUFFERSUBDATAPROC BufferSubData; /* can be NULL */
PFNGLBUFFERSTORAGEPROC BufferStorage; /* can be NULL */
PFNGLMAPBUFFERRANGEPROC MapBufferRange; /* can be NULL */
PFNGLFLUSHMAPPEDBUFFERRANGEPROC FlushMappedBufferRange; /* can be NULL */
+ PFNGLMAPBUFFERPROC MapBuffer;
PFNGLUNMAPBUFFERPROC UnmapBuffer; /* can be NULL */
PFNGLFENCESYNCPROC FenceSync; /* can be NULL */
PFNGLDELETESYNCPROC DeleteSync; /* can be NULL */
--
2.30.1
More information about the vlc-devel
mailing list