[vlc-devel] [PATCH 3/6] video_filter: add egl_pbuffer filter plugin

[Alexandre Janniaux]

Hi,

On Tue, Feb 23, 2021 at 07:28:44PM +0200, Rémi Denis-Courmont wrote:
> Le tiistaina 23. helmikuuta 2021, 18.06.11 EET Romain Vimont a écrit :
> > From: Alexandre Janniaux <ajanni at videolabs.io>
> >
> > egl_pbuffer allows running OpenGL filters inside an EGL pixel buffer
> > context, using multiple framebuffers.
> >
> > Co-authored-by: Romain Vimont <rom1v at videolabs.io>
> > ---
> >  modules/video_filter/egl_pbuffer.c      | 470 ++++++++++++++++++++++++
> >  modules/video_output/opengl/Makefile.am |  15 +
> >  modules/video_output/opengl/gl_api.c    |   3 +
> >  modules/video_output/opengl/gl_common.h |  17 +
> >  4 files changed, 505 insertions(+)
> >  create mode 100644 modules/video_filter/egl_pbuffer.c
> >
> > diff --git a/modules/video_filter/egl_pbuffer.c
> > b/modules/video_filter/egl_pbuffer.c new file mode 100644
> > index 0000000000..4a9c2da8c1
> > --- /dev/null
> > +++ b/modules/video_filter/egl_pbuffer.c
> > @@ -0,0 +1,470 @@
> > +/**************************************************************************
> > *** + * egl_pbuffer.c: OpenGL filter in EGL offscreen framebuffer
> > +
> > ***************************************************************************
> > ** + * Copyright (C) 2020 Videolabs
> > + *
> > + * This program is free software; you can redistribute it and/or modify it
> > + * under the terms of the GNU Lesser General Public License as published by
> > + * the Free Software Foundation; either version 2.1 of the License, or + *
> > (at your option) any later version.
> > + *
> > + * This program is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > + * GNU Lesser General Public License for more details.
> > + *
> > + * You should have received a copy of the GNU Lesser General Public License
> > + * along with this program; if not, write to the Free Software Foundation,
> > + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. +
> > ***************************************************************************
> > **/ +#ifdef HAVE_CONFIG_H
> > +# include <config.h>
> > +#endif
> > +
> > +#include <vlc_common.h>
> > +#include <vlc_plugin.h>
> > +#include <vlc_modules.h>
> > +#include <vlc_picture.h>
> > +#include <vlc_filter.h>
> > +#include <vlc_opengl.h>
> > +#include <vlc_vout_display.h>
> > +#include <vlc_atomic.h>
> > +#include "../video_output/opengl/vout_helper.h"
> > +#include "../video_output/opengl/filters.h"
> > +#include "../video_output/opengl/gl_api.h"
> > +#include "../video_output/opengl/gl_common.h"
> > +#include "../video_output/opengl/interop.h"
> > +#include <EGL/egl.h>
> > +#include <EGL/eglext.h>
> > +
> > +#define BUFFER_COUNT 4
> > +
> > +struct pbo_picture_context
> > +{
> > +    struct picture_context_t context;
> > +    void *buffer_mapping;
> > +    int rc;
> > +    vlc_mutex_t *lock;
> > +    vlc_cond_t *cond;
> > +};
> > +
> > +struct vlc_gl_pbuffer
> > +{
> > +    vlc_gl_t                *gl;
> > +    vlc_mutex_t             lock;
> > +    vlc_cond_t              cond;
> > +
> > +    video_format_t          fmt_out;
> > +
> > +    struct vlc_gl_api api;
> > +
> > +    size_t                  current_flip;
> > +    GLuint                  pixelbuffers[BUFFER_COUNT];
> > +    GLuint                  framebuffers[BUFFER_COUNT];
> > +    GLuint                  textures[BUFFER_COUNT];
> > +    struct pbo_picture_context     picture_contexts[BUFFER_COUNT];
> > +
> > +    EGLDisplay display;
> > +    EGLSurface surface;
> > +    EGLContext context;
> > +
> > +    PFNEGLCREATEIMAGEKHRPROC    eglCreateImageKHR;
> > +    PFNEGLDESTROYIMAGEKHRPROC   eglDestroyImageKHR;
> > +
> > +    bool current;
> > +};
> > +
> > +static int MakeCurrent (vlc_gl_t *gl)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    assert(!sys->current);
> > +    if (eglMakeCurrent (sys->display, sys->surface, sys->surface,
> > +                        sys->context) != EGL_TRUE)
> > +        return VLC_EGENERIC;
> > +
> > +    sys->current = true;
> > +    return VLC_SUCCESS;
> > +}
> > +
> > +static void ReleaseCurrent (vlc_gl_t *gl)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    assert(sys->current);
> > +    eglMakeCurrent (sys->display, EGL_NO_SURFACE, EGL_NO_SURFACE,
> > +                    EGL_NO_CONTEXT);
> > +
> > +    sys->current = false;
> > +}
> > +
> > +static void *GetSymbol(vlc_gl_t *gl, const char *procname)
> > +{
> > +    (void) gl;
> > +    return (void *)eglGetProcAddress (procname);
> > +}
> > +
> > +static const char *QueryString(vlc_gl_t *gl, int32_t name)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    return eglQueryString(sys->display, name);
> > +}
> > +
> > +static void *CreateImageKHR(vlc_gl_t *gl, unsigned target, void *buffer,
> > +                            const int32_t *attrib_list)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    return sys->eglCreateImageKHR(sys->display, NULL, target, buffer,
> > +                                  attrib_list);
> > +}
> > +
> > +static bool DestroyImageKHR(vlc_gl_t *gl, void *image)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    return sys->eglDestroyImageKHR(sys->display, image);
> > +}
> > +
> > +static int InitEGL(vlc_gl_t *gl, unsigned width, unsigned height)
> > +{
> > +    struct vlc_gl_pbuffer *sys = gl->sys;
> > +
> > +    sys->display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
> > +    if (sys->display == EGL_NO_DISPLAY)
> > +        return VLC_EGENERIC;
>
> This looks rather suspicious, as it seems like it depends on a real display.
> As such it would presumably fail in a headless cases, or if the display type
> is mismatched (e.g. Wayland when X11 is expected).

Later in the work, I use the GBM platform to provide surfaceless
context in a less suspicious and headless-compatible way, but you're
right on the fact it is suspicious. Especially this would not work
with multiple players using the gl implementation on non-android
since eglTerminate from one filter will kill both filters.

The correct solution would be to open a new display session, which
is possible for both wayland and X11 but would mean that we link
both wayland and X11 or generate the plugin twice. It could be an
acceptable solution though.

There's no failure between wayland and X11 though, we can run this
implementation with wayland as WSI and X11 as pbuffer provider,
since the pbuffer is not even used for rendering, and the output
of this implementation is raw buffer.

Regards,
--
Alexandre Janniaux
Videolabs