[vlc-devel] [PATCH 3/9] libvlc: the setup callback for video surfaces returns a pointer

[Steve Lhomme]

This will be necessary for Direct3D where we need the D3DDeviceContext from the
host to render into. We will need to keep the reference between setup_cb and
finish_cb calls.
---
 doc/libvlc/sdl_opengl_player.cpp  | 4 ++--
 include/vlc/libvlc_media_player.h | 7 +++++--
 modules/video_output/vgl.c        | 4 ++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/doc/libvlc/sdl_opengl_player.cpp b/doc/libvlc/sdl_opengl_player.cpp
index 901adafbe2..ef62dad04e 100644
--- a/doc/libvlc/sdl_opengl_player.cpp
+++ b/doc/libvlc/sdl_opengl_player.cpp
@@ -153,12 +153,12 @@ public:
     }
 
     // This callback is called during initialisation.
-    static bool setup(void* data)
+    static void *setup(void* data)
     {
         VLCVideo* that = static_cast<VLCVideo*>(data);
         that->m_width = 0;
         that->m_height = 0;
-        return true;
+        return that; // the pointer value doesn't matter as long as it's not NULL
     }
 
 
diff --git a/include/vlc/libvlc_media_player.h b/include/vlc/libvlc_media_player.h
index 715972ccdf..6f9eb0e59f 100644
--- a/include/vlc/libvlc_media_player.h
+++ b/include/vlc/libvlc_media_player.h
@@ -425,10 +425,13 @@ void libvlc_video_set_callbacks( libvlc_media_player_t *mp,
  * Callback prototype called to initialize user data.
  *
  * \param opaque private pointer passed to the @a libvlc_video_set_output_callbacks() [IN]
- * \return true on success
+ * \return NULL on failure or a context pointer that depends on the engine type
+ *
+ * For \ref libvlc_video_engine_opengl and \ref libvlc_video_engine_gles2 it's either NULL
+ * on failure to setup or any other value in success.
  * \version LibVLC 4.0.0 or later
  */
-typedef bool (*libvlc_video_setup_cb)(void* opaque);
+typedef void *(*libvlc_video_setup_cb)(void* opaque);
 
 
 /**
diff --git a/modules/video_output/vgl.c b/modules/video_output/vgl.c
index 7079985d2a..b9406fc7e2 100644
--- a/modules/video_output/vgl.c
+++ b/modules/video_output/vgl.c
@@ -32,7 +32,7 @@
 struct vout_display_sys_t
 {
     void (*cleanupCb)(void* opaque);
-    bool (*setupCb)(void* opaque);
+    void *(*setupCb)(void* opaque);
     bool (*resizeCb)(void* opaque, unsigned, unsigned);
     void (*swapCb)(void* opaque);
     bool (*makeCurrentCb)(void* opaque, bool);
@@ -128,7 +128,7 @@ static int Open(vlc_object_t *object)
     gl->getProcAddress = OurGetProcAddress;
 
     if( sys->setupCb )
-        if( !sys->setupCb(sys->opaque) )
+        if( sys->setupCb(sys->opaque) == NULL )
         {
             msg_Err( gl, "user setup failed" );
             return VLC_EGENERIC;
-- 
2.17.1