[vlc-devel] [PATCH 7/9] direct3d11: handle external callbacks to render into an external surface

[Steve Lhomme]

Instead of a swapchain we create.
---
 modules/video_output/win32/direct3d11.c | 58 ++++++++++++++++++-------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index 2dc41eabfc..5663070d33 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -123,6 +123,8 @@ struct vout_display_sys_t
     void (*endRenderCb)(void* opaque);
     bool (*starRenderCb)(void* opaque);
     bool (*resizeCb)(void* opaque, unsigned, unsigned);
+    void *(*startCb)(void* opaque);
+    void (*finishCb)(void* opaque);
 };
 
 #define RECTWidth(r)   (int)((r).right - (r).left)
@@ -384,19 +386,13 @@ static int Open(vout_display_t *vd, const vout_display_cfg_t *cfg,
     if (ret != VLC_SUCCESS)
         return ret;
 
-    ID3D11DeviceContext *d3d11_ctx = NULL;
-#if VLC_WINSTORE_APP
-    if (d3d11_ctx == NULL)
-        d3d11_ctx = var_InheritInteger(vd, "winrt-d3dcontext");
-#endif
-    if (CommonInit(vd, d3d11_ctx != NULL, cfg))
-        goto error;
-
-#if VLC_WINSTORE_APP
-    sys->sys.pf_GetRect = GetRect;
-#endif
-    sys->sys.pf_GetPictureWidth  = GetPictureWidth;
-    sys->sys.pf_GetPictureHeight = GetPictureHeight;
+    sys->outside_opaque = var_InheritAddress(vd, "vout-cb-opaque");
+    sys->swapCb         = var_InheritAddress(vd, "vout-cb-swap");
+    sys->starRenderCb   = var_InheritAddress(vd, "vout-cb-start-render");
+    sys->endRenderCb    = var_InheritAddress(vd, "vout-cb-end-render");
+    sys->resizeCb       = var_InheritAddress(vd, "vout-cb-update-output");
+    sys->startCb        = var_InheritAddress(vd, "vout-cb-setup");
+    sys->finishCb       = var_InheritAddress(vd, "vout-cb-cleanup");
 
     if (!sys->swapCb || !sys->starRenderCb || !sys->endRenderCb || !sys->resizeCb)
     {
@@ -407,6 +403,20 @@ static int Open(vout_display_t *vd, const vout_display_cfg_t *cfg,
         sys->resizeCb     = Resize;
     }
 
+    bool windowless = sys->startCb != NULL;
+#if VLC_WINSTORE_APP
+    if ( !windowless )
+        windowless = var_InheritInteger(vd, "winrt-d3dcontext") != 0; /* LEGACY */
+#endif
+    if (CommonInit(vd, windowless, cfg))
+        goto error;
+
+#if VLC_WINSTORE_APP
+    sys->sys.pf_GetRect = GetRect;
+#endif
+    sys->sys.pf_GetPictureWidth  = GetPictureWidth;
+    sys->sys.pf_GetPictureHeight = GetPictureHeight;
+
     if (Direct3D11Open(vd, fmtp)) {
         msg_Err(vd, "Direct3D11 could not be opened");
         assert(!vd->info.is_slow); /* vd->info was not modified */
@@ -890,7 +900,7 @@ static void PreparePicture(vout_display_t *vd, picture_t *picture, subpicture_t
     {
         D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, GetFormatLuminance(VLC_OBJECT(vd), &picture->format) / (float)sys->display.luminance_peak);
 
-        if (sys->dxgiswapChain4)
+        if (sys->dxgiswapChain4) /* TODO pass the HDR metadata through a callback in windowless mode */
         {
             DXGI_HDR_METADATA_HDR10 hdr10 = {0};
             hdr10.GreenPrimary[0] = picture->format.mastering.primaries[0];
@@ -1194,9 +1204,15 @@ static int Direct3D11Open(vout_display_t *vd, video_format_t *fmtp)
     DXGI_FORMAT windowlessFormat = DXGI_FORMAT_R8G8B8A8_UNORM;
 
     ID3D11DeviceContext *d3d11_ctx = NULL;
+    if ( sys->startCb != NULL )
+    {
+        d3d11_ctx = sys->startCb( sys->outside_opaque );
+        if ( d3d11_ctx == NULL)
+            return VLC_EGENERIC;
+    }
 #if VLC_WINSTORE_APP
     if (d3d11_ctx == NULL)
-        d3d11_ctx = var_InheritInteger(vd, "winrt-d3dcontext");
+        d3d11_ctx = var_InheritInteger(vd, "winrt-d3dcontext"); /* LEGACY */
     if (d3d11_ctx == NULL)
         return VLC_EGENERIC;
 #endif /* VLC_WINSTORE_APP */
@@ -1234,6 +1250,7 @@ static int Direct3D11Open(vout_display_t *vd, video_format_t *fmtp)
 #endif /* VLC_WINSTORE_APP */
     if (sys->sys.b_windowless)
     {
+        /* TODO query the output format using callbacks */
         for (const d3d_format_t *output_format = GetRenderFormatList();
              output_format->name != NULL; ++output_format)
         {
@@ -1298,10 +1315,16 @@ static int Direct3D11Open(vout_display_t *vd, video_format_t *fmtp)
            msg_Err(vd, "Could not create the SwapChain. (hr=0x%lX)", hr);
            return VLC_EGENERIC;
         }
+    }
+#endif /* !VLC_WINSTORE_APP */
 
+    if (sys->dxgiswapChain)
+    {
+        /* we still handle a swapchain in the legacy winrt mode even though
+         * we don't have a window
+         */
         IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain4, (void **)&sys->dxgiswapChain4);
     }
-#endif /* !VLC_WINSTORE_APP */
 
     D3D11SetColorSpace(vd);
 
@@ -1459,6 +1482,9 @@ static void Direct3D11Close(vout_display_t *vd)
 
     D3D11_ReleaseDevice( &sys->d3d_dev );
 
+    if ( sys->finishCb != NULL )
+        sys->finishCb( sys->outside_opaque );
+
     msg_Dbg(vd, "Direct3D11 device adapter closed");
 }
 
-- 
2.17.1