[vlc-devel] [PATCH] d3d11_player: use a different ID3D11Device between the app and VLC

[Steve Lhomme]

The texture we render to is shared between the 2 devices and locked internally
by the driver when needed.
---
 doc/libvlc/d3d11_player.cpp | 55 ++++++++++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 13 deletions(-)

diff --git a/doc/libvlc/d3d11_player.cpp b/doc/libvlc/d3d11_player.cpp
index a94fe793f3d..2ae4967f5e4 100644
--- a/doc/libvlc/d3d11_player.cpp
+++ b/doc/libvlc/d3d11_player.cpp
@@ -4,6 +4,9 @@
 #include <d3d11.h>
 #include <d3dcompiler.h>
 
+#include <d3d11_1.h>
+#include <dxgi1_2.h>
+
 #include <vlc/vlc.h>
 
 #define SCREEN_WIDTH  1500
@@ -15,6 +18,13 @@
 
 struct render_context
 {
+    /* resources shared by VLC */
+    ID3D11Device            *d3deviceVLC;
+    ID3D11DeviceContext     *d3dctxVLC;
+    ID3D11Texture2D         *textureVLC; // shared between VLC and the app
+    HANDLE                  sharedHandled; // handle of the texture used by VLC and the app
+    ID3D11RenderTargetView  *textureRenderTarget;
+
     /* Direct3D11 device/context */
     ID3D11Device        *d3device;
     ID3D11DeviceContext *d3dctx;
@@ -38,7 +48,6 @@ struct render_context
     /* texture VLC renders into */
     ID3D11Texture2D          *texture;
     ID3D11ShaderResourceView *textureShaderInput;
-    ID3D11RenderTargetView   *textureRenderTarget;
 
     CRITICAL_SECTION sizeLock; // the ReportSize callback cannot be called during/after the Cleanup_cb is called
     unsigned width, height;
@@ -59,6 +68,11 @@ static bool UpdateOutput_cb( void *opaque, const libvlc_video_direct3d_cfg_t *cf
         ctx->texture->Release();
         ctx->texture = NULL;
     }
+    if (ctx->textureVLC)
+    {
+        ctx->textureVLC->Release();
+        ctx->textureVLC = NULL;
+    }
     if (ctx->textureShaderInput)
     {
         ctx->textureShaderInput->Release();
@@ -82,10 +96,21 @@ static bool UpdateOutput_cb( void *opaque, const libvlc_video_direct3d_cfg_t *cf
     texDesc.Format = renderFormat;
     texDesc.Height = cfg->height;
     texDesc.Width  = cfg->width;
+    texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED | D3D11_RESOURCE_MISC_SHARED_NTHANDLE;
 
     hr = ctx->d3device->CreateTexture2D( &texDesc, NULL, &ctx->texture );
     if (FAILED(hr)) return false;
 
+    IDXGIResource1* sharedResource = NULL;
+    ctx->texture->QueryInterface(__uuidof(IDXGIResource1), (LPVOID*) &sharedResource);
+    hr = sharedResource->CreateSharedHandle(NULL, DXGI_SHARED_RESOURCE_READ, NULL, &ctx->sharedHandled);
+    sharedResource->Release();
+
+    ID3D11Device1* d3d11VLC1;
+    ctx->d3deviceVLC->QueryInterface(__uuidof(ID3D11Device1), (LPVOID*) &d3d11VLC1);
+    hr = d3d11VLC1->OpenSharedResource1(ctx->sharedHandled, __uuidof(ID3D11Texture2D), (void**)&ctx->textureVLC);
+    d3d11VLC1->Release();
+
     D3D11_SHADER_RESOURCE_VIEW_DESC resviewDesc;
     ZeroMemory(&resviewDesc, sizeof(D3D11_SHADER_RESOURCE_VIEW_DESC));
     resviewDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
@@ -98,7 +123,7 @@ static bool UpdateOutput_cb( void *opaque, const libvlc_video_direct3d_cfg_t *cf
         .Format = texDesc.Format,
         .ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D,
     };
-    hr = ctx->d3device->CreateRenderTargetView(ctx->texture, &renderTargetViewDesc, &ctx->textureRenderTarget);
+    hr = ctx->d3deviceVLC->CreateRenderTargetView(ctx->textureVLC, &renderTargetViewDesc, &ctx->textureRenderTarget);
     if (FAILED(hr)) return false;
 
 
@@ -162,14 +187,7 @@ static bool StartRendering_cb( void *opaque, bool enter, const libvlc_video_dire
     if ( enter )
     {
         static const FLOAT blackRGBA[4] = {0.5f, 0.5f, 0.0f, 1.0f};
-
-        /* force unbinding the input texture, otherwise we get:
-         * OMSetRenderTargets: Resource being set to OM RenderTarget slot 0 is still bound on input! */
-        ID3D11ShaderResourceView *reset = NULL;
-        ctx->d3dctx->PSSetShaderResources(0, 1, &reset);
-        //ctx->d3dctx->Flush();
-
-        ctx->d3dctx->ClearRenderTargetView( ctx->textureRenderTarget, blackRGBA);
+        ctx->d3dctxVLC->ClearRenderTargetView( ctx->textureRenderTarget, blackRGBA);
         return true;
     }
 
@@ -182,14 +200,14 @@ static bool SelectPlane_cb( void *opaque, size_t plane )
     struct render_context *ctx = static_cast<struct render_context *>( opaque );
     if ( plane != 0 ) // we only support one packed RGBA plane (DXGI_FORMAT_R8G8B8A8_UNORM)
         return false;
-    ctx->d3dctx->OMSetRenderTargets( 1, &ctx->textureRenderTarget, NULL );
+    ctx->d3dctxVLC->OMSetRenderTargets( 1, &ctx->textureRenderTarget, NULL );
     return true;
 }
 
 static bool Setup_cb( void **opaque, const libvlc_video_direct3d_device_cfg_t *cfg, libvlc_video_direct3d_device_setup_t *out )
 {
     struct render_context *ctx = static_cast<struct render_context *>(*opaque);
-    out->device_context = ctx->d3dctx;
+    out->device_context = ctx->d3dctxVLC;
     return true;
 }
 
@@ -275,7 +293,7 @@ static void init_direct3d(struct render_context *ctx, HWND hWnd)
     scd.Windowed = TRUE;
     scd.Flags = DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH;
 
-    UINT creationFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT; /* needed for hardware decoding */
+    UINT creationFlags = 0;
 #ifndef NDEBUG
     creationFlags |= D3D11_CREATE_DEVICE_DEBUG;
 #endif
@@ -301,6 +319,14 @@ static void init_direct3d(struct render_context *ctx, HWND hWnd)
         pMultithread->Release();
     }
 
+    D3D11CreateDevice(NULL,
+                      D3D_DRIVER_TYPE_HARDWARE,
+                      NULL,
+                      creationFlags | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, /* needed for hardware decoding */
+                      NULL, 0,
+                      D3D11_SDK_VERSION,
+                      &ctx->d3deviceVLC, NULL, &ctx->d3dctxVLC);
+
     ID3D11Texture2D *pBackBuffer;
     ctx->swapchain->GetBuffer(0, __uuidof(ID3D11Texture2D), (LPVOID*)&pBackBuffer);
 
@@ -384,6 +410,9 @@ static void init_direct3d(struct render_context *ctx, HWND hWnd)
 
 static void release_direct3d(struct render_context *ctx)
 {
+    ctx->d3deviceVLC->Release();
+    ctx->d3dctxVLC->Release();
+
     ctx->samplerState->Release();
     ctx->textureRenderTarget->Release();
     ctx->textureShaderInput->Release();
-- 
2.19.1.windows.1