[vlc-devel] [PATCH 4/7] qt: allow video to be sent to a detached windows with DComp

[Pierre Lamot]

---
 .../gui/qt/maininterface/compositor_dcomp.cpp | 94 +++++++++++++++++--
 .../gui/qt/maininterface/compositor_dcomp.hpp |  8 ++
 2 files changed, 92 insertions(+), 10 deletions(-)

diff --git a/modules/gui/qt/maininterface/compositor_dcomp.cpp b/modules/gui/qt/maininterface/compositor_dcomp.cpp
index 1b0b3d6517..06dedc1a4b 100644
--- a/modules/gui/qt/maininterface/compositor_dcomp.cpp
+++ b/modules/gui/qt/maininterface/compositor_dcomp.cpp
@@ -29,6 +29,7 @@
 #include <QApplication>
 #include <QDesktopWidget>
 #include <QQuickWidget>
+#include <QScreen>
 
 #include <QOpenGLFunctions>
 #include <QOpenGLFramebufferObject>
@@ -57,9 +58,9 @@ int CompositorDirectComposition::window_enable(struct vout_window_t * p_wnd, con
 
     try
     {
+        //default new window to EMBED for now
         that->m_qmlVideoSurfaceProvider->enable(p_wnd);
-        HR(that->m_rootVisual->AddVisual(that->m_videoVisual.Get(), FALSE, that->m_uiVisual.Get()), "add video visual to root");
-        HR(that->m_dcompDevice->Commit(), "commit");
+        that->setVideoSurface(VideoSurfaceTarget::EMBED);
     }
     catch (const DXError& err)
     {
@@ -74,11 +75,10 @@ void CompositorDirectComposition::window_disable(struct vout_window_t * p_wnd)
     CompositorDirectComposition* that = static_cast<CompositorDirectComposition*>(p_wnd->sys);
     try
     {
+        msg_Dbg(that->m_intf, "window_disable");
+        that->setVideoSurface(VideoSurfaceTarget::UNDEF);
         that->m_qmlVideoSurfaceProvider->disable();
         that->m_videoWindowHandler->disable();
-        msg_Dbg(that->m_intf, "window_disable");
-        HR(that->m_rootVisual->RemoveVisual(that->m_videoVisual.Get()), "remove video visual from root");
-        HR(that->m_dcompDevice->Commit(), "commit");
     }
     catch (const DXError& err)
     {
@@ -123,7 +123,7 @@ void CompositorDirectComposition::window_set_fullscreen(struct vout_window_t * p
 }
 
 CompositorDirectComposition::CompositorDirectComposition( intf_thread_t* p_intf,  QObject *parent)
-    : QObject(parent)
+    : Compositor(parent)
     , m_intf(p_intf)
 {
 }
@@ -309,19 +309,93 @@ bool CompositorDirectComposition::setupVoutWindow(vout_window_t *p_wnd)
 
 Compositor::VideoSurfaceTarget CompositorDirectComposition::getSupportedVideoSurfaceModes() const
 {
-    return VideoSurfaceTarget::UNDEF;
+    return static_cast<VideoSurfaceTarget>(VideoSurfaceTarget::EMBED | VideoSurfaceTarget::WINDOWED | VideoSurfaceTarget::FULLSCREEN);
 }
 
 void CompositorDirectComposition::setVideoSurface(VideoSurfaceTarget target, QVariant targetParam)
 {
-    //N/A
+    if (!m_videoVisual)
+        return; //no video to attach
+
+    try {
+
+    switch (m_currentVideoSurface)
+    {
+        case VideoSurfaceTarget::EMBED:
+            //no longer respond to events from the QML surface
+            m_qmlVideoSurfaceProvider->setVideoEmbed(false);
+            m_videoWindowHandler->setWindow(nullptr);
+            HR(m_rootVisual->RemoveVisual(m_videoVisual.Get()), "remove video visual from root");
+            msg_Dbg(m_intf, "disable EMBED video surface");
+        break;
+        case VideoSurfaceTarget::FULLSCREEN:
+        case VideoSurfaceTarget::WINDOWED:
+            m_videoWindowHandler->setWindow(nullptr);
+            HR(m_dcompDetachedVideoTarget->SetRoot(nullptr), "remove root video visual");
+            m_dcompDetachedVideoTarget.Reset();
+            m_detachedVideoWindow.reset();
+            msg_Dbg(m_intf, "disable WINDOWED video surface");
+        break;
+        default:
+        //N/A
+        break;
+
+    }
+
+    switch (target)
+    {
+        case VideoSurfaceTarget::EMBED:
+            HR(m_rootVisual->AddVisual(m_videoVisual.Get(), FALSE, m_uiVisual.Get()), "add video visual to root");
+            m_videoWindowHandler->setWindow(m_rootWindow->windowHandle());
+            m_qmlVideoSurfaceProvider->setVideoEmbed(true);
+            msg_Dbg(m_intf, "enable EMBED video surface");
+        break;
+        case VideoSurfaceTarget::FULLSCREEN:
+        case VideoSurfaceTarget::WINDOWED:
+        {
+            m_detachedVideoWindow = std::make_unique<DetachedVideoWindow>(m_intf, m_window);
+            m_detachedVideoWindow->winId();
+            HR(m_dcompDevice->CreateTargetForHwnd((HWND)m_detachedVideoWindow->winId(), TRUE, &m_dcompDetachedVideoTarget), "create target");
+            HR(m_dcompDetachedVideoTarget->SetRoot(m_videoVisual.Get()), "root video visual");
+            m_videoWindowHandler->setWindow(m_detachedVideoWindow->windowHandle());
+            connect(m_detachedVideoWindow.get(), &DetachedVideoWindow::windowClosed,
+                    this, [this]() {
+                this->setVideoSurface(vlc::Compositor::VideoSurfaceTarget::EMBED);
+            }, Qt::QueuedConnection);
+            if (target == VideoSurfaceTarget::FULLSCREEN)
+            {
+                QString screename = targetParam.toString();
+                m_detachedVideoWindow->showFSOnScreen(screename, m_rootWindow->windowHandle());
+            }
+            else
+            {
+                m_detachedVideoWindow->show();
+            }
+            msg_Dbg(m_intf, "enable WINDOWED video surface");
+            break;
+        }
+        default:
+        //N/A
+        break;
+    }
+
+    HR(m_dcompDevice->Commit(), "commit");
+    m_currentVideoSurface = target;
+    m_currentVideoSurfaceParam = targetParam;
+    emit videoSurfaceChanged(m_currentVideoSurface, m_currentVideoSurfaceParam);
+
+} catch ( const DXError& err)
+{
+    msg_Err(m_intf, "failed to change video surface: %s, code 0x%lX", err.what(), err.code());
+}
 }
 
 Compositor::VideoSurfaceTarget CompositorDirectComposition::getVideoSurface(QVariant* param)
 {
     if (param)
-        *param = QVariant();
-    return VideoSurfaceTarget::UNDEF;
+        *param = m_currentVideoSurfaceParam;
+    return m_currentVideoSurface;
 }
 
+
 }
diff --git a/modules/gui/qt/maininterface/compositor_dcomp.hpp b/modules/gui/qt/maininterface/compositor_dcomp.hpp
index 47ab504fb2..ec54a97ea9 100644
--- a/modules/gui/qt/maininterface/compositor_dcomp.hpp
+++ b/modules/gui/qt/maininterface/compositor_dcomp.hpp
@@ -31,6 +31,7 @@
 #include "videosurface.hpp"
 #include "interface_window_handler.hpp"
 #include "video_window_handler.hpp"
+#include "detached_video_window.hpp"
 
 #include <QOpenGLContext>
 
@@ -83,6 +84,13 @@ private:
     Microsoft::WRL::ComPtr<IDCompositionVisual> m_rootVisual;
     Microsoft::WRL::ComPtr<IDCompositionVisual> m_uiVisual;
     Microsoft::WRL::ComPtr<IDCompositionVisual> m_videoVisual;
+
+    VideoSurfaceTarget m_currentVideoSurface = VideoSurfaceTarget::UNDEF;
+    QVariant m_currentVideoSurfaceParam;
+
+    //detached video window
+    std::unique_ptr<DetachedVideoWindow> m_detachedVideoWindow;
+    Microsoft::WRL::ComPtr<IDCompositionTarget> m_dcompDetachedVideoTarget;
 };
 
 }
-- 
2.25.1