[vlc-devel] [PATCH 11/48] video_output: do the format cleaning in vout_Request and vout_GetDevice

[Steve Lhomme]

vout_GetDevice doesn't need to check the DPB, it only needs to get a decoder
device from a usable window. Only the display needs to DPB size (for now, since
it creates the decoder pool from it).

Only stop the current display if a new one is going to be created (otherwise
the current one is fine or there is none because there's no window).

In vout_GetDevice we don't care about the current display. So we don't need to
use sys->original, we use a locally cleaned video format instead to match what will
be used in the following call to vout_Request.

Try to enable the window before checking if the original format matches. If the
window is not enabled, the sys->original doesn't making sense anyway.
---
 src/video_output/video_output.c | 67 ++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 30 deletions(-)

diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index c4744037f99..3b6a733272f 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -1943,7 +1943,8 @@ vout_thread_t *vout_Hold(vout_thread_t *vout)
     return vout;
 }
 
-static int vout_EnableWindow(const vout_configuration_t *cfg, vlc_decoder_device **pp_dec_device)
+static int vout_EnableWindow(const vout_configuration_t *cfg, const video_format_t *original,
+                             vlc_decoder_device **pp_dec_device)
 {
     vout_thread_t *vout = cfg->vout;
     vout_thread_sys_t *sys = vout->p;
@@ -1952,30 +1953,6 @@ static int vout_EnableWindow(const vout_configuration_t *cfg, vlc_decoder_device
     assert(vout != NULL);
     assert(cfg->clock != NULL);
 
-    video_format_t original;
-    VoutFixFormat(&original, cfg->fmt);
-
-    /* TODO: If dimensions are equal or slightly smaller, update the aspect
-     * ratio and crop settings, instead of recreating a display.
-     */
-    if (video_format_IsSimilar(&original, &sys->original)) {
-        if (cfg->dpb_size <= sys->dpb_size) {
-            video_format_Clean(&original);
-            /* It is assumed that the SPU input matches input already. */
-            if (pp_dec_device)
-                *pp_dec_device = sys->dec_device ? vlc_decoder_device_Hold( sys->dec_device ) : NULL;
-            return 0;
-        }
-        msg_Warn(vout, "DPB need to be increased");
-    }
-
-    if (sys->display != NULL)
-        vout_StopDisplay(vout);
-
-    vout_ReinitInterlacingSupport(vout);
-
-    sys->original = original;
-
     vlc_mutex_lock(&sys->window_lock);
     if (!sys->window_enabled) {
         vout_window_cfg_t wcfg = {
@@ -1988,13 +1965,12 @@ static int vout_EnableWindow(const vout_configuration_t *cfg, vlc_decoder_device
 #endif
         };
 
-        VoutGetDisplayCfg(vout, &original, &sys->display_cfg);
-        vout_SizeWindow(vout, &original, &wcfg.width, &wcfg.height);
+        VoutGetDisplayCfg(vout, original, &sys->display_cfg);
+        vout_SizeWindow(vout, original, &wcfg.width, &wcfg.height);
 
         if (vout_window_Enable(sys->display_cfg.window, &wcfg)) {
             vlc_mutex_unlock(&sys->window_lock);
             msg_Err(vout, "failed to enable window");
-            video_format_Clean(&sys->original);
             return -1;
         }
         sys->window_enabled = true;
@@ -2022,8 +1998,34 @@ int vout_Request(const vout_configuration_t *cfg, input_thread_t *input)
         /* don't stop the display and keep sys->original */
         return -1;
 
-    if (vout_EnableWindow(cfg, NULL) != 0)
+    video_format_t original;
+    VoutFixFormat(&original, cfg->fmt);
+
+    /* TODO: If dimensions are equal or slightly smaller, update the aspect
+     * ratio and crop settings, instead of recreating a display.
+     */
+    if (video_format_IsSimilar(&original, &sys->original)) {
+        if (cfg->dpb_size <= sys->dpb_size) {
+            video_format_Clean(&original);
+            return 0;
+        }
+        msg_Warn(vout, "DPB need to be increased");
+    }
+
+    if (vout_EnableWindow(cfg, &original, NULL) != 0)
+    {
+        /* the window was not enabled, nor the display started */
+        msg_Err(vout, "failed to enable window");
+        video_format_Clean(&original);
         return -1;
+    }
+
+    if (sys->display != NULL)
+        vout_StopDisplay(vout);
+
+    vout_ReinitInterlacingSupport(vout);
+
+    sys->original = original;
 
     sys->delay = 0;
     sys->rate = 1.f;
@@ -2058,7 +2060,12 @@ vlc_decoder_device *vout_GetDevice(const vout_configuration_t *cfg)
     if (!VoutCheckFormat(cfg->fmt))
         return NULL;
 
-    if (vout_EnableWindow(cfg, &dec_device) != 0)
+    video_format_t original;
+    VoutFixFormat(&original, cfg->fmt);
+
+    int res = vout_EnableWindow(cfg, &original, &dec_device);
+    video_format_Clean(&original);
+    if (res != 0)
         return NULL;
     return dec_device;
 }
-- 
2.17.1