[vlc-devel] [PATCH 38/39] video_output: do the format cleaning in vout_Request and vout_GetDevice

[Steve Lhomme]

vout_GetDevice doesn't need to check the DPB, it only needs to get a decoder
device from a usable window. Only the display needs to DPB size (for now, since
it creates the decoder pool from it).

Only stop the current display if a new one is going to be created.
In vout_GetDevice we don't care about the current display.

use a locally cleaned video format in vout_GetDevice

No need to touch sys->original which is only used with the display module.
---
 src/video_output/video_output.c | 77 ++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 34 deletions(-)

diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index 005ed3006aa..a2ef6877871 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -1915,41 +1915,14 @@ vout_thread_t *vout_Hold(vout_thread_t *vout)
     return vout;
 }
 
-static int VoutEnableWindow(const vout_configuration_t *cfg, vlc_decoder_device **pp_dec_device)
+static int VoutEnableWindow(const vout_configuration_t *cfg, const video_format_t *original,
+                            vlc_decoder_device **pp_dec_device)
 {
     vout_thread_t *vout = cfg->vout;
     vout_thread_sys_t *sys = vout->p;
 
     assert(!sys->dummy);
     assert(vout != NULL);
-    assert(cfg->fmt != NULL);
-
-    if (!VoutCheckFormat(cfg->fmt))
-        return -1;
-
-    video_format_t original;
-    VoutFixFormat(&original, cfg->fmt);
-
-    /* TODO: If dimensions are equal or slightly smaller, update the aspect
-     * ratio and crop settings, instead of recreating a display.
-     */
-    if (video_format_IsSimilar(&original, &sys->original)) {
-        if (cfg->dpb_size <= sys->dpb_size) {
-            video_format_Clean(&original);
-            /* It is assumed that the SPU input matches input already. */
-            if (pp_dec_device)
-                *pp_dec_device = sys->dec_device ? vlc_decoder_device_Hold( sys->dec_device ) : NULL;
-            return 0;
-        }
-        msg_Warn(vout, "DPB need to be increased");
-    }
-
-    if (sys->display != NULL)
-        vout_StopDisplay(vout);
-
-    vout_ReinitInterlacingSupport(vout);
-
-    sys->original = original;
 
     vlc_mutex_lock(&sys->window_lock);
     if (!sys->window_enabled) {
@@ -1964,13 +1937,12 @@ static int VoutEnableWindow(const vout_configuration_t *cfg, vlc_decoder_device
         };
 
         VoutGetDisplayCfg(vout, &sys->display_cfg);
-        sys->display_cfg.viewpoint = sys->original.pose;
-        vout_SizeWindow(vout, &sys->original, &wcfg.width, &wcfg.height);
+        sys->display_cfg.viewpoint = original->pose;
+        vout_SizeWindow(vout, original, &wcfg.width, &wcfg.height);
 
         if (vout_window_Enable(sys->display_cfg.window, &wcfg)) {
             vlc_mutex_unlock(&sys->window_lock);
             msg_Err(vout, "window enabling failed");
-            video_format_Clean(&sys->original);
             return -1;
         }
         sys->window_enabled = true;
@@ -1993,10 +1965,36 @@ int vout_Request(const vout_configuration_t *cfg, vlc_decoder_device *dec_dev, i
     vout_thread_sys_t *sys = vout->p;
 
     assert(cfg->clock != NULL);
+    assert(cfg->fmt != NULL);
 
-    if (VoutEnableWindow(cfg, NULL) != 0)
+    if (!VoutCheckFormat(cfg->fmt))
         return -1;
 
+    video_format_t original;
+    VoutFixFormat(&original, cfg->fmt);
+
+    /* TODO: If dimensions are equal or slightly smaller, update the aspect
+     * ratio and crop settings, instead of recreating a display.
+     */
+    if (video_format_IsSimilar(&original, &sys->original)) {
+        if (cfg->dpb_size <= sys->dpb_size) {
+            video_format_Clean(&original);
+            /* It is assumed that the SPU input matches input already. */
+            return 0;
+        }
+        msg_Warn(vout, "DPB need to be increased");
+    }
+
+    if (VoutEnableWindow(cfg, &original, NULL) != 0)
+        return -1;
+
+    vout_ReinitInterlacingSupport(vout);
+
+    if (sys->display != NULL)
+        vout_StopDisplay(vout);
+
+    sys->original = original;
+
     sys->delay = 0;
     sys->rate = 1.f;
     sys->clock = cfg->clock;
@@ -2031,8 +2029,19 @@ error:
 
 vlc_decoder_device *vout_GetDevice(const vout_configuration_t *cfg)
 {
+    assert(cfg->fmt != NULL);
+
+    if (!VoutCheckFormat(cfg->fmt))
+        return NULL;
+
+    video_format_t original;
+    VoutFixFormat(&original, cfg->fmt);
+
     vlc_decoder_device *dec_device = NULL;
-    if (VoutEnableWindow(cfg, &dec_device) != 0)
+    int res = VoutEnableWindow(cfg, &original, &dec_device);
+    video_format_Clean(&original);
+    if (res != 0)
         return NULL;
+
     return dec_device;
 }
-- 
2.17.1