[vlc-devel] [PATCH] core-vout: increase maximum format size from 8k to 16k

[Victorien Le Couviour--Tuffet]

This allows the vout to handle more than 8k formats, and partially fixes the
issue #18215, as the vout failed to open due to this restriction. We now have
to check for the maximum size the HW can handle, and resize to it if bigger.
---
Forgot the include for INT_MAX, my bad.

 src/video_output/video_output.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index 2f11ebeec3..c76ef338c5 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -36,6 +36,7 @@
 
 #include <vlc_common.h>
 
+#include <limits.h>
 #include <stdlib.h>                                                /* free() */
 #include <string.h>
 #include <assert.h>
@@ -78,9 +79,14 @@ static void VoutDestructor(vlc_object_t *);
 static int VoutValidateFormat(video_format_t *dst,
                               const video_format_t *src)
 {
-    if (src->i_width == 0  || src->i_width  > 8192 ||
-        src->i_height == 0 || src->i_height > 8192)
+    int64_t max_res =
+        (int64_t)src->i_width * src->i_height * src->i_bits_per_pixel;
+
+    if (max_res > INT_MAX ||
+        src->i_width == 0 || src->i_width > 16384 ||
+        src->i_height == 0 || src->i_height > 16384)
         return VLC_EGENERIC;
+
     if (src->i_sar_num <= 0 || src->i_sar_den <= 0)
         return VLC_EGENERIC;
 
-- 
2.13.1