[vlc-devel] [PATCH] core-vout: increase maximum format size from 8k to 16k

[Victorien Le Couviour--Tuffet]

This allows the vout to handle more than 8k formats, and partially fixes the
issue #18215, as the vout failed to open due to this restriction. We now have
to check for the maximum size the HW can handle, and resize to it if bigger.
---
 src/video_output/video_output.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index 2f11ebeec3..9c862c0c9c 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -78,9 +78,14 @@ static void VoutDestructor(vlc_object_t *);
 static int VoutValidateFormat(video_format_t *dst,
                               const video_format_t *src)
 {
-    if (src->i_width == 0  || src->i_width  > 8192 ||
-        src->i_height == 0 || src->i_height > 8192)
+    int64_t max_res =
+        (int64_t)src->i_width * src->i_height * src->i_bits_per_pixel;
+
+    if (max_res > INT_MAX ||
+        src->i_width == 0 || src->i_width > 16384 ||
+        src->i_height == 0 || src->i_height > 16384)
         return VLC_EGENERIC;
+
     if (src->i_sar_num <= 0 || src->i_sar_den <= 0)
         return VLC_EGENERIC;
 
-- 
2.13.1