[vlc-commits] [Git][videolan/vlc][3.0.x] mediacodec: validate crop values to prevent integer overflow

[Steve Lhomme (@robUx4)]

Steve Lhomme pushed to branch 3.0.x at VideoLAN / VLC


Commits:
77befcc4 by Martin Finkel at 2026-05-03T03:06:16+00:00
mediacodec: validate crop values to prevent integer overflow

Backport of master commit 12500a0c95deb2b3375aee39e0fedbf330627262
adapted to 3.0.x (no fmt_out i_x_offset/i_y_offset fields).

- - - - -


1 changed file:

- modules/codec/omxil/mediacodec.c


Changes:

=====================================
modules/codec/omxil/mediacodec.c
=====================================
@@ -1044,14 +1044,34 @@ static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
                 p_out->conf.video.crop_left, p_out->conf.video.crop_top,
                 p_out->conf.video.crop_right, p_out->conf.video.crop_bottom);
 
-        int i_width  = p_out->conf.video.crop_right + 1
+        bool valid_crop =
+            p_out->conf.video.crop_left >= 0 &&
+            p_out->conf.video.crop_top >= 0 &&
+            p_out->conf.video.crop_right >= p_out->conf.video.crop_left &&
+            p_out->conf.video.crop_bottom >= p_out->conf.video.crop_top &&
+            p_out->conf.video.crop_right < INT_MAX &&
+            p_out->conf.video.crop_bottom < INT_MAX &&
+            (unsigned)p_out->conf.video.crop_right < p_out->conf.video.width &&
+            (unsigned)p_out->conf.video.crop_bottom < p_out->conf.video.height;
+
+        int i_width, i_height;
+        if (valid_crop)
+        {
+            i_width  = p_out->conf.video.crop_right + 1
                      - p_out->conf.video.crop_left;
-        int i_height = p_out->conf.video.crop_bottom + 1
+            i_height = p_out->conf.video.crop_bottom + 1
                      - p_out->conf.video.crop_top;
+        }
+        else
+        {
+            i_width = p_out->conf.video.width;
+            i_height = p_out->conf.video.height;
+        }
         if (i_width <= 1 || i_height <= 1)
         {
             i_width = p_out->conf.video.width;
             i_height = p_out->conf.video.height;
+            valid_crop = false;
         }
 
         if (!(p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_IGNORE_SIZE))
@@ -1077,8 +1097,13 @@ static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
         if (p_sys->video.i_slice_height <= 0)
             p_sys->video.i_slice_height = p_out->conf.video.height;
 
-        if (p_sys->video.i_pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
-            p_sys->video.i_slice_height -= p_out->conf.video.crop_top/2;
+        if (p_sys->video.i_pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar
+            && valid_crop)
+        {
+            unsigned int crop_adj = p_out->conf.video.crop_top / 2;
+            if (crop_adj < p_sys->video.i_slice_height)
+                p_sys->video.i_slice_height -= crop_adj;
+        }
         if ((p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_IGNORE_PADDING))
         {
             p_sys->video.i_slice_height = 0;



View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/77befcc4ff8ca64ba3837732dae3dca44473ce4a

-- 
View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/77befcc4ff8ca64ba3837732dae3dca44473ce4a
You're receiving this email because of your account on code.videolan.org.