[vlc-commits] [Git][videolan/vlc][master] 4 commits: access/vnc: fix endianness handling

[Steve Lhomme (@robUx4)]

Steve Lhomme pushed to branch master at VideoLAN / VLC


Commits:
99948ec4 by Jeffrey Knockel at 2024-02-14T13:24:13+00:00
access/vnc: fix endianness handling

Outside of keeping bits per pixel consistent, we never made any attempt
to match the server's native pixel format in the first place, always
requesting our own hardcoded format for any color depth, so we might as
well also always ask the server for LE byte order (regardless of client
endianness) and code around that.  This change has been tested with LE
and BE clients.  Previous to this commit colors were wrong in the case
of (e.g.) LE <-> LE clients and servers.

- - - - -
3c2c46a7 by Jeffrey Knockel at 2024-02-14T13:24:13+00:00
access/vnc: set trueColour flag

In the unlikely event that the server's native pixel format is not true
color, this flag will be received as zero, and so we cannot assume that
it has already been set.  Since we only support true color pixel
formats, we therefore need to ensure that the flag is set ourselves.

- - - - -
032fdd30 by Jeffrey Knockel at 2024-02-14T13:24:13+00:00
access/vnc: upgrade color depth for tight encoding

libvnc does not support decoding 8 bpp + tight encoding, so upgrade the
requested color depth to 16 to prevent libvnc returning an error when we
try to decode.

- - - - -
bf913aaf by Jeffrey Knockel at 2024-02-14T13:24:13+00:00
access/vnc: do not conflate depth with bpp

We should not assume that the depth is the bits per pixel.  The RFB spec
says that bpp must be either 8, 16, or 32 and that depth must be <= to
bpp.

- - - - -


1 changed file:

- modules/access/vnc.c


Changes:

=====================================
modules/access/vnc.c
=====================================
@@ -152,30 +152,30 @@ static rfbBool mallocFrameBufferHandler( rfbClient* p_client )
     assert(!(p_client->height & ~0xffff)); // fits in 16 bits
     uint16_t i_height = p_client->height;
 
-    uint8_t i_bits_per_pixel = p_client->format.bitsPerPixel;
-    assert((i_bits_per_pixel & 0x7) == 0); // multiple of 8
+    if (p_client->format.depth <= 8 &&
+            !strstr(p_client->appData.encodingsString, "tight")) // libvnc does not support tight at 8 bpp
+        i_chroma = VLC_CODEC_BGR233;
+    else if (p_client->format.depth <= 16)
+        i_chroma = VLC_CODEC_RGB565LE;
+    else // skip 24 bpp pixfmt since rfb protocol forbids 24 bpp pixfmts
+        i_chroma = VLC_CODEC_BGRX;
 
-    switch( i_bits_per_pixel )
-    {
-        case 8:
-            i_chroma = VLC_CODEC_RGB233;
-            break;
-        default:
-        case 16:
-            i_chroma = p_client->format.bigEndian ? VLC_CODEC_RGB565BE : VLC_CODEC_RGB565LE;
-            break;
-        case 24:
-            i_chroma = VLC_CODEC_RGB24;
-            break;
-        case 32:
-            i_chroma = VLC_CODEC_XRGB;
-            break;
-    }
+    p_client->format.bigEndian = 0; // we expect LE byte order regardless of native endianness
+    p_client->format.trueColour = 1; // we do not support color maps
 
     switch( i_chroma )
     {
-        case VLC_CODEC_RGB565BE:
+        case VLC_CODEC_BGR233:
+            p_client->format.depth = p_client->format.bitsPerPixel = 8;
+            p_client->format.redShift   =  0;
+            p_client->format.greenShift =  3;
+            p_client->format.blueShift  =  6;
+            p_client->format.redMax     = 0x7;
+            p_client->format.greenMax   = 0x7;
+            p_client->format.blueMax    = 0x3;
+            break;
         case VLC_CODEC_RGB565LE:
+            p_client->format.depth = p_client->format.bitsPerPixel = 16;
             p_client->format.redShift   = 11;
             p_client->format.greenShift =  5;
             p_client->format.blueShift  =  0;
@@ -183,8 +183,9 @@ static rfbBool mallocFrameBufferHandler( rfbClient* p_client )
             p_client->format.greenMax   = 0x3f;
             p_client->format.blueMax    = 0x1f;
             break;
-        case VLC_CODEC_RGB24:
-        case VLC_CODEC_XRGB:
+        case VLC_CODEC_BGRX:
+            p_client->format.depth = 24;
+            p_client->format.bitsPerPixel = 32;
             p_client->format.redShift   = 16;
             p_client->format.greenShift =  8;
             p_client->format.blueShift  =  0;
@@ -195,7 +196,7 @@ static rfbBool mallocFrameBufferHandler( rfbClient* p_client )
     }
 
     /* Set up framebuffer */
-    if (mul_overflow(i_width, i_height * (i_bits_per_pixel / 8), &p_sys->i_framebuffersize)) {
+    if (mul_overflow(i_width, i_height * (p_client->format.bitsPerPixel / 8), &p_sys->i_framebuffersize)) {
         msg_Err(p_demux, "VNC framebuffersize overflow");
         return FALSE;
     }



View it on GitLab: https://code.videolan.org/videolan/vlc/-/compare/29fbfc0d75e361293ed8c86b837ac4c761f0839a...bf913aaf9c525c45b55a10674d7739f7b4a253d9

-- 
View it on GitLab: https://code.videolan.org/videolan/vlc/-/compare/29fbfc0d75e361293ed8c86b837ac4c761f0839a...bf913aaf9c525c45b55a10674d7739f7b4a253d9
You're receiving this email because of your account on code.videolan.org.


VideoLAN code repository instance