[Qemu-devel] [PATCH] vnc: Add support for color map

[Alexander Graf <agraf@suse.de>]

Our current VNC code does not handle color maps (aka non-true-color) at all
and aborts if a client requests them. There are 2 major issues with this:

 1) A VNC viewer on an 8-bit X11 system may request color maps
 2) RealVNC _always_ starts requesting color maps, then moves on to full color

In order to support these 2 use cases, let's just create a fake color map
that covers exactly our normal true color 8 bit color space. That way we don't
lose anything over a client that wants true color.

Reported-by: Sascha Wehnert <swehnert@suse.com>
Signed-off-by: Alexander Graf <agraf@suse.de>
---
 ui/vnc.c | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ui/vnc.c b/ui/vnc.c
index 5601cc3..e60d118 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -2013,6 +2013,24 @@ static void set_pixel_conversion(VncState *vs)
     }
 }
 
+static void send_color_map(VncState *vs)
+{
+    int i;
+
+    vnc_write_u8(vs, VNC_MSG_SERVER_SET_COLOUR_MAP_ENTRIES);
+    vnc_write_u8(vs,  0);    /* padding     */
+    vnc_write_u16(vs, 0);    /* first color */
+    vnc_write_u16(vs, 256);  /* # of colors */
+
+    for (i = 0; i < 256; i++) {
+        PixelFormat *pf = &vs->client_pf;
+
+        vnc_write_u16(vs, (((i >> pf->rshift) & pf->rmax) << (16 - pf->rbits)));
+        vnc_write_u16(vs, (((i >> pf->gshift) & pf->gmax) << (16 - pf->gbits)));
+        vnc_write_u16(vs, (((i >> pf->bshift) & pf->bmax) << (16 - pf->bbits)));
+    }
+}
+
 static void set_pixel_format(VncState *vs,
                              int bits_per_pixel, int depth,
                              int big_endian_flag, int true_color_flag,
@@ -2020,8 +2038,15 @@ static void set_pixel_format(VncState *vs,
                              int red_shift, int green_shift, int blue_shift)
 {
     if (!true_color_flag) {
-        vnc_client_error(vs);
-        return;
+        /* Expose a reasonable default 256 color map */
+        bits_per_pixel = 8;
+        depth = 8;
+        red_max = 7;
+        green_max = 7;
+        blue_max = 3;
+        red_shift = 0;
+        green_shift = 3;
+        blue_shift = 6;
     }
 
     vs->client_pf.rmax = red_max;
@@ -2041,6 +2066,10 @@ static void set_pixel_format(VncState *vs,
     vs->client_pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
     vs->client_be = big_endian_flag;
 
+    if (!true_color_flag) {
+        send_color_map(vs);
+    }
+
     set_pixel_conversion(vs);
 
     graphic_hw_invalidate(NULL);
-- 
1.7.12.4