[Qemu-devel] [PATCH] vnc fixes and improvements

[Stefano Stabellini <stefano.stabellini@eu.citrix.com>]

Hi all,
this patch fixes a bug and improves the generic pixel conversion
function in vnc.c.
The bug is that when a new vnc client connects we need to reset the flag
has_WMVi but currently we don't.
The generic pixel conversion function is vnc_convert_pixel and currently
is not very efficient since uses the division and multiplication
operators.
To make it more efficient I changed to use bit shift operators instead.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>


diff --git a/console.c b/console.c
index 4bcdac1..68ac970 100644
--- a/console.c
+++ b/console.c
@@ -1470,6 +1470,9 @@ PixelFormat qemu_different_endianness_pixelformat(int bpp)
             pf.rshift = 0;
             pf.gshift = 8;
             pf.bshift = 16;
+            pf.rbits = 8;
+            pf.gbits = 8;
+            pf.bbits = 8;
             break;
         case 32:
             pf.rmask = 0x0000FF00;
@@ -1484,6 +1487,10 @@ PixelFormat qemu_different_endianness_pixelformat(int bpp)
             pf.rshift = 8;
             pf.gshift = 16;
             pf.bshift = 24;
+            pf.rbits = 8;
+            pf.gbits = 8;
+            pf.bbits = 8;
+            pf.abits = 8;
             break;
         default:
             break;
@@ -1512,6 +1519,9 @@ PixelFormat qemu_default_pixelformat(int bpp)
             pf.rshift = 11;
             pf.gshift = 5;
             pf.bshift = 0;
+            pf.rbits = 5;
+            pf.gbits = 6;
+            pf.bbits = 5;
             break;
         case 24:
             pf.rmask = 0x00FF0000;
@@ -1523,6 +1533,9 @@ PixelFormat qemu_default_pixelformat(int bpp)
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;
+            pf.rbits = 8;
+            pf.gbits = 8;
+            pf.bbits = 8;
         case 32:
             pf.rmask = 0x00FF0000;
             pf.gmask = 0x0000FF00;
@@ -1535,6 +1548,10 @@ PixelFormat qemu_default_pixelformat(int bpp)
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;
+            pf.rbits = 8;
+            pf.gbits = 8;
+            pf.bbits = 8;
+            pf.abits = 8;
             break;
         default:
             break;
diff --git a/console.h b/console.h
index 6e764b3..4a2f06f 100644
--- a/console.h
+++ b/console.h
@@ -83,6 +83,7 @@ struct PixelFormat {
     uint32_t rmask, gmask, bmask, amask;
     uint8_t rshift, gshift, bshift, ashift;
     uint8_t rmax, gmax, bmax, amax;
+    uint8_t rbits, gbits, bbits, abits;
 };
 
 struct DisplaySurface {
diff --git a/vnc.c b/vnc.c
index 17ea9a2..2b3a6eb 100644
--- a/vnc.c
+++ b/vnc.c
@@ -56,6 +56,12 @@ static void vnc_debug_gnutls_log(int level, const char* str) {
 #define VNC_DEBUG(fmt, ...) do { } while (0)
 #endif
 
+#define count_bits(c, v) { \
+    for (c = 0; v; v >>= 1) \
+    { \
+        c += v & 1; \
+    } \
+}
 
 typedef struct Buffer
 {
@@ -329,12 +335,12 @@ static void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 {
     uint8_t r, g, b;
 
-    r = ((v >> vs->serverds.pf.rshift) & vs->serverds.pf.rmax) * (vs->clientds.pf.rmax + 1) /
-        (vs->serverds.pf.rmax + 1);
-    g = ((v >> vs->serverds.pf.gshift) & vs->serverds.pf.gmax) * (vs->clientds.pf.gmax + 1) /
-        (vs->serverds.pf.gmax + 1);
-    b = ((v >> vs->serverds.pf.bshift) & vs->serverds.pf.bmax) * (vs->clientds.pf.bmax + 1) /
-        (vs->serverds.pf.bmax + 1);
+    r = ((((v & vs->serverds.pf.rmask) >> vs->serverds.pf.rshift) << vs->clientds.pf.rbits) >>
+        vs->serverds.pf.rbits);
+    g = ((((v & vs->serverds.pf.gmask) >> vs->serverds.pf.gshift) << vs->clientds.pf.gbits) >>
+        vs->serverds.pf.gbits);
+    b = ((((v & vs->serverds.pf.bmask) >> vs->serverds.pf.bshift) << vs->clientds.pf.bbits) >>
+        vs->serverds.pf.bbits);
     v = (r << vs->clientds.pf.rshift) |
         (g << vs->clientds.pf.gshift) |
         (b << vs->clientds.pf.bshift);
@@ -1272,12 +1278,15 @@ static void set_pixel_format(VncState *vs,
 
     vs->clientds = vs->serverds;
     vs->clientds.pf.rmax = red_max;
+    count_bits(vs->clientds.pf.rbits, red_max);
     vs->clientds.pf.rshift = red_shift;
     vs->clientds.pf.rmask = red_max << red_shift;
     vs->clientds.pf.gmax = green_max;
+    count_bits(vs->clientds.pf.gbits, green_max);
     vs->clientds.pf.gshift = green_shift;
     vs->clientds.pf.gmask = green_max << green_shift;
     vs->clientds.pf.bmax = blue_max;
+    count_bits(vs->clientds.pf.bbits, blue_max);
     vs->clientds.pf.bshift = blue_shift;
     vs->clientds.pf.bmask = blue_max << blue_shift;
     vs->clientds.pf.bits_per_pixel = bits_per_pixel;
@@ -2106,6 +2115,7 @@ static void vnc_connect(VncState *vs)
     memset(vs->dirty_row, 0xFF, sizeof(vs->dirty_row));
     vs->has_resize = 0;
     vs->has_hextile = 0;
+    vs->has_WMVi = 0;
     dcl->dpy_copy = NULL;
     vnc_update_client(vs);
     reset_keys(vs);