Re: [Qemu-devel] PATCH 3/8: VNC password authentication

["Daniel P. Berrange" <berrange@redhat.com>]

This patch introduces support for VNC protocols upto 3.8 and with
it, support for password based authentication. VNC's password based
authentication is not entirely secure, but it is a standard and the
RFB spec requires that all clients support it. The password can be
provided by using the monitor 'change vnc :1' and it will prompt for
a password to be entered. Passwords have upto 8 letters of context.
Pressing 'enter' without entering any characters disables password
auth in the server. NB, we need a custom copy of d3des here because
VNC uses a 'special' modification of the algorithm. This d3des code
is public domain & in all other VNC servers & clients.

diff -r fccd2d79f407 Makefile.target
--- a/Makefile.target	Tue Jul 31 14:48:18 2007 -0400
+++ b/Makefile.target	Tue Jul 31 14:48:19 2007 -0400
@@ -479,7 +479,7 @@ ifdef CONFIG_SDL
 ifdef CONFIG_SDL
 VL_OBJS+=sdl.o x_keymap.o
 endif
-VL_OBJS+=vnc.o
+VL_OBJS+=vnc.o d3des.o
 ifdef CONFIG_COCOA
 VL_OBJS+=cocoa.o
 COCOA_LIBS=-F/System/Library/Frameworks -framework Cocoa -framework IOKit
@@ -540,7 +540,7 @@ sdl.o: sdl.c keymaps.c sdl_keysym.h
 sdl.o: sdl.c keymaps.c sdl_keysym.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(SDL_CFLAGS) $(BASE_CFLAGS) -c -o $@ $<
 
-vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h
+vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h d3des.c d3des.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(BASE_CFLAGS) -c -o $@ $<
 
 sdlaudio.o: sdlaudio.c
diff -r fccd2d79f407 d3des.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/d3des.c	Tue Jul 31 14:48:19 2007 -0400
@@ -0,0 +1,434 @@
+/*
+ * This is D3DES (V5.09) by Richard Outerbridge with the double and
+ * triple-length support removed for use in VNC.  Also the bytebit[] array
+ * has been reversed so that the most significant bit in each byte of the
+ * key is ignored, not the least significant.
+ *
+ * These changes are:
+ *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* D3DES (V5.09) -
+ *
+ * A portable, public domain, version of the Data Encryption Standard.
+ *
+ * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
+ * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
+ * code;  Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
+ * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
+ * for humouring me on.
+ *
+ * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
+ * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
+ */
+
+#include "d3des.h"
+
+static void scrunch(unsigned char *, unsigned long *);
+static void unscrun(unsigned long *, unsigned char *);
+static void desfunc(unsigned long *, unsigned long *);
+static void cookey(unsigned long *);
+
+static unsigned long KnL[32] = { 0L };
+
+static unsigned short bytebit[8]	= {
+	01, 02, 04, 010, 020, 040, 0100, 0200 };
+
+static unsigned long bigbyte[24] = {
+	0x800000L,	0x400000L,	0x200000L,	0x100000L,
+	0x80000L,	0x40000L,	0x20000L,	0x10000L,
+	0x8000L,	0x4000L,	0x2000L,	0x1000L,
+	0x800L, 	0x400L, 	0x200L, 	0x100L,
+	0x80L,		0x40L,		0x20L,		0x10L,
+	0x8L,		0x4L,		0x2L,		0x1L	};
+
+/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
+
+static unsigned char pc1[56] = {
+	56, 48, 40, 32, 24, 16,  8,	 0, 57, 49, 41, 33, 25, 17,
+	 9,  1, 58, 50, 42, 34, 26,	18, 10,  2, 59, 51, 43, 35,
+	62, 54, 46, 38, 30, 22, 14,	 6, 61, 53, 45, 37, 29, 21,
+	13,  5, 60, 52, 44, 36, 28,	20, 12,  4, 27, 19, 11,  3 };
+
+static unsigned char totrot[16] = {
+	1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
+
+static unsigned char pc2[48] = {
+	13, 16, 10, 23,  0,  4,  2, 27, 14,  5, 20,  9,
+	22, 18, 11,  3, 25,  7, 15,  6, 26, 19, 12,  1,
+	40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
+	43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
+
+void deskey(key, edf)	/* Thanks to James Gillogly & Phil Karn! */
+unsigned char *key;
+int edf;
+{
+	register int i, j, l, m, n;
+	unsigned char pc1m[56], pcr[56];
+	unsigned long kn[32];
+
+	for ( j = 0; j < 56; j++ ) {
+		l = pc1[j];
+		m = l & 07;
+		pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
+		}
+	for( i = 0; i < 16; i++ ) {
+		if( edf == DE1 ) m = (15 - i) << 1;
+		else m = i << 1;
+		n = m + 1;
+		kn[m] = kn[n] = 0L;
+		for( j = 0; j < 28; j++ ) {
+			l = j + totrot[i];
+			if( l < 28 ) pcr[j] = pc1m[l];
+			else pcr[j] = pc1m[l - 28];
+			}
+		for( j = 28; j < 56; j++ ) {
+		    l = j + totrot[i];
+		    if( l < 56 ) pcr[j] = pc1m[l];
+		    else pcr[j] = pc1m[l - 28];
+		    }
+		for( j = 0; j < 24; j++ ) {
+			if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
+			if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
+			}
+		}
+	cookey(kn);
+	return;
+	}
+
+static void cookey(raw1)
+register unsigned long *raw1;
+{
+	register unsigned long *cook, *raw0;
+	unsigned long dough[32];
+	register int i;
+
+	cook = dough;
+	for( i = 0; i < 16; i++, raw1++ ) {
+		raw0 = raw1++;
+		*cook	 = (*raw0 & 0x00fc0000L) << 6;
+		*cook	|= (*raw0 & 0x00000fc0L) << 10;
+		*cook	|= (*raw1 & 0x00fc0000L) >> 10;
+		*cook++ |= (*raw1 & 0x00000fc0L) >> 6;
+		*cook	 = (*raw0 & 0x0003f000L) << 12;
+		*cook	|= (*raw0 & 0x0000003fL) << 16;
+		*cook	|= (*raw1 & 0x0003f000L) >> 4;
+		*cook++ |= (*raw1 & 0x0000003fL);
+		}
+	usekey(dough);
+	return;
+	}
+
+void cpkey(into)
+register unsigned long *into;
+{
+	register unsigned long *from, *endp;
+
+	from = KnL, endp = &KnL[32];
+	while( from < endp ) *into++ = *from++;
+	return;
+	}
+
+void usekey(from)
+register unsigned long *from;
+{
+	register unsigned long *to, *endp;
+
+	to = KnL, endp = &KnL[32];
+	while( to < endp ) *to++ = *from++;
+	return;
+	}
+
+void des(inblock, outblock)
+unsigned char *inblock, *outblock;
+{
+	unsigned long work[2];
+
+	scrunch(inblock, work);
+	desfunc(work, KnL);
+	unscrun(work, outblock);
+	return;
+	}
+
+static void scrunch(outof, into)
+register unsigned char *outof;
+register unsigned long *into;
+{
+	*into	 = (*outof++ & 0xffL) << 24;
+	*into	|= (*outof++ & 0xffL) << 16;
+	*into	|= (*outof++ & 0xffL) << 8;
+	*into++ |= (*outof++ & 0xffL);
+	*into	 = (*outof++ & 0xffL) << 24;
+	*into	|= (*outof++ & 0xffL) << 16;
+	*into	|= (*outof++ & 0xffL) << 8;
+	*into	|= (*outof   & 0xffL);
+	return;
+	}
+
+static void unscrun(outof, into)
+register unsigned long *outof;
+register unsigned char *into;
+{
+	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
+	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
+	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
+	*into++ = (unsigned char)(*outof++	 & 0xffL);
+	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
+	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
+	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
+	*into	=  (unsigned char)(*outof	 & 0xffL);
+	return;
+	}
+
+static unsigned long SP1[64] = {
+	0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
+	0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
+	0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
+	0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
+	0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
+	0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
+	0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
+	0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
+	0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
+	0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
+	0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
+	0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
+	0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
+	0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
+	0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
+	0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
+
+static unsigned long SP2[64] = {
+	0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
+	0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
+	0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
+	0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
+	0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
+	0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
+	0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
+	0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
+	0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
+	0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
+	0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
+	0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
+	0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
+	0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
+	0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
+	0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
+
+static unsigned long SP3[64] = {
+	0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
+	0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
+	0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
+	0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
+	0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
+	0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
+	0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
+	0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
+	0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
+	0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
+	0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
+	0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
+	0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
+	0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
+	0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
+	0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
+
+static unsigned long SP4[64] = {
+	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
+	0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
+	0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
+	0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
+	0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
+	0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
+	0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
+	0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
+	0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
+	0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
+	0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
+	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
+	0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
+	0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
+	0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
+	0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
+
+static unsigned long SP5[64] = {
+	0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
+	0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
+	0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
+	0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
+	0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
+	0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
+	0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
+	0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
+	0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
+	0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
+	0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
+	0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
+	0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
+	0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
+	0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
+	0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
+
+static unsigned long SP6[64] = {
+	0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
+	0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
+	0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
+	0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
+	0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
+	0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
+	0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
+	0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
+	0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
+	0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
+	0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
+	0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
+	0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
+	0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
+	0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
+	0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
+
+static unsigned long SP7[64] = {
+	0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
+	0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
+	0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
+	0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
+	0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
+	0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
+	0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
+	0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
+	0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
+	0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
+	0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
+	0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
+	0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
+	0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
+	0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
+	0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
+
+static unsigned long SP8[64] = {
+	0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
+	0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
+	0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
+	0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
+	0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
+	0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
+	0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
+	0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
+	0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
+	0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
+	0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
+	0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
+	0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
+	0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
+	0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
+	0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
+
+static void desfunc(block, keys)
+register unsigned long *block, *keys;
+{
+	register unsigned long fval, work, right, leftt;
+	register int round;
+
+	leftt = block[0];
+	right = block[1];
+	work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
+	right ^= work;
+	leftt ^= (work << 4);
+	work = ((leftt >> 16) ^ right) & 0x0000ffffL;
+	right ^= work;
+	leftt ^= (work << 16);
+	work = ((right >> 2) ^ leftt) & 0x33333333L;
+	leftt ^= work;
+	right ^= (work << 2);
+	work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
+	leftt ^= work;
+	right ^= (work << 8);
+	right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
+	work = (leftt ^ right) & 0xaaaaaaaaL;
+	leftt ^= work;
+	right ^= work;
+	leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
+
+	for( round = 0; round < 8; round++ ) {
+		work  = (right << 28) | (right >> 4);
+		work ^= *keys++;
+		fval  = SP7[ work		 & 0x3fL];
+		fval |= SP5[(work >>  8) & 0x3fL];
+		fval |= SP3[(work >> 16) & 0x3fL];
+		fval |= SP1[(work >> 24) & 0x3fL];
+		work  = right ^ *keys++;
+		fval |= SP8[ work		 & 0x3fL];
+		fval |= SP6[(work >>  8) & 0x3fL];
+		fval |= SP4[(work >> 16) & 0x3fL];
+		fval |= SP2[(work >> 24) & 0x3fL];
+		leftt ^= fval;
+		work  = (leftt << 28) | (leftt >> 4);
+		work ^= *keys++;
+		fval  = SP7[ work		 & 0x3fL];
+		fval |= SP5[(work >>  8) & 0x3fL];
+		fval |= SP3[(work >> 16) & 0x3fL];
+		fval |= SP1[(work >> 24) & 0x3fL];
+		work  = leftt ^ *keys++;
+		fval |= SP8[ work		 & 0x3fL];
+		fval |= SP6[(work >>  8) & 0x3fL];
+		fval |= SP4[(work >> 16) & 0x3fL];
+		fval |= SP2[(work >> 24) & 0x3fL];
+		right ^= fval;
+		}
+
+	right = (right << 31) | (right >> 1);
+	work = (leftt ^ right) & 0xaaaaaaaaL;
+	leftt ^= work;
+	right ^= work;
+	leftt = (leftt << 31) | (leftt >> 1);
+	work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
+	right ^= work;
+	leftt ^= (work << 8);
+	work = ((leftt >> 2) ^ right) & 0x33333333L;
+	right ^= work;
+	leftt ^= (work << 2);
+	work = ((right >> 16) ^ leftt) & 0x0000ffffL;
+	leftt ^= work;
+	right ^= (work << 16);
+	work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
+	leftt ^= work;
+	right ^= (work << 4);
+	*block++ = right;
+	*block = leftt;
+	return;
+	}
+
+/* Validation sets:
+ *
+ * Single-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : c957 4425 6a5e d31d
+ *
+ * Double-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : 7f1d 0a77 826b 8aff
+ *
+ * Double-length key, double-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
+ * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
+ * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
+ *
+ * Triple-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : de0b 7c06 ae5e 0ed5
+ *
+ * Triple-length key, double-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
+ * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
+ * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
+ *
+ * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
+ **********************************************************************/
diff -r fccd2d79f407 d3des.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/d3des.h	Tue Jul 31 14:48:19 2007 -0400
@@ -0,0 +1,51 @@
+/*
+ * This is D3DES (V5.09) by Richard Outerbridge with the double and
+ * triple-length support removed for use in VNC.
+ *
+ * These changes are:
+ *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* d3des.h -
+ *
+ *	Headers and defines for d3des.c
+ *	Graven Imagery, 1992.
+ *
+ * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge
+ *	(GEnie : OUTER; CIS : [71755,204])
+ */
+
+#define EN0	0	/* MODE == encrypt */
+#define DE1	1	/* MODE == decrypt */
+
+extern void deskey(unsigned char *, int);
+/*		      hexkey[8]     MODE
+ * Sets the internal key register according to the hexadecimal
+ * key contained in the 8 bytes of hexkey, according to the DES,
+ * for encryption or decryption according to MODE.
+ */
+
+extern void usekey(unsigned long *);
+/*		    cookedkey[32]
+ * Loads the internal key register with the data in cookedkey.
+ */
+
+extern void cpkey(unsigned long *);
+/*		   cookedkey[32]
+ * Copies the contents of the internal key register into the storage
+ * located at &cookedkey[0].
+ */
+
+extern void des(unsigned char *, unsigned char *);
+/*		    from[8]	      to[8]
+ * Encrypts/Decrypts (according to the key currently loaded in the
+ * internal key register) one block of eight bytes at address 'from'
+ * into the block at address 'to'.  They can be the same.
+ */
+
+/* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery
+ ********************************************************************/
diff -r fccd2d79f407 monitor.c
--- a/monitor.c	Tue Jul 31 14:48:18 2007 -0400
+++ b/monitor.c	Tue Jul 31 14:48:19 2007 -0400
@@ -403,7 +403,10 @@ static void do_change_block(const char *
 
 static void do_change_vnc(const char *target)
 {
-    if (vnc_display_open(NULL, target) < 0)
+    char password[9];
+    monitor_readline("Password: ", 1, password, sizeof(password)-1);
+    password[sizeof(password)-1] = '\0';
+    if (vnc_display_open(NULL, target, password) < 0)
 	term_printf("could not start VNC server on %s\n", target);
 }
 
diff -r fccd2d79f407 vl.c
--- a/vl.c	Tue Jul 31 14:48:18 2007 -0400
+++ b/vl.c	Tue Jul 31 14:48:19 2007 -0400
@@ -7945,7 +7945,7 @@ int main(int argc, char **argv)
         dumb_display_init(ds);
     } else if (vnc_display != NULL) {
         vnc_display_init(ds);
-        if (vnc_display_open(ds, vnc_display) < 0)
+        if (vnc_display_open(ds, vnc_display, NULL) < 0)
             exit(1);
     } else {
 #if defined(CONFIG_SDL)
diff -r fccd2d79f407 vl.h
--- a/vl.h	Tue Jul 31 14:48:18 2007 -0400
+++ b/vl.h	Tue Jul 31 14:48:19 2007 -0400
@@ -969,7 +969,7 @@ void cocoa_display_init(DisplayState *ds
 /* vnc.c */
 void vnc_display_init(DisplayState *ds);
 void vnc_display_close(DisplayState *ds);
-int vnc_display_open(DisplayState *ds, const char *display);
+int vnc_display_open(DisplayState *ds, const char *display, const char *password);
 void do_info_vnc(void);
 
 /* x_keymap.c */
diff -r fccd2d79f407 vnc.c
--- a/vnc.c	Tue Jul 31 14:48:18 2007 -0400
+++ b/vnc.c	Tue Jul 31 14:48:19 2007 -0400
@@ -30,6 +30,15 @@
 
 #include "vnc_keysym.h"
 #include "keymaps.c"
+#include "d3des.h"
+
+// #define _VNC_DEBUG
+
+#ifdef _VNC_DEBUG
+#define VNC_DEBUG(fmt, ...) do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#else
+#define VNC_DEBUG(fmt, ...) do { } while (0)
+#endif
 
 typedef struct Buffer
 {
@@ -53,6 +62,20 @@ typedef void VncSendHextileTile(VncState
 #define VNC_MAX_WIDTH 2048
 #define VNC_MAX_HEIGHT 2048
 #define VNC_DIRTY_WORDS (VNC_MAX_WIDTH / (16 * 32))
+
+#define VNC_AUTH_CHALLENGE_SIZE 16
+
+enum {
+    VNC_AUTH_INVALID = 0,
+    VNC_AUTH_NONE = 1,
+    VNC_AUTH_VNC = 2,
+    VNC_AUTH_RA2 = 5,
+    VNC_AUTH_RA2NE = 6,
+    VNC_AUTH_TIGHT = 16,
+    VNC_AUTH_ULTRA = 17,
+    VNC_AUTH_TLS = 18,
+    VNC_AUTH_VENCRYPT = 19
+};
 
 struct VncState
 {
@@ -73,7 +96,13 @@ struct VncState
     int last_x;
     int last_y;
 
+    int major;
+    int minor;
+
     char *display;
+    char *password;
+    int auth;
+    char challenge[VNC_AUTH_CHALLENGE_SIZE];
 
     Buffer output;
     Buffer input;
@@ -1125,23 +1154,152 @@ static int protocol_client_init(VncState
     return 0;
 }
 
+static void make_challenge(VncState *vs)
+{
+    int i;
+
+    srand(time(NULL)+getpid()+getpid()*987654+rand());
+
+    for (i = 0 ; i < sizeof(vs->challenge) ; i++)
+        vs->challenge[i] = (int) (256.0*rand()/(RAND_MAX+1.0));
+}
+
+static int protocol_client_auth_vnc(VncState *vs, char *data, size_t len)
+{
+    char response[VNC_AUTH_CHALLENGE_SIZE];
+    int i, j, pwlen;
+    char key[8];
+
+    memcpy(response, vs->challenge, VNC_AUTH_CHALLENGE_SIZE);
+
+    /* Calculate the expected challenge response */
+    pwlen = strlen(vs->password);
+    for (i=0; i<sizeof(key); i++)
+        key[i] = i<pwlen ? vs->password[i] : 0;
+    deskey(key, EN0);
+    for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8)
+        des(response+j, response+j);
+
+    /* Compare expected vs actual challenge response */
+    if (memcmp(response, data, VNC_AUTH_CHALLENGE_SIZE) != 0) {
+	VNC_DEBUG("Client challenge reponse did not match\n");
+	vnc_write_u32(vs, 1); /* Reject auth */
+	if (vs->minor >= 8) {
+	    static const char err[] = "Authentication failed";
+	    vnc_write_u32(vs, sizeof(err));
+	    vnc_write(vs, err, sizeof(err));
+	}
+	vnc_flush(vs);
+	vnc_client_error(vs);
+    } else {
+	VNC_DEBUG("Accepting VNC challenge response\n");
+	vnc_write_u32(vs, 0); /* Accept auth */
+	vnc_flush(vs);
+
+	vnc_read_when(vs, protocol_client_init, 1);
+    }
+    return 0;
+}
+
+static int start_auth_vnc(VncState *vs)
+{
+    make_challenge(vs);
+    /* Send client a 'random' challenge */
+    vnc_write(vs, vs->challenge, sizeof(vs->challenge));
+    vnc_flush(vs);
+
+    vnc_read_when(vs, protocol_client_auth_vnc, sizeof(vs->challenge));
+    return 0;
+}
+
+static int protocol_client_auth(VncState *vs, char *data, size_t len)
+{
+    /* We only advertise 1 auth scheme at a time, so client
+     * must pick the one we sent. Verify this */
+    if (data[0] != vs->auth) { /* Reject auth */
+       VNC_DEBUG("Reject auth %d\n", (int)data[0]);
+       vnc_write_u32(vs, 1);
+       if (vs->minor >= 8) {
+           static const char err[] = "Authentication failed";
+           vnc_write_u32(vs, sizeof(err));
+           vnc_write(vs, err, sizeof(err));
+       }
+       vnc_client_error(vs);
+    } else { /* Accept requested auth */
+       VNC_DEBUG("Client requested auth %d\n", (int)data[0]);
+       switch (vs->auth) {
+       case VNC_AUTH_NONE:
+           VNC_DEBUG("Accept auth none\n");
+           vnc_write_u32(vs, 0); /* Accept auth completion */
+           vnc_read_when(vs, protocol_client_init, 1);
+           break;
+
+       case VNC_AUTH_VNC:
+           VNC_DEBUG("Start VNC auth\n");
+           return start_auth_vnc(vs);
+
+       default: /* Should not be possible, but just in case */
+           VNC_DEBUG("Reject auth %d\n", vs->auth);
+           vnc_write_u8(vs, 1);
+           if (vs->minor >= 8) {
+               static const char err[] = "Authentication failed";
+               vnc_write_u32(vs, sizeof(err));
+               vnc_write(vs, err, sizeof(err));
+           }
+           vnc_client_error(vs);
+       }
+    }
+    return 0;
+}
+
 static int protocol_version(VncState *vs, char *version, size_t len)
 {
     char local[13];
-    int maj, min;
 
     memcpy(local, version, 12);
     local[12] = 0;
 
-    if (sscanf(local, "RFB %03d.%03d\n", &maj, &min) != 2) {
+    if (sscanf(local, "RFB %03d.%03d\n", &vs->major, &vs->minor) != 2) {
+	VNC_DEBUG("Malformed protocol version %s\n", local);
 	vnc_client_error(vs);
 	return 0;
     }
-
-    vnc_write_u32(vs, 1); /* None */
-    vnc_flush(vs);
-
-    vnc_read_when(vs, protocol_client_init, 1);
+    VNC_DEBUG("Client request protocol version %d.%d\n", vs->major, vs->minor);
+    if (vs->major != 3 ||
+	(vs->minor != 3 &&
+	 vs->minor != 7 &&
+	 vs->minor != 8)) {
+	VNC_DEBUG("Unsupported client version\n");
+	vnc_write_u32(vs, VNC_AUTH_INVALID);
+	vnc_flush(vs);
+	vnc_client_error(vs);
+	return 0;
+    }
+
+    if (vs->minor == 3) {
+	if (vs->auth == VNC_AUTH_NONE) {
+            VNC_DEBUG("Tell client auth none\n");
+            vnc_write_u32(vs, vs->auth);
+            vnc_flush(vs);
+            vnc_read_when(vs, protocol_client_init, 1);
+       } else if (vs->auth == VNC_AUTH_VNC) {
+            VNC_DEBUG("Tell client VNC auth\n");
+            vnc_write_u32(vs, vs->auth);
+            vnc_flush(vs);
+            start_auth_vnc(vs);
+       } else {
+            VNC_DEBUG("Unsupported auth %d for protocol 3.3\n", vs->auth);
+            vnc_write_u32(vs, VNC_AUTH_INVALID);
+            vnc_flush(vs);
+            vnc_client_error(vs);
+       }
+    } else {
+	VNC_DEBUG("Telling client we support auth %d\n", vs->auth);
+	vnc_write_u8(vs, 1); /* num auth */
+	vnc_write_u8(vs, vs->auth);
+	vnc_read_when(vs, protocol_client_auth, 1);
+	vnc_flush(vs);
+    }
 
     return 0;
 }
@@ -1156,7 +1314,7 @@ static void vnc_listen_read(void *opaque
     if (vs->csock != -1) {
         socket_set_nonblock(vs->csock);
 	qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, opaque);
-	vnc_write(vs, "RFB 003.003\n", 12);
+	vnc_write(vs, "RFB 003.008\n", 12);
 	vnc_flush(vs);
 	vnc_read_when(vs, protocol_version, 12);
 	memset(vs->old_data, 0, vs->ds->linesize * vs->ds->height);
@@ -1180,6 +1338,7 @@ void vnc_display_init(DisplayState *ds)
     ds->opaque = vs;
     vnc_state = vs;
     vs->display = NULL;
+    vs->password = NULL;
 
     vs->lsock = -1;
     vs->csock = -1;
@@ -1214,13 +1373,18 @@ void vnc_display_close(DisplayState *ds)
 	qemu_free(vs->display);
 	vs->display = NULL;
     }
+    if (vs->password) {
+	qemu_free(vs->password);
+	vs->password = NULL;
+    }
+    vs->auth = VNC_AUTH_INVALID;
     if (vs->lsock != -1) {
 	close(vs->lsock);
 	vs->lsock = -1;
     }
 }
 
-int vnc_display_open(DisplayState *ds, const char *arg)
+int vnc_display_open(DisplayState *ds, const char *display, const char *password)
 {
     struct sockaddr *addr;
     struct sockaddr_in iaddr;
@@ -1233,13 +1397,25 @@ int vnc_display_open(DisplayState *ds, c
     VncState *vs = ds ? (VncState *)ds->opaque : vnc_state;
 
     vnc_display_close(ds);
-    if (strcmp(arg, "none") == 0)
+    if (strcmp(display, "none") == 0)
 	return 0;
 
-    if (!(vs->display = strdup(arg)))
+    if (!(vs->display = strdup(display)))
 	return -1;
+    if (password && password[0]) {
+	if (!(vs->password = qemu_strdup(password))) {
+	    qemu_free(vs->display);
+	    vs->display = NULL;
+	    return -1;
+	}
+	VNC_DEBUG("Initializing VNC server with password auth\n");
+	vs->auth = VNC_AUTH_VNC;
+    } else {
+	VNC_DEBUG("Initializing VNC server with no auth\n");
+	vs->auth = VNC_AUTH_NONE;
+    }
 #ifndef _WIN32
-    if (strstart(arg, "unix:", &p)) {
+    if (strstart(display, "unix:", &p)) {
 	addr = (struct sockaddr *)&uaddr;
 	addrlen = sizeof(uaddr);
 
@@ -1262,7 +1438,7 @@ int vnc_display_open(DisplayState *ds, c
 	addr = (struct sockaddr *)&iaddr;
 	addrlen = sizeof(iaddr);
 
-	if (parse_host_port(&iaddr, arg) < 0) {
+	if (parse_host_port(&iaddr, display) < 0) {
 	    fprintf(stderr, "Could not parse VNC address\n");
 	    free(vs->display);
 	    vs->display = NULL;

-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=|