[Qemu-devel] mini-HOWTO: Using qemu to lock down tetrinetx server

[Rusty Russell <rusty@rustcorp.com.au>]

[-- Attachment #1: Type: text/plain, Size: 323 bytes --]

Hi all,

	I recently set up a tetrinet server inside a qemu image, for a
layer of additional security.  The image is under 2MB, and below is a
mini-howto in setting up such a server in case anyone else is
interested.

You will need:

1) A qemu-compatible disk image: I have a simple script to make one,
  (see attachment).

[-- Attachment #2: script to make qemu root --]
[-- Type: text/plain, Size: 2114 bytes --]

#! /bin/sh
# Script to make image of given size (in MB)

if [ $# -ne 2 ]; then
    echo Usage: "$0 size-in-MB image" >&2
    echo "eg $0 150 qemu-disk" >&2
    exit 1
fi

SIZE=$1
IMAGE=$2

HEADS=16
SECTORS=63
# 512 bytes in a sector: cancel the 512 with one of the 1024s...
CYLINDERS=$(( $SIZE * 1024 * 2 / ($HEADS * $SECTORS) ))

# Create a filesystem: one track for partition table.
dd bs=$(($SECTORS * 512)) if=/dev/zero of=$IMAGE.raw count=$(($CYLINDERS * $HEADS - 1))
mke2fs -q -m1 -F -j $IMAGE.raw

# Prepend partiation table
# Create file with partition table.
uudecode -o- << "EOF" | gunzip > $IMAGE
begin 664 partition-table.gz
M'XL("*_<##\"`W!A<G1I=&EO;BUT86)L90#LT#$-`"`0!,&']D6A`D6XP1T&
M"%B@))FIMKGF(OA9C;%;EENYZO.Z3P\"````!P``__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0``
M``#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:
M!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````
M__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`
M````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__
M&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0``
M``#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:
M!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````
M__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`
M````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__
M&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/__0@``````__\#`%&_
&<90`?@``
`
end
EOF
cat $IMAGE.raw >> $IMAGE
rm $IMAGE.raw

# Repartition so one partition covers entire disk.
echo '63,' | sfdisk -uS -H$HEADS -S$SECTORS -C$CYLINDERS $IMAGE

[-- Attachment #3: Type: text/plain, Size: 286 bytes --]

2) The "vl" binary, and a qemu-compatible kernel bzImage.
   I used qemu 0.4.3, modified with the -tun-fd patch and the closed
   stdin patch.  If you use the softmm version of qemu, any kernel
   bzImage should work.

3) The root-running version of the tundev program (see attachment)

[-- Attachment #4: tundev C source: root-running version --]
[-- Type: text/plain, Size: 1675 bytes --]

#define _GNU_SOURCE /* asprintf */
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <pwd.h>
#include <grp.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <net/if.h>
#include <linux/if_tun.h>

/* Tiny code to open tap/tun device, and hand the fd to vl.
   Run as root, drops to given user. */
int main(int argc, char *argv[])
{
	struct ifreq ifr;
	struct passwd *p;
	unsigned int i;
	char *newargs[argc];
	int fd;

	if (argc < 4) {
		fprintf(stderr,
			"Usage: switch user logfile vl <vl options>...\n");
		exit(1);
	}

	fd = open("/dev/net/tun", O_RDWR);
	if (fd < 0) {
		perror("Could not open /dev/net/tun");
		exit(1);
	}

	memset(&ifr, 0, sizeof(ifr));
	ifr.ifr_flags = IFF_TAP | IFF_NO_PI;
	strncpy(ifr.ifr_name, "tun%d", IFNAMSIZ);
	if (ioctl(fd, TUNSETIFF, (void *) &ifr) != 0) {
		perror("Could not get tun device");
		exit(1);
	}

	/* Set userid. */
	p = getpwnam(argv[1]);
	if (!p) {
		fprintf(stderr, "No user '%s'\n", argv[1]);
		exit(1);
	}
	setgroups(0, NULL);
	setgid(p->pw_gid);
	if (setuid(p->pw_uid) != 0) {
		perror("setting uid");
		exit(1);
	}

	/* Insert -tun-fd, delete other args */
	newargs[0] = argv[3];
	asprintf(&newargs[1], "-tun-fd=%d", fd);
	for (i = 4; i <= argc; i++)
		newargs[i-2] = argv[i];
		
	if (strcmp(argv[2], "-") == 0) {
		execvp(newargs[0], newargs);
		exit(1);
	}

	switch (fork()) {
	case 0: {
		close(1);
		close(2);
		open(argv[2], O_WRONLY|O_APPEND);
		open(argv[2], O_WRONLY|O_APPEND);
		close(0);
		execvp(newargs[0], newargs);
		exit(1);
	}
	case -1:
		perror("fork failed");
		exit(1);
	}
	printf("%s\n", ifr.ifr_name);
	exit(0);
}

[-- Attachment #5: Type: text/plain, Size: 436 bytes --]

Steps:
1) Mount the filesystem.  If you've already prepended a partition
   table, use
	losetup -o $((63 * 512)) /dev/loop0 <FILE>
	mount /dev/loop0 /mnt/qemu
   Otherwise you can just do:
	mount -o loop <FILE> /mnt/qemu

2) Remove unneccessary files, add files you need, and lock down
   directories.  This is best done by creating an oldroot dir and
   moving things into that.  For tetrinet, I ended up under 2MB
   (see attachment)

[-- Attachment #6: ls-R of qemu image with tetrinet server --]
[-- Type: text/plain, Size: 3044 bytes --]

/mnt/qemu:
total 222
dr-x--x--x    2 root     root         1024 Sep 17 21:10 dev
dr-x--x--x    3 root     root         1024 Sep 17 20:48 etc
-rwx------    1 root     root        53516 Jun 28 12:09 ifconfig
d--x--xr-x    2 root     root         1024 Sep 17 21:08 lib
drwx------    2 root     root        12288 Sep 18 13:30 lost+found
-rwx------    1 root     root        41848 Jun 28 12:09 route
-rwxrwxr-x    1 rusty    rusty        6748 Sep 18 13:27 run
-rwx--x---    1 root     games      104184 Jan 25  2003 tetrinetx
dr-x--x--x    3 root     root         1024 Sep 17 20:54 var

/mnt/qemu/dev:
total 0
crw-------    1 root     root       5,   1 Jul 10 19:32 console
crw-rw----    1 root     games      4,  64 Sep 16 18:48 ttyS0

/mnt/qemu/etc:
total 6
-rw-r-----    1 root     games          41 Sep 17 20:48 hosts
-rw-r-----    1 root     games         465 Mar 12  1999 nsswitch.conf
-rw-r-----    1 root     games        2030 Jun 16 03:40 protocols
-rw-r-----    1 root     games          64 Jul 10 10:49 resolv.conf
dr-x--x--x    2 root     root         1024 Sep 17 20:34 tetrinetx

/mnt/qemu/etc/tetrinetx:
total 7
-rw-r-----    1 root     games        4042 Sep 17 20:34 game.conf
-rw-r-----    1 root     games          87 Jan 25  2003 game.motd
-rw-r-----    1 root     games         180 Jan 25  2003 game.pmotd
-rw-r-----    1 root     games         768 Jan 25  2003 game.secure

/mnt/qemu/lib:
total 1574
-rwxr-xr-x    1 root     root        82456 Apr 20 04:57 ld-2.3.1.so
lrwxrwxrwx    1 root     root           11 Jul 10 19:32 ld-linux.so.2 -> ld-2.3.1.so
lrwxr-xr-x    1 root     root           14 Sep 16 17:27 libadns.so.1 -> libadns.so.1.0
-rwxr-xr-x    1 root     root        61044 Sep  9  2002 libadns.so.1.0
-rwxr-xr-x    1 root     root      1103880 Apr 20 04:57 libc-2.3.1.so
lrwxrwxrwx    1 root     root           13 Jul 10 19:32 libc.so.6 -> libc-2.3.1.so
-rwxr-xr-x    1 root     root         7992 Apr 20 04:57 libdl-2.3.1.so
lrwxrwxrwx    1 root     root           14 Jul 10 19:32 libdl.so.2 -> libdl-2.3.1.so
lrwxrwxrwx    1 root     root           17 Jul 10 19:32 libncurses.so.5 -> libncurses.so.5.3
-rwxr-xr-x    1 root     root       238160 Jun 15 02:13 libncurses.so.5.3
-rwxr-xr-x    1 root     root        12828 Apr 20 04:57 libnss_dns-2.3.1.so
lrwxrwxrwx    1 root     root           19 Jul 10 19:32 libnss_dns.so.2 -> libnss_dns-2.3.1.so
-rwxr-xr-x    1 root     root        32204 Apr 20 04:57 libnss_files-2.3.1.so
lrwxrwxrwx    1 root     root           21 Jul 10 19:32 libnss_files.so.2 -> libnss_files-2.3.1.so
-rwxr-xr-x    1 root     root        56652 Apr 20 04:57 libresolv-2.3.1.so
lrwxrwxrwx    1 root     root           18 Jul 10 19:32 libresolv.so.2 -> libresolv-2.3.1.so

/mnt/qemu/lost+found:
total 0

/mnt/qemu/var:
total 1
dr-x--x--x    3 root     root         1024 Sep 17 20:54 log

/mnt/qemu/var/log:
total 1
dr-x--x--x    2 root     root         1024 Sep 17 20:54 tetrinetx

/mnt/qemu/var/log/tetrinetx:
total 0
lrwxrwxrwx    1 root     root           10 Sep 17 20:54 game.log -> /dev/ttyS0

[-- Attachment #7: Type: text/plain, Size: 342 bytes --]

3) I wanted the filesystem to be read-only (too bad about the tetrinet
   high scores file: I don't care), but I wanted the logs, so I made
   /var/log/tetrinetx/game.log a symlink to /dev/ttyS0, and allowed
   the games group to write to it.

4) I wrote a simply launcher program called run to avoid the need for
   a shell (see attachment)

[-- Attachment #8: run.c source for tetrinet server --]
[-- Type: text/plain, Size: 1570 bytes --]

/* Simply C program to act as init. */
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <grp.h>
#include <sys/reboot.h>
#include <linux/reboot.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>

static int wait_for(const char *prog)
{
	int status;

	wait(&status);
	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
		fprintf(stderr, "run: %s failed\n", prog);
		return 0;
	}
	return 1;
}

int main(int argc, char *argv[])
{
	if (fork() == 0) {
		execl("/ifconfig", "ifconfig", "eth0", "192.168.1.2", NULL);
		fprintf(stderr, "run: ifconfig exec failed\n");
		exit(1);
	}
	if (!wait_for("ifconfig"))
		goto reboot;

	if (fork() == 0) {
		execl("/route", "route", "add", "default", "gw", "192.168.1.1",
		      NULL);
		fprintf(stderr, "run: route exec failed\n");
		exit(1);
	}
	if (!wait_for("route"))
		goto reboot;

	/* tetrinet does DNS lookup on own hostname */
	sethostname("ozlabs.org", strlen("ozlabs.org"));

	if (fork() == 0) {
		setgroups(0, NULL);
		/* Games/games */
		setgid(60);
		setuid(5);
		close(0);
		close(1);
		close(2);
		open("/dev/null", O_RDWR);
		open("/dev/null", O_RDWR);
		open("/dev/null", O_RDWR);
		execl("/tetrinetx", "tetrinetx", NULL);
		exit(1);
	}
	if (!wait_for("tetrinetx"))
		goto reboot;

	/* Drop privs, why not? */
	setgroups(0, NULL);
	setgid(65534);
	setuid(65534);
	close(0);
	close(1);
	close(2);

	while (wait(NULL) >= 0);

reboot:
	sync();
	reboot(LINUX_REBOOT_CMD_RESTART);
	sleep(60);
	return 1;
}

[-- Attachment #9: Type: text/plain, Size: 1185 bytes --]

5) Unmount the qemu image.  Once I'd trimmed out most of the code, I
   created a new smaller one one, mounted that, and used tar to
   transfer everything across.

6) My startup script looks like so:

	#! /bin/sh
	# Set up forwarding and masquerading, and make sure tun module is loaded.

	set -e

	INTERFACE=`./tundev games /var/log/qemu-tetrinet ./vl -hda tetrinet-filesystem bzImage root=/dev/hda1 ide1=noprobe ide2=noprobe ide3=noprobe ide4=noprobe ide5=noprobe init=/run`

	ifdown $INTERFACE 2>/dev/null || true
	ifup $INTERFACE

   You need to reconfigure the tun devices everytime they get opened,
   hence the ifdown/ifup stuff.

8) You will also need to forward incoming connections to the server.
   In this case, the server makes no outgoing connections, so this
   works:

   iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 31457 -j DNAT --to 192.168.1.2
   iptables -A FORWARD -o tun0 -m state --state NEW,INVALID -j DROP
   iptables -N LOGDROP
   iptables -A LOGDROP -j LOG
   iptables -A LOGDROP -j DROP
   iptables -A FORWARD -i tun0 -m state --state NEW,INVALID -j LOGDROP

Good luck!
Rusty.
--
  Anyone who quotes me in their sig is an idiot. -- Rusty Russell.