[PATCH 0/5] kvm-unittests: add pci PORT IO and MMIO speed tests

[PATCH 0/5] kvm-unittests: add pci PORT IO and MMIO speed tests

[Michael S. Tsirkin]

These patches add ability to measure speed of MMIO versus PIO, in
different configurations. As I didn't want to reserve a hardcoded range
of memory, I added pci bus scan support instead.  Used together with the
new pci-testdev backend.  When not present, the new tests are skipped.

Example output:
	mmio-no-eventfd:pci-mem 8796
	mmio-wildcard-eventfd:pci-mem 3609
	mmio-datamatch-eventfd:pci-mem 3685
	portio-no-eventfd:pci-io 5287
	portio-wildcard-eventfd:pci-io 1762
	portio-datamatch-eventfd:pci-io 1777

First interesting conclusion is that the overhead of MMIO
exit to QEMU as compared to PIO is double that of
MMIO ioeventfd as compared to PIO eventfd. Is this a known fact?

Michael S. Tsirkin (5):
  kvm-unittest: add printf %c support
  kvm-unittest: libcflat: add offsetof
  libcflat.h: define NULL
  lib: add pci bus scan support
  vmexit: add pci io and memory speed tests

 config-x86-common.mak |   1 +
 lib/libcflat.h        |   3 +
 lib/printf.c          |   3 +
 lib/x86/pci.c         |  55 ++++++++++++
 lib/x86/pci.h         |  16 ++++
 x86-run               |  13 ++-
 x86/vmexit.c          | 226 +++++++++++++++++++++++++++++++++++++++++++++++---
 7 files changed, 302 insertions(+), 15 deletions(-)
 create mode 100644 lib/x86/pci.c
 create mode 100644 lib/x86/pci.h

-- 
MST

[PATCH 1/5] kvm-unittest: add printf %c support

[Michael S. Tsirkin]

Handy for printing debug strings from hardware.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 lib/printf.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/printf.c b/lib/printf.c
index 3bb9e3d..867eb19 100644
--- a/lib/printf.c
+++ b/lib/printf.c
@@ -104,6 +104,9 @@ int vsnprintf(char *buf, int size, const char *fmt, va_list va)
 	case '%':
 	    addchar(&s, '%');
 	    break;
+	case 'c':
+            addchar(&s, va_arg(va, int));
+	    break;
 	case '\0':
 	    --fmt;
 	    break;
-- 
MST

[PATCH 2/5] kvm-unittest: libcflat: add offsetof

[Michael S. Tsirkin]

implement offsetof using gcc builtin.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 lib/libcflat.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/libcflat.h b/lib/libcflat.h
index 0c62e4c..e005d3a 100644
--- a/lib/libcflat.h
+++ b/lib/libcflat.h
@@ -54,4 +54,6 @@ extern void *memset(void *s, int c, size_t n);
 extern long atol(const char *ptr);
 #define ARRAY_SIZE(_a)  (sizeof(_a)/sizeof((_a)[0]))
 
+#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
+
 #endif
-- 
MST

[PATCH 3/5] libcflat.h: define NULL

[Michael S. Tsirkin]

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 lib/libcflat.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/libcflat.h b/lib/libcflat.h
index e005d3a..0875bd9 100644
--- a/lib/libcflat.h
+++ b/lib/libcflat.h
@@ -56,4 +56,5 @@ extern long atol(const char *ptr);
 
 #define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
 
+#define NULL ((void *)0UL)
 #endif
-- 
MST

[PATCH 4/5] lib: add pci bus scan support

[Michael S. Tsirkin]

Add API to scan bus for a given device,
as well as API to access BAR registers.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 config-x86-common.mak |  1 +
 lib/x86/pci.c         | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/x86/pci.h         | 16 +++++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 lib/x86/pci.c
 create mode 100644 lib/x86/pci.h

diff --git a/config-x86-common.mak b/config-x86-common.mak
index 1f0f1ba..455032b 100644
--- a/config-x86-common.mak
+++ b/config-x86-common.mak
@@ -14,6 +14,7 @@ cflatobjs += lib/x86/apic.o
 cflatobjs += lib/x86/atomic.o
 cflatobjs += lib/x86/desc.o
 cflatobjs += lib/x86/isr.o
+cflatobjs += lib/x86/pci.o
 
 $(libcflat): LDFLAGS += -nostdlib
 $(libcflat): CFLAGS += -ffreestanding -I lib
diff --git a/lib/x86/pci.c b/lib/x86/pci.c
new file mode 100644
index 0000000..f95cd88
--- /dev/null
+++ b/lib/x86/pci.c
@@ -0,0 +1,55 @@
+#include <linux/pci_regs.h>
+#include "pci.h"
+
+static void outl(unsigned short port, unsigned val)
+{
+    asm volatile("outl %d0, %w1" : : "a"(val), "Nd"(port));
+}
+
+static unsigned inl(unsigned short port)
+{
+    unsigned data;
+    asm volatile("inl %w1, %d0" : "=a"(data) : "Nd"(port));
+    return data;
+}
+static uint32_t pci_config_read(pcidevaddr_t dev, uint8_t reg)
+{
+    uint32_t index = reg | (dev << 8) | (0x1 << 31); 
+    outl(0xCF8, index);
+    return inl(0xCFC);
+}
+
+/* Scan bus look for a specific device. Only bus 0 scanned for now. */
+pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id)
+{
+    unsigned dev;
+    for (dev = 0; dev < 256; ++dev) {
+    uint32_t id = pci_config_read(dev, 0);
+    if ((id & 0xFFFF) == vendor_id && (id >> 16) == device_id) {
+        return dev;
+    }
+    }
+    return PCIDEVADDR_INVALID;
+}
+
+unsigned long pci_bar_addr(pcidevaddr_t dev, int bar_num)
+{
+    uint32_t bar = pci_config_read(dev, PCI_BASE_ADDRESS_0 + bar_num * 4);
+    if (bar & PCI_BASE_ADDRESS_SPACE_IO) {
+        return bar & PCI_BASE_ADDRESS_IO_MASK;
+    } else {
+        return bar & PCI_BASE_ADDRESS_MEM_MASK;
+    }
+}
+
+bool pci_bar_is_memory(pcidevaddr_t dev, int bar_num)
+{
+    uint32_t bar = pci_config_read(dev, PCI_BASE_ADDRESS_0 + bar_num * 4);
+    return !(bar & PCI_BASE_ADDRESS_SPACE_IO);
+}
+
+bool pci_bar_is_valid(pcidevaddr_t dev, int bar_num)
+{
+    uint32_t bar = pci_config_read(dev, PCI_BASE_ADDRESS_0 + bar_num * 4);
+    return bar;
+}
diff --git a/lib/x86/pci.h b/lib/x86/pci.h
new file mode 100644
index 0000000..0f69ef0
--- /dev/null
+++ b/lib/x86/pci.h
@@ -0,0 +1,16 @@
+#ifndef PCI_H
+#define PCI_H
+
+#include <inttypes.h>
+#include "libcflat.h"
+
+typedef uint16_t pcidevaddr_t;
+enum {
+    PCIDEVADDR_INVALID = 0x0
+};
+pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id);
+unsigned long pci_bar_addr(pcidevaddr_t dev, int bar_num);
+bool pci_bar_is_memory(pcidevaddr_t dev, int bar_num);
+bool pci_bar_is_valid(pcidevaddr_t dev, int bar_num);
+
+#endif
-- 
MST

[PATCH 5/5] vmexit: add pci io and memory speed tests

[Michael S. Tsirkin]

These tests measure speed of mmio and port io
exits using the new pci-testdev.
Skipped when running on old qemu.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 x86-run      |  13 +++-
 x86/vmexit.c | 226 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 224 insertions(+), 15 deletions(-)

diff --git a/x86-run b/x86-run
index e395a70..f87b8c7 100755
--- a/x86-run
+++ b/x86-run
@@ -17,11 +17,20 @@ else
 	fi
 fi
 
+if ${qemu} -device '?' 2>&1 | fgrep pci-testdev > /dev/null;
+then
+	pci_testdev="-device pci-testdev"
+else
+	pci_testdev=""
+fi
+
 if
 	${qemu} -device '?' 2>&1 | fgrep "pc-testdev" > /dev/null;
 then
-	command="${qemu} -enable-kvm -device pc-testdev -serial stdio -device isa-debug-exit,iobase=0xf4,iosize=0x4 -kernel"
+	pc_testdev="-device pc-testdev -device isa-debug-exit,iobase=0xf4,iosize=0x4"
 else
-	command="${qemu} -device testdev,chardev=testlog -chardev file,id=testlog,path=msr.out -serial stdio -kernel"
+	pc_testdev="-device testdev,chardev=testlog -chardev file,id=testlog,path=msr.out"
 fi
+
+command="${qemu} -enable-kvm $pc_testdev -serial stdio $pci_testdev -kernel"
 exec ${command} "$@"
diff --git a/x86/vmexit.c b/x86/vmexit.c
index f49a993..3b945de 100644
--- a/x86/vmexit.c
+++ b/x86/vmexit.c
@@ -1,12 +1,32 @@
-
 #include "libcflat.h"
 #include "smp.h"
 #include "processor.h"
 #include "atomic.h"
+#include "x86/vm.h"
+#include "x86/desc.h"
+#include "x86/pci.h"
+
+struct test {
+	void (*func)(void);
+	const char *name;
+	int (*valid)(void);
+	int parallel;
+	bool (*next)(struct test *);
+};
+
+static void outb(unsigned short port, unsigned val)
+{
+    asm volatile("outb %b0, %w1" : : "a"(val), "Nd"(port));
+}
 
-static void outb(unsigned short port, int val)
+static void outw(unsigned short port, unsigned val)
 {
-    asm volatile("outb %b0, %w1" : "=a"(val) : "Nd"(port));
+    asm volatile("outw %w0, %w1" : : "a"(val), "Nd"(port));
+}
+
+static void outl(unsigned short port, unsigned val)
+{
+    asm volatile("outl %d0, %w1" : : "a"(val), "Nd"(port));
 }
 
 static unsigned int inb(unsigned short port)
@@ -141,12 +161,153 @@ static void wr_tsc_adjust_msr(void)
 	wrmsr(MSR_TSC_ADJUST, 0x0);
 }
 
-static struct test {
-	void (*func)(void);
-	const char *name;
-	int (*valid)(void);
-	int parallel;
-} tests[] = {
+struct pci_test_dev_hdr {
+    uint8_t test;
+    uint8_t width;
+    uint8_t pad0[2];
+    uint32_t offset;
+    uint32_t data;
+    uint32_t count;
+    uint8_t name[];
+};
+
+static struct pci_test {
+	unsigned iobar;
+	unsigned ioport;
+	volatile void *memaddr;
+	volatile void *mem;
+	int test_idx;
+	uint32_t data;
+	uint32_t offset;
+} pci_test = {
+	.test_idx = -1
+};
+
+static void pci_mem_testb(void)
+{
+	*(volatile uint8_t *)pci_test.mem = pci_test.data;
+}
+
+static void pci_mem_testw(void)
+{
+	*(volatile uint16_t *)pci_test.mem = pci_test.data;
+}
+
+static void pci_mem_testl(void)
+{
+	*(volatile uint32_t *)pci_test.mem = pci_test.data;
+}
+
+static void pci_io_testb(void)
+{
+	outb(pci_test.ioport, pci_test.data);
+}
+
+static void pci_io_testw(void)
+{
+	outw(pci_test.ioport, pci_test.data);
+}
+
+static void pci_io_testl(void)
+{
+	outl(pci_test.ioport, pci_test.data);
+}
+
+static uint8_t ioreadb(unsigned long addr, bool io)
+{
+	if (io) {
+		return inb(addr);
+	} else {
+		return *(volatile uint8_t *)addr;
+	}
+}
+
+static uint32_t ioreadl(unsigned long addr, bool io)
+{
+	/* Note: assumes little endian */
+	if (io) {
+		return inl(addr);
+	} else {
+		return *(volatile uint32_t *)addr;
+	}
+}
+
+static void iowriteb(unsigned long addr, uint8_t data, bool io)
+{
+	if (io) {
+		outb(addr, data);
+	} else {
+		*(volatile uint8_t *)addr = data;
+	}
+}
+
+static bool pci_next(struct test *test, unsigned long addr, bool io)
+{
+	int i;
+	uint8_t width;
+
+	if (!pci_test.memaddr) {
+		test->func = NULL;
+		return true;
+	}
+	pci_test.test_idx++;
+	iowriteb(addr + offsetof(struct pci_test_dev_hdr, test),
+		 pci_test.test_idx, io);
+	width = ioreadb(addr + offsetof(struct pci_test_dev_hdr, width),
+			io);
+	switch (width) {
+		case 1:
+			test->func = io ? pci_io_testb : pci_mem_testb;
+			break;
+		case 2:
+			test->func = io ? pci_io_testw : pci_mem_testw;
+			break;
+		case 4:
+			test->func = io ? pci_io_testl : pci_mem_testl;
+			break;
+		default:
+			/* Reset index for purposes of the next test */
+			pci_test.test_idx = -1;
+			test->func = NULL;
+			return false;
+	}
+	pci_test.data = ioreadl(addr + offsetof(struct pci_test_dev_hdr, data),
+				io);
+	pci_test.offset = ioreadl(addr + offsetof(struct pci_test_dev_hdr,
+						  offset), io);
+	for (i = 0; i < pci_test.offset; ++i) {
+		char c = ioreadb(addr + offsetof(struct pci_test_dev_hdr,
+						 name) + i, io);
+		if (!c) {
+			break;
+		}
+		printf("%c",c);
+	}
+	printf(":");
+	return true;
+}
+
+static bool pci_mem_next(struct test *test)
+{
+	bool ret;
+	ret = pci_next(test, ((unsigned long)pci_test.memaddr), false);
+	if (ret) {
+		pci_test.mem = pci_test.memaddr + pci_test.offset;
+	}
+	return ret;
+}
+
+static bool pci_io_next(struct test *test)
+{
+	bool ret;
+	ret = pci_next(test, ((unsigned long)pci_test.iobar), true);
+	if (ret) {
+		pci_test.ioport = pci_test.iobar + pci_test.offset;
+	}
+	return ret;
+}
+
+static struct test tests[] = {
 	{ cpuid_test, "cpuid", .parallel = 1,  },
 	{ vmcall, "vmcall", .parallel = 1, },
 #ifdef __x86_64__
@@ -162,6 +323,8 @@ static struct test {
 	{ ple_round_robin, "ple-round-robin", .parallel = 1 },
 	{ wr_tsc_adjust_msr, "wr_tsc_adjust_msr", .parallel = 1 },
 	{ rd_tsc_adjust_msr, "rd_tsc_adjust_msr", .parallel = 1 },
+	{ NULL, "pci-mem", .parallel = 0, .next = pci_mem_next },
+	{ NULL, "pci-io", .parallel = 0, .next = pci_io_next },
 };
 
 unsigned iterations;
@@ -178,17 +341,27 @@ static void run_test(void *_func)
     atomic_inc(&nr_cpus_done);
 }
 
-static void do_test(struct test *test)
+static bool do_test(struct test *test)
 {
 	int i;
 	unsigned long long t1, t2;
-        void (*func)(void) = test->func;
+        void (*func)(void);
 
         iterations = 32;
 
         if (test->valid && !test->valid()) {
 		printf("%s (skipped)\n", test->name);
-		return;
+		return false;
+	}
+
+	if (test->next && !test->next(test)) {
+		return false;
+	}
+
+	func = test->func;
+        if (!func) {
+		printf("%s (skipped)\n", test->name);
+		return false;
 	}
 
 	do {
@@ -208,6 +381,7 @@ static void do_test(struct test *test)
 		t2 = rdtsc();
 	} while ((t2 - t1) < GOAL);
 	printf("%s %d\n", test->name, (int)((t2 - t1) / iterations));
+	return test->next;
 }
 
 static void enable_nx(void *junk)
@@ -233,16 +407,42 @@ bool test_wanted(struct test *test, char *wanted[], int nwanted)
 int main(int ac, char **av)
 {
 	int i;
+	unsigned long membar = 0, base, offset;
+	void *m;
+	pcidevaddr_t pcidev;
 
 	smp_init();
+	setup_vm();
 	nr_cpus = cpu_count();
 
 	for (i = cpu_count(); i > 0; i--)
 		on_cpu(i-1, enable_nx, 0);
 
+	pcidev = pci_find_dev(0x1b36, 0x0005);
+	if (pcidev) {
+		for (i = 0; i < 2; i++) {
+			if (!pci_bar_is_valid(pcidev, i)) {
+				continue;
+			}
+			if (pci_bar_is_memory(pcidev, i)) {
+				membar = pci_bar_addr(pcidev, i);
+				base = membar & ~4095;
+				offset = membar - base;
+				m = alloc_vpages(1);
+				
+				install_page((void *)read_cr3(), base, m);
+				pci_test.memaddr = m + offset;
+			} else {
+				pci_test.iobar = pci_bar_addr(pcidev, i);
+			}
+		}
+		printf("pci-testdev at 0x%x membar %lx iobar %x\n",
+		       pcidev, membar, pci_test.iobar);
+	}
+
 	for (i = 0; i < ARRAY_SIZE(tests); ++i)
 		if (test_wanted(&tests[i], av + 1, ac - 1))
-			do_test(&tests[i]);
+			while (do_test(&tests[i])) {}
 
 	return 0;
 }
-- 
MST

Re: [PATCH 0/5] kvm-unittests: add pci PORT IO and MMIO speed tests

[Gleb Natapov]

On Wed, Apr 03, 2013 at 11:52:19AM +0300, Michael S. Tsirkin wrote:
> These patches add ability to measure speed of MMIO versus PIO, in
> different configurations. As I didn't want to reserve a hardcoded range
> of memory, I added pci bus scan support instead.  Used together with the
> new pci-testdev backend.  When not present, the new tests are skipped.
> 
> Example output:
> 	mmio-no-eventfd:pci-mem 8796
> 	mmio-wildcard-eventfd:pci-mem 3609
> 	mmio-datamatch-eventfd:pci-mem 3685
> 	portio-no-eventfd:pci-io 5287
> 	portio-wildcard-eventfd:pci-io 1762
> 	portio-datamatch-eventfd:pci-io 1777
> 
> First interesting conclusion is that the overhead of MMIO
> exit to QEMU as compared to PIO is double that of
> MMIO ioeventfd as compared to PIO eventfd. Is this a known fact?
> 
Looks fine to me. Will apply when QEMU part will land in upstream.

> Michael S. Tsirkin (5):
>   kvm-unittest: add printf %c support
>   kvm-unittest: libcflat: add offsetof
>   libcflat.h: define NULL
>   lib: add pci bus scan support
>   vmexit: add pci io and memory speed tests
> 
>  config-x86-common.mak |   1 +
>  lib/libcflat.h        |   3 +
>  lib/printf.c          |   3 +
>  lib/x86/pci.c         |  55 ++++++++++++
>  lib/x86/pci.h         |  16 ++++
>  x86-run               |  13 ++-
>  x86/vmexit.c          | 226 +++++++++++++++++++++++++++++++++++++++++++++++---
>  7 files changed, 302 insertions(+), 15 deletions(-)
>  create mode 100644 lib/x86/pci.c
>  create mode 100644 lib/x86/pci.h
> 
> -- 
> MST

--
			Gleb.

Re: [PATCH 0/5] kvm-unittests: add pci PORT IO and MMIO speed tests

[Gleb Natapov]

On Wed, Apr 03, 2013 at 11:52:19AM +0300, Michael S. Tsirkin wrote:
> These patches add ability to measure speed of MMIO versus PIO, in
> different configurations. As I didn't want to reserve a hardcoded range
> of memory, I added pci bus scan support instead.  Used together with the
> new pci-testdev backend.  When not present, the new tests are skipped.
> 
> Example output:
> 	mmio-no-eventfd:pci-mem 8796
> 	mmio-wildcard-eventfd:pci-mem 3609
> 	mmio-datamatch-eventfd:pci-mem 3685
> 	portio-no-eventfd:pci-io 5287
> 	portio-wildcard-eventfd:pci-io 1762
> 	portio-datamatch-eventfd:pci-io 1777
> 
> First interesting conclusion is that the overhead of MMIO
> exit to QEMU as compared to PIO is double that of
> MMIO ioeventfd as compared to PIO eventfd. Is this a known fact?
> 
Applied, thanks.

> Michael S. Tsirkin (5):
>   kvm-unittest: add printf %c support
>   kvm-unittest: libcflat: add offsetof
>   libcflat.h: define NULL
>   lib: add pci bus scan support
>   vmexit: add pci io and memory speed tests
> 
>  config-x86-common.mak |   1 +
>  lib/libcflat.h        |   3 +
>  lib/printf.c          |   3 +
>  lib/x86/pci.c         |  55 ++++++++++++
>  lib/x86/pci.h         |  16 ++++
>  x86-run               |  13 ++-
>  x86/vmexit.c          | 226 +++++++++++++++++++++++++++++++++++++++++++++++---
>  7 files changed, 302 insertions(+), 15 deletions(-)
>  create mode 100644 lib/x86/pci.c
>  create mode 100644 lib/x86/pci.h
> 
> -- 
> MST

--
			Gleb.