[PATCH v9 01/19] xen: add PV/PVH kernel entry point

[PATCH v9 01/19] xen: add PV/PVH kernel entry point

[Roger Pau Monne]

Add the PV/PVH entry point and the low level functions for PVH
initialization.
---
 sys/amd64/amd64/locore.S     |    1 +
 sys/amd64/amd64/xen-locore.S |   83 +++++++++++++++++++++++++++++
 sys/amd64/include/asmacros.h |   26 +++++++++
 sys/conf/files.amd64         |    2 +
 sys/i386/xen/xen_machdep.c   |    2 +
 sys/x86/xen/hvm.c            |    1 +
 sys/x86/xen/pv.c             |  119 ++++++++++++++++++++++++++++++++++++++++++
 sys/xen/xen-os.h             |    4 ++
 8 files changed, 238 insertions(+), 0 deletions(-)
 create mode 100644 sys/amd64/amd64/xen-locore.S
 create mode 100644 sys/x86/xen/pv.c

diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S
index 55cda3a..4acef97 100644
--- a/sys/amd64/amd64/locore.S
+++ b/sys/amd64/amd64/locore.S
@@ -84,5 +84,6 @@ NON_GPROF_ENTRY(btext)
 
 	.bss
 	ALIGN_DATA			/* just to be sure */
+	.globl	bootstack
 	.space	0x1000			/* space for bootstack - temporary stack */
 bootstack:
diff --git a/sys/amd64/amd64/xen-locore.S b/sys/amd64/amd64/xen-locore.S
new file mode 100644
index 0000000..84287c4
--- /dev/null
+++ b/sys/amd64/amd64/xen-locore.S
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 2003 Peter Wemm <peter@FreeBSD.org>
+ * Copyright (c) 2013 Roger Pau Monne <royger@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/asmacros.h>
+#include <machine/psl.h>
+#include <machine/pmap.h>
+#include <machine/specialreg.h>
+
+#include <xen/xen-os.h>
+#define __ASSEMBLY__
+#include <xen/interface/elfnote.h>
+
+#include "assym.s"
+
+.section __xen_guest
+	ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS,       .asciz, "FreeBSD")
+	ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION,  .asciz, "HEAD")
+	ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION,    .asciz, "xen-3.0")
+	ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE,      .quad,  KERNBASE)
+	ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET,   .quad,  KERNBASE) /* Xen honours elf->p_paddr; compensate for this */
+	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          .quad,  xen_start)
+	ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .quad,	 hypercall_page)
+	ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW,   .quad,  HYPERVISOR_VIRT_START)
+	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz, "writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel|hvm_callback_vector")
+	ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE,       .asciz, "yes")
+	ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID,   .long,  PG_V, PG_V)
+	ELFNOTE(Xen, XEN_ELFNOTE_LOADER,         .asciz, "generic")
+	ELFNOTE(Xen, XEN_ELFNOTE_SUSPEND_CANCEL, .long,  0)
+	ELFNOTE(Xen, XEN_ELFNOTE_BSD_SYMTAB,	 .asciz, "yes")
+
+	.text
+.p2align PAGE_SHIFT, 0x90	/* Hypercall_page needs to be PAGE aligned */
+
+NON_GPROF_ENTRY(hypercall_page)
+	.skip	0x1000, 0x90	/* Fill with "nop"s */
+
+NON_GPROF_ENTRY(xen_start)
+	/* Don't trust what the loader gives for rflags. */
+	pushq	$PSL_KERNEL
+	popfq
+
+	/* Parameters for the xen init function */
+	movq	%rsi, %rdi		/* shared_info (arg 1) */
+	movq	%rsp, %rsi		/* xenstack    (arg 2) */
+
+	/* Use our own stack */
+	movq	$bootstack,%rsp
+	xorl	%ebp, %ebp
+
+	/* u_int64_t hammer_time_xen(start_info_t *si, u_int64_t xenstack); */
+	call	hammer_time_xen
+	movq	%rax, %rsp		/* set up kstack for mi_startup() */
+	call	mi_startup		/* autoconfiguration, mountroot etc */
+
+	/* NOTREACHED */
+0:	hlt
+	jmp 	0b
diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h
index 1fb592a..ce8dce4 100644
--- a/sys/amd64/include/asmacros.h
+++ b/sys/amd64/include/asmacros.h
@@ -201,4 +201,30 @@
 
 #endif /* LOCORE */
 
+#ifdef __STDC__
+#define ELFNOTE(name, type, desctype, descdata...) \
+.pushsection .note.name                 ;       \
+  .align 4                              ;       \
+  .long 2f - 1f         /* namesz */    ;       \
+  .long 4f - 3f         /* descsz */    ;       \
+  .long type                            ;       \
+1:.asciz #name                          ;       \
+2:.align 4                              ;       \
+3:desctype descdata                     ;       \
+4:.align 4                              ;       \
+.popsection
+#else /* !__STDC__, i.e. -traditional */
+#define ELFNOTE(name, type, desctype, descdata) \
+.pushsection .note.name                 ;       \
+  .align 4                              ;       \
+  .long 2f - 1f         /* namesz */    ;       \
+  .long 4f - 3f         /* descsz */    ;       \
+  .long type                            ;       \
+1:.asciz "name"                         ;       \
+2:.align 4                              ;       \
+3:desctype descdata                     ;       \
+4:.align 4                              ;       \
+.popsection
+#endif /* __STDC__ */
+
 #endif /* !_MACHINE_ASMACROS_H_ */
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index d1bdcd9..16029d8 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -119,6 +119,7 @@ amd64/amd64/in_cksum.c		optional	inet | inet6
 amd64/amd64/initcpu.c		standard
 amd64/amd64/io.c		optional	io
 amd64/amd64/locore.S		standard	no-obj
+amd64/amd64/xen-locore.S	optional	xenhvm
 amd64/amd64/machdep.c		standard
 amd64/amd64/mem.c		optional	mem
 amd64/amd64/minidump_machdep.c	standard
@@ -566,3 +567,4 @@ x86/x86/nexus.c			standard
 x86/x86/tsc.c			standard
 x86/xen/hvm.c			optional	xenhvm
 x86/xen/xen_intr.c		optional	xen | xenhvm
+x86/xen/pv.c			optional	xenhvm
diff --git a/sys/i386/xen/xen_machdep.c b/sys/i386/xen/xen_machdep.c
index 7049be6..fd575ee 100644
--- a/sys/i386/xen/xen_machdep.c
+++ b/sys/i386/xen/xen_machdep.c
@@ -89,6 +89,7 @@ IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
 
 int xendebug_flags; 
 start_info_t *xen_start_info;
+start_info_t *HYPERVISOR_start_info;
 shared_info_t *HYPERVISOR_shared_info;
 xen_pfn_t *xen_machine_phys = machine_to_phys_mapping;
 xen_pfn_t *xen_phys_machine;
@@ -927,6 +928,7 @@ initvalues(start_info_t *startinfo)
 	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments_notify);	
 #endif	
 	xen_start_info = startinfo;
+	HYPERVISOR_start_info = startinfo;
 	xen_phys_machine = (xen_pfn_t *)startinfo->mfn_list;
 
 	IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index 72811dc..b397721 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -159,6 +159,7 @@ DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
 /** Hypercall table accessed via HYPERVISOR_*_op() methods. */
 char *hypercall_stubs;
 shared_info_t *HYPERVISOR_shared_info;
+start_info_t *HYPERVISOR_start_info;
 
 #ifdef SMP
 /*---------------------------- XEN PV IPI Handlers ---------------------------*/
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
new file mode 100644
index 0000000..5571ecf
--- /dev/null
+++ b/sys/x86/xen/pv.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2004 Christian Limpach.
+ * Copyright (c) 2004-2006,2008 Kip Macy
+ * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/reboot.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/rwlock.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_pager.h>
+#include <vm/vm_param.h>
+
+#include <xen/xen-os.h>
+#include <xen/hypervisor.h>
+
+/* Native initial function */
+extern u_int64_t hammer_time(u_int64_t, u_int64_t);
+/* Xen initial function */
+extern u_int64_t hammer_time_xen(start_info_t *, u_int64_t);
+
+/*
+ * First function called by the Xen PVH boot sequence.
+ *
+ * Set some Xen global variables and prepare the environment so it is
+ * as similar as possible to what native FreeBSD init function expects.
+ */
+u_int64_t
+hammer_time_xen(start_info_t *si, u_int64_t xenstack)
+{
+	u_int64_t physfree;
+	u_int64_t *PT4 = (u_int64_t *)xenstack;
+	u_int64_t *PT3 = (u_int64_t *)(xenstack + PAGE_SIZE);
+	u_int64_t *PT2 = (u_int64_t *)(xenstack + 2 * PAGE_SIZE);
+	int i;
+
+	if ((si == NULL) || (xenstack == 0)) {
+		HYPERVISOR_shutdown(SHUTDOWN_crash);
+	}
+
+	/* We use 3 pages of xen stack for the boot pagetables */
+	physfree = xenstack + 3 * PAGE_SIZE - KERNBASE;
+
+	/* Setup Xen global variables */
+	HYPERVISOR_start_info = si;
+	HYPERVISOR_shared_info =
+		(shared_info_t *)(si->shared_info + KERNBASE);
+
+	/*
+	 * Setup some misc global variables for Xen devices
+	 *
+	 * XXX: devices that need this specific variables should
+	 *      be rewritten to fetch this info by themselves from the
+	 *      start_info page.
+	 */
+	xen_store = (struct xenstore_domain_interface *)
+	            (ptoa(si->store_mfn) + KERNBASE);
+
+	xen_domain_type = XEN_PV_DOMAIN;
+	vm_guest = VM_GUEST_XEN;
+
+	/*
+	 * Use the stack Xen gives us to build the page tables
+	 * as native FreeBSD expects to find them (created
+	 * by the boot trampoline).
+	 */
+	for (i = 0; i < 512; i++) {
+		/* Each slot of the level 4 pages points to the same level 3 page */
+		PT4[i] = ((u_int64_t)&PT3[0]) - KERNBASE;
+		PT4[i] |= PG_V | PG_RW | PG_U;
+
+		/* Each slot of the level 3 pages points to the same level 2 page */
+		PT3[i] = ((u_int64_t)&PT2[0]) - KERNBASE;
+		PT3[i] |= PG_V | PG_RW | PG_U;
+
+		/* The level 2 page slots are mapped with 2MB pages for 1GB. */
+		PT2[i] = i * (2 * 1024 * 1024);
+		PT2[i] |= PG_V | PG_RW | PG_PS | PG_U;
+	}
+	load_cr3(((u_int64_t)&PT4[0]) - KERNBASE);
+
+	/* Now we can jump into the native init function */
+	return (hammer_time(0, physfree));
+}
diff --git a/sys/xen/xen-os.h b/sys/xen/xen-os.h
index 87644e9..c7474d8 100644
--- a/sys/xen/xen-os.h
+++ b/sys/xen/xen-os.h
@@ -51,6 +51,10 @@
 void force_evtchn_callback(void);
 
 extern shared_info_t *HYPERVISOR_shared_info;
+extern start_info_t *HYPERVISOR_start_info;
+
+/* XXX: we need to get rid of this and use HYPERVISOR_start_info directly */
+extern struct xenstore_domain_interface *xen_store;
 
 enum xen_domain_type {
 	XEN_NATIVE,             /* running on bare hardware    */
-- 
1.7.7.5 (Apple Git-26)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v9 02/19] xen: add macro to detect if running as Dom0

[Roger Pau Monne]

---
 sys/xen/xen-os.h |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/sys/xen/xen-os.h b/sys/xen/xen-os.h
index c7474d8..e8a5a99 100644
--- a/sys/xen/xen-os.h
+++ b/sys/xen/xen-os.h
@@ -82,6 +82,13 @@ xen_hvm_domain(void)
 	return (xen_domain_type == XEN_HVM_DOMAIN);
 }
 
+static inline int
+xen_initial_domain(void)
+{
+	return (xen_domain() && HYPERVISOR_start_info &&
+	        HYPERVISOR_start_info->flags & SIF_INITDOMAIN);
+}
+
 #ifndef xen_mb
 #define xen_mb() mb()
 #endif
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 03/19] xen: add and enable Xen console for PVH guests

[Roger Pau Monne]

This adds and enables the console used on XEN kernels.
---
 sys/conf/files                     |    4 +-
 sys/dev/xen/console/console.c      |   37 +++++++++++++++++++++++++++++------
 sys/dev/xen/console/xencons_ring.c |   15 +++++++++----
 sys/i386/include/xen/xen-os.h      |    1 -
 sys/i386/xen/xen_machdep.c         |   17 ----------------
 sys/x86/xen/pv.c                   |    4 +++
 sys/xen/xen-os.h                   |    4 +++
 7 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/sys/conf/files b/sys/conf/files
index a73d31e..f55479d 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -2523,8 +2523,8 @@ dev/xe/if_xe_pccard.c		optional xe pccard
 dev/xen/balloon/balloon.c	optional xen | xenhvm
 dev/xen/blkfront/blkfront.c	optional xen | xenhvm
 dev/xen/blkback/blkback.c	optional xen | xenhvm
-dev/xen/console/console.c	optional xen
-dev/xen/console/xencons_ring.c	optional xen
+dev/xen/console/console.c	optional xen | xenhvm
+dev/xen/console/xencons_ring.c	optional xen | xenhvm
 dev/xen/control/control.c	optional xen | xenhvm
 dev/xen/netback/netback.c	optional xen | xenhvm
 dev/xen/netfront/netfront.c	optional xen | xenhvm
diff --git a/sys/dev/xen/console/console.c b/sys/dev/xen/console/console.c
index 23eaee2..899dffc 100644
--- a/sys/dev/xen/console/console.c
+++ b/sys/dev/xen/console/console.c
@@ -69,11 +69,14 @@ struct mtx              cn_mtx;
 static char wbuf[WBUF_SIZE];
 static char rbuf[RBUF_SIZE];
 static int rc, rp;
-static unsigned int cnsl_evt_reg;
+unsigned int cnsl_evt_reg;
 static unsigned int wc, wp; /* write_cons, write_prod */
 xen_intr_handle_t xen_intr_handle;
 device_t xencons_dev;
 
+/* Virtual address of the shared console page */
+char *console_page;
+
 #ifdef KDB
 static int	xc_altbrk;
 #endif
@@ -110,9 +113,26 @@ static struct ttydevsw xc_ttydevsw = {
         .tsw_outwakeup	= xcoutwakeup,
 };
 
+/*----------------------------- Debug function -------------------------------*/
+#define XC_PRINTF_BUFSIZE 1024
+void
+xc_printf(const char *fmt, ...)
+{
+	static char buf[XC_PRINTF_BUFSIZE];
+	__va_list ap;
+
+	va_start(ap, fmt);
+	vsnprintf(buf, sizeof(buf), fmt, ap);
+	va_end(ap);
+	HYPERVISOR_console_write(buf, strlen(buf));
+}
+
 static void
 xc_cnprobe(struct consdev *cp)
 {
+	if (!xen_pv_domain())
+		return;
+
 	cp->cn_pri = CN_REMOTE;
 	sprintf(cp->cn_name, "%s0", driver_name);
 }
@@ -175,7 +195,7 @@ static void
 xc_cnputc(struct consdev *dev, int c)
 {
 
-	if (xen_start_info->flags & SIF_INITDOMAIN)
+	if (xen_initial_domain())
 		xc_cnputc_dom0(dev, c);
 	else
 		xc_cnputc_domu(dev, c);
@@ -206,8 +226,7 @@ xcons_putc(int c)
 		xcons_force_flush();
 #endif	    	
 	}
-	if (cnsl_evt_reg)
-		__xencons_tx_flush();
+	__xencons_tx_flush();
 	
 	/* inform start path that we're pretty full */
 	return ((wp - wc) >= WBUF_SIZE - 100) ? TRUE : FALSE;
@@ -217,6 +236,10 @@ static void
 xc_identify(driver_t *driver, device_t parent)
 {
 	device_t child;
+
+	if (!xen_pv_domain())
+		return;
+
 	child = BUS_ADD_CHILD(parent, 0, driver_name, 0);
 	device_set_driver(child, driver);
 	device_set_desc(child, "Xen Console");
@@ -245,7 +268,7 @@ xc_attach(device_t dev)
 	cnsl_evt_reg = 1;
 	callout_reset(&xc_callout, XC_POLLTIME, xc_timeout, xccons);
     
-	if (xen_start_info->flags & SIF_INITDOMAIN) {
+	if (xen_initial_domain()) {
 		error = xen_intr_bind_virq(dev, VIRQ_CONSOLE, 0, NULL,
 		                           xencons_priv_interrupt, NULL,
 		                           INTR_TYPE_TTY, &xen_intr_handle);
@@ -309,7 +332,7 @@ __xencons_tx_flush(void)
 		sz = wp - wc;
 		if (sz > (WBUF_SIZE - WBUF_MASK(wc)))
 			sz = WBUF_SIZE - WBUF_MASK(wc);
-		if (xen_start_info->flags & SIF_INITDOMAIN) {
+		if (xen_initial_domain()) {
 			HYPERVISOR_console_io(CONSOLEIO_write, sz, &wbuf[WBUF_MASK(wc)]);
 			wc += sz;
 		} else {
@@ -424,7 +447,7 @@ xcons_force_flush(void)
 {
 	int        sz;
 
-	if (xen_start_info->flags & SIF_INITDOMAIN)
+	if (xen_initial_domain())
 		return;
 
 	/* Spin until console data is flushed through to the domain controller. */
diff --git a/sys/dev/xen/console/xencons_ring.c b/sys/dev/xen/console/xencons_ring.c
index 3701551..d826363 100644
--- a/sys/dev/xen/console/xencons_ring.c
+++ b/sys/dev/xen/console/xencons_ring.c
@@ -32,9 +32,9 @@ __FBSDID("$FreeBSD$");
 
 #define console_evtchn	console.domU.evtchn
 xen_intr_handle_t console_handle;
-extern char *console_page;
 extern struct mtx              cn_mtx;
 extern device_t xencons_dev;
+extern int cnsl_evt_reg;
 
 static inline struct xencons_interface *
 xencons_interface(void)
@@ -60,6 +60,8 @@ xencons_ring_send(const char *data, unsigned len)
 	struct xencons_interface *intf; 
 	XENCONS_RING_IDX cons, prod;
 	int sent;
+	struct evtchn_send send = { .port =
+	                            HYPERVISOR_start_info->console_evtchn };
 
 	intf = xencons_interface();
 	cons = intf->out_cons;
@@ -76,7 +78,10 @@ xencons_ring_send(const char *data, unsigned len)
 	wmb();
 	intf->out_prod = prod;
 
-	xen_intr_signal(console_handle);
+	if (cnsl_evt_reg)
+		xen_intr_signal(console_handle);
+	else
+		HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
 
 	return sent;
 
@@ -125,11 +130,11 @@ xencons_ring_init(void)
 {
 	int err;
 
-	if (!xen_start_info->console_evtchn)
+	if (!HYPERVISOR_start_info->console_evtchn)
 		return 0;
 
 	err = xen_intr_bind_local_port(xencons_dev,
-	    xen_start_info->console_evtchn, NULL, xencons_handle_input, NULL,
+	    HYPERVISOR_start_info->console_evtchn, NULL, xencons_handle_input, NULL,
 	    INTR_TYPE_MISC | INTR_MPSAFE, &console_handle);
 	if (err) {
 		return err;
@@ -145,7 +150,7 @@ void
 xencons_suspend(void)
 {
 
-	if (!xen_start_info->console_evtchn)
+	if (!HYPERVISOR_start_info->console_evtchn)
 		return;
 
 	xen_intr_unbind(&console_handle);
diff --git a/sys/i386/include/xen/xen-os.h b/sys/i386/include/xen/xen-os.h
index a8fba61..3d1ef04 100644
--- a/sys/i386/include/xen/xen-os.h
+++ b/sys/i386/include/xen/xen-os.h
@@ -45,7 +45,6 @@ static inline void rep_nop(void)
 #define cpu_relax() rep_nop()
 
 #ifndef XENHVM
-void xc_printf(const char *fmt, ...);
 
 #ifdef SMP
 extern int gdtset;
diff --git a/sys/i386/xen/xen_machdep.c b/sys/i386/xen/xen_machdep.c
index fd575ee..09c01f1 100644
--- a/sys/i386/xen/xen_machdep.c
+++ b/sys/i386/xen/xen_machdep.c
@@ -186,21 +186,6 @@ xen_boothowto(char *envp)
 	return howto;
 }
 
-#define XC_PRINTF_BUFSIZE 1024
-void
-xc_printf(const char *fmt, ...)
-{
-        __va_list ap;
-        int retval;
-        static char buf[XC_PRINTF_BUFSIZE];
-
-        va_start(ap, fmt);
-        retval = vsnprintf(buf, XC_PRINTF_BUFSIZE - 1, fmt, ap);
-        va_end(ap);
-        buf[retval] = 0;
-        (void)HYPERVISOR_console_write(buf, retval);
-}
-
 
 #define XPQUEUE_SIZE 128
 
@@ -745,8 +730,6 @@ void initvalues(start_info_t *startinfo);
 struct xenstore_domain_interface;
 extern struct xenstore_domain_interface *xen_store;
 
-char *console_page;
-
 void *
 bootmem_alloc(unsigned int size) 
 {
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index 5571ecf..db3b7a3 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -70,9 +70,12 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
 	int i;
 
 	if ((si == NULL) || (xenstack == 0)) {
+		xc_printf("ERROR: invalid start_info or xen stack, halting\n");
 		HYPERVISOR_shutdown(SHUTDOWN_crash);
 	}
 
+	xc_printf("FreeBSD PVH running on %s\n", si->magic);
+
 	/* We use 3 pages of xen stack for the boot pagetables */
 	physfree = xenstack + 3 * PAGE_SIZE - KERNBASE;
 
@@ -90,6 +93,7 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
 	 */
 	xen_store = (struct xenstore_domain_interface *)
 	            (ptoa(si->store_mfn) + KERNBASE);
+	console_page = (char *)(ptoa(si->console.domU.mfn) + KERNBASE);
 
 	xen_domain_type = XEN_PV_DOMAIN;
 	vm_guest = VM_GUEST_XEN;
diff --git a/sys/xen/xen-os.h b/sys/xen/xen-os.h
index e8a5a99..e005ccd 100644
--- a/sys/xen/xen-os.h
+++ b/sys/xen/xen-os.h
@@ -55,6 +55,7 @@ extern start_info_t *HYPERVISOR_start_info;
 
 /* XXX: we need to get rid of this and use HYPERVISOR_start_info directly */
 extern struct xenstore_domain_interface *xen_store;
+extern char *console_page;
 
 enum xen_domain_type {
 	XEN_NATIVE,             /* running on bare hardware    */
@@ -89,6 +90,9 @@ xen_initial_domain(void)
 	        HYPERVISOR_start_info->flags & SIF_INITDOMAIN);
 }
 
+/* Debug function, prints directly to hypervisor console */
+void xc_printf(const char *, ...);
+
 #ifndef xen_mb
 #define xen_mb() mb()
 #endif
-- 
1.7.7.5 (Apple Git-26)

Re: [PATCH v9 03/19] xen: add and enable Xen console for PVH guests

[Julien Grall]

On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
> This adds and enables the console used on XEN kernels.
> ---

[..]

> +/* Debug function, prints directly to hypervisor console */
> +void xc_printf(const char *, ...);
> +

Can you add __printflike(...)? It will be easier to catch bad format.

-- 
Julien Grall

[PATCH v9 04/19] amd64: introduce hook for custom preload metadata parsers

[Roger Pau Monne]

---
 sys/amd64/amd64/machdep.c   |   41 ++++++++++++++++------
 sys/amd64/include/sysarch.h |   12 ++++++
 sys/x86/xen/pv.c            |   82 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 124 insertions(+), 11 deletions(-)

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index eae657b..e073eea 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -126,6 +126,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/reg.h>
 #include <machine/sigframe.h>
 #include <machine/specialreg.h>
+#include <machine/sysarch.h>
 #ifdef PERFMON
 #include <machine/perfmon.h>
 #endif
@@ -165,6 +166,14 @@ static int  set_fpcontext(struct thread *td, const mcontext_t *mcp,
     char *xfpustate, size_t xfpustate_len);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 
+/* Preload data parse function */
+static caddr_t native_parse_preload_data(u_int64_t);
+
+/* Default init_ops implementation. */
+struct init_ops init_ops = {
+	.parse_preload_data =	native_parse_preload_data,
+};
+
 /*
  * The file "conf/ldscript.amd64" defines the symbol "kernphys".  Its value is
  * the physical address at which the kernel is loaded.
@@ -1683,6 +1692,26 @@ do_next:
 	msgbufp = (struct msgbuf *)PHYS_TO_DMAP(phys_avail[pa_indx]);
 }
 
+static caddr_t
+native_parse_preload_data(u_int64_t modulep)
+{
+	caddr_t kmdp;
+
+	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
+	preload_bootstrap_relocate(KERNBASE);
+	kmdp = preload_search_by_type("elf kernel");
+	if (kmdp == NULL)
+		kmdp = preload_search_by_type("elf64 kernel");
+	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
+	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
+#ifdef DDB
+	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
+	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
+#endif
+
+	return (kmdp);
+}
+
 u_int64_t
 hammer_time(u_int64_t modulep, u_int64_t physfree)
 {
@@ -1707,17 +1736,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	 */
 	proc_linkup0(&proc0, &thread0);
 
-	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
-	preload_bootstrap_relocate(KERNBASE);
-	kmdp = preload_search_by_type("elf kernel");
-	if (kmdp == NULL)
-		kmdp = preload_search_by_type("elf64 kernel");
-	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
-	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
-#ifdef DDB
-	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
-	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
-#endif
+	kmdp = init_ops.parse_preload_data(modulep);
 
 	/* Init basic tunables, hz etc */
 	init_param1();
diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
index cd380d4..58ac8cd 100644
--- a/sys/amd64/include/sysarch.h
+++ b/sys/amd64/include/sysarch.h
@@ -4,3 +4,15 @@
 /* $FreeBSD$ */
 
 #include <x86/sysarch.h>
+
+/*
+ * Struct containing pointers to init functions whose
+ * implementation is run time selectable.  Selection can be made,
+ * for example, based on detection of a BIOS variant or
+ * hypervisor environment.
+ */
+struct init_ops {
+	caddr_t	(*parse_preload_data)(u_int64_t);
+};
+
+extern struct init_ops init_ops;
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index db3b7a3..908b50b 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -46,6 +46,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_pager.h>
 #include <vm/vm_param.h>
 
+#include <machine/sysarch.h>
+
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
 
@@ -54,6 +56,36 @@ extern u_int64_t hammer_time(u_int64_t, u_int64_t);
 /* Xen initial function */
 extern u_int64_t hammer_time_xen(start_info_t *, u_int64_t);
 
+/*--------------------------- Forward Declarations ---------------------------*/
+static caddr_t xen_pv_parse_preload_data(u_int64_t);
+
+static void xen_pv_set_init_ops(void);
+
+/*-------------------------------- Global Data -------------------------------*/
+/* Xen init_ops implementation. */
+struct init_ops xen_init_ops = {
+	.parse_preload_data =	xen_pv_parse_preload_data,
+};
+
+static struct
+{
+	const char	*ev;
+	int		mask;
+} howto_names[] = {
+	{"boot_askname",	RB_ASKNAME},
+	{"boot_single",		RB_SINGLE},
+	{"boot_nosync",		RB_NOSYNC},
+	{"boot_halt",		RB_ASKNAME},
+	{"boot_serial",		RB_SERIAL},
+	{"boot_cdrom",		RB_CDROM},
+	{"boot_gdb",		RB_GDB},
+	{"boot_gdb_pause",	RB_RESERVED1},
+	{"boot_verbose",	RB_VERBOSE},
+	{"boot_multicons",	RB_MULTIPLE},
+	{NULL,	0}
+};
+
+/*-------------------------------- Xen PV init -------------------------------*/
 /*
  * First function called by the Xen PVH boot sequence.
  *
@@ -118,6 +150,56 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
 	}
 	load_cr3(((u_int64_t)&PT4[0]) - KERNBASE);
 
+	/* Set the hooks for early functions that diverge from bare metal */
+	xen_pv_set_init_ops();
+
 	/* Now we can jump into the native init function */
 	return (hammer_time(0, physfree));
 }
+
+/*-------------------------------- PV specific -------------------------------*/
+/*
+ * Functions to convert the "extra" parameters passed by Xen
+ * into FreeBSD boot options (from the i386 Xen port).
+ */
+static char *
+xen_setbootenv(char *cmd_line)
+{
+	char *cmd_line_next;
+
+        /* Skip leading spaces */
+        for (; *cmd_line == ' '; cmd_line++);
+
+	for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
+	return (cmd_line);
+}
+
+static int
+xen_boothowto(char *envp)
+{
+	int i, howto = 0;
+
+	/* get equivalents from the environment */
+	for (i = 0; howto_names[i].ev != NULL; i++)
+		if (getenv(howto_names[i].ev) != NULL)
+			howto |= howto_names[i].mask;
+	return (howto);
+}
+
+static caddr_t
+xen_pv_parse_preload_data(u_int64_t modulep)
+{
+	/* Parse the extra boot information given by Xen */
+	if (HYPERVISOR_start_info->cmd_line)
+		kern_envp = xen_setbootenv(HYPERVISOR_start_info->cmd_line);
+	boothowto |= xen_boothowto(kern_envp);
+
+	return (NULL);
+}
+
+static void
+xen_pv_set_init_ops(void)
+{
+	/* Init ops for Xen PV */
+	init_ops = xen_init_ops;
+}
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 05/19] xen: rework xen timer so it can be used early in boot process

[Roger Pau Monne]

This should not introduce any functional change, and makes the
functions suitable to be called before we have actually mapped the
vcpu_info struct on a per-cpu basis.
---
 sys/dev/xen/timer/timer.c |   29 ++++++++++++++++++++---------
 1 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/sys/dev/xen/timer/timer.c b/sys/dev/xen/timer/timer.c
index 354085b..b2f6bcd 100644
--- a/sys/dev/xen/timer/timer.c
+++ b/sys/dev/xen/timer/timer.c
@@ -230,22 +230,22 @@ xen_fetch_vcpu_tinfo(struct vcpu_time_info *dst, struct vcpu_time_info *src)
 /**
  * \brief Get the current time, in nanoseconds, since the hypervisor booted.
  *
+ * \param vcpu		vcpu_info structure to fetch the time from.
+ *
  * \note This function returns the current CPU's idea of this value, unless
  *       it happens to be less than another CPU's previously determined value.
  */
 static uint64_t
-xen_fetch_vcpu_time(void)
+xen_fetch_vcpu_time(struct vcpu_info *vcpu)
 {
 	struct vcpu_time_info dst;
 	struct vcpu_time_info *src;
 	uint32_t pre_version;
 	uint64_t now;
 	volatile uint64_t last;
-	struct vcpu_info *vcpu = DPCPU_GET(vcpu_info);
 
 	src = &vcpu->time;
 
-	critical_enter();
 	do {
 		pre_version = xen_fetch_vcpu_tinfo(&dst, src);
 		barrier();
@@ -266,16 +266,19 @@ xen_fetch_vcpu_time(void)
 		}
 	} while (!atomic_cmpset_64(&xen_timer_last_time, last, now));
 
-	critical_exit();
-
 	return (now);
 }
 
 static uint32_t
 xentimer_get_timecount(struct timecounter *tc)
 {
+	uint32_t xen_time;
 
-	return ((uint32_t)xen_fetch_vcpu_time() & UINT_MAX);
+	critical_enter();
+	xen_time = (uint32_t)xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)) & UINT_MAX;
+	critical_exit();
+
+	return (xen_time);
 }
 
 /**
@@ -305,7 +308,12 @@ xen_fetch_wallclock(struct timespec *ts)
 static void
 xen_fetch_uptime(struct timespec *ts)
 {
-	uint64_t uptime = xen_fetch_vcpu_time();
+	uint64_t uptime;
+
+	critical_enter();
+	uptime = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info));
+	critical_exit();
+
 	ts->tv_sec = uptime / NSEC_IN_SEC;
 	ts->tv_nsec = uptime % NSEC_IN_SEC;
 }
@@ -354,7 +362,7 @@ xentimer_intr(void *arg)
 	struct xentimer_softc *sc = (struct xentimer_softc *)arg;
 	struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu);
 
-	pcpu->last_processed = xen_fetch_vcpu_time();
+	pcpu->last_processed = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info));
 	if (pcpu->timer != 0 && sc->et.et_active)
 		sc->et.et_event_cb(&sc->et, sc->et.et_arg);
 
@@ -415,7 +423,10 @@ xentimer_et_start(struct eventtimer *et,
 	do {
 		if (++i == 60)
 			panic("can't schedule timer");
-		next_time = xen_fetch_vcpu_time() + first_in_ns;
+		critical_enter();
+		next_time = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)) +
+		            first_in_ns;
+		critical_exit();
 		error = xentimer_vcpu_start_timer(cpu, next_time);
 	} while (error == -ETIME);
 
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 06/19] xen: implement an early timer for Xen PVH

[Roger Pau Monne]

When running as a PVH guest, there's no emulated i8254, so we need to
use the Xen PV timer as the early source for DELAY. This change allows
for different implementations of the early DELAY function and
implements a Xen variant for it.
---
 sys/amd64/amd64/machdep.c   |    6 ++-
 sys/amd64/include/clock.h   |    5 ++
 sys/amd64/include/sysarch.h |    2 +
 sys/conf/files.amd64        |    1 +
 sys/conf/files.i386         |    1 +
 sys/dev/xen/timer/timer.c   |   33 +++++++++++++
 sys/i386/include/clock.h    |    5 ++
 sys/x86/isa/clock.c         |   53 +--------------------
 sys/x86/x86/delay.c         |  112 +++++++++++++++++++++++++++++++++++++++++++
 sys/x86/xen/pv.c            |    3 +
 10 files changed, 167 insertions(+), 54 deletions(-)
 create mode 100644 sys/x86/x86/delay.c

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index e073eea..178d8b3 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -172,6 +172,8 @@ static caddr_t native_parse_preload_data(u_int64_t);
 /* Default init_ops implementation. */
 struct init_ops init_ops = {
 	.parse_preload_data =	native_parse_preload_data,
+	.early_delay_init =	i8254_init,
+	.early_delay =		i8254_delay,
 };
 
 /*
@@ -1820,10 +1822,10 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	lidt(&r_idt);
 
 	/*
-	 * Initialize the i8254 before the console so that console
+	 * Initialize the early delay before the console so that console
 	 * initialization can use DELAY().
 	 */
-	i8254_init();
+	init_ops.early_delay_init();
 
 	/*
 	 * Initialize the console before we print anything out.
diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h
index d7f7d82..ac8818f 100644
--- a/sys/amd64/include/clock.h
+++ b/sys/amd64/include/clock.h
@@ -25,6 +25,11 @@ extern int	smp_tsc;
 #endif
 
 void	i8254_init(void);
+void	i8254_delay(int);
+#ifdef XENHVM
+void	xen_delay_init(void);
+void	xen_delay(int);
+#endif
 
 /*
  * Driver to clock driver interface.
diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
index 58ac8cd..60fa635 100644
--- a/sys/amd64/include/sysarch.h
+++ b/sys/amd64/include/sysarch.h
@@ -13,6 +13,8 @@
  */
 struct init_ops {
 	caddr_t	(*parse_preload_data)(u_int64_t);
+	void	(*early_delay_init)(void);
+	void	(*early_delay)(int);
 };
 
 extern struct init_ops init_ops;
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 16029d8..109a796 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -565,6 +565,7 @@ x86/x86/mptable_pci.c		optional	mptable pci
 x86/x86/msi.c			optional	pci
 x86/x86/nexus.c			standard
 x86/x86/tsc.c			standard
+x86/x86/delay.c			standard
 x86/xen/hvm.c			optional	xenhvm
 x86/xen/xen_intr.c		optional	xen | xenhvm
 x86/xen/pv.c			optional	xenhvm
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index eb8697c..790296d 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -600,5 +600,6 @@ x86/x86/mptable_pci.c		optional apic native pci
 x86/x86/msi.c			optional apic pci
 x86/x86/nexus.c			standard
 x86/x86/tsc.c			standard
+x86/x86/delay.c			standard
 x86/xen/hvm.c			optional xenhvm
 x86/xen/xen_intr.c		optional xen | xenhvm
diff --git a/sys/dev/xen/timer/timer.c b/sys/dev/xen/timer/timer.c
index b2f6bcd..96372ab 100644
--- a/sys/dev/xen/timer/timer.c
+++ b/sys/dev/xen/timer/timer.c
@@ -59,6 +59,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/_inttypes.h>
 #include <machine/smp.h>
 
+/* For the declaration of clock_lock */
+#include <isa/rtc.h>
+
 #include "clock_if.h"
 
 static devclass_t xentimer_devclass;
@@ -584,6 +587,36 @@ xentimer_suspend(device_t dev)
 	return (0);
 }
 
+/*
+ * Xen delay early init
+ */
+void xen_delay_init(void)
+{
+	/* Init the clock lock */
+	mtx_init(&clock_lock, "clk", NULL, MTX_SPIN | MTX_NOPROFILE);
+}
+/*
+ * Xen PV DELAY function
+ *
+ * When running on PVH mode we don't have an emulated i8524, so
+ * make use of the Xen time info in order to code a simple DELAY
+ * function that can be used during early boot.
+ */
+void xen_delay(int n)
+{
+	uint64_t end_ns;
+	uint64_t current;
+
+	end_ns = xen_fetch_vcpu_time(&HYPERVISOR_shared_info->vcpu_info[0]);
+	end_ns += n * NSEC_IN_USEC;
+
+	for (;;) {
+		current = xen_fetch_vcpu_time(&HYPERVISOR_shared_info->vcpu_info[0]);
+		if (current >= end_ns)
+			break;
+	}
+}
+
 static device_method_t xentimer_methods[] = {
 	DEVMETHOD(device_identify, xentimer_identify),
 	DEVMETHOD(device_probe, xentimer_probe),
diff --git a/sys/i386/include/clock.h b/sys/i386/include/clock.h
index d980ec7..b831445 100644
--- a/sys/i386/include/clock.h
+++ b/sys/i386/include/clock.h
@@ -22,6 +22,11 @@ extern int	tsc_is_invariant;
 extern int	tsc_perf_stat;
 
 void	i8254_init(void);
+void	i8254_delay(int);
+#ifdef XENHVM
+void	xen_delay_init(void);
+void	xen_delay(int);
+#endif
 
 /*
  * Driver to clock driver interface.
diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c
index a12e175..a5aed1c 100644
--- a/sys/x86/isa/clock.c
+++ b/sys/x86/isa/clock.c
@@ -247,61 +247,13 @@ getit(void)
 	return ((high << 8) | low);
 }
 
-#ifndef DELAYDEBUG
-static u_int
-get_tsc(__unused struct timecounter *tc)
-{
-
-	return (rdtsc32());
-}
-
-static __inline int
-delay_tc(int n)
-{
-	struct timecounter *tc;
-	timecounter_get_t *func;
-	uint64_t end, freq, now;
-	u_int last, mask, u;
-
-	tc = timecounter;
-	freq = atomic_load_acq_64(&tsc_freq);
-	if (tsc_is_invariant && freq != 0) {
-		func = get_tsc;
-		mask = ~0u;
-	} else {
-		if (tc->tc_quality <= 0)
-			return (0);
-		func = tc->tc_get_timecount;
-		mask = tc->tc_counter_mask;
-		freq = tc->tc_frequency;
-	}
-	now = 0;
-	end = freq * n / 1000000;
-	if (func == get_tsc)
-		sched_pin();
-	last = func(tc) & mask;
-	do {
-		cpu_spinwait();
-		u = func(tc) & mask;
-		if (u < last)
-			now += mask - last + u + 1;
-		else
-			now += u - last;
-		last = u;
-	} while (now < end);
-	if (func == get_tsc)
-		sched_unpin();
-	return (1);
-}
-#endif
-
 /*
  * Wait "n" microseconds.
  * Relies on timer 1 counting down from (i8254_freq / hz)
  * Note: timer had better have been programmed before this is first used!
  */
 void
-DELAY(int n)
+i8254_delay(int n)
 {
 	int delta, prev_tick, tick, ticks_left;
 #ifdef DELAYDEBUG
@@ -317,9 +269,6 @@ DELAY(int n)
 	}
 	if (state == 1)
 		printf("DELAY(%d)...", n);
-#else
-	if (delay_tc(n))
-		return;
 #endif
 	/*
 	 * Read the counter first, so that the rest of the setup overhead is
diff --git a/sys/x86/x86/delay.c b/sys/x86/x86/delay.c
new file mode 100644
index 0000000..d13c727
--- /dev/null
+++ b/sys/x86/x86/delay.c
@@ -0,0 +1,112 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz and Don Ahn.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/* Generic x86 routines to handle delay */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/timetc.h>
+#include <sys/proc.h>
+#include <sys/kernel.h>
+#include <sys/sched.h>
+
+#include <machine/clock.h>
+#include <machine/cpu.h>
+#include <machine/sysarch.h>
+
+static u_int
+get_tsc(__unused struct timecounter *tc)
+{
+
+	return (rdtsc32());
+}
+
+static int
+delay_tc(int n)
+{
+	struct timecounter *tc;
+	timecounter_get_t *func;
+	uint64_t end, freq, now;
+	u_int last, mask, u;
+
+	tc = timecounter;
+	freq = atomic_load_acq_64(&tsc_freq);
+	if (tsc_is_invariant && freq != 0) {
+		func = get_tsc;
+		mask = ~0u;
+	} else {
+		if (tc->tc_quality <= 0)
+			return (0);
+		func = tc->tc_get_timecount;
+		mask = tc->tc_counter_mask;
+		freq = tc->tc_frequency;
+	}
+	now = 0;
+	end = freq * n / 1000000;
+	if (func == get_tsc)
+		sched_pin();
+	last = func(tc) & mask;
+	do {
+		cpu_spinwait();
+		u = func(tc) & mask;
+		if (u < last)
+			now += mask - last + u + 1;
+		else
+			now += u - last;
+		last = u;
+	} while (now < end);
+	if (func == get_tsc)
+		sched_unpin();
+	return (1);
+}
+
+#ifndef XEN
+void
+DELAY(int n)
+{
+
+	if (delay_tc(n))
+		return;
+
+#ifdef __amd64__
+	init_ops.early_delay(n);
+#else
+	i8254_delay(n);
+#endif
+}
+#endif
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index 908b50b..0ec4b54 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_param.h>
 
 #include <machine/sysarch.h>
+#include <machine/clock.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
@@ -65,6 +66,8 @@ static void xen_pv_set_init_ops(void);
 /* Xen init_ops implementation. */
 struct init_ops xen_init_ops = {
 	.parse_preload_data =	xen_pv_parse_preload_data,
+	.early_delay_init =	xen_delay_init,
+	.early_delay =		xen_delay,
 };
 
 static struct
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 07/19] xen: implement hook to fetch e820 memory map

[Roger Pau Monne]

---
 sys/amd64/amd64/machdep.c   |   50 ++++++++++++++++++++++++++----------------
 sys/amd64/include/pc/bios.h |    2 +
 sys/amd64/include/sysarch.h |    1 +
 sys/x86/xen/pv.c            |   25 +++++++++++++++++++++
 4 files changed, 59 insertions(+), 19 deletions(-)

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 178d8b3..f6eef50 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -169,11 +169,15 @@ SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 /* Preload data parse function */
 static caddr_t native_parse_preload_data(u_int64_t);
 
+/* Native function to fetch and parse the e820 map */
+static void native_parse_memmap(caddr_t, vm_paddr_t *, int *);
+
 /* Default init_ops implementation. */
 struct init_ops init_ops = {
 	.parse_preload_data =	native_parse_preload_data,
 	.early_delay_init =	i8254_init,
 	.early_delay =		i8254_delay,
+	.parse_memmap =		native_parse_memmap,
 };
 
 /*
@@ -1401,21 +1405,12 @@ add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
 	return (1);
 }
 
-static void
-add_smap_entries(struct bios_smap *smapbase, vm_paddr_t *physmap,
-    int *physmap_idx)
+void
+bios_add_smap_entries(struct bios_smap *smapbase, u_int32_t smapsize,
+                      vm_paddr_t *physmap, int *physmap_idx)
 {
 	struct bios_smap *smap, *smapend;
-	u_int32_t smapsize;
 
-	/*
-	 * Memory map from INT 15:E820.
-	 *
-	 * subr_module.c says:
-	 * "Consumer may safely assume that size value precedes data."
-	 * ie: an int32_t immediately precedes smap.
-	 */
-	smapsize = *((u_int32_t *)smapbase - 1);
 	smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
 
 	for (smap = smapbase; smap < smapend; smap++) {
@@ -1432,6 +1427,29 @@ add_smap_entries(struct bios_smap *smapbase, vm_paddr_t *physmap,
 	}
 }
 
+static void
+native_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
+{
+	struct bios_smap *smap;
+	u_int32_t size;
+
+	/*
+	 * Memory map from INT 15:E820.
+	 *
+	 * subr_module.c says:
+	 * "Consumer may safely assume that size value precedes data."
+	 * ie: an int32_t immediately precedes smap.
+	 */
+
+	smap = (struct bios_smap *)preload_search_info(kmdp,
+	    MODINFO_METADATA | MODINFOMD_SMAP);
+	if (smap == NULL)
+		panic("No BIOS smap info from loader!");
+	size = *((u_int32_t *)smap - 1);
+
+	bios_add_smap_entries(smap, size, physmap, physmap_idx);
+}
+
 /*
  * Populate the (physmap) array with base/bound pairs describing the
  * available physical memory in the system, then test this memory and
@@ -1449,19 +1467,13 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 	vm_paddr_t pa, physmap[PHYSMAP_SIZE];
 	u_long physmem_start, physmem_tunable, memtest;
 	pt_entry_t *pte;
-	struct bios_smap *smapbase;
 	quad_t dcons_addr, dcons_size;
 
 	bzero(physmap, sizeof(physmap));
 	basemem = 0;
 	physmap_idx = 0;
 
-	smapbase = (struct bios_smap *)preload_search_info(kmdp,
-	    MODINFO_METADATA | MODINFOMD_SMAP);
-	if (smapbase == NULL)
-		panic("No BIOS smap info from loader!");
-
-	add_smap_entries(smapbase, physmap, &physmap_idx);
+	init_ops.parse_memmap(kmdp, physmap, &physmap_idx);
 
 	/*
 	 * Find the 'base memory' segment for SMP
diff --git a/sys/amd64/include/pc/bios.h b/sys/amd64/include/pc/bios.h
index e7d568e..95ef703 100644
--- a/sys/amd64/include/pc/bios.h
+++ b/sys/amd64/include/pc/bios.h
@@ -106,6 +106,8 @@ struct bios_oem {
 int	bios_oem_strings(struct bios_oem *oem, u_char *buffer, size_t maxlen);
 uint32_t bios_sigsearch(uint32_t start, u_char *sig, int siglen, int paralen,
 	    int sigofs);
+void bios_add_smap_entries(struct bios_smap *smapbase, u_int32_t smapsize,
+	    vm_paddr_t *physmap, int *physmap_idx);
 #endif
 
 #endif /* _MACHINE_PC_BIOS_H_ */
diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
index 60fa635..084223e 100644
--- a/sys/amd64/include/sysarch.h
+++ b/sys/amd64/include/sysarch.h
@@ -15,6 +15,7 @@ struct init_ops {
 	caddr_t	(*parse_preload_data)(u_int64_t);
 	void	(*early_delay_init)(void);
 	void	(*early_delay)(int);
+	void	(*parse_memmap)(caddr_t, vm_paddr_t *, int *);
 };
 
 extern struct init_ops init_ops;
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index 0ec4b54..d11bc1a 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/sysarch.h>
 #include <machine/clock.h>
+#include <machine/pc/bios.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
@@ -57,8 +58,11 @@ extern u_int64_t hammer_time(u_int64_t, u_int64_t);
 /* Xen initial function */
 extern u_int64_t hammer_time_xen(start_info_t *, u_int64_t);
 
+#define MAX_E820_ENTRIES	128
+
 /*--------------------------- Forward Declarations ---------------------------*/
 static caddr_t xen_pv_parse_preload_data(u_int64_t);
+static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *);
 
 static void xen_pv_set_init_ops(void);
 
@@ -68,6 +72,7 @@ struct init_ops xen_init_ops = {
 	.parse_preload_data =	xen_pv_parse_preload_data,
 	.early_delay_init =	xen_delay_init,
 	.early_delay =		xen_delay,
+	.parse_memmap =		xen_pv_parse_memmap,
 };
 
 static struct
@@ -88,6 +93,8 @@ static struct
 	{NULL,	0}
 };
 
+static struct bios_smap xen_smap[MAX_E820_ENTRIES];
+
 /*-------------------------------- Xen PV init -------------------------------*/
 /*
  * First function called by the Xen PVH boot sequence.
@@ -201,6 +208,24 @@ xen_pv_parse_preload_data(u_int64_t modulep)
 }
 
 static void
+xen_pv_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
+{
+	struct xen_memory_map memmap;
+	u_int32_t size;
+	int rc;
+
+	/* Fetch the E820 map from Xen */
+	memmap.nr_entries = MAX_E820_ENTRIES;
+	set_xen_guest_handle(memmap.buffer, xen_smap);
+	rc = HYPERVISOR_memory_op(XENMEM_memory_map, &memmap);
+	if (rc)
+		panic("unable to fetch Xen E820 memory map");
+	size = memmap.nr_entries * sizeof(xen_smap[0]);
+
+	bios_add_smap_entries(xen_smap, size, physmap, physmap_idx);
+}
+
+static void
 xen_pv_set_init_ops(void)
 {
 	/* Init ops for Xen PV */
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 08/19] xen: use the same hypercall mechanism for XEN and XENHVM

[Roger Pau Monne]

---
 sys/amd64/include/xen/hypercall.h |    7 -------
 sys/i386/i386/locore.s            |    9 +++++++++
 sys/i386/include/xen/hypercall.h  |    8 --------
 sys/x86/xen/hvm.c                 |   24 ++++++++++--------------
 4 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/sys/amd64/include/xen/hypercall.h b/sys/amd64/include/xen/hypercall.h
index a1b2a5c..499fb4d 100644
--- a/sys/amd64/include/xen/hypercall.h
+++ b/sys/amd64/include/xen/hypercall.h
@@ -51,15 +51,8 @@
 #define CONFIG_XEN_COMPAT	0x030002
 #define __must_check
 
-#ifdef XEN
 #define HYPERCALL_STR(name)					\
 	"call hypercall_page + ("STR(__HYPERVISOR_##name)" * 32)"
-#else
-#define HYPERCALL_STR(name)					\
-	"mov $("STR(__HYPERVISOR_##name)" * 32),%%eax; "\
-	"add hypercall_stubs(%%rip),%%rax; "			\
-	"call *%%rax"
-#endif
 
 #define _hypercall0(type, name)			\
 ({						\
diff --git a/sys/i386/i386/locore.s b/sys/i386/i386/locore.s
index 68cb430..bd136b1 100644
--- a/sys/i386/i386/locore.s
+++ b/sys/i386/i386/locore.s
@@ -898,3 +898,12 @@ done_pde:
 #endif
 
 	ret
+
+#ifdef XENHVM
+/* Xen Hypercall page */
+	.text
+.p2align PAGE_SHIFT, 0x90	/* Hypercall_page needs to be PAGE aligned */
+
+NON_GPROF_ENTRY(hypercall_page)
+	.skip	0x1000, 0x90	/* Fill with "nop"s */
+#endif
diff --git a/sys/i386/include/xen/hypercall.h b/sys/i386/include/xen/hypercall.h
index edc13f4..16b5ee2 100644
--- a/sys/i386/include/xen/hypercall.h
+++ b/sys/i386/include/xen/hypercall.h
@@ -39,16 +39,8 @@
 #define	ENOXENSYS	38
 #define CONFIG_XEN_COMPAT	0x030002
 
-
-#if defined(XEN)
 #define HYPERCALL_STR(name)                                     \
         "call hypercall_page + ("STR(__HYPERVISOR_##name)" * 32)"
-#else
-#define HYPERCALL_STR(name)                                     \
-        "mov hypercall_stubs,%%eax; "                           \
-        "add $("STR(__HYPERVISOR_##name)" * 32),%%eax; "        \
-        "call *%%eax"
-#endif
 
 #define _hypercall0(type, name)                 \
 ({                                              \
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index b397721..9a0411e 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -157,7 +157,7 @@ DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
 
 /*------------------ Hypervisor Access Shared Memory Regions -----------------*/
 /** Hypercall table accessed via HYPERVISOR_*_op() methods. */
-char *hypercall_stubs;
+extern char *hypercall_page;
 shared_info_t *HYPERVISOR_shared_info;
 start_info_t *HYPERVISOR_start_info;
 
@@ -559,7 +559,7 @@ xen_hvm_cpuid_base(void)
  * Allocate and fill in the hypcall page.
  */
 static int
-xen_hvm_init_hypercall_stubs(void)
+xen_hvm_init_hypercall_stubs(enum xen_hvm_init_type init_type)
 {
 	uint32_t base, regs[4];
 	int i;
@@ -568,7 +568,7 @@ xen_hvm_init_hypercall_stubs(void)
 	if (base == 0)
 		return (ENXIO);
 
-	if (hypercall_stubs == NULL) {
+	if (init_type == XEN_HVM_INIT_COLD) {
 		do_cpuid(base + 1, regs);
 		printf("XEN: Hypervisor version %d.%d detected.\n",
 		    regs[0] >> 16, regs[0] & 0xffff);
@@ -578,18 +578,9 @@ xen_hvm_init_hypercall_stubs(void)
 	 * Find the hypercall pages.
 	 */
 	do_cpuid(base + 2, regs);
-	
-	if (hypercall_stubs == NULL) {
-		size_t call_region_size;
-
-		call_region_size = regs[0] * PAGE_SIZE;
-		hypercall_stubs = malloc(call_region_size, M_XENHVM, M_NOWAIT);
-		if (hypercall_stubs == NULL)
-			panic("Unable to allocate Xen hypercall region");
-	}
 
 	for (i = 0; i < regs[0]; i++)
-		wrmsr(regs[1], vtophys(hypercall_stubs + i * PAGE_SIZE) + i);
+		wrmsr(regs[1], vtophys(&hypercall_page + i * PAGE_SIZE) + i);
 
 	return (0);
 }
@@ -692,7 +683,12 @@ xen_hvm_init(enum xen_hvm_init_type init_type)
 	if (init_type == XEN_HVM_INIT_CANCELLED_SUSPEND)
 		return;
 
-	error = xen_hvm_init_hypercall_stubs();
+	if (xen_pv_domain()) {
+		/* hypercall page is already set in the PV case */
+		error = 0;
+	} else {
+		error = xen_hvm_init_hypercall_stubs(init_type);
+	}
 
 	switch (init_type) {
 	case XEN_HVM_INIT_COLD:
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 09/19] xen: add a apic_enumerator for PVH

[Roger Pau Monne]

---
 sys/conf/files.amd64     |    1 +
 sys/x86/xen/pvcpu_enum.c |  136 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 137 insertions(+), 0 deletions(-)
 create mode 100644 sys/x86/xen/pvcpu_enum.c

diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 109a796..a3491da 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -569,3 +569,4 @@ x86/x86/delay.c			standard
 x86/xen/hvm.c			optional	xenhvm
 x86/xen/xen_intr.c		optional	xen | xenhvm
 x86/xen/pv.c			optional	xenhvm
+x86/xen/pvcpu_enum.c		optional	xenhvm
diff --git a/sys/x86/xen/pvcpu_enum.c b/sys/x86/xen/pvcpu_enum.c
new file mode 100644
index 0000000..0384886
--- /dev/null
+++ b/sys/x86/xen/pvcpu_enum.c
@@ -0,0 +1,136 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/smp.h>
+#include <sys/pcpu.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+
+#include <machine/cpu.h>
+#include <machine/smp.h>
+
+#include <xen/xen-os.h>
+#include <xen/hypervisor.h>
+
+#include <xen/interface/vcpu.h>
+
+static int xenpv_probe(void);
+static int xenpv_probe_cpus(void);
+static int xenpv_setup_local(void);
+static int xenpv_setup_io(void);
+
+static struct apic_enumerator xenpv_enumerator = {
+	"Xen PV",
+	xenpv_probe,
+	xenpv_probe_cpus,
+	xenpv_setup_local,
+	xenpv_setup_io
+};
+
+/*
+ * This enumerator will only be registered on PVH
+ */
+static int
+xenpv_probe(void)
+{
+	return (-100);
+}
+
+/*
+ * Test each possible vCPU in order to find the number of vCPUs
+ */
+static int
+xenpv_probe_cpus(void)
+{
+#ifdef SMP
+	int i, ret;
+
+	for (i = 0; i < MAXCPU; i++) {
+		ret = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
+		if (ret >= 0)
+			cpu_add((i * 2), (i == 0));
+	}
+#endif
+	return (0);
+}
+
+/*
+ * Initialize the vCPU id of the BSP
+ */
+static int
+xenpv_setup_local(void)
+{
+	PCPU_SET(vcpu_id, 0);
+	return (0);
+}
+
+/*
+ * On PVH guests there's no IO APIC
+ */
+static int
+xenpv_setup_io(void)
+{
+	return (0);
+}
+
+static void
+xenpv_register(void *dummy __unused)
+{
+	if (xen_pv_domain()) {
+		apic_register_enumerator(&xenpv_enumerator);
+	}
+}
+SYSINIT(xenpv_register, SI_SUB_TUNABLES - 1, SI_ORDER_FIRST, xenpv_register, NULL);
+
+/*
+ * Setup per-CPU vCPU IDs
+ */
+static void
+xenpv_set_ids(void *dummy)
+{
+	struct pcpu *pc;
+	int i;
+
+	CPU_FOREACH(i) {
+		pc = pcpu_find(i);
+		pc->pc_vcpu_id = i;
+	}
+}
+SYSINIT(xenpv_set_ids, SI_SUB_CPU, SI_ORDER_MIDDLE, xenpv_set_ids, NULL);
-- 
1.7.7.5 (Apple Git-26)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v9 10/19] xen: add hook for AP bootstrap memory reservation

[Roger Pau Monne]

This hook will only be implemented for bare metal, Xen doesn't require
any bootstrap code since APs are started in long mode with paging
enabled.
---
 sys/amd64/amd64/machdep.c   |    6 +++++-
 sys/amd64/include/sysarch.h |    1 +
 2 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index f6eef50..babf16d 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -178,6 +178,9 @@ struct init_ops init_ops = {
 	.early_delay_init =	i8254_init,
 	.early_delay =		i8254_delay,
 	.parse_memmap =		native_parse_memmap,
+#ifdef SMP
+	.mp_bootaddress =	mp_bootaddress,
+#endif
 };
 
 /*
@@ -1490,7 +1493,8 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 
 #ifdef SMP
 	/* make hole for AP bootstrap code */
-	physmap[1] = mp_bootaddress(physmap[1] / 1024);
+	if (init_ops.mp_bootaddress)
+		physmap[1] = init_ops.mp_bootaddress(physmap[1] / 1024);
 #endif
 
 	/*
diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
index 084223e..7696064 100644
--- a/sys/amd64/include/sysarch.h
+++ b/sys/amd64/include/sysarch.h
@@ -16,6 +16,7 @@ struct init_ops {
 	void	(*early_delay_init)(void);
 	void	(*early_delay)(int);
 	void	(*parse_memmap)(caddr_t, vm_paddr_t *, int *);
+	u_int	(*mp_bootaddress)(u_int);
 };
 
 extern struct init_ops init_ops;
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 11/19] xen: changes to hvm code in order to support PVH guests

[Roger Pau Monne]

On PVH we don't need to init the shared info page, or disable emulated
devices. Also, make sure PV IPIs are set before starting the APs.
---
 sys/x86/xen/hvm.c |   17 ++++++++++++-----
 1 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index 9a0411e..fb1ed79 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -523,7 +523,7 @@ xen_setup_cpus(void)
 {
 	int i;
 
-	if (!xen_hvm_domain() || !xen_vector_callback_enabled)
+	if (!xen_vector_callback_enabled)
 		return;
 
 #ifdef __amd64__
@@ -712,10 +712,13 @@ xen_hvm_init(enum xen_hvm_init_type init_type)
 	}
 
 	xen_vector_callback_enabled = 0;
-	xen_domain_type = XEN_HVM_DOMAIN;
-	xen_hvm_init_shared_info_page();
 	xen_hvm_set_callback(NULL);
-	xen_hvm_disable_emulated_devices();
+
+	if (!xen_pv_domain()) {
+		xen_domain_type = XEN_HVM_DOMAIN;
+		xen_hvm_init_shared_info_page();
+		xen_hvm_disable_emulated_devices();
+	}
 } 
 
 void
@@ -746,6 +749,9 @@ xen_set_vcpu_id(void)
 	struct pcpu *pc;
 	int i;
 
+	if (!xen_hvm_domain())
+		return;
+
 	/* Set vcpu_id to acpi_id */
 	CPU_FOREACH(i) {
 		pc = pcpu_find(i);
@@ -789,7 +795,8 @@ xen_hvm_cpu_init(void)
 
 SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_sysinit, NULL);
 #ifdef SMP
-SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_FIRST, xen_setup_cpus, NULL);
+/* We need to setup IPIs before APs are started */
+SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL);
 #endif
 SYSINIT(xen_hvm_cpu_init, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_cpu_init, NULL);
 SYSINIT(xen_set_vcpu_id, SI_SUB_CPU, SI_ORDER_ANY, xen_set_vcpu_id, NULL);
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 12/19] xen: add a hook to perform AP startup

[Roger Pau Monne]

AP startup on PVH follows the PV method, so we need to add a hook in
order to diverge from bare metal.
---
 sys/amd64/amd64/mp_machdep.c |   14 +++---
 sys/amd64/include/cpu.h      |    1 +
 sys/amd64/include/smp.h      |    1 +
 sys/x86/xen/hvm.c            |   12 +++++-
 sys/x86/xen/pv.c             |   85 ++++++++++++++++++++++++++++++++++++++++++
 sys/xen/pv.h                 |   32 ++++++++++++++++
 6 files changed, 137 insertions(+), 8 deletions(-)
 create mode 100644 sys/xen/pv.h

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 4ef4b3d..0738a37 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -90,7 +90,7 @@ extern  struct pcpu __pcpu[];
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-static int bootAP;
+int bootAP;
 
 /* Free these after use */
 void *bootstacks[MAXCPU];
@@ -124,7 +124,8 @@ static u_long *ipi_hardclock_counts[MAXCPU];
 
 /* Default cpu_ops implementation. */
 struct cpu_ops cpu_ops = {
-	.ipi_vectored = lapic_ipi_vectored
+	.ipi_vectored = lapic_ipi_vectored,
+	.start_all_aps = native_start_all_aps,
 };
 
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
@@ -138,7 +139,7 @@ extern int pmap_pcid_enabled;
 static volatile cpuset_t ipi_nmi_pending;
 
 /* used to hold the AP's until we are ready to release them */
-static struct mtx ap_boot_mtx;
+struct mtx ap_boot_mtx;
 
 /* Set to 1 once we're ready to let the APs out of the pen. */
 static volatile int aps_ready = 0;
@@ -165,7 +166,6 @@ static int cpu_cores;			/* cores per package */
 
 static void	assign_cpu_ids(void);
 static void	set_interrupt_apic_ids(void);
-static int	start_all_aps(void);
 static int	start_ap(int apic_id);
 static void	release_aps(void *dummy);
 
@@ -569,7 +569,7 @@ cpu_mp_start(void)
 	assign_cpu_ids();
 
 	/* Start each Application Processor */
-	start_all_aps();
+	cpu_ops.start_all_aps();
 
 	set_interrupt_apic_ids();
 }
@@ -908,8 +908,8 @@ assign_cpu_ids(void)
 /*
  * start each AP in our list
  */
-static int
-start_all_aps(void)
+int
+native_start_all_aps(void)
 {
 	vm_offset_t va = boot_address + KERNBASE;
 	u_int64_t *pt4, *pt3, *pt2;
diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h
index 3d9ff531..ed9f1db 100644
--- a/sys/amd64/include/cpu.h
+++ b/sys/amd64/include/cpu.h
@@ -64,6 +64,7 @@ struct cpu_ops {
 	void (*cpu_init)(void);
 	void (*cpu_resume)(void);
 	void (*ipi_vectored)(u_int, int);
+	int  (*start_all_aps)(void);
 };
 
 extern struct	cpu_ops cpu_ops;
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index d1b366b..15bc823 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -79,6 +79,7 @@ void	smp_masked_invlpg_range(cpuset_t mask, struct pmap *pmap,
 	    vm_offset_t startva, vm_offset_t endva);
 void	smp_invltlb(struct pmap *pmap);
 void	smp_masked_invltlb(cpuset_t mask, struct pmap *pmap);
+int	native_start_all_aps(void);
 
 #endif /* !LOCORE */
 #endif /* SMP */
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index fb1ed79..49caacf 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -53,6 +53,9 @@ __FBSDID("$FreeBSD$");
 #include <xen/hypervisor.h>
 #include <xen/hvm.h>
 #include <xen/xen_intr.h>
+#ifdef __amd64__
+#include <xen/pv.h>
+#endif
 
 #include <xen/interface/hvm/params.h>
 #include <xen/interface/vcpu.h>
@@ -119,7 +122,10 @@ enum xen_domain_type xen_domain_type = XEN_NATIVE;
 struct cpu_ops xen_hvm_cpu_ops = {
 	.ipi_vectored	= lapic_ipi_vectored,
 	.cpu_init	= xen_hvm_cpu_init,
-	.cpu_resume	= xen_hvm_cpu_resume
+	.cpu_resume	= xen_hvm_cpu_resume,
+#ifdef __amd64__
+	.start_all_aps = native_start_all_aps,
+#endif
 };
 
 static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support");
@@ -698,6 +704,10 @@ xen_hvm_init(enum xen_hvm_init_type init_type)
 		setup_xen_features();
 		cpu_ops = xen_hvm_cpu_ops;
  		vm_guest = VM_GUEST_XEN;
+#ifdef __amd64__
+		if (xen_pv_domain())
+			cpu_ops.start_all_aps = xen_pv_start_all_aps;
+#endif
 		break;
 	case XEN_HVM_INIT_RESUME:
 		if (error != 0)
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index d11bc1a..22fd6a6 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -34,8 +34,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/reboot.h>
 #include <sys/systm.h>
+#include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/rwlock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -49,9 +52,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/sysarch.h>
 #include <machine/clock.h>
 #include <machine/pc/bios.h>
+#include <machine/smp.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
+#include <xen/pv.h>
+
+#include <xen/interface/vcpu.h>
 
 /* Native initial function */
 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
@@ -65,6 +72,15 @@ static caddr_t xen_pv_parse_preload_data(u_int64_t);
 static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *);
 
 static void xen_pv_set_init_ops(void);
+/*---------------------------- Extern Declarations ---------------------------*/
+/* Variables used by amd64 mp_machdep to start APs */
+extern struct mtx ap_boot_mtx;
+extern void *bootstacks[];
+extern char *doublefault_stack;
+extern char *nmi_stack;
+extern void *dpcpu;
+extern int bootAP;
+extern char *bootSTK;
 
 /*-------------------------------- Global Data -------------------------------*/
 /* Xen init_ops implementation. */
@@ -168,6 +184,75 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
 }
 
 /*-------------------------------- PV specific -------------------------------*/
+
+static int
+start_xen_ap(int cpu)
+{
+	struct vcpu_guest_context *ctxt;
+	int ms, cpus = mp_naps;
+
+	ctxt = malloc(sizeof(*ctxt), M_TEMP, M_NOWAIT | M_ZERO);
+	if (ctxt == NULL)
+		panic("unable to allocate memory");
+
+	ctxt->flags = VGCF_IN_KERNEL;
+	ctxt->user_regs.rip = (unsigned long) init_secondary;
+	ctxt->user_regs.rsp = (unsigned long) bootSTK;
+
+	/* Set the AP to use the same page tables */
+	ctxt->ctrlreg[3] = KPML4phys;
+
+	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
+		panic("unable to initialize AP#%d\n", cpu);
+
+	free(ctxt, M_TEMP);
+
+	/* Launch the vCPU */
+	if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
+		panic("unable to start AP#%d\n", cpu);
+
+	/* Wait up to 5 seconds for it to start. */
+	for (ms = 0; ms < 5000; ms++) {
+		if (mp_naps > cpus)
+			return (1);	/* return SUCCESS */
+		DELAY(1000);
+	}
+
+	return (0);
+}
+
+int
+xen_pv_start_all_aps(void)
+{
+	int cpu;
+
+	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
+
+	for (cpu = 1; cpu < mp_ncpus; cpu++) {
+
+		/* allocate and set up an idle stack data page */
+		bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,
+		    KSTACK_PAGES * PAGE_SIZE, M_WAITOK | M_ZERO);
+		doublefault_stack = (char *)kmem_malloc(kernel_arena,
+		    PAGE_SIZE, M_WAITOK | M_ZERO);
+		nmi_stack = (char *)kmem_malloc(kernel_arena, PAGE_SIZE,
+		    M_WAITOK | M_ZERO);
+		dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
+		    M_WAITOK | M_ZERO);
+
+		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
+		bootAP = cpu;
+
+		/* attempt to start the Application Processor */
+		if (!start_xen_ap(cpu))
+			panic("AP #%d failed to start!", cpu);
+
+		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
+	}
+
+	return (mp_naps);
+}
+
 /*
  * Functions to convert the "extra" parameters passed by Xen
  * into FreeBSD boot options (from the i386 Xen port).
diff --git a/sys/xen/pv.h b/sys/xen/pv.h
new file mode 100644
index 0000000..45b7473
--- /dev/null
+++ b/sys/xen/pv.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	__XEN_PV_H__
+#define	__XEN_PV_H__
+
+int	xen_pv_start_all_aps(void);
+
+#endif	/* __XEN_PV_H__ */
-- 
1.7.7.5 (Apple Git-26)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v9 13/19] xen: introduce flag to disable the local apic

[Roger Pau Monne]

PVH guests don't have an emulated lapic.
---
 sys/amd64/amd64/mp_machdep.c |   10 ++++++----
 sys/amd64/include/apicvar.h  |    1 +
 sys/i386/include/apicvar.h   |    1 +
 sys/i386/xen/xen_machdep.c   |    2 ++
 sys/x86/x86/local_apic.c     |    8 +++++---
 sys/x86/xen/pv.c             |    3 +++
 6 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 0738a37..fd6eace 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -707,7 +707,8 @@ init_secondary(void)
 	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
 
 	/* Disable local APIC just to be sure. */
-	lapic_disable();
+	if (lapic_valid)
+		lapic_disable();
 
 	/* signal our startup to the BSP. */
 	mp_naps++;
@@ -733,7 +734,7 @@ init_secondary(void)
 
 	/* A quick check from sanity claus */
 	cpuid = PCPU_GET(cpuid);
-	if (PCPU_GET(apic_id) != lapic_id()) {
+	if (lapic_valid && (PCPU_GET(apic_id) != lapic_id())) {
 		printf("SMP: cpuid = %d\n", cpuid);
 		printf("SMP: actual apic_id = %d\n", lapic_id());
 		printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
@@ -749,7 +750,8 @@ init_secondary(void)
 	mtx_lock_spin(&ap_boot_mtx);
 
 	/* Init local apic for irq's */
-	lapic_setup(1);
+	if (lapic_valid)
+		lapic_setup(1);
 
 	/* Set memory range attributes for this CPU to match the BSP */
 	mem_range_AP_init();
@@ -764,7 +766,7 @@ init_secondary(void)
 	if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
 		CPU_SET(cpuid, &logical_cpus_mask);
 
-	if (bootverbose)
+	if (lapic_valid && bootverbose)
 		lapic_dump("AP");
 
 	if (smp_cpus == mp_ncpus) {
diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h
index e7423a3..c04a238 100644
--- a/sys/amd64/include/apicvar.h
+++ b/sys/amd64/include/apicvar.h
@@ -169,6 +169,7 @@ inthand_t
 
 extern vm_paddr_t lapic_paddr;
 extern int apic_cpuids[];
+extern bool lapic_valid;
 
 u_int	apic_alloc_vector(u_int apic_id, u_int irq);
 u_int	apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count,
diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h
index df99ebe..ea8a3c3 100644
--- a/sys/i386/include/apicvar.h
+++ b/sys/i386/include/apicvar.h
@@ -168,6 +168,7 @@ inthand_t
 
 extern vm_paddr_t lapic_paddr;
 extern int apic_cpuids[];
+extern bool lapic_valid;
 
 u_int	apic_alloc_vector(u_int apic_id, u_int irq);
 u_int	apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count,
diff --git a/sys/i386/xen/xen_machdep.c b/sys/i386/xen/xen_machdep.c
index 09c01f1..25b9cfc 100644
--- a/sys/i386/xen/xen_machdep.c
+++ b/sys/i386/xen/xen_machdep.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/asmacros.h>
+#include <machine/apicvar.h>
 
 
 
@@ -912,6 +913,7 @@ initvalues(start_info_t *startinfo)
 #endif	
 	xen_start_info = startinfo;
 	HYPERVISOR_start_info = startinfo;
+	lapic_valid = false;
 	xen_phys_machine = (xen_pfn_t *)startinfo->mfn_list;
 
 	IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
index 41bd602..fddf1fb 100644
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -156,6 +156,7 @@ extern inthand_t IDTVEC(rsvd);
 
 volatile lapic_t *lapic;
 vm_paddr_t lapic_paddr;
+bool lapic_valid = true;
 static u_long lapic_timer_divisor;
 static struct eventtimer lapic_et;
 
@@ -1367,9 +1368,10 @@ apic_setup_io(void *dummy __unused)
 	if (retval != 0)
 		printf("%s: Failed to setup I/O APICs: returned %d\n",
 		    best_enum->apic_name, retval);
-#ifdef XEN
-	return;
-#endif
+
+	if (!lapic_valid)
+		return;
+
 	/*
 	 * Finish setting up the local APIC on the BSP once we know how to
 	 * properly program the LINT pins.
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index 22fd6a6..6ea1e2a 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/clock.h>
 #include <machine/pc/bios.h>
 #include <machine/smp.h>
+#include <machine/apicvar.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
@@ -315,4 +316,6 @@ xen_pv_set_init_ops(void)
 {
 	/* Init ops for Xen PV */
 	init_ops = xen_init_ops;
+	/* Disable lapic */
+	lapic_valid = false;
 }
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 14/19] xen: introduce xenpv bus and a dummy pvcpu device

[Roger Pau Monne]

Since Xen PVH guests doesn't have ACPI, we need to create a dummy
bus so top level Xen devices can attach to it (instead of
attaching directly to the nexus) and a pvcpu device that will be used
to fill the pcpu->pc_device field.
---
 sys/conf/files.amd64 |    1 +
 sys/conf/files.i386  |    1 +
 sys/x86/xen/xenpv.c  |  155 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 157 insertions(+), 0 deletions(-)
 create mode 100644 sys/x86/xen/xenpv.c

diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index a3491da..d7c98cc 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -570,3 +570,4 @@ x86/xen/hvm.c			optional	xenhvm
 x86/xen/xen_intr.c		optional	xen | xenhvm
 x86/xen/pv.c			optional	xenhvm
 x86/xen/pvcpu_enum.c		optional	xenhvm
+x86/xen/xenpv.c			optional	xenhvm
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 790296d..81142e3 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -603,3 +603,4 @@ x86/x86/tsc.c			standard
 x86/x86/delay.c			standard
 x86/xen/hvm.c			optional xenhvm
 x86/xen/xen_intr.c		optional xen | xenhvm
+x86/xen/xenpv.c			optional xen | xenhvm
diff --git a/sys/x86/xen/xenpv.c b/sys/x86/xen/xenpv.c
new file mode 100644
index 0000000..41d674f
--- /dev/null
+++ b/sys/x86/xen/xenpv.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/pcpu.h>
+#include <sys/smp.h>
+
+#include <xen/xen-os.h>
+
+static int
+xenpv_probe(device_t dev)
+{
+
+	device_set_desc(dev, "Xen PV bus");
+	device_quiet(dev);
+	return (0);
+}
+
+static int
+xenpv_attach(device_t dev)
+{
+	device_t child;
+
+	/*
+	 * Let our child drivers identify any child devices that they
+	 * can find.  Once that is done attach any devices that we
+	 * found.
+	 */
+	bus_generic_probe(dev);
+	bus_generic_attach(dev);
+
+	if (!devclass_get_device(devclass_find("isa"), 0)) {
+		child = BUS_ADD_CHILD(dev, 0, "isa", 0);
+		if (child == NULL)
+			panic("xenpv_attach isa");
+		device_probe_and_attach(child);
+	}
+
+	return 0;
+}
+
+static device_method_t xenpv_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		xenpv_probe),
+	DEVMETHOD(device_attach,	xenpv_attach),
+	DEVMETHOD(device_suspend,	bus_generic_suspend),
+	DEVMETHOD(device_resume,	bus_generic_resume),
+
+	/* Bus interface */
+	DEVMETHOD(bus_add_child,	bus_generic_add_child),
+
+	DEVMETHOD_END
+};
+
+static driver_t xenpv_driver = {
+	"xenpv",
+	xenpv_methods,
+	1,			/* no softc */
+};
+static devclass_t xenpv_devclass;
+
+DRIVER_MODULE(xenpv, nexus, xenpv_driver, xenpv_devclass, 0, 0);
+
+/*
+ * Dummy Xen cpu device
+ *
+ * Since there's no ACPI on PVH guests, we need to create a dummy
+ * CPU device in order to fill the pcpu->pc_device field.
+ */
+
+static void
+xenpvcpu_identify(driver_t *driver, device_t parent)
+{
+	device_t child;
+	int i;
+
+	/* Only attach to PV guests, HVM guests use the ACPI CPU devices */
+	if (!xen_pv_domain())
+		return;
+
+	CPU_FOREACH(i) {
+		child = BUS_ADD_CHILD(parent, 0, "pvcpu", i);
+		if (child == NULL)
+			panic("xenpvcpu_identify add pvcpu");
+	}
+}
+
+static int
+xenpvcpu_probe(device_t dev)
+{
+
+	device_set_desc(dev, "Xen PV CPU");
+	return (0);
+}
+
+static int
+xenpvcpu_attach(device_t dev)
+{
+	struct pcpu *pc;
+	int cpu;
+
+	cpu = device_get_unit(dev);
+	pc = pcpu_find(cpu);
+	pc->pc_device = dev;
+	return (0);
+}
+
+static device_method_t xenpvcpu_methods[] = {
+	DEVMETHOD(device_identify, xenpvcpu_identify),
+	DEVMETHOD(device_probe, xenpvcpu_probe),
+	DEVMETHOD(device_attach, xenpvcpu_attach),
+
+	DEVMETHOD_END
+};
+
+static driver_t xenpvcpu_driver = {
+	"pvcpu",
+	xenpvcpu_methods,
+	0,
+};
+
+devclass_t xenpvcpu_devclass;
+
+DRIVER_MODULE(xenpvcpu, xenpv, xenpvcpu_driver, xenpvcpu_devclass, 0, 0);
+MODULE_DEPEND(xenpvcpu, xenpv, 1, 1, 1);
-- 
1.7.7.5 (Apple Git-26)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 14/19] xen: introduce xenpv bus and a dummy pvcpu device

[Julien Grall]

On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
> Since Xen PVH guests doesn't have ACPI, we need to create a dummy
> bus so top level Xen devices can attach to it (instead of
> attaching directly to the nexus) and a pvcpu device that will be used
> to fill the pcpu->pc_device field.
> ---
>   sys/conf/files.amd64 |    1 +
>   sys/conf/files.i386  |    1 +
>   sys/x86/xen/xenpv.c  |  155 ++++++++++++++++++++++++++++++++++++++++++++++++++

I think it makes more sense to have 2 files: one for xenpv bus and one 
for a dummy pvcpu device. It would allow us to move xenpv bus to common 
code (sys/xen or sys/dev/xen).

[..]

> +
> +static int
> +xenpv_probe(device_t dev)
> +{
> +
> +	device_set_desc(dev, "Xen PV bus");
> +	device_quiet(dev);
> +	return (0);

As I understand, 0 means I can "handle" the current device, in this case 
if a device is probing, because it doesn't have yet a driver, we will 
use xenpv and end up with 2 (or even more) xenpv buses.

As we only want to probe xenpv bus once, when the bus was added 
manually, returning BUS_PROBE_NO_WILDCARD would suit better.

[..]

> +static int
> +xenpvcpu_probe(device_t dev)
> +{
> +
> +	device_set_desc(dev, "Xen PV CPU");
> +	return (0);

Same here: BUS_PROBE_NOWILDCARD.

-- 
Julien Grall

Re: [PATCH v9 14/19] xen: introduce xenpv bus and a dummy pvcpu device

[Roger Pau Monné]

On 05/01/14 22:52, Julien Grall wrote:
> 
> 
> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>> Since Xen PVH guests doesn't have ACPI, we need to create a dummy
>> bus so top level Xen devices can attach to it (instead of
>> attaching directly to the nexus) and a pvcpu device that will be used
>> to fill the pcpu->pc_device field.
>> ---
>>   sys/conf/files.amd64 |    1 +
>>   sys/conf/files.i386  |    1 +
>>   sys/x86/xen/xenpv.c  |  155
>> ++++++++++++++++++++++++++++++++++++++++++++++++++
> 
> I think it makes more sense to have 2 files: one for xenpv bus and one
> for a dummy pvcpu device. It would allow us to move xenpv bus to common
> code (sys/xen or sys/dev/xen).

Ack. I wasn't thinking other arches will probably use the xenpv bus but
not the dummy cpu device. Would you agree to leave xenpv bus inside
x86/xen for now and move the dummy PV cpu device to dev/xen/pvcpu/?

> 
> [..]
> 
>> +
>> +static int
>> +xenpv_probe(device_t dev)
>> +{
>> +
>> +    device_set_desc(dev, "Xen PV bus");
>> +    device_quiet(dev);
>> +    return (0);
> 
> As I understand, 0 means I can "handle" the current device, in this case
> if a device is probing, because it doesn't have yet a driver, we will
> use xenpv and end up with 2 (or even more) xenpv buses.
>
> As we only want to probe xenpv bus once, when the bus was added
> manually, returning BUS_PROBE_NO_WILDCARD would suit better.
> 
> [..]
> 
>> +static int
>> +xenpvcpu_probe(device_t dev)
>> +{
>> +
>> +    device_set_desc(dev, "Xen PV CPU");
>> +    return (0);
> 
> Same here: BUS_PROBE_NOWILDCARD.

Ack for both, will change it to BUS_PROBE_NOWILDCARD. While at it, we
should also change xenstore probe function to return BUS_PROBE_NOWILDCARD.

Re: [PATCH v9 14/19] xen: introduce xenpv bus and a dummy pvcpu device

[Julien Grall]

On 01/06/2014 09:46 AM, Roger Pau Monné wrote:
> On 05/01/14 22:52, Julien Grall wrote:
>>
>>
>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>> Since Xen PVH guests doesn't have ACPI, we need to create a dummy
>>> bus so top level Xen devices can attach to it (instead of
>>> attaching directly to the nexus) and a pvcpu device that will be used
>>> to fill the pcpu->pc_device field.
>>> ---
>>>    sys/conf/files.amd64 |    1 +
>>>    sys/conf/files.i386  |    1 +
>>>    sys/x86/xen/xenpv.c  |  155
>>> ++++++++++++++++++++++++++++++++++++++++++++++++++
>>
>> I think it makes more sense to have 2 files: one for xenpv bus and one
>> for a dummy pvcpu device. It would allow us to move xenpv bus to common
>> code (sys/xen or sys/dev/xen).
>
> Ack. I wasn't thinking other arches will probably use the xenpv bus but
> not the dummy cpu device. Would you agree to leave xenpv bus inside
> x86/xen for now and move the dummy PV cpu device to dev/xen/pvcpu/?

As we will attach every xen device to xenpv, it makes more sense to have 
xenpv bus used on ARM. It will avoid duplication code and keep it nicer.

I'm fine with this solution for now. I will update/move the code when I 
will send the patch series to support FreeBSD on Xen on ARM.

>>
>> [..]
>>
>>> +
>>> +static int
>>> +xenpv_probe(device_t dev)
>>> +{
>>> +
>>> +    device_set_desc(dev, "Xen PV bus");
>>> +    device_quiet(dev);
>>> +    return (0);
>>
>> As I understand, 0 means I can "handle" the current device, in this case
>> if a device is probing, because it doesn't have yet a driver, we will
>> use xenpv and end up with 2 (or even more) xenpv buses.
>>
>> As we only want to probe xenpv bus once, when the bus was added
>> manually, returning BUS_PROBE_NO_WILDCARD would suit better.
>>
>> [..]
>>
>>> +static int
>>> +xenpvcpu_probe(device_t dev)
>>> +{
>>> +
>>> +    device_set_desc(dev, "Xen PV CPU");
>>> +    return (0);
>>
>> Same here: BUS_PROBE_NOWILDCARD.
>
> Ack for both, will change it to BUS_PROBE_NOWILDCARD. While at it, we
> should also change xenstore probe function to return BUS_PROBE_NOWILDCARD.
>

Right, I have a patch for xenstore. Do you want me to send it?

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 14/19] xen: introduce xenpv bus and a dummy pvcpu device

[Roger Pau Monné]

On 06/01/14 12:28, Julien Grall wrote:
> 
> 
> On 01/06/2014 09:46 AM, Roger Pau Monné wrote:
>> On 05/01/14 22:52, Julien Grall wrote:
>>>
>>>
>>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>>> Since Xen PVH guests doesn't have ACPI, we need to create a dummy
>>>> bus so top level Xen devices can attach to it (instead of
>>>> attaching directly to the nexus) and a pvcpu device that will be used
>>>> to fill the pcpu->pc_device field.
>>>> ---
>>>>    sys/conf/files.amd64 |    1 +
>>>>    sys/conf/files.i386  |    1 +
>>>>    sys/x86/xen/xenpv.c  |  155
>>>> ++++++++++++++++++++++++++++++++++++++++++++++++++
>>>
>>> I think it makes more sense to have 2 files: one for xenpv bus and one
>>> for a dummy pvcpu device. It would allow us to move xenpv bus to common
>>> code (sys/xen or sys/dev/xen).
>>
>> Ack. I wasn't thinking other arches will probably use the xenpv bus but
>> not the dummy cpu device. Would you agree to leave xenpv bus inside
>> x86/xen for now and move the dummy PV cpu device to dev/xen/pvcpu/?
> 
> As we will attach every xen device to xenpv, it makes more sense to have
> xenpv bus used on ARM. It will avoid duplication code and keep it nicer.
> 
> I'm fine with this solution for now. I will update/move the code when I
> will send the patch series to support FreeBSD on Xen on ARM.
> 
>>>
>>> [..]
>>>
>>>> +
>>>> +static int
>>>> +xenpv_probe(device_t dev)
>>>> +{
>>>> +
>>>> +    device_set_desc(dev, "Xen PV bus");
>>>> +    device_quiet(dev);
>>>> +    return (0);
>>>
>>> As I understand, 0 means I can "handle" the current device, in this case
>>> if a device is probing, because it doesn't have yet a driver, we will
>>> use xenpv and end up with 2 (or even more) xenpv buses.
>>>
>>> As we only want to probe xenpv bus once, when the bus was added
>>> manually, returning BUS_PROBE_NO_WILDCARD would suit better.
>>>
>>> [..]
>>>
>>>> +static int
>>>> +xenpvcpu_probe(device_t dev)
>>>> +{
>>>> +
>>>> +    device_set_desc(dev, "Xen PV CPU");
>>>> +    return (0);
>>>
>>> Same here: BUS_PROBE_NOWILDCARD.
>>
>> Ack for both, will change it to BUS_PROBE_NOWILDCARD. While at it, we
>> should also change xenstore probe function to return
>> BUS_PROBE_NOWILDCARD.
>>
> 
> Right, I have a patch for xenstore. Do you want me to send it?

Sure, send it!


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Roger Pau Monne]

Introduce a Xen specific nexus that is going to be in charge for
attaching Xen specific devices.
---
 sys/conf/files.amd64          |    1 +
 sys/conf/files.i386           |    1 +
 sys/dev/xen/console/console.c |    2 +-
 sys/dev/xen/timer/timer.c     |    4 +-
 sys/dev/xen/xenpci/xenpci.c   |    4 ++
 sys/x86/xen/xen_nexus.c       |   82 +++++++++++++++++++++++++++++++++++++++++
 sys/x86/xen/xenpv.c           |    1 +
 sys/xen/xenstore/xenstore.c   |    6 +--
 8 files changed, 93 insertions(+), 8 deletions(-)
 create mode 100644 sys/x86/xen/xen_nexus.c

diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index d7c98cc..f378983 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -571,3 +571,4 @@ x86/xen/xen_intr.c		optional	xen | xenhvm
 x86/xen/pv.c			optional	xenhvm
 x86/xen/pvcpu_enum.c		optional	xenhvm
 x86/xen/xenpv.c			optional	xenhvm
+x86/xen/xen_nexus.c		optional	xenhvm
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 81142e3..02887a33 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -604,3 +604,4 @@ x86/x86/delay.c			standard
 x86/xen/hvm.c			optional xenhvm
 x86/xen/xen_intr.c		optional xen | xenhvm
 x86/xen/xenpv.c			optional xen | xenhvm
+x86/xen/xen_nexus.c		optional xen | xenhvm
diff --git a/sys/dev/xen/console/console.c b/sys/dev/xen/console/console.c
index 899dffc..91538fe 100644
--- a/sys/dev/xen/console/console.c
+++ b/sys/dev/xen/console/console.c
@@ -462,4 +462,4 @@ xcons_force_flush(void)
 	}
 }
 
-DRIVER_MODULE(xc, nexus, xc_driver, xc_devclass, 0, 0);
+DRIVER_MODULE(xc, xenpv, xc_driver, xc_devclass, 0, 0);
diff --git a/sys/dev/xen/timer/timer.c b/sys/dev/xen/timer/timer.c
index 96372ab..f16f5a5 100644
--- a/sys/dev/xen/timer/timer.c
+++ b/sys/dev/xen/timer/timer.c
@@ -636,5 +636,5 @@ static driver_t xentimer_driver = {
 	sizeof(struct xentimer_softc),
 };
 
-DRIVER_MODULE(xentimer, nexus, xentimer_driver, xentimer_devclass, 0, 0);
-MODULE_DEPEND(xentimer, nexus, 1, 1, 1);
+DRIVER_MODULE(xentimer, xenpv, xentimer_driver, xentimer_devclass, 0, 0);
+MODULE_DEPEND(xentimer, xenpv, 1, 1, 1);
diff --git a/sys/dev/xen/xenpci/xenpci.c b/sys/dev/xen/xenpci/xenpci.c
index dd2ad92..a27b54f 100644
--- a/sys/dev/xen/xenpci/xenpci.c
+++ b/sys/dev/xen/xenpci/xenpci.c
@@ -270,6 +270,10 @@ xenpci_attach(device_t dev)
 		goto errexit;
 	}
 
+	/* Add the xenpv device so top level Xen devices can attach */
+	if (BUS_ADD_CHILD(dev, 0, "xenpv", 0) == NULL)
+		panic("xenpci: unable to add xenpv device");
+
 	return (bus_generic_attach(dev));
 
 errexit:
diff --git a/sys/x86/xen/xen_nexus.c b/sys/x86/xen/xen_nexus.c
new file mode 100644
index 0000000..d34e333
--- /dev/null
+++ b/sys/x86/xen/xen_nexus.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+
+#include <machine/nexusvar.h>
+
+#include <xen/xen-os.h>
+
+/*
+ * Xen nexus(4) driver.
+ */
+static int
+nexus_xen_probe(device_t dev)
+{
+	if (!xen_pv_domain())
+		return (ENXIO);
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+nexus_xen_attach(device_t dev)
+{
+
+	nexus_init_resources();
+	bus_generic_probe(dev);
+
+	/*
+	 * Explicitly add the xenpv device here. Other top level
+	 * Xen devices will attach to this.
+	 */
+	if (BUS_ADD_CHILD(dev, 0, "xenpv", 0) == NULL)
+		panic("xenpv: could not attach");
+	bus_generic_attach(dev);
+	return 0;
+}
+
+static device_method_t nexus_xen_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		nexus_xen_probe),
+	DEVMETHOD(device_attach,	nexus_xen_attach),
+
+	{ 0, 0 }
+};
+
+DEFINE_CLASS_1(nexus, nexus_xen_driver, nexus_xen_methods, 1, nexus_driver);
+static devclass_t nexus_devclass;
+
+DRIVER_MODULE(nexus_xen, root, nexus_xen_driver, nexus_devclass, 0, 0);
diff --git a/sys/x86/xen/xenpv.c b/sys/x86/xen/xenpv.c
index 41d674f..7bd810b 100644
--- a/sys/x86/xen/xenpv.c
+++ b/sys/x86/xen/xenpv.c
@@ -90,6 +90,7 @@ static driver_t xenpv_driver = {
 static devclass_t xenpv_devclass;
 
 DRIVER_MODULE(xenpv, nexus, xenpv_driver, xenpv_devclass, 0, 0);
+DRIVER_MODULE(xenpv, xenpci, xenpv_driver, xenpv_devclass, 0, 0);
 
 /*
  * Dummy Xen cpu device
diff --git a/sys/xen/xenstore/xenstore.c b/sys/xen/xenstore/xenstore.c
index d404862..b5cf413 100644
--- a/sys/xen/xenstore/xenstore.c
+++ b/sys/xen/xenstore/xenstore.c
@@ -1261,11 +1261,7 @@ static device_method_t xenstore_methods[] = {
 DEFINE_CLASS_0(xenstore, xenstore_driver, xenstore_methods, 0);
 static devclass_t xenstore_devclass; 
  
-#ifdef XENHVM
-DRIVER_MODULE(xenstore, xenpci, xenstore_driver, xenstore_devclass, 0, 0);
-#else
-DRIVER_MODULE(xenstore, nexus, xenstore_driver, xenstore_devclass, 0, 0);
-#endif
+DRIVER_MODULE(xenstore, xenpv, xenstore_driver, xenstore_devclass, 0, 0);
 
 /*------------------------------- Sysctl Data --------------------------------*/
 /* XXX Shouldn't the node be somewhere else? */
-- 
1.7.7.5 (Apple Git-26)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v9 16/19] xen: add shutdown hook for PVH

[Roger Pau Monne]

Add the PV shutdown hook to PVH.
---
 sys/dev/xen/control/control.c |   37 ++++++++++++++++++-------------------
 1 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/sys/dev/xen/control/control.c b/sys/dev/xen/control/control.c
index bc0609d..78894ba 100644
--- a/sys/dev/xen/control/control.c
+++ b/sys/dev/xen/control/control.c
@@ -316,21 +316,6 @@ xctrl_suspend()
 	EVENTHANDLER_INVOKE(power_resume);
 }
 
-static void
-xen_pv_shutdown_final(void *arg, int howto)
-{
-	/*
-	 * Inform the hypervisor that shutdown is complete.
-	 * This is not necessary in HVM domains since Xen
-	 * emulates ACPI in that mode and FreeBSD's ACPI
-	 * support will request this transition.
-	 */
-	if (howto & (RB_HALT | RB_POWEROFF))
-		HYPERVISOR_shutdown(SHUTDOWN_poweroff);
-	else
-		HYPERVISOR_shutdown(SHUTDOWN_reboot);
-}
-
 #else
 
 /* HVM mode suspension. */
@@ -440,6 +425,21 @@ xctrl_crash()
 	panic("Xen directed crash");
 }
 
+static void
+xen_pv_shutdown_final(void *arg, int howto)
+{
+	/*
+	 * Inform the hypervisor that shutdown is complete.
+	 * This is not necessary in HVM domains since Xen
+	 * emulates ACPI in that mode and FreeBSD's ACPI
+	 * support will request this transition.
+	 */
+	if (howto & (RB_HALT | RB_POWEROFF))
+		HYPERVISOR_shutdown(SHUTDOWN_poweroff);
+	else
+		HYPERVISOR_shutdown(SHUTDOWN_reboot);
+}
+
 /*------------------------------ Event Reception -----------------------------*/
 static void
 xctrl_on_watch_event(struct xs_watch *watch, const char **vec, unsigned int len)
@@ -522,10 +522,9 @@ xctrl_attach(device_t dev)
 	xctrl->xctrl_watch.callback_data = (uintptr_t)xctrl;
 	xs_register_watch(&xctrl->xctrl_watch);
 
-#ifndef XENHVM
-	EVENTHANDLER_REGISTER(shutdown_final, xen_pv_shutdown_final, NULL,
-			      SHUTDOWN_PRI_LAST);
-#endif
+	if (xen_pv_domain())
+		EVENTHANDLER_REGISTER(shutdown_final, xen_pv_shutdown_final, NULL,
+		                      SHUTDOWN_PRI_LAST);
 
 	return (0);
 }
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 17/19] xen: xenstore changes to support PVH

[Roger Pau Monne]

---
 sys/xen/xenstore/xenstore.c |   18 +++++++++---------
 1 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/sys/xen/xenstore/xenstore.c b/sys/xen/xenstore/xenstore.c
index b5cf413..7fa08cc 100644
--- a/sys/xen/xenstore/xenstore.c
+++ b/sys/xen/xenstore/xenstore.c
@@ -229,13 +229,11 @@ struct xs_softc {
 	 */
 	struct sx xenwatch_mutex;
 
-#ifdef XENHVM
 	/**
 	 * The HVM guest pseudo-physical frame number.  This is Xen's mapping
 	 * of the true machine frame number into our "physical address space".
 	 */
 	unsigned long gpfn;
-#endif
 
 	/**
 	 * The event channel for communicating with the
@@ -1147,13 +1145,15 @@ xs_attach(device_t dev)
 	/* Initialize the interface to xenstore. */
 	struct proc *p;
 
-#ifdef XENHVM
-	xs.evtchn = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN);
-	xs.gpfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
-	xen_store = pmap_mapdev(xs.gpfn * PAGE_SIZE, PAGE_SIZE);
-#else
-	xs.evtchn = xen_start_info->store_evtchn;
-#endif
+	if (xen_hvm_domain()) {
+		xs.evtchn = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN);
+		xs.gpfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
+		xen_store = pmap_mapdev(xs.gpfn * PAGE_SIZE, PAGE_SIZE);
+	} else if (xen_pv_domain()) {
+		xs.evtchn = HYPERVISOR_start_info->store_evtchn;
+	} else {
+		panic("Unknown domain type, cannot initialize xenstore\n");
+	}
 
 	TAILQ_INIT(&xs.reply_list);
 	TAILQ_INIT(&xs.watch_events);
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 18/19] xen: changes to gnttab for PVH

[Roger Pau Monne]

---
 sys/xen/gnttab.c |   26 +++++++++++++++++++++-----
 1 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/sys/xen/gnttab.c b/sys/xen/gnttab.c
index 03c32b7..6949be5 100644
--- a/sys/xen/gnttab.c
+++ b/sys/xen/gnttab.c
@@ -25,6 +25,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
+#include <sys/limits.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
@@ -607,6 +608,7 @@ gnttab_resume(void)
 {
 	int error;
 	unsigned int max_nr_gframes, nr_gframes;
+	void *alloc_mem;
 
 	nr_gframes = nr_grant_frames;
 	max_nr_gframes = max_nr_grant_frames();
@@ -614,11 +616,25 @@ gnttab_resume(void)
 		return (ENOSYS);
 
 	if (!resume_frames) {
-		error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
-		    &resume_frames);
-		if (error) {
-			printf("error mapping gnttab share frames\n");
-			return (error);
+		if (xen_pv_domain()) {
+			/*
+			 * This is a waste of physical memory,
+			 * we should use ballooned pages instead,
+			 * but it will do for now.
+			 */
+			alloc_mem = contigmalloc(max_nr_gframes * PAGE_SIZE,
+			                         M_DEVBUF, M_NOWAIT, 0,
+			                         ULONG_MAX, PAGE_SIZE, 0);
+			KASSERT((alloc_mem != NULL),
+				("unable to alloc memory for gnttab"));
+			resume_frames = vtophys(alloc_mem);
+		} else {
+			error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
+			    &resume_frames);
+			if (error) {
+				printf("error mapping gnttab share frames\n");
+				return (error);
+			}
 		}
 	}
 
-- 
1.7.7.5 (Apple Git-26)

[PATCH v9 19/19] isa: allow ISA bus to attach to xenpv device

[Roger Pau Monne]

---
 sys/x86/isa/isa.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/sys/x86/isa/isa.c b/sys/x86/isa/isa.c
index 1a57137..9287ff2 100644
--- a/sys/x86/isa/isa.c
+++ b/sys/x86/isa/isa.c
@@ -241,3 +241,6 @@ isa_release_resource(device_t bus, device_t child, int type, int rid,
  * On this platform, isa can also attach to the legacy bus.
  */
 DRIVER_MODULE(isa, legacy, isa_driver, isa_devclass, 0, 0);
+#ifdef XENHVM
+DRIVER_MODULE(isa, xenpv, isa_driver, isa_devclass, 0, 0);
+#endif
-- 
1.7.7.5 (Apple Git-26)

Re: [PATCH v9 19/19] isa: allow ISA bus to attach to xenpv device

[Julian Elischer]

On 1/2/14, 4:43 PM, Roger Pau Monne wrote:
> ---
>   sys/x86/isa/isa.c |    3 +++
>   1 files changed, 3 insertions(+), 0 deletions(-)
>
> diff --git a/sys/x86/isa/isa.c b/sys/x86/isa/isa.c
> index 1a57137..9287ff2 100644
> --- a/sys/x86/isa/isa.c
> +++ b/sys/x86/isa/isa.c
> @@ -241,3 +241,6 @@ isa_release_resource(device_t bus, device_t child, int type, int rid,
>    * On this platform, isa can also attach to the legacy bus.
>    */
>   DRIVER_MODULE(isa, legacy, isa_driver, isa_devclass, 0, 0);
> +#ifdef XENHVM
> +DRIVER_MODULE(isa, xenpv, isa_driver, isa_devclass, 0, 0);
> +#endif
read all 19 patches. I'm glad you split them up.. makes it 
understandable.. even by me :-)
no real negative comments except a question as to whether there is any 
noticable performance impact on real hardware?

Re: [PATCH v9 19/19] isa: allow ISA bus to attach to xenpv device

[Roger Pau Monné]

On 03/01/14 01:22, Julian Elischer wrote:
> On 1/2/14, 4:43 PM, Roger Pau Monne wrote:
>> ---
>>   sys/x86/isa/isa.c |    3 +++
>>   1 files changed, 3 insertions(+), 0 deletions(-)
>>
>> diff --git a/sys/x86/isa/isa.c b/sys/x86/isa/isa.c
>> index 1a57137..9287ff2 100644
>> --- a/sys/x86/isa/isa.c
>> +++ b/sys/x86/isa/isa.c
>> @@ -241,3 +241,6 @@ isa_release_resource(device_t bus, device_t child,
>> int type, int rid,
>>    * On this platform, isa can also attach to the legacy bus.
>>    */
>>   DRIVER_MODULE(isa, legacy, isa_driver, isa_devclass, 0, 0);
>> +#ifdef XENHVM
>> +DRIVER_MODULE(isa, xenpv, isa_driver, isa_devclass, 0, 0);
>> +#endif
> read all 19 patches. I'm glad you split them up.. makes it
> understandable.. even by me :-)
> no real negative comments except a question as to whether there is any
> noticable performance impact on real hardware?

Thanks for taking a look. I haven't seen any performance impact when
running a PVH capable kernel (a kernel with this patch series applied)
on real hardware. I'm not adding hooks to any hot paths, most of the
code added in this series is only used during boot time.

Roger.

Re: [PATCH v9 04/19] amd64: introduce hook for custom preload metadata parsers

[Konrad Rzeszutek Wilk]

On Thu, Jan 02, 2014 at 04:43:38PM +0100, Roger Pau Monne wrote:
> ---
>  sys/amd64/amd64/machdep.c   |   41 ++++++++++++++++------
>  sys/amd64/include/sysarch.h |   12 ++++++
>  sys/x86/xen/pv.c            |   82 +++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 124 insertions(+), 11 deletions(-)
> 
> diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
> index eae657b..e073eea 100644
> --- a/sys/amd64/amd64/machdep.c
> +++ b/sys/amd64/amd64/machdep.c
> @@ -126,6 +126,7 @@ __FBSDID("$FreeBSD$");
>  #include <machine/reg.h>
>  #include <machine/sigframe.h>
>  #include <machine/specialreg.h>
> +#include <machine/sysarch.h>
>  #ifdef PERFMON
>  #include <machine/perfmon.h>
>  #endif
> @@ -165,6 +166,14 @@ static int  set_fpcontext(struct thread *td, const mcontext_t *mcp,
>      char *xfpustate, size_t xfpustate_len);
>  SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
>  
> +/* Preload data parse function */
> +static caddr_t native_parse_preload_data(u_int64_t);
> +
> +/* Default init_ops implementation. */
> +struct init_ops init_ops = {
> +	.parse_preload_data =	native_parse_preload_data,

Extra space there.

> +};
> +
>  /*
>   * The file "conf/ldscript.amd64" defines the symbol "kernphys".  Its value is
>   * the physical address at which the kernel is loaded.
> @@ -1683,6 +1692,26 @@ do_next:
>  	msgbufp = (struct msgbuf *)PHYS_TO_DMAP(phys_avail[pa_indx]);
>  }
>  
> +static caddr_t
> +native_parse_preload_data(u_int64_t modulep)
> +{
> +	caddr_t kmdp;
> +
> +	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);

Two casts? Could it be done via one?

> +	preload_bootstrap_relocate(KERNBASE);
> +	kmdp = preload_search_by_type("elf kernel");
> +	if (kmdp == NULL)
> +		kmdp = preload_search_by_type("elf64 kernel");
> +	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
> +	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
> +#ifdef DDB
> +	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
> +	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
> +#endif
> +
> +	return (kmdp);
> +}
> +
>  u_int64_t
>  hammer_time(u_int64_t modulep, u_int64_t physfree)
>  {
> @@ -1707,17 +1736,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
>  	 */
>  	proc_linkup0(&proc0, &thread0);
>  
> -	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);

Oh, you just moved the code - right, lets not modify it in this patch.

> -	preload_bootstrap_relocate(KERNBASE);
> -	kmdp = preload_search_by_type("elf kernel");
> -	if (kmdp == NULL)
> -		kmdp = preload_search_by_type("elf64 kernel");
> -	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
> -	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
> -#ifdef DDB
> -	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
> -	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
> -#endif
> +	kmdp = init_ops.parse_preload_data(modulep);
>  
>  	/* Init basic tunables, hz etc */
>  	init_param1();
> diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
> index cd380d4..58ac8cd 100644
> --- a/sys/amd64/include/sysarch.h
> +++ b/sys/amd64/include/sysarch.h
> @@ -4,3 +4,15 @@
>  /* $FreeBSD$ */
>  
>  #include <x86/sysarch.h>
> +
> +/*
> + * Struct containing pointers to init functions whose
> + * implementation is run time selectable.  Selection can be made,
> + * for example, based on detection of a BIOS variant or
> + * hypervisor environment.
> + */
> +struct init_ops {
> +	caddr_t	(*parse_preload_data)(u_int64_t);
> +};
> +
> +extern struct init_ops init_ops;
> diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
> index db3b7a3..908b50b 100644
> --- a/sys/x86/xen/pv.c
> +++ b/sys/x86/xen/pv.c
> @@ -46,6 +46,8 @@ __FBSDID("$FreeBSD$");
>  #include <vm/vm_pager.h>
>  #include <vm/vm_param.h>
>  
> +#include <machine/sysarch.h>
> +
>  #include <xen/xen-os.h>
>  #include <xen/hypervisor.h>
>  
> @@ -54,6 +56,36 @@ extern u_int64_t hammer_time(u_int64_t, u_int64_t);
>  /* Xen initial function */
>  extern u_int64_t hammer_time_xen(start_info_t *, u_int64_t);
>  
> +/*--------------------------- Forward Declarations ---------------------------*/
> +static caddr_t xen_pv_parse_preload_data(u_int64_t);
> +
> +static void xen_pv_set_init_ops(void);
> +
> +/*-------------------------------- Global Data -------------------------------*/
> +/* Xen init_ops implementation. */
> +struct init_ops xen_init_ops = {
> +	.parse_preload_data =	xen_pv_parse_preload_data,
> +};
> +
> +static struct
> +{
> +	const char	*ev;
> +	int		mask;
> +} howto_names[] = {
> +	{"boot_askname",	RB_ASKNAME},
> +	{"boot_single",		RB_SINGLE},
> +	{"boot_nosync",		RB_NOSYNC},
> +	{"boot_halt",		RB_ASKNAME},
> +	{"boot_serial",		RB_SERIAL},
> +	{"boot_cdrom",		RB_CDROM},
> +	{"boot_gdb",		RB_GDB},
> +	{"boot_gdb_pause",	RB_RESERVED1},
> +	{"boot_verbose",	RB_VERBOSE},
> +	{"boot_multicons",	RB_MULTIPLE},
> +	{NULL,	0}
> +};
> +
> +/*-------------------------------- Xen PV init -------------------------------*/
>  /*
>   * First function called by the Xen PVH boot sequence.
>   *
> @@ -118,6 +150,56 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
>  	}
>  	load_cr3(((u_int64_t)&PT4[0]) - KERNBASE);
>  
> +	/* Set the hooks for early functions that diverge from bare metal */
> +	xen_pv_set_init_ops();
> +
>  	/* Now we can jump into the native init function */
>  	return (hammer_time(0, physfree));
>  }
> +
> +/*-------------------------------- PV specific -------------------------------*/
> +/*
> + * Functions to convert the "extra" parameters passed by Xen
> + * into FreeBSD boot options (from the i386 Xen port).
> + */
> +static char *
> +xen_setbootenv(char *cmd_line)
> +{
> +	char *cmd_line_next;
> +
> +        /* Skip leading spaces */
> +        for (; *cmd_line == ' '; cmd_line++);

Spaces?

> +
> +	for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
> +	return (cmd_line);
> +}
> +
> +static int
> +xen_boothowto(char *envp)
> +{
> +	int i, howto = 0;
> +
> +	/* get equivalents from the environment */
> +	for (i = 0; howto_names[i].ev != NULL; i++)
> +		if (getenv(howto_names[i].ev) != NULL)
> +			howto |= howto_names[i].mask;

You don't believe in '{}' do you ? :-)

> +	return (howto);
> +}
> +
> +static caddr_t
> +xen_pv_parse_preload_data(u_int64_t modulep)
> +{
> +	/* Parse the extra boot information given by Xen */
> +	if (HYPERVISOR_start_info->cmd_line)
> +		kern_envp = xen_setbootenv(HYPERVISOR_start_info->cmd_line);
> +	boothowto |= xen_boothowto(kern_envp);
> +
> +	return (NULL);
> +}
> +
> +static void
> +xen_pv_set_init_ops(void)
> +{
> +	/* Init ops for Xen PV */
> +	init_ops = xen_init_ops;
> +}
> -- 
> 1.7.7.5 (Apple Git-26)
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel

Re: [PATCH v9 04/19] amd64: introduce hook for custom preload metadata parsers

[Roger Pau Monné]

On 03/01/14 21:59, Konrad Rzeszutek Wilk wrote:
> On Thu, Jan 02, 2014 at 04:43:38PM +0100, Roger Pau Monne wrote:
>> ---
>>  sys/amd64/amd64/machdep.c   |   41 ++++++++++++++++------
>>  sys/amd64/include/sysarch.h |   12 ++++++
>>  sys/x86/xen/pv.c            |   82 +++++++++++++++++++++++++++++++++++++++++++
>>  3 files changed, 124 insertions(+), 11 deletions(-)
>>
>> diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
>> index eae657b..e073eea 100644
>> --- a/sys/amd64/amd64/machdep.c
>> +++ b/sys/amd64/amd64/machdep.c
>> @@ -126,6 +126,7 @@ __FBSDID("$FreeBSD$");
>>  #include <machine/reg.h>
>>  #include <machine/sigframe.h>
>>  #include <machine/specialreg.h>
>> +#include <machine/sysarch.h>
>>  #ifdef PERFMON
>>  #include <machine/perfmon.h>
>>  #endif
>> @@ -165,6 +166,14 @@ static int  set_fpcontext(struct thread *td, const mcontext_t *mcp,
>>      char *xfpustate, size_t xfpustate_len);
>>  SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
>>  
>> +/* Preload data parse function */
>> +static caddr_t native_parse_preload_data(u_int64_t);
>> +
>> +/* Default init_ops implementation. */
>> +struct init_ops init_ops = {
>> +	.parse_preload_data =	native_parse_preload_data,
> 
> Extra space there.

It's for alignment, looks strange now because there's only one hook.

>> +};
>> +
>>  /*
>>   * The file "conf/ldscript.amd64" defines the symbol "kernphys".  Its value is
>>   * the physical address at which the kernel is loaded.
>> @@ -1683,6 +1692,26 @@ do_next:
>>  	msgbufp = (struct msgbuf *)PHYS_TO_DMAP(phys_avail[pa_indx]);
>>  }
>>  
>> +static caddr_t
>> +native_parse_preload_data(u_int64_t modulep)
>> +{
>> +	caddr_t kmdp;
>> +
>> +	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
> 
> Two casts? Could it be done via one?
> 
>> +	preload_bootstrap_relocate(KERNBASE);
>> +	kmdp = preload_search_by_type("elf kernel");
>> +	if (kmdp == NULL)
>> +		kmdp = preload_search_by_type("elf64 kernel");
>> +	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
>> +	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
>> +#ifdef DDB
>> +	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
>> +	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
>> +#endif
>> +
>> +	return (kmdp);
>> +}
>> +
>>  u_int64_t
>>  hammer_time(u_int64_t modulep, u_int64_t physfree)
>>  {
>> @@ -1707,17 +1736,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
>>  	 */
>>  	proc_linkup0(&proc0, &thread0);
>>  
>> -	preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
> 
> Oh, you just moved the code - right, lets not modify it in this patch.

Yes, it's just code movement.

> 
>> -	preload_bootstrap_relocate(KERNBASE);
>> -	kmdp = preload_search_by_type("elf kernel");
>> -	if (kmdp == NULL)
>> -		kmdp = preload_search_by_type("elf64 kernel");
>> -	boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
>> -	kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + KERNBASE;
>> -#ifdef DDB
>> -	ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
>> -	ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
>> -#endif
>> +	kmdp = init_ops.parse_preload_data(modulep);
>>  
>>  	/* Init basic tunables, hz etc */
>>  	init_param1();
>> diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
>> index cd380d4..58ac8cd 100644
>> --- a/sys/amd64/include/sysarch.h
>> +++ b/sys/amd64/include/sysarch.h
>> @@ -4,3 +4,15 @@
>>  /* $FreeBSD$ */
>>  
>>  #include <x86/sysarch.h>
>> +
>> +/*
>> + * Struct containing pointers to init functions whose
>> + * implementation is run time selectable.  Selection can be made,
>> + * for example, based on detection of a BIOS variant or
>> + * hypervisor environment.
>> + */
>> +struct init_ops {
>> +	caddr_t	(*parse_preload_data)(u_int64_t);
>> +};
>> +
>> +extern struct init_ops init_ops;
>> diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
>> index db3b7a3..908b50b 100644
>> --- a/sys/x86/xen/pv.c
>> +++ b/sys/x86/xen/pv.c
>> @@ -46,6 +46,8 @@ __FBSDID("$FreeBSD$");
>>  #include <vm/vm_pager.h>
>>  #include <vm/vm_param.h>
>>  
>> +#include <machine/sysarch.h>
>> +
>>  #include <xen/xen-os.h>
>>  #include <xen/hypervisor.h>
>>  
>> @@ -54,6 +56,36 @@ extern u_int64_t hammer_time(u_int64_t, u_int64_t);
>>  /* Xen initial function */
>>  extern u_int64_t hammer_time_xen(start_info_t *, u_int64_t);
>>  
>> +/*--------------------------- Forward Declarations ---------------------------*/
>> +static caddr_t xen_pv_parse_preload_data(u_int64_t);
>> +
>> +static void xen_pv_set_init_ops(void);
>> +
>> +/*-------------------------------- Global Data -------------------------------*/
>> +/* Xen init_ops implementation. */
>> +struct init_ops xen_init_ops = {
>> +	.parse_preload_data =	xen_pv_parse_preload_data,
>> +};
>> +
>> +static struct
>> +{
>> +	const char	*ev;
>> +	int		mask;
>> +} howto_names[] = {
>> +	{"boot_askname",	RB_ASKNAME},
>> +	{"boot_single",		RB_SINGLE},
>> +	{"boot_nosync",		RB_NOSYNC},
>> +	{"boot_halt",		RB_ASKNAME},
>> +	{"boot_serial",		RB_SERIAL},
>> +	{"boot_cdrom",		RB_CDROM},
>> +	{"boot_gdb",		RB_GDB},
>> +	{"boot_gdb_pause",	RB_RESERVED1},
>> +	{"boot_verbose",	RB_VERBOSE},
>> +	{"boot_multicons",	RB_MULTIPLE},
>> +	{NULL,	0}
>> +};
>> +
>> +/*-------------------------------- Xen PV init -------------------------------*/
>>  /*
>>   * First function called by the Xen PVH boot sequence.
>>   *
>> @@ -118,6 +150,56 @@ hammer_time_xen(start_info_t *si, u_int64_t xenstack)
>>  	}
>>  	load_cr3(((u_int64_t)&PT4[0]) - KERNBASE);
>>  
>> +	/* Set the hooks for early functions that diverge from bare metal */
>> +	xen_pv_set_init_ops();
>> +
>>  	/* Now we can jump into the native init function */
>>  	return (hammer_time(0, physfree));
>>  }
>> +
>> +/*-------------------------------- PV specific -------------------------------*/
>> +/*
>> + * Functions to convert the "extra" parameters passed by Xen
>> + * into FreeBSD boot options (from the i386 Xen port).
>> + */
>> +static char *
>> +xen_setbootenv(char *cmd_line)
>> +{
>> +	char *cmd_line_next;
>> +
>> +        /* Skip leading spaces */
>> +        for (; *cmd_line == ' '; cmd_line++);
> 
> Spaces?
> 
>> +
>> +	for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
>> +	return (cmd_line);
>> +}
>> +
>> +static int
>> +xen_boothowto(char *envp)
>> +{
>> +	int i, howto = 0;
>> +
>> +	/* get equivalents from the environment */
>> +	for (i = 0; howto_names[i].ev != NULL; i++)
>> +		if (getenv(howto_names[i].ev) != NULL)
>> +			howto |= howto_names[i].mask;
> 
> You don't believe in '{}' do you ? :-)

All this code has also been taken from the FreeBSD Xen i386 PV port, but
maybe some refactoring wouldn't be bad :).

>> +	return (howto);
>> +}
>> +
>> +static caddr_t
>> +xen_pv_parse_preload_data(u_int64_t modulep)
>> +{
>> +	/* Parse the extra boot information given by Xen */
>> +	if (HYPERVISOR_start_info->cmd_line)
>> +		kern_envp = xen_setbootenv(HYPERVISOR_start_info->cmd_line);
>> +	boothowto |= xen_boothowto(kern_envp);
>> +
>> +	return (NULL);
>> +}
>> +
>> +static void
>> +xen_pv_set_init_ops(void)
>> +{
>> +	/* Init ops for Xen PV */
>> +	init_ops = xen_init_ops;
>> +}
>> -- 
>> 1.7.7.5 (Apple Git-26)
>>
>>
>> _______________________________________________
>> Xen-devel mailing list
>> Xen-devel@lists.xen.org
>> http://lists.xen.org/xen-devel

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Julien Grall]

On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
> Introduce a Xen specific nexus that is going to be in charge for
> attaching Xen specific devices.

Now that we have a xenpv bus, do we really need a specific nexus for Xen?
We should be able to use the identify callback of xenpv to create the bus.

The other part of this patch can be merged in the patch #14 "Introduce 
xenpv bus and a dummy pvcpu device".

-- 
Julien Grall

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Roger Pau Monné]

On 05/01/14 22:55, Julien Grall wrote:
> 
> 
> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>> Introduce a Xen specific nexus that is going to be in charge for
>> attaching Xen specific devices.
> 
> Now that we have a xenpv bus, do we really need a specific nexus for Xen?
> We should be able to use the identify callback of xenpv to create the bus.
> 
> The other part of this patch can be merged in the patch #14 "Introduce
> xenpv bus and a dummy pvcpu device".

On x86 at least we need the Xen specific nexus, or we will fall back to
use the legacy nexus which is not what we really want.

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Julien Grall]

On 01/06/2014 09:35 AM, Roger Pau Monné wrote:
> On 05/01/14 22:55, Julien Grall wrote:
>>
>>
>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>> Introduce a Xen specific nexus that is going to be in charge for
>>> attaching Xen specific devices.
>>
>> Now that we have a xenpv bus, do we really need a specific nexus for Xen?
>> We should be able to use the identify callback of xenpv to create the bus.
>>
>> The other part of this patch can be merged in the patch #14 "Introduce
>> xenpv bus and a dummy pvcpu device".
>
> On x86 at least we need the Xen specific nexus, or we will fall back to
> use the legacy nexus which is not what we really want.
>

Oh right, in any case can we use the identify callback of xenpv to add 
the bus?

With this solution xenpv can add itself no matter FreeBSD use the 
generic nexus or the nexus Xen, of course with a check if we are running 
on Xen :).

For instance, on ARM side I don't plan to have a specific Xen nexus.

Cheers,

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Roger Pau Monné]

On 06/01/14 12:33, Julien Grall wrote:
> 
> 
> On 01/06/2014 09:35 AM, Roger Pau Monné wrote:
>> On 05/01/14 22:55, Julien Grall wrote:
>>>
>>>
>>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>>> Introduce a Xen specific nexus that is going to be in charge for
>>>> attaching Xen specific devices.
>>>
>>> Now that we have a xenpv bus, do we really need a specific nexus for
>>> Xen?
>>> We should be able to use the identify callback of xenpv to create the
>>> bus.
>>>
>>> The other part of this patch can be merged in the patch #14 "Introduce
>>> xenpv bus and a dummy pvcpu device".
>>
>> On x86 at least we need the Xen specific nexus, or we will fall back to
>> use the legacy nexus which is not what we really want.
>>
> 
> Oh right, in any case can we use the identify callback of xenpv to add
> the bus?

AFAICT this kind of bus devices don't have a identify routine, and they
are usually added manually from the specific nexus, see acpi or legacy.
Could you add the device on ARM when you detect that you are running as
a Xen guest, or in the generic ARM nexus if Xen is detected?

Roger.

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Julien Grall]

On 01/07/2014 08:29 AM, Roger Pau Monné wrote:
> On 06/01/14 12:33, Julien Grall wrote:
>>
>>
>> On 01/06/2014 09:35 AM, Roger Pau Monné wrote:
>>> On 05/01/14 22:55, Julien Grall wrote:
>>>>
>>>>
>>>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>>>> Introduce a Xen specific nexus that is going to be in charge for
>>>>> attaching Xen specific devices.
>>>>
>>>> Now that we have a xenpv bus, do we really need a specific nexus for
>>>> Xen?
>>>> We should be able to use the identify callback of xenpv to create the
>>>> bus.
>>>>
>>>> The other part of this patch can be merged in the patch #14 "Introduce
>>>> xenpv bus and a dummy pvcpu device".
>>>
>>> On x86 at least we need the Xen specific nexus, or we will fall back to
>>> use the legacy nexus which is not what we really want.
>>>
>>
>> Oh right, in any case can we use the identify callback of xenpv to add
>> the bus?
> 
> AFAICT this kind of bus devices don't have a identify routine, and they
> are usually added manually from the specific nexus, see acpi or legacy.
> Could you add the device on ARM when you detect that you are running as
> a Xen guest, or in the generic ARM nexus if Xen is detected?

Is there any reason to not add identify callback? If it's possible, I
would like to avoid as much as possible #ifdef XENHVM in ARM code.

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Roger Pau Monné]

On 07/01/14 15:27, Julien Grall wrote:
> On 01/07/2014 08:29 AM, Roger Pau Monné wrote:
>> On 06/01/14 12:33, Julien Grall wrote:
>>>
>>>
>>> On 01/06/2014 09:35 AM, Roger Pau Monné wrote:
>>>> On 05/01/14 22:55, Julien Grall wrote:
>>>>>
>>>>>
>>>>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>>>>> Introduce a Xen specific nexus that is going to be in charge for
>>>>>> attaching Xen specific devices.
>>>>>
>>>>> Now that we have a xenpv bus, do we really need a specific nexus for
>>>>> Xen?
>>>>> We should be able to use the identify callback of xenpv to create the
>>>>> bus.
>>>>>
>>>>> The other part of this patch can be merged in the patch #14 "Introduce
>>>>> xenpv bus and a dummy pvcpu device".
>>>>
>>>> On x86 at least we need the Xen specific nexus, or we will fall back to
>>>> use the legacy nexus which is not what we really want.
>>>>
>>>
>>> Oh right, in any case can we use the identify callback of xenpv to add
>>> the bus?
>>
>> AFAICT this kind of bus devices don't have a identify routine, and they
>> are usually added manually from the specific nexus, see acpi or legacy.
>> Could you add the device on ARM when you detect that you are running as
>> a Xen guest, or in the generic ARM nexus if Xen is detected?
> 
> Is there any reason to not add identify callback? If it's possible, I
> would like to avoid as much as possible #ifdef XENHVM in ARM code.

Maybe the x86 world is really different from the ARM world in how nexus
works, but I rather prefer to have a #ifdef XENHVM and a BUS_ADD_CHILD
that attaches the xenpv bus in the generic ARM nexus rather than having
something that completely diverges from what buses usually do in
FreeBSD. It's going to be much more difficult to track in case of bugs,
and it's not what people expects, but that's just my opinion. I can
certainly add the identify routine if there's an agreement that it's the
best way to deal with it.

Roger.

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v9 15/19] xen: create a Xen nexus to use in PV/PVH

[Nathan Whitehorn]

On 01/09/14 11:30, Roger Pau Monné wrote:
> On 07/01/14 15:27, Julien Grall wrote:
>> On 01/07/2014 08:29 AM, Roger Pau Monné wrote:
>>> On 06/01/14 12:33, Julien Grall wrote:
>>>>
>>>> On 01/06/2014 09:35 AM, Roger Pau Monné wrote:
>>>>> On 05/01/14 22:55, Julien Grall wrote:
>>>>>>
>>>>>> On 01/02/2014 03:43 PM, Roger Pau Monne wrote:
>>>>>>> Introduce a Xen specific nexus that is going to be in charge for
>>>>>>> attaching Xen specific devices.
>>>>>> Now that we have a xenpv bus, do we really need a specific nexus for
>>>>>> Xen?
>>>>>> We should be able to use the identify callback of xenpv to create the
>>>>>> bus.
>>>>>>
>>>>>> The other part of this patch can be merged in the patch #14 "Introduce
>>>>>> xenpv bus and a dummy pvcpu device".
>>>>> On x86 at least we need the Xen specific nexus, or we will fall back to
>>>>> use the legacy nexus which is not what we really want.
>>>>>
>>>> Oh right, in any case can we use the identify callback of xenpv to add
>>>> the bus?
>>> AFAICT this kind of bus devices don't have a identify routine, and they
>>> are usually added manually from the specific nexus, see acpi or legacy.
>>> Could you add the device on ARM when you detect that you are running as
>>> a Xen guest, or in the generic ARM nexus if Xen is detected?
>> Is there any reason to not add identify callback? If it's possible, I
>> would like to avoid as much as possible #ifdef XENHVM in ARM code.
> Maybe the x86 world is really different from the ARM world in how nexus
> works, but I rather prefer to have a #ifdef XENHVM and a BUS_ADD_CHILD
> that attaches the xenpv bus in the generic ARM nexus rather than having
> something that completely diverges from what buses usually do in
> FreeBSD. It's going to be much more difficult to track in case of bugs,
> and it's not what people expects, but that's just my opinion. I can
> certainly add the identify routine if there's an agreement that it's the
> best way to deal with it.
>
> Roger.
> _______________________________________________
> freebsd-current@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-current
> To unsubscribe, send any mail to "freebsd-current-unsubscribe@freebsd.org"
>

Attaching sub-devices to nexus using device_identify() is the usual way
to do this kind of thing. Note that if you do this, your device_probe()
routine should return BUS_PROBE_NOWILDCARD to deal with platforms (ARM,
MIPS, PowerPC, sparc64) with real autoconfigured devices hanging
directly from nexus.
-Nathan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel