[PATCH V10 01/14] xen/pvh: Support ParaVirtualized Hardware extensions.

[PATCH V10 01/14] xen/pvh: Support ParaVirtualized Hardware extensions.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

PVH allows PV linux guest to utilize hardware extended capabilities, such
as running MMU updates in a HVM container.

This patch allows it to be configured and enabled. Also, basic header file
changes to add new subcalls to physmap hypercall.

Lastly, mfn_to_local_pfn must return mfn for paging mode translate
(since we let the hypervisor - or CPU - do them for us).

Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/include/asm/xen/page.h |    3 +++
 arch/x86/xen/Kconfig            |   10 ++++++++++
 arch/x86/xen/xen-head.S         |   11 ++++++++++-
 include/xen/interface/memory.h  |    5 ++++-
 include/xen/interface/physdev.h |   10 ++++++++++
 5 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index b913915..65194f7 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -168,6 +168,9 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine)
 static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 {
 	unsigned long pfn = mfn_to_pfn(mfn);
+
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return mfn;
 	if (get_phys_to_machine(pfn) != mfn)
 		return -1; /* force !pfn_valid() */
 	return pfn;
diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
index 1a3c765..4d890c3 100644
--- a/arch/x86/xen/Kconfig
+++ b/arch/x86/xen/Kconfig
@@ -51,3 +51,13 @@ config XEN_DEBUG_FS
 	  Enable statistics output and various tuning options in debugfs.
 	  Enabling this option may incur a significant performance overhead.
 
+config XEN_X86_PVH
+	bool "Support for running as a PVH guest (EXPERIMENTAL)"
+	depends on X86_64 && XEN && EXPERIMENTAL
+	default n
+	help
+	   This option enables support for running as a PVH guest (PV guest
+	   using hardware extensions) under a suitably capable hypervisor.
+	   This option is EXPERIMENTAL because the hypervisor interfaces
+	   which it uses is not yet considered stable therefore backwards and
+	   forwards compatibility is not yet guaranteed.  If unsure, say N.
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 7faed58..1a6bca1 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -13,6 +13,15 @@
 #include <xen/interface/elfnote.h>
 #include <asm/xen/interface.h>
 
+#ifdef CONFIG_XEN_X86_PVH
+#define FEATURES_PVH "|writable_descriptor_tables" \
+		     "|auto_translated_physmap" \
+		     "|supervisor_mode_kernel" \
+		     "|hvm_callback_vector"
+#else
+#define FEATURES_PVH /* Not supported */
+#endif
+
 	__INIT
 ENTRY(startup_xen)
 	cld
@@ -95,7 +104,7 @@ NEXT_HYPERCALL(arch_6)
 #endif
 	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          _ASM_PTR startup_xen)
 	ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, _ASM_PTR hypercall_page)
-	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb")
+	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb"FEATURES_PVH)
 	ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE,       .asciz "yes")
 	ELFNOTE(Xen, XEN_ELFNOTE_LOADER,         .asciz "generic")
 	ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID,
diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
index 2ecfe4f..a122164 100644
--- a/include/xen/interface/memory.h
+++ b/include/xen/interface/memory.h
@@ -175,7 +175,10 @@ struct xen_add_to_physmap {
     uint16_t    size;
 
     /* Source mapping space. */
-    unsigned int space;
+    uint16_t space;
+    domid_t foreign_domid;         /* IFF XENMAPSPACE_gmfn_foreign */
+
+#define XENMAPIDX_grant_table_status 0x80000000
 
     /* Index into source mapping space. */
     xen_ulong_t idx;
diff --git a/include/xen/interface/physdev.h b/include/xen/interface/physdev.h
index 42721d1..19d30ad 100644
--- a/include/xen/interface/physdev.h
+++ b/include/xen/interface/physdev.h
@@ -291,6 +291,16 @@ struct physdev_dbgp_op {
     } u;
 };
 
+#define PHYSDEVOP_map_iomem        30
+struct physdev_map_iomem {
+    /* IN */
+    uint64_t first_gfn;
+    uint64_t first_mfn;
+    uint32_t nr_mfns;
+    uint32_t add_mapping; /* 1 == add mapping;  0 == unmap */
+
+};
+
 /*
  * Notify that some PIRQ-bound event channels have been unmasked.
  * ** This command is obsolete since interface version 0x00030202 and is **
-- 
1.7.7.6

[PATCH V10 02/14] xen/pvh: Extend vcpu_guest_context, p2m, event, and XenBus.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

Make gdt_frames[]/gdt_ents into a union with {gdtaddr, gdtsz},
as PVH only needs to send down gdtaddr and gdtsz in the
vcpu_guest_context structure..

For interrupts, PVH uses native_irq_ops so we can skip most of the
PV ones. In the future we can support the pirq_eoi_map..
Also VCPU hotplug is currently not available for PVH.

For events (and IRQs) we follow what PVHVM does - so use callback
vector.  Lastly, for XenBus we use the same logic that is used in
the PVHVM case.

Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
[v2: Rebased it]
[v3: Move 64-bit ifdef and based on Stefan add extra comments.]
[v4: Rebased it once more]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

Conflicts:
	arch/x86/xen/p2m.c
	arch/x86/xen/smp.c
[Conflict due to xen/smp: Update pv_lock_ops functions before alternative code starts under PVHVM
squashing PV and HVM code together.]
---
 arch/x86/include/asm/xen/interface.h |   11 +++++++++-
 arch/x86/xen/irq.c                   |    5 +++-
 arch/x86/xen/p2m.c                   |    4 ++-
 arch/x86/xen/smp.c                   |   38 +++++++++++++++++++++++++---------
 drivers/xen/cpu_hotplug.c            |    4 ++-
 drivers/xen/events.c                 |    9 +++++++-
 drivers/xen/xenbus/xenbus_client.c   |    3 +-
 7 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h
index fd9cb76..20e738a 100644
--- a/arch/x86/include/asm/xen/interface.h
+++ b/arch/x86/include/asm/xen/interface.h
@@ -145,7 +145,16 @@ struct vcpu_guest_context {
     struct cpu_user_regs user_regs;         /* User-level CPU registers     */
     struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
     unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
-    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
+    union {
+	struct {
+		/* PV: GDT (machine frames, # ents).*/
+		unsigned long gdt_frames[16], gdt_ents;
+	} pv;
+	struct {
+		/* PVH: GDTR addr and size */
+		unsigned long gdtaddr, gdtsz;
+	} pvh;
+    } u;
     unsigned long kernel_ss, kernel_sp;     /* Virtual TSS (only SS1/SP1)   */
     /* NB. User pagetable on x86/64 is placed in ctrlreg[1]. */
     unsigned long ctrlreg[8];               /* CR0-CR7 (control registers)  */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 0da7f86..4f7f351 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -5,6 +5,7 @@
 #include <xen/interface/xen.h>
 #include <xen/interface/sched.h>
 #include <xen/interface/vcpu.h>
+#include <xen/features.h>
 #include <xen/events.h>
 
 #include <asm/xen/hypercall.h>
@@ -128,6 +129,8 @@ static const struct pv_irq_ops xen_irq_ops __initconst = {
 
 void __init xen_init_irq_ops(void)
 {
-	pv_irq_ops = xen_irq_ops;
+	/* For PVH we use default pv_irq_ops settings */
+	if (!xen_feature(XENFEAT_hvm_callback_vector))
+		pv_irq_ops = xen_irq_ops;
 	x86_init.irqs.intr_init = xen_init_IRQ;
 }
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 2ae8699..93655e8 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -800,8 +800,10 @@ bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 	unsigned topidx, mididx, idx;
 
 	/* don't track P2M changes in autotranslate guests */
-	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
+		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 		return true;
+	}
 
 	if (unlikely(pfn >= MAX_P2M_PFN)) {
 		BUG_ON(mfn != INVALID_P2M_ENTRY);
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index c36b325..afffa07 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -73,9 +73,11 @@ static void cpu_bringup(void)
 	touch_softlockup_watchdog();
 	preempt_disable();
 
-	xen_enable_sysenter();
-	xen_enable_syscall();
-
+	/* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
+	if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
+		xen_enable_sysenter();
+		xen_enable_syscall();
+	}
 	cpu = smp_processor_id();
 	smp_store_cpu_info(cpu);
 	cpu_data(cpu).x86_max_cores = 1;
@@ -274,9 +276,10 @@ static void __init xen_smp_prepare_boot_cpu(void)
 	native_smp_prepare_boot_cpu();
 
 	if (xen_pv_domain()) {
-		/* We've switched to the "real" per-cpu gdt, so make sure the
-		   old memory can be recycled */
-		make_lowmem_page_readwrite(xen_initial_gdt);
+		if (!xen_feature(XENFEAT_writable_page_tables))
+			/* We've switched to the "real" per-cpu gdt, so make sure the
+			 * old memory can be recycled */
+			make_lowmem_page_readwrite(xen_initial_gdt);
 
 #ifdef CONFIG_X86_32
 		/*
@@ -286,7 +289,6 @@ static void __init xen_smp_prepare_boot_cpu(void)
 		loadsegment(ds, __USER_DS);
 		loadsegment(es, __USER_DS);
 #endif
-
 		xen_filter_cpu_maps();
 		xen_setup_vcpu_info_placement();
 	}
@@ -372,7 +374,23 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 
 	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
 
-	{
+	if (xen_feature(XENFEAT_auto_translated_physmap) &&
+	    xen_feature(XENFEAT_supervisor_mode_kernel)) {
+		/* Note: PVH is not supported on x86_32. */
+#ifdef CONFIG_X86_64
+		ctxt->user_regs.ds = __KERNEL_DS;
+		ctxt->user_regs.es = 0;
+		ctxt->user_regs.gs = 0;
+
+		/* GUEST_GDTR_BASE and */
+		ctxt->u.pvh.gdtaddr = (unsigned long)gdt;
+		/* GUEST_GDTR_LIMIT in the VMCS. */
+		ctxt->u.pvh.gdtsz = (unsigned long)(GDT_SIZE - 1);
+
+		ctxt->gs_base_user = (unsigned long)
+					per_cpu(irq_stack_union.gs_base, cpu);
+#endif
+	} else {
 		ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
 		ctxt->user_regs.ds = __USER_DS;
 		ctxt->user_regs.es = __USER_DS;
@@ -387,8 +405,8 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 		make_lowmem_page_readonly(gdt);
 		make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
 
-		ctxt->gdt_frames[0] = gdt_mfn;
-		ctxt->gdt_ents      = GDT_ENTRIES;
+		ctxt->u.pv.gdt_frames[0] = gdt_mfn;
+		ctxt->u.pv.gdt_ents      = GDT_ENTRIES;
 
 		ctxt->kernel_ss = __KERNEL_DS;
 		ctxt->kernel_sp = idle->thread.sp0;
diff --git a/drivers/xen/cpu_hotplug.c b/drivers/xen/cpu_hotplug.c
index cc6513a..cbb02af 100644
--- a/drivers/xen/cpu_hotplug.c
+++ b/drivers/xen/cpu_hotplug.c
@@ -4,6 +4,7 @@
 
 #include <xen/xen.h>
 #include <xen/xenbus.h>
+#include <xen/features.h>
 
 #include <asm/xen/hypervisor.h>
 #include <asm/cpu.h>
@@ -102,7 +103,8 @@ static int __init setup_vcpu_hotplug_event(void)
 	static struct notifier_block xsn_cpu = {
 		.notifier_call = setup_cpu_watcher };
 
-	if (!xen_pv_domain())
+	/* PVH TBD/FIXME: future work */
+	if (!xen_pv_domain() || xen_feature(XENFEAT_auto_translated_physmap))
 		return -ENODEV;
 
 	register_xenstore_notifier(&xsn_cpu);
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 4035e83..38d56a2 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -1862,7 +1862,7 @@ int xen_set_callback_via(uint64_t via)
 }
 EXPORT_SYMBOL_GPL(xen_set_callback_via);
 
-#ifdef CONFIG_XEN_PVHVM
+#ifdef CONFIG_X86
 /* Vector callbacks are better than PCI interrupts to receive event
  * channel notifications because we can receive vector callbacks on any
  * vcpu and we don't need PCI support or APIC interactions. */
@@ -1922,6 +1922,13 @@ void __init xen_init_IRQ(void)
 		if (xen_initial_domain())
 			pci_xen_initial_domain();
 
+		if (xen_feature(XENFEAT_hvm_callback_vector)) {
+			xen_callback_vector();
+			return;
+		}
+
+		/* PVH: TBD/FIXME: debug and fix eio map to work with pvh */
+
 		pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
 		eoi_gmfn.gmfn = virt_to_mfn(pirq_eoi_map);
 		rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c
index ec097d6..7f7c454 100644
--- a/drivers/xen/xenbus/xenbus_client.c
+++ b/drivers/xen/xenbus/xenbus_client.c
@@ -45,6 +45,7 @@
 #include <xen/grant_table.h>
 #include <xen/xenbus.h>
 #include <xen/xen.h>
+#include <xen/features.h>
 
 #include "xenbus_probe.h"
 
@@ -743,7 +744,7 @@ static const struct xenbus_ring_ops ring_ops_hvm = {
 
 void __init xenbus_ring_ops_init(void)
 {
-	if (xen_pv_domain())
+	if (xen_pv_domain() && !xen_feature(XENFEAT_auto_translated_physmap))
 		ring_ops = &ring_ops_pv;
 	else
 		ring_ops = &ring_ops_hvm;
-- 
1.7.7.6

[PATCH V10 03/14] xen/pvh: Implement MMU changes for PVH.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

.. which are surprinsingly small compared to the amount for PV.
First the set/clear mmio pte function make a hypercall to update the
P2M in Xen with 1:1 mapping. Since PVH uses mostly native mmu ops, we
leave the generic (native_*) for the majority and just overwrite the
baremetal with the ones we need.

Two local functions are introduced to add to Xen physmap for Xen remap
interface. Xen unmap interface is introduced so that the privcmd PTe entries
can be cleared in Xen P2M table.

ijc -- rebase on top of ARM privcmd changes rather than vice versa,
xen_remap_domain_mfn_range (and unmap) split into a separate patch
earlier in the series.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
[v1: Rebase on x86, mm, Xen: Remove mapping_pagetable_reserve()]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

Conflicts:
	arch/x86/xen/mmu.c
[due to xen/mmu: On early bootup, flush the TLB when changing RO->RW
bits Xen provided pagetables.]
---
 arch/x86/xen/mmu.c |  147 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 arch/x86/xen/mmu.h |    2 +
 2 files changed, 144 insertions(+), 5 deletions(-)

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index ce563be..c7de065 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -74,6 +74,7 @@
 #include <xen/interface/version.h>
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
+#include <xen/balloon.h>
 
 #include "multicalls.h"
 #include "mmu.h"
@@ -332,6 +333,20 @@ static void xen_set_pte(pte_t *ptep, pte_t pteval)
 	__xen_set_pte(ptep, pteval);
 }
 
+void xen_set_clr_mmio_pvh_pte(unsigned long pfn, unsigned long mfn,
+			      int nr_mfns, int add_mapping)
+{
+	struct physdev_map_iomem iomem;
+
+	iomem.first_gfn = pfn;
+	iomem.first_mfn = mfn;
+	iomem.nr_mfns = nr_mfns;
+	iomem.add_mapping = add_mapping;
+
+	if (HYPERVISOR_physdev_op(PHYSDEVOP_map_iomem, &iomem))
+		BUG();
+}
+
 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
 		    pte_t *ptep, pte_t pteval)
 {
@@ -1207,6 +1222,8 @@ static void __init xen_pagetable_init(void)
 #endif
 	paging_init();
 	xen_setup_shared_info();
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return;
 #ifdef CONFIG_X86_64
 	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
 		unsigned long new_mfn_list;
@@ -1556,6 +1573,10 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
 static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
 {
 	struct mmuext_op op;
+
+	if (xen_feature(XENFEAT_writable_page_tables))
+		return;
+
 	op.cmd = cmd;
 	op.arg1.mfn = pfn_to_mfn(pfn);
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
@@ -1753,6 +1774,10 @@ static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags)
 	unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
 	pte_t pte = pfn_pte(pfn, prot);
 
+	/* recall for PVH, page tables are native. */
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return;
+
 	if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
 		BUG();
 }
@@ -1834,6 +1859,9 @@ static void convert_pfn_mfn(void *v)
 	pte_t *pte = v;
 	int i;
 
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return;
+
 	/* All levels are converted the same way, so just treat them
 	   as ptes. */
 	for (i = 0; i < PTRS_PER_PTE; i++)
@@ -1853,6 +1881,7 @@ static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
 		(*pt_end)--;
 	}
 }
+
 /*
  * Set up the initial kernel pagetable.
  *
@@ -1863,6 +1892,7 @@ static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
  * but that's enough to get __va working.  We need to fill in the rest
  * of the physical mapping once some sort of allocator has been set
  * up.
+ * NOTE: for PVH, the page tables are native.
  */
 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 {
@@ -1940,10 +1970,13 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 	 * structure to attach it to, so make sure we just set kernel
 	 * pgd.
 	 */
-	xen_mc_batch();
-	__xen_write_cr3(true, __pa(init_level4_pgt));
-	xen_mc_issue(PARAVIRT_LAZY_CPU);
-
+	if (xen_feature(XENFEAT_writable_page_tables)) {
+		native_write_cr3(__pa(init_level4_pgt));
+	} else {
+		xen_mc_batch();
+		__xen_write_cr3(true, __pa(init_level4_pgt));
+		xen_mc_issue(PARAVIRT_LAZY_CPU);
+	}
 	/* We can't that easily rip out L3 and L2, as the Xen pagetables are
 	 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ...  for
 	 * the initial domain. For guests using the toolstack, they are in:
@@ -2207,6 +2240,11 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
 void __init xen_init_mmu_ops(void)
 {
 	x86_init.paging.pagetable_init = xen_pagetable_init;
+
+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+		pv_mmu_ops.flush_tlb_others = xen_flush_tlb_others;
+		return;
+	}
 	pv_mmu_ops = xen_mmu_ops;
 
 	memset(dummy_mapping, 0xff, PAGE_SIZE);
@@ -2487,6 +2525,89 @@ void __init xen_hvm_init_mmu_ops(void)
 }
 #endif
 
+/* Map foreign gmfn, fgmfn, to local pfn, lpfn. This for the user space
+ * creating new guest on PVH dom0 and needs to map domU pages.
+ */
+static int pvh_add_to_xen_p2m(unsigned long lpfn, unsigned long fgmfn,
+			      unsigned int domid)
+{
+	int rc;
+	struct xen_add_to_physmap xatp = { .foreign_domid = domid };
+
+	xatp.gpfn = lpfn;
+	xatp.idx = fgmfn;
+	xatp.domid = DOMID_SELF;
+	xatp.space = XENMAPSPACE_gmfn_foreign;
+	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
+	if (rc)
+		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d\n",
+			lpfn, fgmfn, rc);
+	return rc;
+}
+
+static int pvh_rem_xen_p2m(unsigned long spfn, int count)
+{
+	struct xen_remove_from_physmap xrp;
+	int i, rc;
+
+	for (i = 0; i < count; i++) {
+		xrp.domid = DOMID_SELF;
+		xrp.gpfn = spfn+i;
+		rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
+		if (rc) {
+			pr_warn("Failed to unmap pfn:%lx rc:%d done:%d\n",
+				spfn+i, rc, i);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+struct pvh_remap_data {
+	unsigned long fgmfn;		/* foreign domain's gmfn */
+	pgprot_t prot;
+	domid_t  domid;
+	int	 index;
+	struct page **pages;
+};
+
+static int pvh_map_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr,
+			void *data)
+{
+	int rc;
+	struct pvh_remap_data *remap = data;
+	unsigned long pfn = page_to_pfn(remap->pages[remap->index++]);
+	pte_t pteval = pte_mkspecial(pfn_pte(pfn, remap->prot));
+
+	rc = pvh_add_to_xen_p2m(pfn, remap->fgmfn, remap->domid);
+	if (rc)
+		return rc;
+	native_set_pte(ptep, pteval);
+
+	return 0;
+}
+
+static int pvh_remap_gmfn_range(struct vm_area_struct *vma,
+				unsigned long addr, unsigned long mfn, int nr,
+				pgprot_t prot, unsigned domid,
+				struct page **pages)
+{
+	int err;
+	struct pvh_remap_data pvhdata;
+
+	BUG_ON(!pages);
+
+	pvhdata.fgmfn = mfn;
+	pvhdata.prot = prot;
+	pvhdata.domid = domid;
+	pvhdata.index = 0;
+	pvhdata.pages = pages;
+	err = apply_to_page_range(vma->vm_mm, addr, nr << PAGE_SHIFT,
+				  pvh_map_pte_fn, &pvhdata);
+	flush_tlb_all();
+	return err;
+}
+
 #define REMAP_BATCH_SIZE 16
 
 struct remap_data {
@@ -2528,6 +2649,10 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
 
 	BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
 
+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+		/* We need to update the local page tables and the xen HAP */
+		return pvh_remap_gmfn_range(vma, addr, mfn, nr, prot, domid, pages);
+	}
 	rmd.mfn = mfn;
 	rmd.prot = prot;
 
@@ -2565,6 +2690,18 @@ int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
 	if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
 		return 0;
 
-	return -EINVAL;
+	while (numpgs--) {
+
+		/* the mmu has already cleaned up the process mmu resources at
+		 * this point (lookup_address will return NULL). */
+		unsigned long pfn = page_to_pfn(pages[numpgs]);
+
+		pvh_rem_xen_p2m(pfn, 1);
+	}
+	/* We don't need to flush tlbs because as part of pvh_rem_xen_p2m(),
+	 * the hypervisor will do tlb flushes after removing the p2m entries
+	 * from the EPT/NPT */
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range);
diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h
index 73809bb..6d0bb56 100644
--- a/arch/x86/xen/mmu.h
+++ b/arch/x86/xen/mmu.h
@@ -23,4 +23,6 @@ unsigned long xen_read_cr2_direct(void);
 
 extern void xen_init_mmu_ops(void);
 extern void xen_hvm_init_mmu_ops(void);
+extern void xen_set_clr_mmio_pvh_pte(unsigned long pfn, unsigned long mfn,
+				     int nr_mfns, int add_mapping);
 #endif	/* _XEN_MMU_H */
-- 
1.7.7.6

[PATCH V10 04/14] xen/pvh: bootup and setup (E820) related changes.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

In the bootup code for PVH we can trap cpuid via vmexit, so don't
need to use emulated prefix call. We also check for vector callback
early on, as it is a required feature. PVH also runs at default kernel
IOPL.

In setup.c which deals with E820, in xen_add_extra_mem() we can skip
updating P2M as it's managed by Xen. PVH maps the entire IO space,
but only RAM pages need to be repopulated.

Finally, pure PV settings are moved to a separate function that are
only called for pure PV, ie, pv with pvmmu.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[ ijc -- rebase onto xen PVonHVM: use E820_Reserved area for
         shared_info ]
[v2: Rebase on v3.9-rc1 with MMIO/Kexec reverted]

Conflicts:
	arch/x86/xen/setup.c
[due to "xen: Support 64-bit PV guest receiving NMIs"]
---
 arch/x86/xen/enlighten.c |   77 ++++++++++++++++++++++++++++++++++-----------
 arch/x86/xen/setup.c     |   63 ++++++++++++++++++++++++++++++-------
 2 files changed, 109 insertions(+), 31 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index fa6ade7..500508d 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -46,6 +46,7 @@
 #include <xen/hvm.h>
 #include <xen/hvc-console.h>
 #include <xen/acpi.h>
+#include <xen/features.h>
 
 #include <asm/paravirt.h>
 #include <asm/apic.h>
@@ -129,6 +130,9 @@ RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
 __read_mostly int xen_have_vector_callback;
 EXPORT_SYMBOL_GPL(xen_have_vector_callback);
 
+#define xen_pvh_domain() (xen_pv_domain() && \
+			  xen_feature(XENFEAT_auto_translated_physmap) && \
+			  xen_have_vector_callback)
 /*
  * Point at some empty memory to start with. We map the real shared_info
  * page as soon as fixmap is up and running.
@@ -262,8 +266,9 @@ static void __init xen_banner(void)
 	struct xen_extraversion extra;
 	HYPERVISOR_xen_version(XENVER_extraversion, &extra);
 
-	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
-	       pv_info.name);
+	pr_info("Booting paravirtualized kernel %son %s\n",
+		xen_feature(XENFEAT_auto_translated_physmap) ?
+			"with PVH extensions " : "", pv_info.name);
 	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
 	       version >> 16, version & 0xffff, extra.extraversion,
 	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
@@ -331,12 +336,15 @@ static void xen_cpuid(unsigned int *ax, unsigned int *bx,
 		break;
 	}
 
-	asm(XEN_EMULATE_PREFIX "cpuid"
-		: "=a" (*ax),
-		  "=b" (*bx),
-		  "=c" (*cx),
-		  "=d" (*dx)
-		: "0" (*ax), "2" (*cx));
+	if (xen_pvh_domain())
+		native_cpuid(ax, bx, cx, dx);
+	else
+		asm(XEN_EMULATE_PREFIX "cpuid"
+			: "=a" (*ax),
+			"=b" (*bx),
+			"=c" (*cx),
+			"=d" (*dx)
+			: "0" (*ax), "2" (*cx));
 
 	*bx &= maskebx;
 	*cx &= maskecx;
@@ -1125,6 +1133,10 @@ void xen_setup_shared_info(void)
 		HYPERVISOR_shared_info =
 			(struct shared_info *)__va(xen_start_info->shared_info);
 
+	/* PVH TBD/FIXME: vcpu info placement in phase 2 */
+	if (xen_pvh_domain())
+		return;
+
 #ifndef CONFIG_SMP
 	/* In UP this is as good a place as any to set up shared info */
 	xen_setup_vcpu_info_placement();
@@ -1410,6 +1422,11 @@ static void __init xen_boot_params_init_edd(void)
  */
 static void __init xen_setup_stackprotector(void)
 {
+	/* PVH TBD/FIXME: investigate setup_stack_canary_segment */
+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+		switch_to_new_gdt(0);
+		return;
+	}
 	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
 	pv_cpu_ops.load_gdt = xen_load_gdt_boot;
 
@@ -1420,6 +1437,19 @@ static void __init xen_setup_stackprotector(void)
 	pv_cpu_ops.load_gdt = xen_load_gdt;
 }
 
+static void __init xen_pvh_early_guest_init(void)
+{
+	if (xen_feature(XENFEAT_hvm_callback_vector))
+		xen_have_vector_callback = 1;
+
+#ifdef CONFIG_X86_32
+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+		xen_raw_printk("ERROR: 32bit PVH guests are not supported\n");
+		BUG();
+	}
+#endif
+}
+
 /* First C function to be called on Xen boot */
 asmlinkage void __init xen_start_kernel(void)
 {
@@ -1431,13 +1461,18 @@ asmlinkage void __init xen_start_kernel(void)
 
 	xen_domain_type = XEN_PV_DOMAIN;
 
+	xen_setup_features();
+	xen_pvh_early_guest_init();
 	xen_setup_machphys_mapping();
 
 	/* Install Xen paravirt ops */
 	pv_info = xen_info;
 	pv_init_ops = xen_init_ops;
-	pv_cpu_ops = xen_cpu_ops;
 	pv_apic_ops = xen_apic_ops;
+	if (xen_pvh_domain())
+		pv_cpu_ops.cpuid = xen_cpuid;
+	else
+		pv_cpu_ops = xen_cpu_ops;
 
 	x86_init.resources.memory_setup = xen_memory_setup;
 	x86_init.oem.arch_setup = xen_arch_setup;
@@ -1469,8 +1504,6 @@ asmlinkage void __init xen_start_kernel(void)
 	/* Work out if we support NX */
 	x86_configure_nx();
 
-	xen_setup_features();
-
 	/* Get mfn list */
 	if (!xen_feature(XENFEAT_auto_translated_physmap))
 		xen_build_dynamic_phys_to_machine();
@@ -1548,14 +1581,18 @@ asmlinkage void __init xen_start_kernel(void)
 	/* set the limit of our address space */
 	xen_reserve_top();
 
-	/* We used to do this in xen_arch_setup, but that is too late on AMD
-	 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
-	 * which pokes 0xcf8 port.
-	 */
-	set_iopl.iopl = 1;
-	rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
-	if (rc != 0)
-		xen_raw_printk("physdev_op failed %d\n", rc);
+	/* PVH: runs at default kernel iopl of 0 */
+	if (!xen_pvh_domain()) {
+		/*
+		 * We used to do this in xen_arch_setup, but that is too late
+		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
+		 * early_amd_init which pokes 0xcf8 port.
+		 */
+		set_iopl.iopl = 1;
+		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
+		if (rc != 0)
+			xen_raw_printk("physdev_op failed %d\n", rc);
+	}
 
 #ifdef CONFIG_X86_32
 	/* set up basic CPUID stuff */
@@ -1625,6 +1662,8 @@ asmlinkage void __init xen_start_kernel(void)
 }
 
 void __ref xen_hvm_init_shared_info(void)
+/* Use a pfn in RAM, may move to MMIO before kexec.
+ * This function also called for PVH dom0 */
 {
 	int cpu;
 	struct xen_add_to_physmap xatp;
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 68c054f..e3dcd8c 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -27,6 +27,7 @@
 #include <xen/interface/memory.h>
 #include <xen/interface/physdev.h>
 #include <xen/features.h>
+#include "mmu.h"
 #include "xen-ops.h"
 #include "vdso.h"
 
@@ -81,6 +82,9 @@ static void __init xen_add_extra_mem(u64 start, u64 size)
 
 	memblock_reserve(start, size);
 
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return;
+
 	xen_max_p2m_pfn = PFN_DOWN(start + size);
 	for (pfn = PFN_DOWN(start); pfn < xen_max_p2m_pfn; pfn++) {
 		unsigned long mfn = pfn_to_mfn(pfn);
@@ -103,6 +107,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
 		.domid        = DOMID_SELF
 	};
 	unsigned long len = 0;
+	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);
 	unsigned long pfn;
 	int ret;
 
@@ -116,7 +121,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
 				continue;
 			frame = mfn;
 		} else {
-			if (mfn != INVALID_P2M_ENTRY)
+			if (!xlated_phys && mfn != INVALID_P2M_ENTRY)
 				continue;
 			frame = pfn;
 		}
@@ -239,6 +244,27 @@ static void __init xen_set_identity_and_release_chunk(
 	*identity += set_phys_range_identity(start_pfn, end_pfn);
 }
 
+/* For PVH, the pfns [0..MAX] are mapped to mfn's in the EPT/NPT. The mfns
+ * are released as part of this 1:1 mapping hypercall back to the dom heap.
+ * Also, we map the entire IO space, ie, beyond max_pfn_mapped.
+ */
+static void __init xen_pvh_identity_map_chunk(unsigned long start_pfn,
+		unsigned long end_pfn, unsigned long *released,
+		unsigned long *identity, unsigned long max_pfn)
+{
+	unsigned long pfn;
+	int numpfns = 1, add_mapping = 1;
+
+	for (pfn = start_pfn; pfn < end_pfn; pfn++)
+		xen_set_clr_mmio_pvh_pte(pfn, pfn, numpfns, add_mapping);
+
+	if (start_pfn <= max_pfn) {
+		unsigned long end = min(max_pfn_mapped, end_pfn);
+		*released += end - start_pfn;
+	}
+	*identity += end_pfn - start_pfn;
+}
+
 static unsigned long __init xen_set_identity_and_release(
 	const struct e820entry *list, size_t map_size, unsigned long nr_pages)
 {
@@ -247,6 +273,7 @@ static unsigned long __init xen_set_identity_and_release(
 	unsigned long identity = 0;
 	const struct e820entry *entry;
 	int i;
+	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);
 
 	/*
 	 * Combine non-RAM regions and gaps until a RAM region (or the
@@ -268,11 +295,17 @@ static unsigned long __init xen_set_identity_and_release(
 			if (entry->type == E820_RAM)
 				end_pfn = PFN_UP(entry->addr);
 
-			if (start_pfn < end_pfn)
-				xen_set_identity_and_release_chunk(
-					start_pfn, end_pfn, nr_pages,
-					&released, &identity);
-
+			if (start_pfn < end_pfn) {
+				if (xlated_phys) {
+					xen_pvh_identity_map_chunk(start_pfn,
+						end_pfn, &released, &identity,
+						nr_pages);
+				} else {
+					xen_set_identity_and_release_chunk(
+						start_pfn, end_pfn, nr_pages,
+						&released, &identity);
+				}
+			}
 			start = end;
 		}
 	}
@@ -563,16 +596,13 @@ void xen_enable_nmi(void)
 		BUG();
 #endif
 }
-void __init xen_arch_setup(void)
+void __init xen_pvmmu_arch_setup(void)
 {
-	xen_panic_handler_init();
-
 	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);
 	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables);
 
-	if (!xen_feature(XENFEAT_auto_translated_physmap))
-		HYPERVISOR_vm_assist(VMASST_CMD_enable,
-				     VMASST_TYPE_pae_extended_cr3);
+	HYPERVISOR_vm_assist(VMASST_CMD_enable,
+			     VMASST_TYPE_pae_extended_cr3);
 
 	if (register_callback(CALLBACKTYPE_event, xen_hypervisor_callback) ||
 	    register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback))
@@ -581,6 +611,15 @@ void __init xen_arch_setup(void)
 	xen_enable_sysenter();
 	xen_enable_syscall();
 	xen_enable_nmi();
+}
+
+/* This function not called for HVM domain */
+void __init xen_arch_setup(void)
+{
+	xen_panic_handler_init();
+
+	if (!xen_feature(XENFEAT_auto_translated_physmap))
+		xen_pvmmu_arch_setup();
 #ifdef CONFIG_ACPI
 	if (!(xen_start_info->flags & SIF_INITDOMAIN)) {
 		printk(KERN_INFO "ACPI in unprivileged domain disabled\n");
-- 
1.7.7.6

[PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

For balloon changes we skip setting of local P2M as it's updated
in Xen. For grant, the shared grant frame is the PFN and not MFN,
hence its mapped via the same code path as HVM.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
[v1: Rebase on v3.7-rc3]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
CC: <stefano.stabellini@eu.citrix.com>
Conflicts:
	drivers/xen/balloon.c
[v2: Rebase on v3.13-rc3 has lot of ARM changes.]
---
 drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
 drivers/xen/gntdev.c      |    3 ++-
 drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
 3 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index 55ea73f..0e9875a 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
 
 #ifdef CONFIG_XEN_HAVE_PVMMU
 		/* Link back into the page tables if not highmem. */
-		if (xen_pv_domain() && !PageHighMem(page)) {
+		if (xen_pv_domain() && !PageHighMem(page) &&
+		    !xen_feature(XENFEAT_auto_translated_physmap)) {
+
 			int ret;
 			ret = HYPERVISOR_update_va_mapping(
 				(unsigned long)__va(pfn << PAGE_SHIFT),
@@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
 
 		scrub_page(page);
 
-		/*
-		 * Ballooned out frames are effectively replaced with
-		 * a scratch frame.  Ensure direct mappings and the
-		 * p2m are consistent.
-		 */
-		scratch_page = get_balloon_scratch_page();
+		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+			unsigned long p;
+
+			/*
+			 * Ballooned out frames are effectively replaced with
+			 * a scratch frame.  Ensure direct mappings and the
+			 * p2m are consistent.
+			 */
+			scratch_page = get_balloon_scratch_page();
 #ifdef CONFIG_XEN_HAVE_PVMMU
-		if (xen_pv_domain() && !PageHighMem(page)) {
-			ret = HYPERVISOR_update_va_mapping(
-				(unsigned long)__va(pfn << PAGE_SHIFT),
-				pfn_pte(page_to_pfn(scratch_page),
+			if (xen_pv_domain() && !PageHighMem(page)) {
+				ret = HYPERVISOR_update_va_mapping(
+					(unsigned long)__va(pfn << PAGE_SHIFT),
+					pfn_pte(page_to_pfn(scratch_page),
 					PAGE_KERNEL_RO), 0);
-			BUG_ON(ret);
-		}
+				BUG_ON(ret);
+			}
 #endif
-		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
-			unsigned long p;
 			p = page_to_pfn(scratch_page);
 			__set_phys_to_machine(pfn, pfn_to_mfn(p));
-		}
-		put_balloon_scratch_page();
 
+			put_balloon_scratch_page();
+		}
 		balloon_append(pfn_to_page(pfn));
 	}
 
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index e41c79c..c1ae3ce 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -846,7 +846,8 @@ static int __init gntdev_init(void)
 	if (!xen_domain())
 		return -ENODEV;
 
-	use_ptemod = xen_pv_domain();
+	use_ptemod = xen_pv_domain() &&
+		     !xen_feature(XENFEAT_auto_translated_physmap);
 
 	err = misc_register(&gntdev_miscdev);
 	if (err != 0) {
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 0283871..1fcf75b 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -1056,14 +1056,20 @@ static void gnttab_unmap_frames_v2(void)
 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct gnttab_setup_table setup;
+	unsigned long start_gpfn;
 	xen_pfn_t *frames;
 	unsigned int nr_gframes = end_idx + 1;
 	int rc;
 
-	if (xen_hvm_domain()) {
+	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap)) {
 		struct xen_add_to_physmap xatp;
 		unsigned int i = end_idx;
 		rc = 0;
+
+		if (xen_hvm_domain())
+			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
+		else
+			start_gpfn = virt_to_pfn(gnttab_shared.addr);
 		/*
 		 * Loop backwards, so that the first hypercall has the largest
 		 * index, ensuring that the table will grow only once.
@@ -1072,7 +1078,7 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 			xatp.domid = DOMID_SELF;
 			xatp.idx = i;
 			xatp.space = XENMAPSPACE_grant_table;
-			xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
+			xatp.gpfn = start_gpfn + i;
 			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 			if (rc != 0) {
 				pr_warn("grant table add_to_physmap failed, err=%d\n",
@@ -1135,7 +1141,7 @@ static void gnttab_request_version(void)
 	int rc;
 	struct gnttab_set_version gsv;
 
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))
 		gsv.version = 1;
 	else
 		gsv.version = 2;
@@ -1164,11 +1170,24 @@ static void gnttab_request_version(void)
 static int gnttab_setup(void)
 {
 	unsigned int max_nr_gframes;
+	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
 
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
+	/* PVH note: xen will free existing kmalloc'd mfn in
+	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
+	 * kmalloc(). */
+	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap) &&
+	    !gnttab_shared.addr) {
+		gnttab_shared.addr =
+			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
+		if (!gnttab_shared.addr) {
+			pr_warn("%s", kmsg);
+			return -ENOMEM;
+		}
+	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);
 
-- 
1.7.7.6

[PATCH V10 06/14] xen: x86 pvh: use XENMEM_add_to_physmap_range for foreign gmfn mappings

[Konrad Rzeszutek Wilk]

From: Ian Campbell <ian.campbell@citrix.com>

Squeezing the necessary fields into the existing XENMEM_add_to_physmap
interface was proving to be a bit tricky so we have decided to go with
a new interface upstream (the XENMAPSPACE_gmfn_foreign interface using
XENMEM_add_to_physmap was never committed anywhere). This interface
also allows for batching which was impossible to support at the same
time as foreign mfns in the old interface.

This reverts the relevant parts of "PVH: basic and header changes,
elfnote changes, ..." and followups and trivially converts
pvh_add_to_xen_p2m over.

Acked-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/mmu.c             |   18 ++++++++++++------
 include/xen/interface/memory.h |    5 +----
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index c7de065..e334134 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -2532,13 +2532,19 @@ static int pvh_add_to_xen_p2m(unsigned long lpfn, unsigned long fgmfn,
 			      unsigned int domid)
 {
 	int rc;
-	struct xen_add_to_physmap xatp = { .foreign_domid = domid };
+	struct xen_add_to_physmap_range xatp = {
+		.domid = DOMID_SELF,
+		.foreign_domid = domid,
+		.size = 1,
+		.space = XENMAPSPACE_gmfn_foreign,
+	};
+	xen_ulong_t idx = fgmfn;
+	xen_pfn_t gpfn = lpfn;
+
+	set_xen_guest_handle(xatp.idxs, &idx);
+	set_xen_guest_handle(xatp.gpfns, &gpfn);
 
-	xatp.gpfn = lpfn;
-	xatp.idx = fgmfn;
-	xatp.domid = DOMID_SELF;
-	xatp.space = XENMAPSPACE_gmfn_foreign;
-	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
+	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
 	if (rc)
 		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d\n",
 			lpfn, fgmfn, rc);
diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
index a122164..2ecfe4f 100644
--- a/include/xen/interface/memory.h
+++ b/include/xen/interface/memory.h
@@ -175,10 +175,7 @@ struct xen_add_to_physmap {
     uint16_t    size;
 
     /* Source mapping space. */
-    uint16_t space;
-    domid_t foreign_domid;         /* IFF XENMAPSPACE_gmfn_foreign */
-
-#define XENMAPIDX_grant_table_status 0x80000000
+    unsigned int space;
 
     /* Index into source mapping space. */
     xen_ulong_t idx;
-- 
1.7.7.6

[PATCH V10 07/14] xen/x86: remove duplicated include from enlighten.c

[Konrad Rzeszutek Wilk]

From: Wei Yongjun <yongjun_wei@trendmicro.com.cn>

Remove duplicated include.

dpatch engine is used to auto generate this patch.
(https://github.com/weiyj/dpatch)

Signed-off-by: Wei Yongjun <yongjun_wei@trendmicro.com.cn>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/enlighten.c |    1 -
 1 files changed, 0 insertions(+), 1 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 500508d..0bfbe92 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -46,7 +46,6 @@
 #include <xen/hvm.h>
 #include <xen/hvc-console.h>
 #include <xen/acpi.h>
-#include <xen/features.h>
 
 #include <asm/paravirt.h>
 #include <asm/apic.h>
-- 
1.7.7.6

[PATCH V10 08/14] x86/xen: Use __pa_symbol instead of __pa on C visible symbols

[Konrad Rzeszutek Wilk]

From: Alexander Duyck <alexander.h.duyck@intel.com>

This change updates a few of the functions to use __pa_symbol when
translating C visible symbols instead of __pa.  By using __pa_symbol we are
able to drop a few extra lines of code as don't have to test to see if the
virtual pointer is a part of the kernel text or just standard virtual memory.

Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/mmu.c |   21 +++++++++++----------
 1 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index e334134..1d1fbdb 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1508,7 +1508,8 @@ static int xen_pgd_alloc(struct mm_struct *mm)
 
 		if (user_pgd != NULL) {
 			user_pgd[pgd_index(VSYSCALL_START)] =
-				__pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
+				__pgd(__pa_symbol(level3_user_vsyscall) |
+				      _PAGE_TABLE);
 			ret = 0;
 		}
 
@@ -1971,10 +1972,10 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 	 * pgd.
 	 */
 	if (xen_feature(XENFEAT_writable_page_tables)) {
-		native_write_cr3(__pa(init_level4_pgt));
+		native_write_cr3(__pa_symbol(init_level4_pgt));
 	} else {
 		xen_mc_batch();
-		__xen_write_cr3(true, __pa(init_level4_pgt));
+		__xen_write_cr3(true, __pa_symbol(init_level4_pgt));
 		xen_mc_issue(PARAVIRT_LAZY_CPU);
 	}
 	/* We can't that easily rip out L3 and L2, as the Xen pagetables are
@@ -1997,10 +1998,10 @@ static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
 
 static void __init xen_write_cr3_init(unsigned long cr3)
 {
-	unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
+	unsigned long pfn = PFN_DOWN(__pa_symbol(swapper_pg_dir));
 
-	BUG_ON(read_cr3() != __pa(initial_page_table));
-	BUG_ON(cr3 != __pa(swapper_pg_dir));
+	BUG_ON(read_cr3() != __pa_symbol(initial_page_table));
+	BUG_ON(cr3 != __pa_symbol(swapper_pg_dir));
 
 	/*
 	 * We are switching to swapper_pg_dir for the first time (from
@@ -2024,7 +2025,7 @@ static void __init xen_write_cr3_init(unsigned long cr3)
 	pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
 
 	pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
-			  PFN_DOWN(__pa(initial_page_table)));
+			  PFN_DOWN(__pa_symbol(initial_page_table)));
 	set_page_prot(initial_page_table, PAGE_KERNEL);
 	set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
 
@@ -2049,7 +2050,7 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 
 	copy_page(initial_page_table, pgd);
 	initial_page_table[KERNEL_PGD_BOUNDARY] =
-		__pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
+		__pgd(__pa_symbol(initial_kernel_pmd) | _PAGE_PRESENT);
 
 	set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
 	set_page_prot(initial_page_table, PAGE_KERNEL_RO);
@@ -2058,8 +2059,8 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 	pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
 
 	pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
-			  PFN_DOWN(__pa(initial_page_table)));
-	xen_write_cr3(__pa(initial_page_table));
+			  PFN_DOWN(__pa_symbol(initial_page_table)));
+	xen_write_cr3(__pa_symbol(initial_page_table));
 
 	memblock_reserve(__pa(xen_start_info->pt_base),
 			 xen_start_info->nr_pt_frames * PAGE_SIZE);
-- 
1.7.7.6

[PATCH V10 09/14] xen: implement updated XENMEM_add_to_physmap_range ABI

[Konrad Rzeszutek Wilk]

From: Ian Campbell <Ian.Campbell@citrix.com>

Allows for more fine grained error reporting. Only used by PVH and
ARM both of which are marked EXPERIMENTAL precisely because the ABI
is not yet stable

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/mmu.c |    8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 1d1fbdb..a31449f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -2541,14 +2541,16 @@ static int pvh_add_to_xen_p2m(unsigned long lpfn, unsigned long fgmfn,
 	};
 	xen_ulong_t idx = fgmfn;
 	xen_pfn_t gpfn = lpfn;
+	int err = 0;
 
 	set_xen_guest_handle(xatp.idxs, &idx);
 	set_xen_guest_handle(xatp.gpfns, &gpfn);
+	set_xen_guest_handle(xatp.errs, &err);
 
 	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
-	if (rc)
-		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d\n",
-			lpfn, fgmfn, rc);
+	if (rc || err)
+		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d:%d\n",
+			lpfn, fgmfn, rc, err);
 	return rc;
 }
 
-- 
1.7.7.6

[PATCH V10 10/14] xen/pvh: specify xen features strings cleanly for PVH

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

Use .ascii and .asciz to define xen feature string. Note, the PVH
string must be in a single line (not multiple lines with \) to keep the
assembler from putting null char after each string before \.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/xen-head.S |   11 +++++------
 1 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 1a6bca1..45226cb 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -14,12 +14,11 @@
 #include <asm/xen/interface.h>
 
 #ifdef CONFIG_XEN_X86_PVH
-#define FEATURES_PVH "|writable_descriptor_tables" \
-		     "|auto_translated_physmap" \
-		     "|supervisor_mode_kernel" \
-		     "|hvm_callback_vector"
+
+#define PVH_FEATURES_STR  "|writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel|hvm_callback_vector"
+
 #else
-#define FEATURES_PVH /* Not supported */
+#define PVH_FEATURES_STR  ""
 #endif
 
 	__INIT
@@ -104,7 +103,7 @@ NEXT_HYPERCALL(arch_6)
 #endif
 	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          _ASM_PTR startup_xen)
 	ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, _ASM_PTR hypercall_page)
-	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb"FEATURES_PVH)
+        ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .ascii "!writable_page_tables|pae_pgdir_above_4gb"; .asciz PVH_FEATURES_STR);
 	ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE,       .asciz "yes")
 	ELFNOTE(Xen, XEN_ELFNOTE_LOADER,         .asciz "generic")
 	ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID,
-- 
1.7.7.6

[PATCH V10 11/14] xen/pvh: remove code to map iomem from guest

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

It was decided during xen patch review that xen map the iomem
transparently, so remove xen_set_clr_mmio_pvh_pte() and the sub
hypercall PHYSDEVOP_map_iomem.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
[v1: Fixed up the comment]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/mmu.c              |   14 --------------
 arch/x86/xen/setup.c            |   17 ++++++-----------
 include/xen/interface/physdev.h |   10 ----------
 3 files changed, 6 insertions(+), 35 deletions(-)

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index a31449f..108fb4a 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -333,20 +333,6 @@ static void xen_set_pte(pte_t *ptep, pte_t pteval)
 	__xen_set_pte(ptep, pteval);
 }
 
-void xen_set_clr_mmio_pvh_pte(unsigned long pfn, unsigned long mfn,
-			      int nr_mfns, int add_mapping)
-{
-	struct physdev_map_iomem iomem;
-
-	iomem.first_gfn = pfn;
-	iomem.first_mfn = mfn;
-	iomem.nr_mfns = nr_mfns;
-	iomem.add_mapping = add_mapping;
-
-	if (HYPERVISOR_physdev_op(PHYSDEVOP_map_iomem, &iomem))
-		BUG();
-}
-
 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
 		    pte_t *ptep, pte_t pteval)
 {
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index e3dcd8c..94c9b49 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -244,20 +244,15 @@ static void __init xen_set_identity_and_release_chunk(
 	*identity += set_phys_range_identity(start_pfn, end_pfn);
 }
 
-/* For PVH, the pfns [0..MAX] are mapped to mfn's in the EPT/NPT. The mfns
- * are released as part of this 1:1 mapping hypercall back to the dom heap.
- * Also, we map the entire IO space, ie, beyond max_pfn_mapped.
+
+/*
+ * PVH: xen has already mapped the IO space in the EPT/NPT for us, so we
+ * just need to adjust the released and identity count.
  */
-static void __init xen_pvh_identity_map_chunk(unsigned long start_pfn,
+static void __init xen_pvh_adjust_stats(unsigned long start_pfn,
 		unsigned long end_pfn, unsigned long *released,
 		unsigned long *identity, unsigned long max_pfn)
 {
-	unsigned long pfn;
-	int numpfns = 1, add_mapping = 1;
-
-	for (pfn = start_pfn; pfn < end_pfn; pfn++)
-		xen_set_clr_mmio_pvh_pte(pfn, pfn, numpfns, add_mapping);
-
 	if (start_pfn <= max_pfn) {
 		unsigned long end = min(max_pfn_mapped, end_pfn);
 		*released += end - start_pfn;
@@ -297,7 +292,7 @@ static unsigned long __init xen_set_identity_and_release(
 
 			if (start_pfn < end_pfn) {
 				if (xlated_phys) {
-					xen_pvh_identity_map_chunk(start_pfn,
+					xen_pvh_adjust_stats(start_pfn,
 						end_pfn, &released, &identity,
 						nr_pages);
 				} else {
diff --git a/include/xen/interface/physdev.h b/include/xen/interface/physdev.h
index 19d30ad..42721d1 100644
--- a/include/xen/interface/physdev.h
+++ b/include/xen/interface/physdev.h
@@ -291,16 +291,6 @@ struct physdev_dbgp_op {
     } u;
 };
 
-#define PHYSDEVOP_map_iomem        30
-struct physdev_map_iomem {
-    /* IN */
-    uint64_t first_gfn;
-    uint64_t first_mfn;
-    uint32_t nr_mfns;
-    uint32_t add_mapping; /* 1 == add mapping;  0 == unmap */
-
-};
-
 /*
  * Notify that some PIRQ-bound event channels have been unmasked.
  * ** This command is obsolete since interface version 0x00030202 and is **
-- 
1.7.7.6

[PATCH V10 12/14] xen/pvh: Use ballooning to allocate grant table pages [v2]

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
allocate pfns for grant table pages instead of kmalloc. This also
simplifies add to physmap on the xen side a bit.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[v1: Rebase on xen/grant-table: correctly initialize grant table version 1]

Conflicts:
	drivers/xen/grant-table.c
[v2: Rebase on xen: Convert printks to pr_<level>]
---
 drivers/xen/grant-table.c |   67 +++++++++++++++++++++++++++++++++++---------
 1 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 1fcf75b..4467d54 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -50,6 +50,7 @@
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
 #include <xen/swiotlb-xen.h>
+#include <xen/balloon.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
@@ -1053,10 +1054,22 @@ static void gnttab_unmap_frames_v2(void)
 	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 }
 
+static xen_pfn_t pvh_get_grant_pfn(int grant_idx)
+{
+	unsigned long vaddr;
+	unsigned int level;
+	pte_t *pte;
+
+	vaddr = (unsigned long)(gnttab_shared.addr) + grant_idx * PAGE_SIZE;
+	pte = lookup_address(vaddr, &level);
+	BUG_ON(pte == NULL);
+	return pte_mfn(*pte);
+}
+
 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct gnttab_setup_table setup;
-	unsigned long start_gpfn;
+	unsigned long start_gpfn = 0;
 	xen_pfn_t *frames;
 	unsigned int nr_gframes = end_idx + 1;
 	int rc;
@@ -1068,8 +1081,6 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 
 		if (xen_hvm_domain())
 			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
-		else
-			start_gpfn = virt_to_pfn(gnttab_shared.addr);
 		/*
 		 * Loop backwards, so that the first hypercall has the largest
 		 * index, ensuring that the table will grow only once.
@@ -1078,7 +1089,11 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 			xatp.domid = DOMID_SELF;
 			xatp.idx = i;
 			xatp.space = XENMAPSPACE_grant_table;
-			xatp.gpfn = start_gpfn + i;
+			if (xen_hvm_domain())
+				xatp.gpfn = start_gpfn + i;
+			else
+				xatp.gpfn = pvh_get_grant_pfn(i);
+
 			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 			if (rc != 0) {
 				pr_warn("grant table add_to_physmap failed, err=%d\n",
@@ -1167,26 +1182,50 @@ static void gnttab_request_version(void)
 	pr_info("Grant tables using version %d layout\n", grant_table_version);
 }
 
+/*
+ * PVH: we need three things: virtual address, pfns, and mfns. The pfns
+ * are allocated via ballooning, then we call arch_gnttab_map_shared to
+ * allocate the VA and put pfn's in the pte's for the VA. The mfn's are
+ * finally allocated in gnttab_map() by xen which also populates the P2M.
+ */
+static int xlated_setup_gnttab_pages(unsigned long numpages, void **addr)
+{
+	int i, rc;
+	unsigned long pfns[numpages];
+	struct page *pages[numpages];
+
+	rc = alloc_xenballooned_pages(numpages, pages, 0);
+	if (rc != 0) {
+		pr_warn("%s Couldn't balloon alloc %ld pfns rc:%d\n", __func__,
+			numpages, rc);
+		return rc;
+	}
+	for (i = 0; i < numpages; i++)
+		pfns[i] = page_to_pfn(pages[i]);
+
+	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
+	if (rc != 0)
+		free_xenballooned_pages(numpages, pages);
+
+	return rc;
+}
+
 static int gnttab_setup(void)
 {
+	int rc;
 	unsigned int max_nr_gframes;
-	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
 
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
-	/* PVH note: xen will free existing kmalloc'd mfn in
-	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
-	 * kmalloc(). */
 	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap) &&
 	    !gnttab_shared.addr) {
-		gnttab_shared.addr =
-			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
-		if (!gnttab_shared.addr) {
-			pr_warn("%s", kmsg);
-			return -ENOMEM;
-		}
+
+		rc = xlated_setup_gnttab_pages((unsigned long)max_nr_gframes,
+					       &gnttab_shared.addr);
+		if (rc != 0)
+			return rc;
 	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);
-- 
1.7.7.6

[PATCH V10 13/14] x86/xen: remove depends on CONFIG_EXPERIMENTAL

[Konrad Rzeszutek Wilk]

From: Kees Cook <keescook@chromium.org>

The CONFIG_EXPERIMENTAL config item has not carried much meaning for a
while now and is almost always enabled by default. As agreed during the
Linux kernel summit, remove it from any "depends on" lines in Kconfigs.

Signed-off-by: Kees Cook <keescook@chromium.org>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Mukesh Rathor <mukesh.rathor@oracle.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/Kconfig |    8 +++-----
 1 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
index 4d890c3..47316b9 100644
--- a/arch/x86/xen/Kconfig
+++ b/arch/x86/xen/Kconfig
@@ -52,12 +52,10 @@ config XEN_DEBUG_FS
 	  Enabling this option may incur a significant performance overhead.
 
 config XEN_X86_PVH
-	bool "Support for running as a PVH guest (EXPERIMENTAL)"
-	depends on X86_64 && XEN && EXPERIMENTAL
+	bool "Support for running as a PVH guest"
+	depends on X86_64 && XEN
 	default n
 	help
 	   This option enables support for running as a PVH guest (PV guest
 	   using hardware extensions) under a suitably capable hypervisor.
-	   This option is EXPERIMENTAL because the hypervisor interfaces
-	   which it uses is not yet considered stable therefore backwards and
-	   forwards compatibility is not yet guaranteed.  If unsure, say N.
+	   If unsure, say N.
-- 
1.7.7.6

[PATCH V10 14/14] xen/pvh: vcpu info placement, load CS selector, and remove debug printk.

[Konrad Rzeszutek Wilk]

From: Mukesh Rathor <mukesh.rathor@oracle.com>

This patch addresses 3 things:
   - Resolve vcpu info placement fixme.
   - Load CS selector for PVH after switching to new gdt.
   - Remove printk in case of failure to map pnfs in p2m. This because qemu
     has lot of expected failures when mapping HVM pages.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
[v1: Added #ifdef for 64-bit]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/enlighten.c |   23 +++++++++++++++++++----
 arch/x86/xen/mmu.c       |    3 ---
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 0bfbe92..329dad9 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1132,14 +1132,12 @@ void xen_setup_shared_info(void)
 		HYPERVISOR_shared_info =
 			(struct shared_info *)__va(xen_start_info->shared_info);
 
-	/* PVH TBD/FIXME: vcpu info placement in phase 2 */
-	if (xen_pvh_domain())
-		return;
-
 #ifndef CONFIG_SMP
 	/* In UP this is as good a place as any to set up shared info */
 	xen_setup_vcpu_info_placement();
 #endif
+	if (xen_pvh_domain())
+		return;
 
 	xen_setup_mfn_list_list();
 }
@@ -1152,6 +1150,10 @@ void xen_setup_vcpu_info_placement(void)
 	for_each_possible_cpu(cpu)
 		xen_vcpu_setup(cpu);
 
+	/* PVH always uses native IRQ ops */
+	if (xen_pvh_domain())
+		return;
+
 	/* xen_vcpu_setup managed to place the vcpu_info within the
 	   percpu area for all cpus, so make use of it */
 	if (have_vcpu_info_placement) {
@@ -1423,7 +1425,20 @@ static void __init xen_setup_stackprotector(void)
 {
 	/* PVH TBD/FIXME: investigate setup_stack_canary_segment */
 	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+		unsigned long dummy;
+
 		switch_to_new_gdt(0);
+#ifdef CONFIG_X86_64
+		asm volatile ("pushq %0\n"
+			      "leaq 1f(%%rip),%0\n"
+			      "pushq %0\n"
+			      "lretq\n"
+			      "1:\n"
+			      : "=&r" (dummy) : "0" (__KERNEL_CS));
+#else
+		/* PVH: TODO Implement. */
+		BUG();
+#endif
 		return;
 	}
 	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 108fb4a..5499689 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -2534,9 +2534,6 @@ static int pvh_add_to_xen_p2m(unsigned long lpfn, unsigned long fgmfn,
 	set_xen_guest_handle(xatp.errs, &err);
 
 	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
-	if (rc || err)
-		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d:%d\n",
-			lpfn, fgmfn, rc, err);
 	return rc;
 }
 
-- 
1.7.7.6

Re: [PATCH V10 01/14] xen/pvh: Support ParaVirtualized Hardware extensions.

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> PVH allows PV linux guest to utilize hardware extended capabilities, such
> as running MMU updates in a HVM container.
> 
> This patch allows it to be configured and enabled. Also, basic header file
> changes to add new subcalls to physmap hypercall.
> 
> Lastly, mfn_to_local_pfn must return mfn for paging mode translate
> (since we let the hypervisor - or CPU - do them for us).
[...]
> --- a/arch/x86/include/asm/xen/page.h
> +++ b/arch/x86/include/asm/xen/page.h
> @@ -168,6 +168,9 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine)
>  static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
>  {
>  	unsigned long pfn = mfn_to_pfn(mfn);
> +
> +	if (xen_feature(XENFEAT_auto_translated_physmap))
> +		return mfn;

This does the mfn_to_pfn() call first and then ignores the result.

>  	if (get_phys_to_machine(pfn) != mfn)
>  		return -1; /* force !pfn_valid() */
>  	return pfn;
> diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
> index 1a3c765..4d890c3 100644
> --- a/arch/x86/xen/Kconfig
> +++ b/arch/x86/xen/Kconfig
> @@ -51,3 +51,13 @@ config XEN_DEBUG_FS
>  	  Enable statistics output and various tuning options in debugfs.
>  	  Enabling this option may incur a significant performance overhead.
>  
> +config XEN_X86_PVH
> +	bool "Support for running as a PVH guest (EXPERIMENTAL)"
> +	depends on X86_64 && XEN && EXPERIMENTAL

Drop EXPERIMENTAL here (not in the follow up patch).

> +	default n

default y or don't provide this as a user selectable choice.

> diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
> index 7faed58..1a6bca1 100644
> --- a/arch/x86/xen/xen-head.S
> +++ b/arch/x86/xen/xen-head.S
> @@ -13,6 +13,15 @@
>  #include <xen/interface/elfnote.h>
>  #include <asm/xen/interface.h>
>  
> +#ifdef CONFIG_XEN_X86_PVH
> +#define FEATURES_PVH "|writable_descriptor_tables" \
> +		     "|auto_translated_physmap" \
> +		     "|supervisor_mode_kernel" \
> +		     "|hvm_callback_vector"
> +#else
> +#define FEATURES_PVH /* Not supported */
> +#endif
> +
>  	__INIT
>  ENTRY(startup_xen)
>  	cld
> @@ -95,7 +104,7 @@ NEXT_HYPERCALL(arch_6)
>  #endif
>  	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          _ASM_PTR startup_xen)
>  	ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, _ASM_PTR hypercall_page)
> -	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb")
> +	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb"FEATURES_PVH)
>  	ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE,       .asciz "yes")
>  	ELFNOTE(Xen, XEN_ELFNOTE_LOADER,         .asciz "generic")
>  	ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID,

Not clear how this can be part of the first patch.  A guest with only
this patch cannot actually work in PVH mode, right?

> diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
> index 2ecfe4f..a122164 100644
> --- a/include/xen/interface/memory.h
> +++ b/include/xen/interface/memory.h
> @@ -175,7 +175,10 @@ struct xen_add_to_physmap {
>      uint16_t    size;
>  
>      /* Source mapping space. */
> -    unsigned int space;
> +    uint16_t space;
> +    domid_t foreign_domid;         /* IFF XENMAPSPACE_gmfn_foreign */
> +
> +#define XENMAPIDX_grant_table_status 0x80000000

What's this change?  It doesn't match the hypervisor.

> --- a/include/xen/interface/physdev.h
> +++ b/include/xen/interface/physdev.h
> @@ -291,6 +291,16 @@ struct physdev_dbgp_op {
>      } u;
>  };
>  
> +#define PHYSDEVOP_map_iomem        30
> +struct physdev_map_iomem {
> +    /* IN */
> +    uint64_t first_gfn;
> +    uint64_t first_mfn;
> +    uint32_t nr_mfns;
> +    uint32_t add_mapping; /* 1 == add mapping;  0 == unmap */
> +
> +};
> +

Not provided by the hypervisor.

David

Re: [PATCH V10 02/14] xen/pvh: Extend vcpu_guest_context, p2m, event, and XenBus.

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> Make gdt_frames[]/gdt_ents into a union with {gdtaddr, gdtsz},
> as PVH only needs to send down gdtaddr and gdtsz in the
> vcpu_guest_context structure..
> 
> For interrupts, PVH uses native_irq_ops so we can skip most of the
> PV ones. In the future we can support the pirq_eoi_map..
> Also VCPU hotplug is currently not available for PVH.
> 
> For events (and IRQs) we follow what PVHVM does - so use callback
> vector.  Lastly, for XenBus we use the same logic that is used in
> the PVHVM case.
[...]
> --- a/arch/x86/include/asm/xen/interface.h
> +++ b/arch/x86/include/asm/xen/interface.h
> @@ -145,7 +145,16 @@ struct vcpu_guest_context {
>      struct cpu_user_regs user_regs;         /* User-level CPU registers     */
>      struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
>      unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
> -    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
> +    union {
> +	struct {
> +		/* PV: GDT (machine frames, # ents).*/
> +		unsigned long gdt_frames[16], gdt_ents;
> +	} pv;
> +	struct {
> +		/* PVH: GDTR addr and size */
> +		unsigned long gdtaddr, gdtsz;
> +	} pvh;
> +    } u;

Doesn't match the hypervisor.

> --- a/arch/x86/xen/p2m.c
> +++ b/arch/x86/xen/p2m.c
> @@ -800,8 +800,10 @@ bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
>  	unsigned topidx, mididx, idx;
>  
>  	/* don't track P2M changes in autotranslate guests */
> -	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
> +	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
> +		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
>  		return true;
> +	}

Isn't this undoing a recent change that removed this BUG_ON()?

David

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> For balloon changes we skip setting of local P2M as it's updated
> in Xen. For grant, the shared grant frame is the PFN and not MFN,
> hence its mapped via the same code path as HVM.

It's difficult to see how these are related.  Please split.

I also think Stefano recently submitted an equivalent fix for the
balloon driver.

> --- a/drivers/xen/grant-table.c
> +++ b/drivers/xen/grant-table.c
> @@ -1056,14 +1056,20 @@ static void gnttab_unmap_frames_v2(void)
>  static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
>  {
>  	struct gnttab_setup_table setup;
> +	unsigned long start_gpfn;
>  	xen_pfn_t *frames;
>  	unsigned int nr_gframes = end_idx + 1;
>  	int rc;
>  
> -	if (xen_hvm_domain()) {
> +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap)) {
>  		struct xen_add_to_physmap xatp;
>  		unsigned int i = end_idx;
>  		rc = 0;
> +
> +		if (xen_hvm_domain())
> +			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
> +		else
> +			start_gpfn = virt_to_pfn(gnttab_shared.addr);

Not really clear why PVH diverges from HVM here.

> @@ -1135,7 +1141,7 @@ static void gnttab_request_version(void)
>  	int rc;
>  	struct gnttab_set_version gsv;
>  
> -	if (xen_hvm_domain())
> +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))
>  		gsv.version = 1;
>  	else
>  		gsv.version = 2;

Why is PVH limited to version 1?

David

Re: [PATCH V10 08/14] x86/xen: Use __pa_symbol instead of __pa on C visible symbols

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Alexander Duyck <alexander.h.duyck@intel.com>
> 
> This change updates a few of the functions to use __pa_symbol when
> translating C visible symbols instead of __pa.  By using __pa_symbol we are
> able to drop a few extra lines of code as don't have to test to see if the
> virtual pointer is a part of the kernel text or just standard virtual memory.

Why is this part of the PVH series?

David

Re: [PATCH V10 14/14] xen/pvh: vcpu info placement, load CS selector, and remove debug printk.

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> This patch addresses 3 things:
>    - Resolve vcpu info placement fixme.
>    - Load CS selector for PVH after switching to new gdt.
>    - Remove printk in case of failure to map pnfs in p2m. This because qemu
>      has lot of expected failures when mapping HVM pages.
[...]
> @@ -1423,7 +1425,20 @@ static void __init xen_setup_stackprotector(void)
>  {
>  	/* PVH TBD/FIXME: investigate setup_stack_canary_segment */
>  	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> +		unsigned long dummy;
> +
>  		switch_to_new_gdt(0);
> +#ifdef CONFIG_X86_64
> +		asm volatile ("pushq %0\n"
> +			      "leaq 1f(%%rip),%0\n"
> +			      "pushq %0\n"
> +			      "lretq\n"
> +			      "1:\n"
> +			      : "=&r" (dummy) : "0" (__KERNEL_CS));

Why does CS need to be reloaded here?

David

Re: [PATCH V10 09/14] xen: implement updated XENMEM_add_to_physmap_range ABI

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> From: Ian Campbell <Ian.Campbell@citrix.com>
> 
> Allows for more fine grained error reporting. Only used by PVH and
> ARM both of which are marked EXPERIMENTAL precisely because the ABI
> is not yet stable

This should be folded into patch 7.

David

Re: [PATCH v10] Repost of PVH patches for Linux.

[David Vrabel]

On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> Hey,
> 
> Mukesh had posted them quite a while ago - they got reviewed, acked, etc,
> put in #linux-next, got patches to fix some things from folks.
> 
> Then everybody focused on the Xen patches .. and now that is mostly
> baked this is what is needed on the Linux side to make it work
> with PVH. I had them based on v3.6, then merged on v3.11, then..
> 
> Decided to rebase all of them on v3.13-rc3 to make it easier.

In general the series is a bit of mess, early patches doing things one
way (using hypercalls that no longer exist for example) and later
patches fixing things up.  I think this should be sorted out as it makes
it too difficult to review carefully (I've only given the series a
quick, initial review).

> Couple of things:
> 
> a) the patch:
>  [PATCH V10 14/14] xen/pvh: vcpu info placement, load CS selector,
> 
> could be split in three - and I think Mukesh did have a version like that.
> I can do it myself and will - it should be obvious of how it will be done.
> 
> b). Regression testing  - I ran this with a Linux kernel dom0 (32 and 64)
>     on the latest Xen hypervisor without PVH enabled. It booted all the
>     guests I had:
>     RHEL5, SLES11, SLES12, F15, F16, F17, F18, F19, OEL5, OEL6, NetBSD PV,
>     FreeBSD HVM; and also 32 and 64 PV and PVHVM combination of the same
>     kernel (and save/restore on those). Nothing broke.

That's good to know.

David

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Stefano Stabellini]

On Thu, 12 Dec 2013, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> For balloon changes we skip setting of local P2M as it's updated
> in Xen. For grant, the shared grant frame is the PFN and not MFN,
> hence its mapped via the same code path as HVM.
> 
> Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> [v1: Rebase on v3.7-rc3]
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> CC: <stefano.stabellini@eu.citrix.com>
> Conflicts:
> 	drivers/xen/balloon.c
> [v2: Rebase on v3.13-rc3 has lot of ARM changes.]
> ---
>  drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
>  drivers/xen/gntdev.c      |    3 ++-
>  drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
>  3 files changed, 44 insertions(+), 21 deletions(-)
> 
> diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> index 55ea73f..0e9875a 100644
> --- a/drivers/xen/balloon.c
> +++ b/drivers/xen/balloon.c
> @@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
>  
>  #ifdef CONFIG_XEN_HAVE_PVMMU
>  		/* Link back into the page tables if not highmem. */
> -		if (xen_pv_domain() && !PageHighMem(page)) {
> +		if (xen_pv_domain() && !PageHighMem(page) &&
> +		    !xen_feature(XENFEAT_auto_translated_physmap)) {
> +
>  			int ret;
>  			ret = HYPERVISOR_update_va_mapping(
>  				(unsigned long)__va(pfn << PAGE_SHIFT),
> @@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
>  
>  		scrub_page(page);
>  
> -		/*
> -		 * Ballooned out frames are effectively replaced with
> -		 * a scratch frame.  Ensure direct mappings and the
> -		 * p2m are consistent.
> -		 */
> -		scratch_page = get_balloon_scratch_page();
> +		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> +			unsigned long p;
> +
> +			/*
> +			 * Ballooned out frames are effectively replaced with
> +			 * a scratch frame.  Ensure direct mappings and the
> +			 * p2m are consistent.
> +			 */
> +			scratch_page = get_balloon_scratch_page();
>  #ifdef CONFIG_XEN_HAVE_PVMMU
> -		if (xen_pv_domain() && !PageHighMem(page)) {
> -			ret = HYPERVISOR_update_va_mapping(
> -				(unsigned long)__va(pfn << PAGE_SHIFT),
> -				pfn_pte(page_to_pfn(scratch_page),
> +			if (xen_pv_domain() && !PageHighMem(page)) {
> +				ret = HYPERVISOR_update_va_mapping(
> +					(unsigned long)__va(pfn << PAGE_SHIFT),
> +					pfn_pte(page_to_pfn(scratch_page),
>  					PAGE_KERNEL_RO), 0);
> -			BUG_ON(ret);
> -		}
> +				BUG_ON(ret);
> +			}
>  #endif
> -		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> -			unsigned long p;
>  			p = page_to_pfn(scratch_page);
>  			__set_phys_to_machine(pfn, pfn_to_mfn(p));
> -		}
> -		put_balloon_scratch_page();
>  
> +			put_balloon_scratch_page();
> +		}
>  		balloon_append(pfn_to_page(pfn));
>  	}

These changes to balloon.c are similar to my patch but:

- they don't prevent set_phys_to_machine from being called by
increase_reservation for auto_translated guests

- they don't prevent wasteful balloon_scratch_page from being allocated
for auto_translated guests

I think that it would be best to take my patch instead.



> diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
> index e41c79c..c1ae3ce 100644
> --- a/drivers/xen/gntdev.c
> +++ b/drivers/xen/gntdev.c
> @@ -846,7 +846,8 @@ static int __init gntdev_init(void)
>  	if (!xen_domain())
>  		return -ENODEV;
>  
> -	use_ptemod = xen_pv_domain();
> +	use_ptemod = xen_pv_domain() &&
> +		     !xen_feature(XENFEAT_auto_translated_physmap);
>  
>  	err = misc_register(&gntdev_miscdev);
>  	if (err != 0) {

you might actually just want to have:
    
use_ptemod = !xen_feature(XENFEAT_auto_translated_physmap);




> diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
> index 0283871..1fcf75b 100644
> --- a/drivers/xen/grant-table.c
> +++ b/drivers/xen/grant-table.c
> @@ -1056,14 +1056,20 @@ static void gnttab_unmap_frames_v2(void)
>  static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
>  {
>  	struct gnttab_setup_table setup;
> +	unsigned long start_gpfn;
>  	xen_pfn_t *frames;
>  	unsigned int nr_gframes = end_idx + 1;
>  	int rc;
>  
> -	if (xen_hvm_domain()) {
> +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap)) {
>  		struct xen_add_to_physmap xatp;
>  		unsigned int i = end_idx;
>  		rc = 0;
> +
> +		if (xen_hvm_domain())
> +			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
> +		else
> +			start_gpfn = virt_to_pfn(gnttab_shared.addr);
>  		/*
>  		 * Loop backwards, so that the first hypercall has the largest
>  		 * index, ensuring that the table will grow only once.
> @@ -1072,7 +1078,7 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
>  			xatp.domid = DOMID_SELF;
>  			xatp.idx = i;
>  			xatp.space = XENMAPSPACE_grant_table;
> -			xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
> +			xatp.gpfn = start_gpfn + i;
>  			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
>  			if (rc != 0) {
>  				pr_warn("grant table add_to_physmap failed, err=%d\n",
> @@ -1135,7 +1141,7 @@ static void gnttab_request_version(void)
>  	int rc;
>  	struct gnttab_set_version gsv;
>  
> -	if (xen_hvm_domain())
> +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))
>  		gsv.version = 1;
>  	else
>  		gsv.version = 2;
> @@ -1164,11 +1170,24 @@ static void gnttab_request_version(void)
>  static int gnttab_setup(void)
>  {
>  	unsigned int max_nr_gframes;
> +	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
>  
>  	max_nr_gframes = gnttab_max_grant_frames();
>  	if (max_nr_gframes < nr_grant_frames)
>  		return -ENOSYS;
>  
> +	/* PVH note: xen will free existing kmalloc'd mfn in
> +	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
> +	 * kmalloc(). */
> +	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap) &&
> +	    !gnttab_shared.addr) {
> +		gnttab_shared.addr =
> +			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
> +		if (!gnttab_shared.addr) {
> +			pr_warn("%s", kmsg);
> +			return -ENOMEM;
> +		}
> +	}
>  	if (xen_pv_domain())
>  		return gnttab_map(0, nr_grant_frames - 1);

Can't you just use xen_hvm_resume_frames instead? Just rename it
xen_auto_translated_frames and allocate memory to it from
arch/x86/xen/enlighten.c.
Then you can completely unify the hvm and pvh code paths in this file,
as a result all the

if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))

tests, could be reduced to

if (xen_feature(XENFEAT_auto_translated_physmap))

Re: [PATCH V10 08/14] x86/xen: Use __pa_symbol instead of __pa on C visible symbols

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 11:04:29AM +0000, David Vrabel wrote:
> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > From: Alexander Duyck <alexander.h.duyck@intel.com>
> > 
> > This change updates a few of the functions to use __pa_symbol when
> > translating C visible symbols instead of __pa.  By using __pa_symbol we are
> > able to drop a few extra lines of code as don't have to test to see if the
> > virtual pointer is a part of the kernel text or just standard virtual memory.
> 
> Why is this part of the PVH series?

B/c this patch was sent when the #linux-next had the PVH patches and it
was based on that.

> 
> David

Re: [PATCH V10 02/14] xen/pvh: Extend vcpu_guest_context, p2m, event, and XenBus.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 10:56:24AM +0000, David Vrabel wrote:
> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > 
> > Make gdt_frames[]/gdt_ents into a union with {gdtaddr, gdtsz},
> > as PVH only needs to send down gdtaddr and gdtsz in the
> > vcpu_guest_context structure..
> > 
> > For interrupts, PVH uses native_irq_ops so we can skip most of the
> > PV ones. In the future we can support the pirq_eoi_map..
> > Also VCPU hotplug is currently not available for PVH.
> > 
> > For events (and IRQs) we follow what PVHVM does - so use callback
> > vector.  Lastly, for XenBus we use the same logic that is used in
> > the PVHVM case.
> [...]
> > --- a/arch/x86/include/asm/xen/interface.h
> > +++ b/arch/x86/include/asm/xen/interface.h
> > @@ -145,7 +145,16 @@ struct vcpu_guest_context {
> >      struct cpu_user_regs user_regs;         /* User-level CPU registers     */
> >      struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
> >      unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
> > -    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
> > +    union {
> > +	struct {
> > +		/* PV: GDT (machine frames, # ents).*/
> > +		unsigned long gdt_frames[16], gdt_ents;
> > +	} pv;
> > +	struct {
> > +		/* PVH: GDTR addr and size */
> > +		unsigned long gdtaddr, gdtsz;
> > +	} pvh;
> > +    } u;
> 
> Doesn't match the hypervisor.

Right. Mukesh is going to post follow on patches to fix that.

> 
> > --- a/arch/x86/xen/p2m.c
> > +++ b/arch/x86/xen/p2m.c
> > @@ -800,8 +800,10 @@ bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
> >  	unsigned topidx, mididx, idx;
> >  
> >  	/* don't track P2M changes in autotranslate guests */
> > -	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
> > +	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
> > +		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
> >  		return true;
> > +	}
> 
> Isn't this undoing a recent change that removed this BUG_ON()?

This one:
commit 2f558d40911c1b8f929b8a382833ae1da5df3293
Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Date:   Wed Oct 9 20:39:01 2013 +0000

    xen/x86: allow __set_phys_to_machine for autotranslate guests
    
    Allow __set_phys_to_machine to be called for autotranslate guests.
    It can be used to keep track of phys_to_machine changes, however we
    don't do anything with the information at the moment.
    
    Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>

I am not actually sure why the BUG_ON was removed in Stefano's patch.
Stefano?

> 
> David

Re: [PATCH V10 09/14] xen: implement updated XENMEM_add_to_physmap_range ABI

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 11:35:28AM +0000, David Vrabel wrote:
> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > From: Ian Campbell <Ian.Campbell@citrix.com>
> > 
> > Allows for more fine grained error reporting. Only used by PVH and
> > ARM both of which are marked EXPERIMENTAL precisely because the ABI
> > is not yet stable
> 
> This should be folded into patch 7.

In "07/14] xen/x86: remove duplicated include from enlighten.c" ?
> 
> David

Re: [PATCH V10 09/14] xen: implement updated XENMEM_add_to_physmap_range ABI

[David Vrabel]

On 13/12/13 14:27, Konrad Rzeszutek Wilk wrote:
> On Fri, Dec 13, 2013 at 11:35:28AM +0000, David Vrabel wrote:
>> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
>>> From: Ian Campbell <Ian.Campbell@citrix.com>
>>>
>>> Allows for more fine grained error reporting. Only used by PVH and
>>> ARM both of which are marked EXPERIMENTAL precisely because the ABI
>>> is not yet stable
>>
>> This should be folded into patch 7.
> 
> In "07/14] xen/x86: remove duplicated include from enlighten.c" ?

Sorry. I meant:

[06/14] xen: x86 pvh: use XENMEM_add_to_physmap_range for foreign gmfn
mappings

I stopped calling out places where patches should be folded/refactored
at this point.

David

Re: [PATCH v10] Repost of PVH patches for Linux.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 11:37:26AM +0000, David Vrabel wrote:
> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > Hey,
> > 
> > Mukesh had posted them quite a while ago - they got reviewed, acked, etc,
> > put in #linux-next, got patches to fix some things from folks.
> > 
> > Then everybody focused on the Xen patches .. and now that is mostly
> > baked this is what is needed on the Linux side to make it work
> > with PVH. I had them based on v3.6, then merged on v3.11, then..
> > 
> > Decided to rebase all of them on v3.13-rc3 to make it easier.
> 
> In general the series is a bit of mess, early patches doing things one
> way (using hypercalls that no longer exist for example) and later
> patches fixing things up.  I think this should be sorted out as it makes
> it too difficult to review carefully (I've only given the series a
> quick, initial review).

I need from Mukesh the follow on patches that "fix" this
so I can rebase, squash, etc them them up a bit to make it easier.


> 
> > Couple of things:
> > 
> > a) the patch:
> >  [PATCH V10 14/14] xen/pvh: vcpu info placement, load CS selector,
> > 
> > could be split in three - and I think Mukesh did have a version like that.
> > I can do it myself and will - it should be obvious of how it will be done.
> > 
> > b). Regression testing  - I ran this with a Linux kernel dom0 (32 and 64)
> >     on the latest Xen hypervisor without PVH enabled. It booted all the
> >     guests I had:
> >     RHEL5, SLES11, SLES12, F15, F16, F17, F18, F19, OEL5, OEL6, NetBSD PV,
> >     FreeBSD HVM; and also 32 and 64 PV and PVHVM combination of the same
> >     kernel (and save/restore on those). Nothing broke.
> 
> That's good to know.
> 
> David
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel

Re: [PATCH V10 01/14] xen/pvh: Support ParaVirtualized Hardware extensions.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 10:44:31AM +0000, David Vrabel wrote:
> On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > 
> > PVH allows PV linux guest to utilize hardware extended capabilities, such
> > as running MMU updates in a HVM container.
> > 
> > This patch allows it to be configured and enabled. Also, basic header file
> > changes to add new subcalls to physmap hypercall.
> > 
> > Lastly, mfn_to_local_pfn must return mfn for paging mode translate
> > (since we let the hypervisor - or CPU - do them for us).
> [...]
> > --- a/arch/x86/include/asm/xen/page.h
> > +++ b/arch/x86/include/asm/xen/page.h
> > @@ -168,6 +168,9 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine)
> >  static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
> >  {
> >  	unsigned long pfn = mfn_to_pfn(mfn);
> > +
> > +	if (xen_feature(XENFEAT_auto_translated_physmap))
> > +		return mfn;
> 
> This does the mfn_to_pfn() call first and then ignores the result.
> 
> >  	if (get_phys_to_machine(pfn) != mfn)
> >  		return -1; /* force !pfn_valid() */
> >  	return pfn;
> > diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
> > index 1a3c765..4d890c3 100644
> > --- a/arch/x86/xen/Kconfig
> > +++ b/arch/x86/xen/Kconfig
> > @@ -51,3 +51,13 @@ config XEN_DEBUG_FS
> >  	  Enable statistics output and various tuning options in debugfs.
> >  	  Enabling this option may incur a significant performance overhead.
> >  
> > +config XEN_X86_PVH
> > +	bool "Support for running as a PVH guest (EXPERIMENTAL)"
> > +	depends on X86_64 && XEN && EXPERIMENTAL
> 
> Drop EXPERIMENTAL here (not in the follow up patch).
> 
> > +	default n
> 
> default y or don't provide this as a user selectable choice.
> 
> > diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
> > index 7faed58..1a6bca1 100644
> > --- a/arch/x86/xen/xen-head.S
> > +++ b/arch/x86/xen/xen-head.S
> > @@ -13,6 +13,15 @@
> >  #include <xen/interface/elfnote.h>
> >  #include <asm/xen/interface.h>
> >  
> > +#ifdef CONFIG_XEN_X86_PVH
> > +#define FEATURES_PVH "|writable_descriptor_tables" \
> > +		     "|auto_translated_physmap" \
> > +		     "|supervisor_mode_kernel" \
> > +		     "|hvm_callback_vector"
> > +#else
> > +#define FEATURES_PVH /* Not supported */
> > +#endif
> > +
> >  	__INIT
> >  ENTRY(startup_xen)
> >  	cld
> > @@ -95,7 +104,7 @@ NEXT_HYPERCALL(arch_6)
> >  #endif
> >  	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          _ASM_PTR startup_xen)
> >  	ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, _ASM_PTR hypercall_page)
> > -	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb")
> > +	ELFNOTE(Xen, XEN_ELFNOTE_FEATURES,       .asciz "!writable_page_tables|pae_pgdir_above_4gb"FEATURES_PVH)
> >  	ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE,       .asciz "yes")
> >  	ELFNOTE(Xen, XEN_ELFNOTE_LOADER,         .asciz "generic")
> >  	ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID,
> 
> Not clear how this can be part of the first patch.  A guest with only
> this patch cannot actually work in PVH mode, right?

Good point. We should make this one of the last patches or split
this part out.

> 
> > diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
> > index 2ecfe4f..a122164 100644
> > --- a/include/xen/interface/memory.h
> > +++ b/include/xen/interface/memory.h
> > @@ -175,7 +175,10 @@ struct xen_add_to_physmap {
> >      uint16_t    size;
> >  
> >      /* Source mapping space. */
> > -    unsigned int space;
> > +    uint16_t space;
> > +    domid_t foreign_domid;         /* IFF XENMAPSPACE_gmfn_foreign */
> > +
> > +#define XENMAPIDX_grant_table_status 0x80000000
> 
> What's this change?  It doesn't match the hypervisor.
> 
> > --- a/include/xen/interface/physdev.h
> > +++ b/include/xen/interface/physdev.h
> > @@ -291,6 +291,16 @@ struct physdev_dbgp_op {
> >      } u;
> >  };
> >  
> > +#define PHYSDEVOP_map_iomem        30
> > +struct physdev_map_iomem {
> > +    /* IN */
> > +    uint64_t first_gfn;
> > +    uint64_t first_mfn;
> > +    uint32_t nr_mfns;
> > +    uint32_t add_mapping; /* 1 == add mapping;  0 == unmap */
> > +
> > +};
> > +
> 
> Not provided by the hypervisor.

Mukesh, could you send a followup patch to the entire patchset which
fixes the change so I can make sure to have the right hypervisor ABI please?


> 
> David

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 11:56:31AM +0000, Stefano Stabellini wrote:
> On Thu, 12 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > 
> > For balloon changes we skip setting of local P2M as it's updated
> > in Xen. For grant, the shared grant frame is the PFN and not MFN,
> > hence its mapped via the same code path as HVM.
> > 
> > Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> > [v1: Rebase on v3.7-rc3]
> > Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> > CC: <stefano.stabellini@eu.citrix.com>
> > Conflicts:
> > 	drivers/xen/balloon.c
> > [v2: Rebase on v3.13-rc3 has lot of ARM changes.]
> > ---
> >  drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
> >  drivers/xen/gntdev.c      |    3 ++-
> >  drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
> >  3 files changed, 44 insertions(+), 21 deletions(-)
> > 
> > diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> > index 55ea73f..0e9875a 100644
> > --- a/drivers/xen/balloon.c
> > +++ b/drivers/xen/balloon.c
> > @@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
> >  
> >  #ifdef CONFIG_XEN_HAVE_PVMMU
> >  		/* Link back into the page tables if not highmem. */
> > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > +		if (xen_pv_domain() && !PageHighMem(page) &&
> > +		    !xen_feature(XENFEAT_auto_translated_physmap)) {
> > +
> >  			int ret;
> >  			ret = HYPERVISOR_update_va_mapping(
> >  				(unsigned long)__va(pfn << PAGE_SHIFT),
> > @@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
> >  
> >  		scrub_page(page);
> >  
> > -		/*
> > -		 * Ballooned out frames are effectively replaced with
> > -		 * a scratch frame.  Ensure direct mappings and the
> > -		 * p2m are consistent.
> > -		 */
> > -		scratch_page = get_balloon_scratch_page();
> > +		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > +			unsigned long p;
> > +
> > +			/*
> > +			 * Ballooned out frames are effectively replaced with
> > +			 * a scratch frame.  Ensure direct mappings and the
> > +			 * p2m are consistent.
> > +			 */
> > +			scratch_page = get_balloon_scratch_page();
> >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > -			ret = HYPERVISOR_update_va_mapping(
> > -				(unsigned long)__va(pfn << PAGE_SHIFT),
> > -				pfn_pte(page_to_pfn(scratch_page),
> > +			if (xen_pv_domain() && !PageHighMem(page)) {
> > +				ret = HYPERVISOR_update_va_mapping(
> > +					(unsigned long)__va(pfn << PAGE_SHIFT),
> > +					pfn_pte(page_to_pfn(scratch_page),
> >  					PAGE_KERNEL_RO), 0);
> > -			BUG_ON(ret);
> > -		}
> > +				BUG_ON(ret);
> > +			}
> >  #endif
> > -		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > -			unsigned long p;
> >  			p = page_to_pfn(scratch_page);
> >  			__set_phys_to_machine(pfn, pfn_to_mfn(p));
> > -		}
> > -		put_balloon_scratch_page();
> >  
> > +			put_balloon_scratch_page();
> > +		}
> >  		balloon_append(pfn_to_page(pfn));
> >  	}
> 
> These changes to balloon.c are similar to my patch but:
> 
> - they don't prevent set_phys_to_machine from being called by
> increase_reservation for auto_translated guests

<nods>
> 
> - they don't prevent wasteful balloon_scratch_page from being allocated
> for auto_translated guests

<nods>
> 
> I think that it would be best to take my patch instead.

Lets focus the review on your patch then and have it part of this
series. 
> 
> 
> 
> > diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
> > index e41c79c..c1ae3ce 100644
> > --- a/drivers/xen/gntdev.c
> > +++ b/drivers/xen/gntdev.c
> > @@ -846,7 +846,8 @@ static int __init gntdev_init(void)
> >  	if (!xen_domain())
> >  		return -ENODEV;
> >  
> > -	use_ptemod = xen_pv_domain();
> > +	use_ptemod = xen_pv_domain() &&
> > +		     !xen_feature(XENFEAT_auto_translated_physmap);
> >  
> >  	err = misc_register(&gntdev_miscdev);
> >  	if (err != 0) {
> 
> you might actually just want to have:
>     
> use_ptemod = !xen_feature(XENFEAT_auto_translated_physmap);
> 
> 
> 
> 
> > diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
> > index 0283871..1fcf75b 100644
> > --- a/drivers/xen/grant-table.c
> > +++ b/drivers/xen/grant-table.c
> > @@ -1056,14 +1056,20 @@ static void gnttab_unmap_frames_v2(void)
> >  static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
> >  {
> >  	struct gnttab_setup_table setup;
> > +	unsigned long start_gpfn;
> >  	xen_pfn_t *frames;
> >  	unsigned int nr_gframes = end_idx + 1;
> >  	int rc;
> >  
> > -	if (xen_hvm_domain()) {
> > +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap)) {
> >  		struct xen_add_to_physmap xatp;
> >  		unsigned int i = end_idx;
> >  		rc = 0;
> > +
> > +		if (xen_hvm_domain())
> > +			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
> > +		else
> > +			start_gpfn = virt_to_pfn(gnttab_shared.addr);
> >  		/*
> >  		 * Loop backwards, so that the first hypercall has the largest
> >  		 * index, ensuring that the table will grow only once.
> > @@ -1072,7 +1078,7 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
> >  			xatp.domid = DOMID_SELF;
> >  			xatp.idx = i;
> >  			xatp.space = XENMAPSPACE_grant_table;
> > -			xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
> > +			xatp.gpfn = start_gpfn + i;
> >  			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
> >  			if (rc != 0) {
> >  				pr_warn("grant table add_to_physmap failed, err=%d\n",
> > @@ -1135,7 +1141,7 @@ static void gnttab_request_version(void)
> >  	int rc;
> >  	struct gnttab_set_version gsv;
> >  
> > -	if (xen_hvm_domain())
> > +	if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))
> >  		gsv.version = 1;
> >  	else
> >  		gsv.version = 2;
> > @@ -1164,11 +1170,24 @@ static void gnttab_request_version(void)
> >  static int gnttab_setup(void)
> >  {
> >  	unsigned int max_nr_gframes;
> > +	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
> >  
> >  	max_nr_gframes = gnttab_max_grant_frames();
> >  	if (max_nr_gframes < nr_grant_frames)
> >  		return -ENOSYS;
> >  
> > +	/* PVH note: xen will free existing kmalloc'd mfn in
> > +	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
> > +	 * kmalloc(). */
> > +	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap) &&
> > +	    !gnttab_shared.addr) {
> > +		gnttab_shared.addr =
> > +			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
> > +		if (!gnttab_shared.addr) {
> > +			pr_warn("%s", kmsg);
> > +			return -ENOMEM;
> > +		}
> > +	}
> >  	if (xen_pv_domain())
> >  		return gnttab_map(0, nr_grant_frames - 1);
> 
> Can't you just use xen_hvm_resume_frames instead? Just rename it
> xen_auto_translated_frames and allocate memory to it from
> arch/x86/xen/enlighten.c.
> Then you can completely unify the hvm and pvh code paths in this file,
> as a result all the
> 
> if (xen_hvm_domain() || xen_feature(XENFEAT_auto_translated_physmap))
> 
> tests, could be reduced to
> 
> if (xen_feature(XENFEAT_auto_translated_physmap))

Mukesh? That should work right?

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Stefano Stabellini]

On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> On Fri, Dec 13, 2013 at 11:56:31AM +0000, Stefano Stabellini wrote:
> > On Thu, 12 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > 
> > > For balloon changes we skip setting of local P2M as it's updated
> > > in Xen. For grant, the shared grant frame is the PFN and not MFN,
> > > hence its mapped via the same code path as HVM.
> > > 
> > > Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > [v1: Rebase on v3.7-rc3]
> > > Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> > > CC: <stefano.stabellini@eu.citrix.com>
> > > Conflicts:
> > > 	drivers/xen/balloon.c
> > > [v2: Rebase on v3.13-rc3 has lot of ARM changes.]
> > > ---
> > >  drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
> > >  drivers/xen/gntdev.c      |    3 ++-
> > >  drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
> > >  3 files changed, 44 insertions(+), 21 deletions(-)
> > > 
> > > diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> > > index 55ea73f..0e9875a 100644
> > > --- a/drivers/xen/balloon.c
> > > +++ b/drivers/xen/balloon.c
> > > @@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
> > >  
> > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > >  		/* Link back into the page tables if not highmem. */
> > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > +		if (xen_pv_domain() && !PageHighMem(page) &&
> > > +		    !xen_feature(XENFEAT_auto_translated_physmap)) {
> > > +
> > >  			int ret;
> > >  			ret = HYPERVISOR_update_va_mapping(
> > >  				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > @@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
> > >  
> > >  		scrub_page(page);
> > >  
> > > -		/*
> > > -		 * Ballooned out frames are effectively replaced with
> > > -		 * a scratch frame.  Ensure direct mappings and the
> > > -		 * p2m are consistent.
> > > -		 */
> > > -		scratch_page = get_balloon_scratch_page();
> > > +		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > +			unsigned long p;
> > > +
> > > +			/*
> > > +			 * Ballooned out frames are effectively replaced with
> > > +			 * a scratch frame.  Ensure direct mappings and the
> > > +			 * p2m are consistent.
> > > +			 */
> > > +			scratch_page = get_balloon_scratch_page();
> > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > -			ret = HYPERVISOR_update_va_mapping(
> > > -				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > -				pfn_pte(page_to_pfn(scratch_page),
> > > +			if (xen_pv_domain() && !PageHighMem(page)) {
> > > +				ret = HYPERVISOR_update_va_mapping(
> > > +					(unsigned long)__va(pfn << PAGE_SHIFT),
> > > +					pfn_pte(page_to_pfn(scratch_page),
> > >  					PAGE_KERNEL_RO), 0);
> > > -			BUG_ON(ret);
> > > -		}
> > > +				BUG_ON(ret);
> > > +			}
> > >  #endif
> > > -		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > -			unsigned long p;
> > >  			p = page_to_pfn(scratch_page);
> > >  			__set_phys_to_machine(pfn, pfn_to_mfn(p));
> > > -		}
> > > -		put_balloon_scratch_page();
> > >  
> > > +			put_balloon_scratch_page();
> > > +		}
> > >  		balloon_append(pfn_to_page(pfn));
> > >  	}
> > 
> > These changes to balloon.c are similar to my patch but:
> > 
> > - they don't prevent set_phys_to_machine from being called by
> > increase_reservation for auto_translated guests
> 
> <nods>
> > 
> > - they don't prevent wasteful balloon_scratch_page from being allocated
> > for auto_translated guests
> 
> <nods>
> > 
> > I think that it would be best to take my patch instead.
> 
> Lets focus the review on your patch then and have it part of this
> series. 

OK, but keep in mind that Xen on ARMv8 is currently broken without that
patch.

Re: [PATCH V10 02/14] xen/pvh: Extend vcpu_guest_context, p2m, event, and XenBus.

[Stefano Stabellini]

On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> On Fri, Dec 13, 2013 at 10:56:24AM +0000, David Vrabel wrote:
> > On 13/12/13 02:10, Konrad Rzeszutek Wilk wrote:
> > > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > 
> > > Make gdt_frames[]/gdt_ents into a union with {gdtaddr, gdtsz},
> > > as PVH only needs to send down gdtaddr and gdtsz in the
> > > vcpu_guest_context structure..
> > > 
> > > For interrupts, PVH uses native_irq_ops so we can skip most of the
> > > PV ones. In the future we can support the pirq_eoi_map..
> > > Also VCPU hotplug is currently not available for PVH.
> > > 
> > > For events (and IRQs) we follow what PVHVM does - so use callback
> > > vector.  Lastly, for XenBus we use the same logic that is used in
> > > the PVHVM case.
> > [...]
> > > --- a/arch/x86/include/asm/xen/interface.h
> > > +++ b/arch/x86/include/asm/xen/interface.h
> > > @@ -145,7 +145,16 @@ struct vcpu_guest_context {
> > >      struct cpu_user_regs user_regs;         /* User-level CPU registers     */
> > >      struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
> > >      unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
> > > -    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
> > > +    union {
> > > +	struct {
> > > +		/* PV: GDT (machine frames, # ents).*/
> > > +		unsigned long gdt_frames[16], gdt_ents;
> > > +	} pv;
> > > +	struct {
> > > +		/* PVH: GDTR addr and size */
> > > +		unsigned long gdtaddr, gdtsz;
> > > +	} pvh;
> > > +    } u;
> > 
> > Doesn't match the hypervisor.
> 
> Right. Mukesh is going to post follow on patches to fix that.
> 
> > 
> > > --- a/arch/x86/xen/p2m.c
> > > +++ b/arch/x86/xen/p2m.c
> > > @@ -800,8 +800,10 @@ bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
> > >  	unsigned topidx, mididx, idx;
> > >  
> > >  	/* don't track P2M changes in autotranslate guests */
> > > -	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
> > > +	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
> > > +		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
> > >  		return true;
> > > +	}
> > 
> > Isn't this undoing a recent change that removed this BUG_ON()?
> 
> This one:
> commit 2f558d40911c1b8f929b8a382833ae1da5df3293
> Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
> Date:   Wed Oct 9 20:39:01 2013 +0000
> 
>     xen/x86: allow __set_phys_to_machine for autotranslate guests
>     
>     Allow __set_phys_to_machine to be called for autotranslate guests.
>     It can be used to keep track of phys_to_machine changes, however we
>     don't do anything with the information at the moment.
>     
>     Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
> 
> I am not actually sure why the BUG_ON was removed in Stefano's patch.
> Stefano?

The following commit:

commit 3d24bbd7dddbea54358a9795abaf051b0f18973c
Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Date:   Fri Oct 25 10:41:44 2013 +0000

    grant-table: call set_phys_to_machine after mapping grant refs
    
introduces set_phys_to_machine calls for auto_translated guests
(even on x86) in gnttab_map_refs and gnttab_unmap_refs.
They are necessary to allow the kernel to keep track of foreign grant
mappings: without an IOMMU the kernel needs to take special care of
these pages because they can't be used directly in dma operations unless
translated by swiotlb-xen.

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 02:49:15PM +0000, Stefano Stabellini wrote:
> On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > On Fri, Dec 13, 2013 at 11:56:31AM +0000, Stefano Stabellini wrote:
> > > On Thu, 12 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > > > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > > 
> > > > For balloon changes we skip setting of local P2M as it's updated
> > > > in Xen. For grant, the shared grant frame is the PFN and not MFN,
> > > > hence its mapped via the same code path as HVM.
> > > > 
> > > > Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > > [v1: Rebase on v3.7-rc3]
> > > > Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> > > > CC: <stefano.stabellini@eu.citrix.com>
> > > > Conflicts:
> > > > 	drivers/xen/balloon.c
> > > > [v2: Rebase on v3.13-rc3 has lot of ARM changes.]
> > > > ---
> > > >  drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
> > > >  drivers/xen/gntdev.c      |    3 ++-
> > > >  drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
> > > >  3 files changed, 44 insertions(+), 21 deletions(-)
> > > > 
> > > > diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> > > > index 55ea73f..0e9875a 100644
> > > > --- a/drivers/xen/balloon.c
> > > > +++ b/drivers/xen/balloon.c
> > > > @@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
> > > >  
> > > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > > >  		/* Link back into the page tables if not highmem. */
> > > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > > +		if (xen_pv_domain() && !PageHighMem(page) &&
> > > > +		    !xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > +
> > > >  			int ret;
> > > >  			ret = HYPERVISOR_update_va_mapping(
> > > >  				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > @@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
> > > >  
> > > >  		scrub_page(page);
> > > >  
> > > > -		/*
> > > > -		 * Ballooned out frames are effectively replaced with
> > > > -		 * a scratch frame.  Ensure direct mappings and the
> > > > -		 * p2m are consistent.
> > > > -		 */
> > > > -		scratch_page = get_balloon_scratch_page();
> > > > +		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > +			unsigned long p;
> > > > +
> > > > +			/*
> > > > +			 * Ballooned out frames are effectively replaced with
> > > > +			 * a scratch frame.  Ensure direct mappings and the
> > > > +			 * p2m are consistent.
> > > > +			 */
> > > > +			scratch_page = get_balloon_scratch_page();
> > > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > > -			ret = HYPERVISOR_update_va_mapping(
> > > > -				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > -				pfn_pte(page_to_pfn(scratch_page),
> > > > +			if (xen_pv_domain() && !PageHighMem(page)) {
> > > > +				ret = HYPERVISOR_update_va_mapping(
> > > > +					(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > +					pfn_pte(page_to_pfn(scratch_page),
> > > >  					PAGE_KERNEL_RO), 0);
> > > > -			BUG_ON(ret);
> > > > -		}
> > > > +				BUG_ON(ret);
> > > > +			}
> > > >  #endif
> > > > -		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > -			unsigned long p;
> > > >  			p = page_to_pfn(scratch_page);
> > > >  			__set_phys_to_machine(pfn, pfn_to_mfn(p));
> > > > -		}
> > > > -		put_balloon_scratch_page();
> > > >  
> > > > +			put_balloon_scratch_page();
> > > > +		}
> > > >  		balloon_append(pfn_to_page(pfn));
> > > >  	}
> > > 
> > > These changes to balloon.c are similar to my patch but:
> > > 
> > > - they don't prevent set_phys_to_machine from being called by
> > > increase_reservation for auto_translated guests
> > 
> > <nods>
> > > 
> > > - they don't prevent wasteful balloon_scratch_page from being allocated
> > > for auto_translated guests
> > 
> > <nods>
> > > 
> > > I think that it would be best to take my patch instead.
> > 
> > Lets focus the review on your patch then and have it part of this
> > series. 
> 
> OK, but keep in mind that Xen on ARMv8 is currently broken without that
> patch.

Hm. We could have another patch to just disable BALLOON on ARMv8 :-)

The question you have to ask - if your patch causes regression are you
OK tracking/fixing it over the holidays :-) If yes, then yes - we can
put that in v3.13-rc3 to make it work properly on ARMv8.

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Stefano Stabellini]

On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> On Fri, Dec 13, 2013 at 02:49:15PM +0000, Stefano Stabellini wrote:
> > On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > > On Fri, Dec 13, 2013 at 11:56:31AM +0000, Stefano Stabellini wrote:
> > > > On Thu, 12 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > > > > From: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > > > 
> > > > > For balloon changes we skip setting of local P2M as it's updated
> > > > > in Xen. For grant, the shared grant frame is the PFN and not MFN,
> > > > > hence its mapped via the same code path as HVM.
> > > > > 
> > > > > Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> > > > > [v1: Rebase on v3.7-rc3]
> > > > > Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> > > > > CC: <stefano.stabellini@eu.citrix.com>
> > > > > Conflicts:
> > > > > 	drivers/xen/balloon.c
> > > > > [v2: Rebase on v3.13-rc3 has lot of ARM changes.]
> > > > > ---
> > > > >  drivers/xen/balloon.c     |   37 ++++++++++++++++++++-----------------
> > > > >  drivers/xen/gntdev.c      |    3 ++-
> > > > >  drivers/xen/grant-table.c |   25 ++++++++++++++++++++++---
> > > > >  3 files changed, 44 insertions(+), 21 deletions(-)
> > > > > 
> > > > > diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> > > > > index 55ea73f..0e9875a 100644
> > > > > --- a/drivers/xen/balloon.c
> > > > > +++ b/drivers/xen/balloon.c
> > > > > @@ -354,7 +354,9 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
> > > > >  
> > > > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > > > >  		/* Link back into the page tables if not highmem. */
> > > > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > > > +		if (xen_pv_domain() && !PageHighMem(page) &&
> > > > > +		    !xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > > +
> > > > >  			int ret;
> > > > >  			ret = HYPERVISOR_update_va_mapping(
> > > > >  				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > > @@ -411,28 +413,29 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
> > > > >  
> > > > >  		scrub_page(page);
> > > > >  
> > > > > -		/*
> > > > > -		 * Ballooned out frames are effectively replaced with
> > > > > -		 * a scratch frame.  Ensure direct mappings and the
> > > > > -		 * p2m are consistent.
> > > > > -		 */
> > > > > -		scratch_page = get_balloon_scratch_page();
> > > > > +		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > > +			unsigned long p;
> > > > > +
> > > > > +			/*
> > > > > +			 * Ballooned out frames are effectively replaced with
> > > > > +			 * a scratch frame.  Ensure direct mappings and the
> > > > > +			 * p2m are consistent.
> > > > > +			 */
> > > > > +			scratch_page = get_balloon_scratch_page();
> > > > >  #ifdef CONFIG_XEN_HAVE_PVMMU
> > > > > -		if (xen_pv_domain() && !PageHighMem(page)) {
> > > > > -			ret = HYPERVISOR_update_va_mapping(
> > > > > -				(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > > -				pfn_pte(page_to_pfn(scratch_page),
> > > > > +			if (xen_pv_domain() && !PageHighMem(page)) {
> > > > > +				ret = HYPERVISOR_update_va_mapping(
> > > > > +					(unsigned long)__va(pfn << PAGE_SHIFT),
> > > > > +					pfn_pte(page_to_pfn(scratch_page),
> > > > >  					PAGE_KERNEL_RO), 0);
> > > > > -			BUG_ON(ret);
> > > > > -		}
> > > > > +				BUG_ON(ret);
> > > > > +			}
> > > > >  #endif
> > > > > -		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
> > > > > -			unsigned long p;
> > > > >  			p = page_to_pfn(scratch_page);
> > > > >  			__set_phys_to_machine(pfn, pfn_to_mfn(p));
> > > > > -		}
> > > > > -		put_balloon_scratch_page();
> > > > >  
> > > > > +			put_balloon_scratch_page();
> > > > > +		}
> > > > >  		balloon_append(pfn_to_page(pfn));
> > > > >  	}
> > > > 
> > > > These changes to balloon.c are similar to my patch but:
> > > > 
> > > > - they don't prevent set_phys_to_machine from being called by
> > > > increase_reservation for auto_translated guests
> > > 
> > > <nods>
> > > > 
> > > > - they don't prevent wasteful balloon_scratch_page from being allocated
> > > > for auto_translated guests
> > > 
> > > <nods>
> > > > 
> > > > I think that it would be best to take my patch instead.
> > > 
> > > Lets focus the review on your patch then and have it part of this
> > > series. 
> > 
> > OK, but keep in mind that Xen on ARMv8 is currently broken without that
> > patch.
> 
> Hm. We could have another patch to just disable BALLOON on ARMv8 :-)
> 
> The question you have to ask - if your patch causes regression are you
> OK tracking/fixing it over the holidays :-) If yes, then yes - we can
> put that in v3.13-rc3 to make it work properly on ARMv8.

Damn, that was a good argument.
After thinking more about it, I think that I, or a member of my team,
can actually fix possible bugs during the holidays. Especially
considering that the patch doesn't alter the behavior of PV guests.

So in short, yes :)

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Konrad Rzeszutek Wilk]

> > > > > These changes to balloon.c are similar to my patch but:
> > > > > 
> > > > > - they don't prevent set_phys_to_machine from being called by
> > > > > increase_reservation for auto_translated guests
> > > > 
> > > > <nods>
> > > > > 
> > > > > - they don't prevent wasteful balloon_scratch_page from being allocated
> > > > > for auto_translated guests
> > > > 
> > > > <nods>
> > > > > 
> > > > > I think that it would be best to take my patch instead.
> > > > 
> > > > Lets focus the review on your patch then and have it part of this
> > > > series. 
> > > 
> > > OK, but keep in mind that Xen on ARMv8 is currently broken without that
> > > patch.
> > 
> > Hm. We could have another patch to just disable BALLOON on ARMv8 :-)
> > 
> > The question you have to ask - if your patch causes regression are you
> > OK tracking/fixing it over the holidays :-) If yes, then yes - we can
> > put that in v3.13-rc3 to make it work properly on ARMv8.
> 
> Damn, that was a good argument.
> After thinking more about it, I think that I, or a member of my team,
> can actually fix possible bugs during the holidays. Especially
> considering that the patch doesn't alter the behavior of PV guests.
> 
> So in short, yes :)

Great <evil laughther>! Could you repost it with the description changes?
Or if you OK with the changes I had proposed - just tell me "yes" and I will
alter it.

Re: [PATCH V10 05/14] xen/pvh: balloon and grant changes.

[Stefano Stabellini]

On Fri, 13 Dec 2013, Konrad Rzeszutek Wilk wrote:
> > > > > > These changes to balloon.c are similar to my patch but:
> > > > > > 
> > > > > > - they don't prevent set_phys_to_machine from being called by
> > > > > > increase_reservation for auto_translated guests
> > > > > 
> > > > > <nods>
> > > > > > 
> > > > > > - they don't prevent wasteful balloon_scratch_page from being allocated
> > > > > > for auto_translated guests
> > > > > 
> > > > > <nods>
> > > > > > 
> > > > > > I think that it would be best to take my patch instead.
> > > > > 
> > > > > Lets focus the review on your patch then and have it part of this
> > > > > series. 
> > > > 
> > > > OK, but keep in mind that Xen on ARMv8 is currently broken without that
> > > > patch.
> > > 
> > > Hm. We could have another patch to just disable BALLOON on ARMv8 :-)
> > > 
> > > The question you have to ask - if your patch causes regression are you
> > > OK tracking/fixing it over the holidays :-) If yes, then yes - we can
> > > put that in v3.13-rc3 to make it work properly on ARMv8.
> > 
> > Damn, that was a good argument.
> > After thinking more about it, I think that I, or a member of my team,
> > can actually fix possible bugs during the holidays. Especially
> > considering that the patch doesn't alter the behavior of PV guests.
> > 
> > So in short, yes :)
> 
> Great <evil laughther>! Could you repost it with the description changes?
> Or if you OK with the changes I had proposed - just tell me "yes" and I will
> alter it.

I am OK with all the changes

Re: [PATCH V10 03/14] xen/pvh: Implement MMU changes for PVH.

[Boris Ostrovsky]

On 12/12/2013 09:10 PM, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
>
> .. which are surprinsingly small compared to the amount for PV.
> First the set/clear mmio pte function make a hypercall to update the
> P2M in Xen with 1:1 mapping. Since PVH uses mostly native mmu ops, we
> leave the generic (native_*) for the majority and just overwrite the
> baremetal with the ones we need.
>
> Two local functions are introduced to add to Xen physmap for Xen remap
> interface. Xen unmap interface is introduced so that the privcmd PTe entries
> can be cleared in Xen P2M table.
>
> ijc -- rebase on top of ARM privcmd changes rather than vice versa,
> xen_remap_domain_mfn_range (and unmap) split into a separate patch
> earlier in the series.
>
> Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> [v1: Rebase on x86, mm, Xen: Remove mapping_pagetable_reserve()]
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
>
> Conflicts:
> 	arch/x86/xen/mmu.c
> [due to xen/mmu: On early bootup, flush the TLB when changing RO->RW
> bits Xen provided pagetables.]
> ---
>   arch/x86/xen/mmu.c |  147 ++++++++++++++++++++++++++++++++++++++++++++++++++--
>   arch/x86/xen/mmu.h |    2 +
>   2 files changed, 144 insertions(+), 5 deletions(-)
>
> diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
> index ce563be..c7de065 100644
> --- a/arch/x86/xen/mmu.c
> +++ b/arch/x86/xen/mmu.c
> @@ -74,6 +74,7 @@
>   #include <xen/interface/version.h>
>   #include <xen/interface/memory.h>
>   #include <xen/hvc-console.h>
> +#include <xen/balloon.h>
>   
>   #include "multicalls.h"
>   #include "mmu.h"
> @@ -332,6 +333,20 @@ static void xen_set_pte(pte_t *ptep, pte_t pteval)
>   	__xen_set_pte(ptep, pteval);
>   }
>   
> +void xen_set_clr_mmio_pvh_pte(unsigned long pfn, unsigned long mfn,
> +			      int nr_mfns, int add_mapping)

This probably belongs to the next patch since it's there where it is 
used for the first time.

> +{
> +	struct physdev_map_iomem iomem;
> +
> +	iomem.first_gfn = pfn;
> +	iomem.first_mfn = mfn;
> +	iomem.nr_mfns = nr_mfns;
> +	iomem.add_mapping = add_mapping;
> +
> +	if (HYPERVISOR_physdev_op(PHYSDEVOP_map_iomem, &iomem))
> +		BUG();
> +}
> +
>   static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
>   		    pte_t *ptep, pte_t pteval)
>   {
> @@ -1207,6 +1222,8 @@ static void __init xen_pagetable_init(void)
>   #endif
>   	paging_init();
>   	xen_setup_shared_info();
> +	if (xen_feature(XENFEAT_auto_translated_physmap))
> +		return;
>   #ifdef CONFIG_X86_64
>   	if (!xen_feature(XENFEAT_auto_translated_physmap)) {

You can remove this check.

>   		unsigned long new_mfn_list;
> @@ -1556,6 +1573,10 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
>   static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
>   {
>   	struct mmuext_op op;
> +
> +	if (xen_feature(XENFEAT_writable_page_tables))
> +		return;
> +
>   	op.cmd = cmd;
>   	op.arg1.mfn = pfn_to_mfn(pfn);
>   	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
> @@ -1753,6 +1774,10 @@ static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags)
>   	unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
>   	pte_t pte = pfn_pte(pfn, prot);
>   
> +	/* recall for PVH, page tables are native. */
> +	if (xen_feature(XENFEAT_auto_translated_physmap))
> +		return;

pfn/pte don't need to be initialized for PVH.

> +
>   	if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
>   		BUG();
>   }
> @@ -1834,6 +1859,9 @@ static void convert_pfn_mfn(void *v)
>   	pte_t *pte = v;
>   	int i;
>   
> +	if (xen_feature(XENFEAT_auto_translated_physmap))
> +		return;
> +
>   	/* All levels are converted the same way, so just treat them
>   	   as ptes. */
>   	for (i = 0; i < PTRS_PER_PTE; i++)
> @@ -1853,6 +1881,7 @@ static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
>   		(*pt_end)--;
>   	}
>   }
> +
>   /*
>    * Set up the initial kernel pagetable.
>    *
> @@ -1863,6 +1892,7 @@ static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
>    * but that's enough to get __va working.  We need to fill in the rest
>    * of the physical mapping once some sort of allocator has been set
>    * up.
> + * NOTE: for PVH, the page tables are native.
>    */
>   void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
>   {
> @@ -1940,10 +1970,13 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
>   	 * structure to attach it to, so make sure we just set kernel
>   	 * pgd.
>   	 */
> -	xen_mc_batch();
> -	__xen_write_cr3(true, __pa(init_level4_pgt));
> -	xen_mc_issue(PARAVIRT_LAZY_CPU);
> -
> +	if (xen_feature(XENFEAT_writable_page_tables)) {
> +		native_write_cr3(__pa(init_level4_pgt));
> +	} else {
> +		xen_mc_batch();
> +		__xen_write_cr3(true, __pa(init_level4_pgt));
> +		xen_mc_issue(PARAVIRT_LAZY_CPU);
> +	}
>   	/* We can't that easily rip out L3 and L2, as the Xen pagetables are
>   	 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ...  for
>   	 * the initial domain. For guests using the toolstack, they are in:
> @@ -2207,6 +2240,11 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
>   void __init xen_init_mmu_ops(void)
>   {
>   	x86_init.paging.pagetable_init = xen_pagetable_init;
> +
> +	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> +		pv_mmu_ops.flush_tlb_others = xen_flush_tlb_others;
> +		return;
> +	}
>   	pv_mmu_ops = xen_mmu_ops;
>   
>   	memset(dummy_mapping, 0xff, PAGE_SIZE);
> @@ -2487,6 +2525,89 @@ void __init xen_hvm_init_mmu_ops(void)
>   }
>   #endif
>   
> +/* Map foreign gmfn, fgmfn, to local pfn, lpfn. This for the user space
> + * creating new guest on PVH dom0 and needs to map domU pages.

Since we don't yet have dom0 PVH support in the hypervsor, should this 
be deferred (or at least separated into another patch)?

> + */
> +static int pvh_add_to_xen_p2m(unsigned long lpfn, unsigned long fgmfn,
> +			      unsigned int domid)
> +{
> +	int rc;
> +	struct xen_add_to_physmap xatp = { .foreign_domid = domid };

Why initialize foreign_domid with different syntax than the rest?


> +
> +	xatp.gpfn = lpfn;
> +	xatp.idx = fgmfn;
> +	xatp.domid = DOMID_SELF;
> +	xatp.space = XENMAPSPACE_gmfn_foreign;
> +	rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
> +	if (rc)
> +		pr_warn("d0: Failed to map pfn (0x%lx) to mfn (0x%lx) rc:%d\n",
> +			lpfn, fgmfn, rc);
> +	return rc;
> +}
> +
> +static int pvh_rem_xen_p2m(unsigned long spfn, int count)

I don't think "rem" is a common abbreviation for "remove". Perhaps "rm", 
"del" or "remove"?

-boris

> +{
> +	struct xen_remove_from_physmap xrp;
> +	int i, rc;
> +
> +	for (i = 0; i < count; i++) {
> +		xrp.domid = DOMID_SELF;
> +		xrp.gpfn = spfn+i;
> +		rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
> +		if (rc) {
> +			pr_warn("Failed to unmap pfn:%lx rc:%d done:%d\n",
> +				spfn+i, rc, i);
> +			return 1;
> +		}
> +	}
> +	return 0;
> +}
> +
> +struct pvh_remap_data {
> +	unsigned long fgmfn;		/* foreign domain's gmfn */
> +	pgprot_t prot;
> +	domid_t  domid;
> +	int	 index;
> +	struct page **pages;
> +};
> +
> +static int pvh_map_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr,
> +			void *data)
> +{
> +	int rc;
> +	struct pvh_remap_data *remap = data;
> +	unsigned long pfn = page_to_pfn(remap->pages[remap->index++]);
> +	pte_t pteval = pte_mkspecial(pfn_pte(pfn, remap->prot));
> +
> +	rc = pvh_add_to_xen_p2m(pfn, remap->fgmfn, remap->domid);
> +	if (rc)
> +		return rc;
> +	native_set_pte(ptep, pteval);
> +
> +	return 0;
> +}
> +
> +static int pvh_remap_gmfn_range(struct vm_area_struct *vma,
> +				unsigned long addr, unsigned long mfn, int nr,
> +				pgprot_t prot, unsigned domid,
> +				struct page **pages)
> +{
> +	int err;
> +	struct pvh_remap_data pvhdata;
> +
> +	BUG_ON(!pages);
> +
> +	pvhdata.fgmfn = mfn;
> +	pvhdata.prot = prot;
> +	pvhdata.domid = domid;
> +	pvhdata.index = 0;
> +	pvhdata.pages = pages;
> +	err = apply_to_page_range(vma->vm_mm, addr, nr << PAGE_SHIFT,
> +				  pvh_map_pte_fn, &pvhdata);
> +	flush_tlb_all();
> +	return err;
> +}
> +
>   #define REMAP_BATCH_SIZE 16
>   
>   struct remap_data {
> @@ -2528,6 +2649,10 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
>   
>   	BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
>   
> +	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> +		/* We need to update the local page tables and the xen HAP */
> +		return pvh_remap_gmfn_range(vma, addr, mfn, nr, prot, domid, pages);
> +	}
>   	rmd.mfn = mfn;
>   	rmd.prot = prot;
>   
> @@ -2565,6 +2690,18 @@ int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
>   	if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
>   		return 0;
>   
> -	return -EINVAL;
> +	while (numpgs--) {
> +
> +		/* the mmu has already cleaned up the process mmu resources at
> +		 * this point (lookup_address will return NULL). */
> +		unsigned long pfn = page_to_pfn(pages[numpgs]);
> +
> +		pvh_rem_xen_p2m(pfn, 1);
> +	}
> +	/* We don't need to flush tlbs because as part of pvh_rem_xen_p2m(),
> +	 * the hypervisor will do tlb flushes after removing the p2m entries
> +	 * from the EPT/NPT */
> +
> +	return 0;
>   }
>   EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range);
> diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h
> index 73809bb..6d0bb56 100644
> --- a/arch/x86/xen/mmu.h
> +++ b/arch/x86/xen/mmu.h
> @@ -23,4 +23,6 @@ unsigned long xen_read_cr2_direct(void);
>   
>   extern void xen_init_mmu_ops(void);
>   extern void xen_hvm_init_mmu_ops(void);
> +extern void xen_set_clr_mmio_pvh_pte(unsigned long pfn, unsigned long mfn,
> +				     int nr_mfns, int add_mapping);
>   #endif	/* _XEN_MMU_H */

Re: [PATCH V10 04/14] xen/pvh: bootup and setup (E820) related changes.

[Boris Ostrovsky]

On 12/12/2013 09:10 PM, Konrad Rzeszutek Wilk wrote:
> From: Mukesh Rathor <mukesh.rathor@oracle.com>
>
> In the bootup code for PVH we can trap cpuid via vmexit, so don't
> need to use emulated prefix call. We also check for vector callback
> early on, as it is a required feature. PVH also runs at default kernel
> IOPL.
>
> In setup.c which deals with E820, in xen_add_extra_mem() we can skip
> updating P2M as it's managed by Xen. PVH maps the entire IO space,
> but only RAM pages need to be repopulated.
>
> Finally, pure PV settings are moved to a separate function that are
> only called for pure PV, ie, pv with pvmmu.
>
> Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> [ ijc -- rebase onto xen PVonHVM: use E820_Reserved area for
>           shared_info ]
> [v2: Rebase on v3.9-rc1 with MMIO/Kexec reverted]
>
> Conflicts:
> 	arch/x86/xen/setup.c
> [due to "xen: Support 64-bit PV guest receiving NMIs"]
> ---
>   arch/x86/xen/enlighten.c |   77 ++++++++++++++++++++++++++++++++++-----------
>   arch/x86/xen/setup.c     |   63 ++++++++++++++++++++++++++++++-------
>   2 files changed, 109 insertions(+), 31 deletions(-)
>
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index fa6ade7..500508d 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -46,6 +46,7 @@
>   #include <xen/hvm.h>
>   #include <xen/hvc-console.h>
>   #include <xen/acpi.h>
> +#include <xen/features.h>
>   
>   #include <asm/paravirt.h>
>   #include <asm/apic.h>
> @@ -129,6 +130,9 @@ RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
>   __read_mostly int xen_have_vector_callback;
>   EXPORT_SYMBOL_GPL(xen_have_vector_callback);
>   
> +#define xen_pvh_domain() (xen_pv_domain() && \
> +			  xen_feature(XENFEAT_auto_translated_physmap) && \
> +			  xen_have_vector_callback)

Can this be used in earlier patches instead of checking for 
XENFEAT_auto_translated_physmap, when it's clear that we actually mean PVH?

>   /*
>    * Point at some empty memory to start with. We map the real shared_info
>    * page as soon as fixmap is up and running.
> @@ -262,8 +266,9 @@ static void __init xen_banner(void)
>   	struct xen_extraversion extra;
>   	HYPERVISOR_xen_version(XENVER_extraversion, &extra);
>   
> -	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
> -	       pv_info.name);
> +	pr_info("Booting paravirtualized kernel %son %s\n",
> +		xen_feature(XENFEAT_auto_translated_physmap) ?

... and here as well (and possibly elsewhere).

> +			"with PVH extensions " : "", pv_info.name);
>   	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
>   	       version >> 16, version & 0xffff, extra.extraversion,
>   	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
> @@ -331,12 +336,15 @@ static void xen_cpuid(unsigned int *ax, unsigned int *bx,
>   		break;
>   	}
>   
> -	asm(XEN_EMULATE_PREFIX "cpuid"
> -		: "=a" (*ax),
> -		  "=b" (*bx),
> -		  "=c" (*cx),
> -		  "=d" (*dx)
> -		: "0" (*ax), "2" (*cx));
> +	if (xen_pvh_domain())
> +		native_cpuid(ax, bx, cx, dx);
> +	else
> +		asm(XEN_EMULATE_PREFIX "cpuid"
> +			: "=a" (*ax),
> +			"=b" (*bx),
> +			"=c" (*cx),
> +			"=d" (*dx)
> +			: "0" (*ax), "2" (*cx));
>   
>   	*bx &= maskebx;
>   	*cx &= maskecx;
> @@ -1125,6 +1133,10 @@ void xen_setup_shared_info(void)
>   		HYPERVISOR_shared_info =
>   			(struct shared_info *)__va(xen_start_info->shared_info);
>   
> +	/* PVH TBD/FIXME: vcpu info placement in phase 2 */
> +	if (xen_pvh_domain())
> +		return;
> +
>   #ifndef CONFIG_SMP
>   	/* In UP this is as good a place as any to set up shared info */
>   	xen_setup_vcpu_info_placement();
> @@ -1410,6 +1422,11 @@ static void __init xen_boot_params_init_edd(void)
>    */
>   static void __init xen_setup_stackprotector(void)
>   {
> +	/* PVH TBD/FIXME: investigate setup_stack_canary_segment */

setup_stack_canary_segment() is for 32-bit only and since PVH (which I 
assume is what this 'if' is about) is a 64-bit only binary this call is 
a nop.

> +	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> +		switch_to_new_gdt(0);
> +		return;
> +	}
>   	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
>   	pv_cpu_ops.load_gdt = xen_load_gdt_boot;
>   
> @@ -1420,6 +1437,19 @@ static void __init xen_setup_stackprotector(void)
>   	pv_cpu_ops.load_gdt = xen_load_gdt;
>   }
>   
> +static void __init xen_pvh_early_guest_init(void)
> +{
> +	if (xen_feature(XENFEAT_hvm_callback_vector))
> +		xen_have_vector_callback = 1;
> +
> +#ifdef CONFIG_X86_32
> +	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> +		xen_raw_printk("ERROR: 32bit PVH guests are not supported\n");
> +		BUG();
> +	}
> +#endif
> +}
> +
>   /* First C function to be called on Xen boot */
>   asmlinkage void __init xen_start_kernel(void)
>   {
> @@ -1431,13 +1461,18 @@ asmlinkage void __init xen_start_kernel(void)
>   
>   	xen_domain_type = XEN_PV_DOMAIN;
>   
> +	xen_setup_features();
> +	xen_pvh_early_guest_init();
>   	xen_setup_machphys_mapping();
>   
>   	/* Install Xen paravirt ops */
>   	pv_info = xen_info;
>   	pv_init_ops = xen_init_ops;
> -	pv_cpu_ops = xen_cpu_ops;
>   	pv_apic_ops = xen_apic_ops;
> +	if (xen_pvh_domain())
> +		pv_cpu_ops.cpuid = xen_cpuid;
> +	else
> +		pv_cpu_ops = xen_cpu_ops;
>   
>   	x86_init.resources.memory_setup = xen_memory_setup;
>   	x86_init.oem.arch_setup = xen_arch_setup;
> @@ -1469,8 +1504,6 @@ asmlinkage void __init xen_start_kernel(void)
>   	/* Work out if we support NX */
>   	x86_configure_nx();
>   
> -	xen_setup_features();
> -
>   	/* Get mfn list */
>   	if (!xen_feature(XENFEAT_auto_translated_physmap))
>   		xen_build_dynamic_phys_to_machine();
> @@ -1548,14 +1581,18 @@ asmlinkage void __init xen_start_kernel(void)
>   	/* set the limit of our address space */
>   	xen_reserve_top();
>   
> -	/* We used to do this in xen_arch_setup, but that is too late on AMD
> -	 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
> -	 * which pokes 0xcf8 port.
> -	 */
> -	set_iopl.iopl = 1;
> -	rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
> -	if (rc != 0)
> -		xen_raw_printk("physdev_op failed %d\n", rc);
> +	/* PVH: runs at default kernel iopl of 0 */
> +	if (!xen_pvh_domain()) {
> +		/*
> +		 * We used to do this in xen_arch_setup, but that is too late
> +		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
> +		 * early_amd_init which pokes 0xcf8 port.
> +		 */
> +		set_iopl.iopl = 1;
> +		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
> +		if (rc != 0)
> +			xen_raw_printk("physdev_op failed %d\n", rc);
> +	}
>   
>   #ifdef CONFIG_X86_32
>   	/* set up basic CPUID stuff */
> @@ -1625,6 +1662,8 @@ asmlinkage void __init xen_start_kernel(void)
>   }
>   
>   void __ref xen_hvm_init_shared_info(void)
> +/* Use a pfn in RAM, may move to MMIO before kexec.
> + * This function also called for PVH dom0 */
>   {
>   	int cpu;
>   	struct xen_add_to_physmap xatp;
> diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
> index 68c054f..e3dcd8c 100644
> --- a/arch/x86/xen/setup.c
> +++ b/arch/x86/xen/setup.c
> @@ -27,6 +27,7 @@
>   #include <xen/interface/memory.h>
>   #include <xen/interface/physdev.h>
>   #include <xen/features.h>
> +#include "mmu.h"
>   #include "xen-ops.h"
>   #include "vdso.h"
>   
> @@ -81,6 +82,9 @@ static void __init xen_add_extra_mem(u64 start, u64 size)
>   
>   	memblock_reserve(start, size);
>   
> +	if (xen_feature(XENFEAT_auto_translated_physmap))
> +		return;
> +
>   	xen_max_p2m_pfn = PFN_DOWN(start + size);
>   	for (pfn = PFN_DOWN(start); pfn < xen_max_p2m_pfn; pfn++) {
>   		unsigned long mfn = pfn_to_mfn(pfn);
> @@ -103,6 +107,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
>   		.domid        = DOMID_SELF
>   	};
>   	unsigned long len = 0;
> +	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);

This is inconsistent with other uses of 
xen_feature(XENFEAT_auto_translated_physmap), so far xen_feature() has 
always been used.

>   	unsigned long pfn;
>   	int ret;
>   
> @@ -116,7 +121,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
>   				continue;
>   			frame = mfn;
>   		} else {
> -			if (mfn != INVALID_P2M_ENTRY)
> +			if (!xlated_phys && mfn != INVALID_P2M_ENTRY)
>   				continue;
>   			frame = pfn;
>   		}
> @@ -239,6 +244,27 @@ static void __init xen_set_identity_and_release_chunk(
>   	*identity += set_phys_range_identity(start_pfn, end_pfn);
>   }
>   
> +/* For PVH, the pfns [0..MAX] are mapped to mfn's in the EPT/NPT. The mfns
> + * are released as part of this 1:1 mapping hypercall back to the dom heap.
> + * Also, we map the entire IO space, ie, beyond max_pfn_mapped.
> + */
> +static void __init xen_pvh_identity_map_chunk(unsigned long start_pfn,
> +		unsigned long end_pfn, unsigned long *released,
> +		unsigned long *identity, unsigned long max_pfn)
> +{
> +	unsigned long pfn;
> +	int numpfns = 1, add_mapping = 1;

No need for these two variables;

> +
> +	for (pfn = start_pfn; pfn < end_pfn; pfn++)
> +		xen_set_clr_mmio_pvh_pte(pfn, pfn, numpfns, add_mapping);
> +
> +	if (start_pfn <= max_pfn) {
> +		unsigned long end = min(max_pfn_mapped, end_pfn);
> +		*released += end - start_pfn;
> +	}
> +	*identity += end_pfn - start_pfn;
> +}
> +
>   static unsigned long __init xen_set_identity_and_release(
>   	const struct e820entry *list, size_t map_size, unsigned long nr_pages)
>   {
> @@ -247,6 +273,7 @@ static unsigned long __init xen_set_identity_and_release(
>   	unsigned long identity = 0;
>   	const struct e820entry *entry;
>   	int i;
> +	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);

Again xlated_phys.


-boris

>   
>   	/*
>   	 * Combine non-RAM regions and gaps until a RAM region (or the
> @@ -268,11 +295,17 @@ static unsigned long __init xen_set_identity_and_release(
>   			if (entry->type == E820_RAM)
>   				end_pfn = PFN_UP(entry->addr);
>   
> -			if (start_pfn < end_pfn)
> -				xen_set_identity_and_release_chunk(
> -					start_pfn, end_pfn, nr_pages,
> -					&released, &identity);
> -
> +			if (start_pfn < end_pfn) {
> +				if (xlated_phys) {
> +					xen_pvh_identity_map_chunk(start_pfn,
> +						end_pfn, &released, &identity,
> +						nr_pages);
> +				} else {
> +					xen_set_identity_and_release_chunk(
> +						start_pfn, end_pfn, nr_pages,
> +						&released, &identity);
> +				}
> +			}
>   			start = end;
>   		}
>   	}
> @@ -563,16 +596,13 @@ void xen_enable_nmi(void)
>   		BUG();
>   #endif
>   }
> -void __init xen_arch_setup(void)
> +void __init xen_pvmmu_arch_setup(void)
>   {
> -	xen_panic_handler_init();
> -
>   	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);
>   	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables);
>   
> -	if (!xen_feature(XENFEAT_auto_translated_physmap))
> -		HYPERVISOR_vm_assist(VMASST_CMD_enable,
> -				     VMASST_TYPE_pae_extended_cr3);
> +	HYPERVISOR_vm_assist(VMASST_CMD_enable,
> +			     VMASST_TYPE_pae_extended_cr3);
>   
>   	if (register_callback(CALLBACKTYPE_event, xen_hypervisor_callback) ||
>   	    register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback))
> @@ -581,6 +611,15 @@ void __init xen_arch_setup(void)
>   	xen_enable_sysenter();
>   	xen_enable_syscall();
>   	xen_enable_nmi();
> +}
> +
> +/* This function not called for HVM domain */
> +void __init xen_arch_setup(void)
> +{
> +	xen_panic_handler_init();
> +
> +	if (!xen_feature(XENFEAT_auto_translated_physmap))
> +		xen_pvmmu_arch_setup();
>   #ifdef CONFIG_ACPI
>   	if (!(xen_start_info->flags & SIF_INITDOMAIN)) {
>   		printk(KERN_INFO "ACPI in unprivileged domain disabled\n");

Re: [PATCH V10 04/14] xen/pvh: bootup and setup (E820) related changes.

[Konrad Rzeszutek Wilk]

On Fri, Dec 13, 2013 at 12:55:26PM -0500, Boris Ostrovsky wrote:
> On 12/12/2013 09:10 PM, Konrad Rzeszutek Wilk wrote:
> >From: Mukesh Rathor <mukesh.rathor@oracle.com>
> >
> >In the bootup code for PVH we can trap cpuid via vmexit, so don't
> >need to use emulated prefix call. We also check for vector callback
> >early on, as it is a required feature. PVH also runs at default kernel
> >IOPL.
> >
> >In setup.c which deals with E820, in xen_add_extra_mem() we can skip
> >updating P2M as it's managed by Xen. PVH maps the entire IO space,
> >but only RAM pages need to be repopulated.
> >
> >Finally, pure PV settings are moved to a separate function that are
> >only called for pure PV, ie, pv with pvmmu.
> >
> >Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> >Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> >[ ijc -- rebase onto xen PVonHVM: use E820_Reserved area for
> >          shared_info ]
> >[v2: Rebase on v3.9-rc1 with MMIO/Kexec reverted]
> >
> >Conflicts:
> >	arch/x86/xen/setup.c
> >[due to "xen: Support 64-bit PV guest receiving NMIs"]
> >---
> >  arch/x86/xen/enlighten.c |   77 ++++++++++++++++++++++++++++++++++-----------
> >  arch/x86/xen/setup.c     |   63 ++++++++++++++++++++++++++++++-------
> >  2 files changed, 109 insertions(+), 31 deletions(-)
> >
> >diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> >index fa6ade7..500508d 100644
> >--- a/arch/x86/xen/enlighten.c
> >+++ b/arch/x86/xen/enlighten.c
> >@@ -46,6 +46,7 @@
> >  #include <xen/hvm.h>
> >  #include <xen/hvc-console.h>
> >  #include <xen/acpi.h>
> >+#include <xen/features.h>
> >  #include <asm/paravirt.h>
> >  #include <asm/apic.h>
> >@@ -129,6 +130,9 @@ RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
> >  __read_mostly int xen_have_vector_callback;
> >  EXPORT_SYMBOL_GPL(xen_have_vector_callback);
> >+#define xen_pvh_domain() (xen_pv_domain() && \
> >+			  xen_feature(XENFEAT_auto_translated_physmap) && \
> >+			  xen_have_vector_callback)
> 
> Can this be used in earlier patches instead of checking for
> XENFEAT_auto_translated_physmap, when it's clear that we actually
> mean PVH?

As I posted the patches I realized we could actually just remove this
check and piggyback on xen_feature(XENFEAT_auto_translated_physmap).

But then I wasn't sure as I think you can do pure PV with auto-translate
on (so it uses shadow paging). But I think that code is bitrotten.

> 
> >  /*
> >   * Point at some empty memory to start with. We map the real shared_info
> >   * page as soon as fixmap is up and running.
> >@@ -262,8 +266,9 @@ static void __init xen_banner(void)
> >  	struct xen_extraversion extra;
> >  	HYPERVISOR_xen_version(XENVER_extraversion, &extra);
> >-	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
> >-	       pv_info.name);
> >+	pr_info("Booting paravirtualized kernel %son %s\n",
> >+		xen_feature(XENFEAT_auto_translated_physmap) ?
> 
> ... and here as well (and possibly elsewhere).
> 
> >+			"with PVH extensions " : "", pv_info.name);
> >  	printk(KERN_INFO "Xen version: %d.%d%s%s\n",
> >  	       version >> 16, version & 0xffff, extra.extraversion,
> >  	       xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
> >@@ -331,12 +336,15 @@ static void xen_cpuid(unsigned int *ax, unsigned int *bx,
> >  		break;
> >  	}
> >-	asm(XEN_EMULATE_PREFIX "cpuid"
> >-		: "=a" (*ax),
> >-		  "=b" (*bx),
> >-		  "=c" (*cx),
> >-		  "=d" (*dx)
> >-		: "0" (*ax), "2" (*cx));
> >+	if (xen_pvh_domain())
> >+		native_cpuid(ax, bx, cx, dx);
> >+	else
> >+		asm(XEN_EMULATE_PREFIX "cpuid"
> >+			: "=a" (*ax),
> >+			"=b" (*bx),
> >+			"=c" (*cx),
> >+			"=d" (*dx)
> >+			: "0" (*ax), "2" (*cx));
> >  	*bx &= maskebx;
> >  	*cx &= maskecx;
> >@@ -1125,6 +1133,10 @@ void xen_setup_shared_info(void)
> >  		HYPERVISOR_shared_info =
> >  			(struct shared_info *)__va(xen_start_info->shared_info);
> >+	/* PVH TBD/FIXME: vcpu info placement in phase 2 */
> >+	if (xen_pvh_domain())
> >+		return;
> >+
> >  #ifndef CONFIG_SMP
> >  	/* In UP this is as good a place as any to set up shared info */
> >  	xen_setup_vcpu_info_placement();
> >@@ -1410,6 +1422,11 @@ static void __init xen_boot_params_init_edd(void)
> >   */
> >  static void __init xen_setup_stackprotector(void)
> >  {
> >+	/* PVH TBD/FIXME: investigate setup_stack_canary_segment */
> 
> setup_stack_canary_segment() is for 32-bit only and since PVH (which
> I assume is what this 'if' is about) is a 64-bit only binary this
> call is a nop.

<nods>
> 
> >+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> >+		switch_to_new_gdt(0);
> >+		return;
> >+	}
> >  	pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
> >  	pv_cpu_ops.load_gdt = xen_load_gdt_boot;
> >@@ -1420,6 +1437,19 @@ static void __init xen_setup_stackprotector(void)
> >  	pv_cpu_ops.load_gdt = xen_load_gdt;
> >  }
> >+static void __init xen_pvh_early_guest_init(void)
> >+{
> >+	if (xen_feature(XENFEAT_hvm_callback_vector))
> >+		xen_have_vector_callback = 1;
> >+
> >+#ifdef CONFIG_X86_32
> >+	if (xen_feature(XENFEAT_auto_translated_physmap)) {
> >+		xen_raw_printk("ERROR: 32bit PVH guests are not supported\n");
> >+		BUG();
> >+	}
> >+#endif
> >+}
> >+
> >  /* First C function to be called on Xen boot */
> >  asmlinkage void __init xen_start_kernel(void)
> >  {
> >@@ -1431,13 +1461,18 @@ asmlinkage void __init xen_start_kernel(void)
> >  	xen_domain_type = XEN_PV_DOMAIN;
> >+	xen_setup_features();
> >+	xen_pvh_early_guest_init();
> >  	xen_setup_machphys_mapping();
> >  	/* Install Xen paravirt ops */
> >  	pv_info = xen_info;
> >  	pv_init_ops = xen_init_ops;
> >-	pv_cpu_ops = xen_cpu_ops;
> >  	pv_apic_ops = xen_apic_ops;
> >+	if (xen_pvh_domain())
> >+		pv_cpu_ops.cpuid = xen_cpuid;
> >+	else
> >+		pv_cpu_ops = xen_cpu_ops;
> >  	x86_init.resources.memory_setup = xen_memory_setup;
> >  	x86_init.oem.arch_setup = xen_arch_setup;
> >@@ -1469,8 +1504,6 @@ asmlinkage void __init xen_start_kernel(void)
> >  	/* Work out if we support NX */
> >  	x86_configure_nx();
> >-	xen_setup_features();
> >-
> >  	/* Get mfn list */
> >  	if (!xen_feature(XENFEAT_auto_translated_physmap))
> >  		xen_build_dynamic_phys_to_machine();
> >@@ -1548,14 +1581,18 @@ asmlinkage void __init xen_start_kernel(void)
> >  	/* set the limit of our address space */
> >  	xen_reserve_top();
> >-	/* We used to do this in xen_arch_setup, but that is too late on AMD
> >-	 * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
> >-	 * which pokes 0xcf8 port.
> >-	 */
> >-	set_iopl.iopl = 1;
> >-	rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
> >-	if (rc != 0)
> >-		xen_raw_printk("physdev_op failed %d\n", rc);
> >+	/* PVH: runs at default kernel iopl of 0 */
> >+	if (!xen_pvh_domain()) {
> >+		/*
> >+		 * We used to do this in xen_arch_setup, but that is too late
> >+		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
> >+		 * early_amd_init which pokes 0xcf8 port.
> >+		 */
> >+		set_iopl.iopl = 1;
> >+		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
> >+		if (rc != 0)
> >+			xen_raw_printk("physdev_op failed %d\n", rc);
> >+	}
> >  #ifdef CONFIG_X86_32
> >  	/* set up basic CPUID stuff */
> >@@ -1625,6 +1662,8 @@ asmlinkage void __init xen_start_kernel(void)
> >  }
> >  void __ref xen_hvm_init_shared_info(void)
> >+/* Use a pfn in RAM, may move to MMIO before kexec.
> >+ * This function also called for PVH dom0 */
> >  {
> >  	int cpu;
> >  	struct xen_add_to_physmap xatp;
> >diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
> >index 68c054f..e3dcd8c 100644
> >--- a/arch/x86/xen/setup.c
> >+++ b/arch/x86/xen/setup.c
> >@@ -27,6 +27,7 @@
> >  #include <xen/interface/memory.h>
> >  #include <xen/interface/physdev.h>
> >  #include <xen/features.h>
> >+#include "mmu.h"
> >  #include "xen-ops.h"
> >  #include "vdso.h"
> >@@ -81,6 +82,9 @@ static void __init xen_add_extra_mem(u64 start, u64 size)
> >  	memblock_reserve(start, size);
> >+	if (xen_feature(XENFEAT_auto_translated_physmap))
> >+		return;
> >+
> >  	xen_max_p2m_pfn = PFN_DOWN(start + size);
> >  	for (pfn = PFN_DOWN(start); pfn < xen_max_p2m_pfn; pfn++) {
> >  		unsigned long mfn = pfn_to_mfn(pfn);
> >@@ -103,6 +107,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
> >  		.domid        = DOMID_SELF
> >  	};
> >  	unsigned long len = 0;
> >+	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);
> 
> This is inconsistent with other uses of
> xen_feature(XENFEAT_auto_translated_physmap), so far xen_feature()
> has always been used.

.. I think sprinkling those 'xen_pvh_domain' in arch/x86 is OK, but not
in the drivers/xen (as those are also used by ARM).

But perhaps we should just use 'if (xen_feature(XENFEAT..)'. Hmm.
> 
> >  	unsigned long pfn;
> >  	int ret;
> >@@ -116,7 +121,7 @@ static unsigned long __init xen_do_chunk(unsigned long start,
> >  				continue;
> >  			frame = mfn;
> >  		} else {
> >-			if (mfn != INVALID_P2M_ENTRY)
> >+			if (!xlated_phys && mfn != INVALID_P2M_ENTRY)
> >  				continue;
> >  			frame = pfn;
> >  		}
> >@@ -239,6 +244,27 @@ static void __init xen_set_identity_and_release_chunk(
> >  	*identity += set_phys_range_identity(start_pfn, end_pfn);
> >  }
> >+/* For PVH, the pfns [0..MAX] are mapped to mfn's in the EPT/NPT. The mfns
> >+ * are released as part of this 1:1 mapping hypercall back to the dom heap.
> >+ * Also, we map the entire IO space, ie, beyond max_pfn_mapped.
> >+ */
> >+static void __init xen_pvh_identity_map_chunk(unsigned long start_pfn,
> >+		unsigned long end_pfn, unsigned long *released,
> >+		unsigned long *identity, unsigned long max_pfn)
> >+{
> >+	unsigned long pfn;
> >+	int numpfns = 1, add_mapping = 1;
> 
> No need for these two variables;

Code gone.
> 
> >+
> >+	for (pfn = start_pfn; pfn < end_pfn; pfn++)
> >+		xen_set_clr_mmio_pvh_pte(pfn, pfn, numpfns, add_mapping);
> >+
> >+	if (start_pfn <= max_pfn) {
> >+		unsigned long end = min(max_pfn_mapped, end_pfn);
> >+		*released += end - start_pfn;
> >+	}
> >+	*identity += end_pfn - start_pfn;
> >+}
> >+
> >  static unsigned long __init xen_set_identity_and_release(
> >  	const struct e820entry *list, size_t map_size, unsigned long nr_pages)
> >  {
> >@@ -247,6 +273,7 @@ static unsigned long __init xen_set_identity_and_release(
> >  	unsigned long identity = 0;
> >  	const struct e820entry *entry;
> >  	int i;
> >+	int xlated_phys = xen_feature(XENFEAT_auto_translated_physmap);
> 
> Again xlated_phys.

Ripped out.
> 
> 
> -boris
> 
> >  	/*
> >  	 * Combine non-RAM regions and gaps until a RAM region (or the
> >@@ -268,11 +295,17 @@ static unsigned long __init xen_set_identity_and_release(
> >  			if (entry->type == E820_RAM)
> >  				end_pfn = PFN_UP(entry->addr);
> >-			if (start_pfn < end_pfn)
> >-				xen_set_identity_and_release_chunk(
> >-					start_pfn, end_pfn, nr_pages,
> >-					&released, &identity);
> >-
> >+			if (start_pfn < end_pfn) {
> >+				if (xlated_phys) {
> >+					xen_pvh_identity_map_chunk(start_pfn,
> >+						end_pfn, &released, &identity,
> >+						nr_pages);
> >+				} else {
> >+					xen_set_identity_and_release_chunk(
> >+						start_pfn, end_pfn, nr_pages,
> >+						&released, &identity);
> >+				}
> >+			}
> >  			start = end;
> >  		}
> >  	}
> >@@ -563,16 +596,13 @@ void xen_enable_nmi(void)
> >  		BUG();
> >  #endif
> >  }
> >-void __init xen_arch_setup(void)
> >+void __init xen_pvmmu_arch_setup(void)
> >  {
> >-	xen_panic_handler_init();
> >-
> >  	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);
> >  	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables);
> >-	if (!xen_feature(XENFEAT_auto_translated_physmap))
> >-		HYPERVISOR_vm_assist(VMASST_CMD_enable,
> >-				     VMASST_TYPE_pae_extended_cr3);
> >+	HYPERVISOR_vm_assist(VMASST_CMD_enable,
> >+			     VMASST_TYPE_pae_extended_cr3);
> >  	if (register_callback(CALLBACKTYPE_event, xen_hypervisor_callback) ||
> >  	    register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback))
> >@@ -581,6 +611,15 @@ void __init xen_arch_setup(void)
> >  	xen_enable_sysenter();
> >  	xen_enable_syscall();
> >  	xen_enable_nmi();
> >+}
> >+
> >+/* This function not called for HVM domain */
> >+void __init xen_arch_setup(void)
> >+{
> >+	xen_panic_handler_init();
> >+
> >+	if (!xen_feature(XENFEAT_auto_translated_physmap))
> >+		xen_pvmmu_arch_setup();
> >  #ifdef CONFIG_ACPI
> >  	if (!(xen_start_info->flags & SIF_INITDOMAIN)) {
> >  		printk(KERN_INFO "ACPI in unprivileged domain disabled\n");
>