[PATCH 4/4] kvm tools: arm: add support for PSCI firmware in place of spin-tables

[Will Deacon <will.deacon@arm.com>]

ARM has recently published a document describing a firmware interface
for CPU power management, which can be used for booting secondary cores
on an SMP platform, amongst other things. As part of the mach-virt
upstreaming for the kernel (that is, the virtual platform targetted by
kvmtool), it was suggested that we use this interface instead of the
current spin-table based approach.

This patch implements PSCI support in kvmtool for ARM, removing a fair
amount of code in the process.

Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 tools/kvm/Makefile                                 |  5 +-
 tools/kvm/arm/aarch32/cortex-a15.c                 |  8 +--
 tools/kvm/arm/aarch32/include/kvm/kvm-arch.h       |  1 -
 tools/kvm/arm/aarch32/include/kvm/kvm-cpu-arch.h   | 12 +++++
 tools/kvm/arm/aarch32/kvm-cpu.c                    | 59 ++++++++++------------
 tools/kvm/arm/aarch32/smp-pen.S                    | 39 --------------
 tools/kvm/arm/fdt.c                                | 36 +++++--------
 tools/kvm/arm/include/arm-common/gic.h             |  2 -
 tools/kvm/arm/include/arm-common/kvm-arch.h        |  5 --
 .../arm/include/{kvm => arm-common}/kvm-cpu-arch.h |  6 +--
 tools/kvm/arm/kvm-cpu.c                            |  4 +-
 tools/kvm/arm/kvm.c                                |  1 +
 tools/kvm/arm/smp.c                                | 21 --------
 13 files changed, 62 insertions(+), 137 deletions(-)
 create mode 100644 tools/kvm/arm/aarch32/include/kvm/kvm-cpu-arch.h
 delete mode 100644 tools/kvm/arm/aarch32/smp-pen.S
 rename tools/kvm/arm/include/{kvm => arm-common}/kvm-cpu-arch.h (87%)
 delete mode 100644 tools/kvm/arm/smp.c

diff --git a/tools/kvm/Makefile b/tools/kvm/Makefile
index a83dd10..33aa4d8 100644
--- a/tools/kvm/Makefile
+++ b/tools/kvm/Makefile
@@ -160,18 +160,15 @@ endif
 
 # ARM
 OBJS_ARM_COMMON		:= arm/fdt.o arm/gic.o arm/ioport.o arm/irq.o \
-			   arm/kvm.o arm/kvm-cpu.o arm/smp.o
+			   arm/kvm.o arm/kvm-cpu.o
 HDRS_ARM_COMMON		:= arm/include
 ifeq ($(ARCH), arm)
 	DEFINES		+= -DCONFIG_ARM
 	OBJS		+= $(OBJS_ARM_COMMON)
 	OBJS		+= arm/aarch32/cortex-a15.o
 	OBJS		+= arm/aarch32/kvm-cpu.o
-	OBJS       	+= arm/aarch32/smp-pen.o
 	ARCH_INCLUDE	:= $(HDRS_ARM_COMMON)
 	ARCH_INCLUDE	+= -Iarm/aarch32/include
-	ASFLAGS		+= -D__ASSEMBLY__
-	ASFLAGS		+= -I$(ARCH_INCLUDE)
 	CFLAGS		+= -march=armv7-a
 	CFLAGS		+= -I../../scripts/dtc/libfdt
 	OTHEROBJS	+= $(LIBFDT_OBJS)
diff --git a/tools/kvm/arm/aarch32/cortex-a15.c b/tools/kvm/arm/aarch32/cortex-a15.c
index eac0bb9..8031747 100644
--- a/tools/kvm/arm/aarch32/cortex-a15.c
+++ b/tools/kvm/arm/aarch32/cortex-a15.c
@@ -31,12 +31,8 @@ static void generate_cpu_nodes(void *fdt, struct kvm *kvm)
 		_FDT(fdt_property_string(fdt, "device_type", "cpu"));
 		_FDT(fdt_property_string(fdt, "compatible", "arm,cortex-a15"));
 
-		if (kvm->nrcpus > 1) {
-			_FDT(fdt_property_string(fdt, "enable-method",
-						 "spin-table"));
-			_FDT(fdt_property_cell(fdt, "cpu-release-addr",
-					       kvm->arch.smp_jump_guest_start));
-		}
+		if (kvm->nrcpus > 1)
+			_FDT(fdt_property_string(fdt, "enable-method", "psci"));
 
 		_FDT(fdt_property_cell(fdt, "reg", cpu));
 		_FDT(fdt_end_node(fdt));
diff --git a/tools/kvm/arm/aarch32/include/kvm/kvm-arch.h b/tools/kvm/arm/aarch32/include/kvm/kvm-arch.h
index f236895..ca79b24 100644
--- a/tools/kvm/arm/aarch32/include/kvm/kvm-arch.h
+++ b/tools/kvm/arm/aarch32/include/kvm/kvm-arch.h
@@ -15,7 +15,6 @@
 
 #define ARM_KERN_OFFSET		0x8000
 
-#define ARM_SMP_PEN_SIZE	PAGE_SIZE
 #define ARM_VIRTIO_MMIO_SIZE	(ARM_GIC_DIST_BASE - ARM_LOMAP_MMIO_AREA)
 #define ARM_PCI_MMIO_SIZE	(ARM_LOMAP_MEMORY_AREA - ARM_LOMAP_AXI_AREA)
 
diff --git a/tools/kvm/arm/aarch32/include/kvm/kvm-cpu-arch.h b/tools/kvm/arm/aarch32/include/kvm/kvm-cpu-arch.h
new file mode 100644
index 0000000..b9fda07
--- /dev/null
+++ b/tools/kvm/arm/aarch32/include/kvm/kvm-cpu-arch.h
@@ -0,0 +1,12 @@
+#ifndef KVM__KVM_CPU_ARCH_H
+#define KVM__KVM_CPU_ARCH_H
+
+#include "kvm/kvm.h"
+
+#include "arm-common/kvm-cpu-arch.h"
+
+#define ARM_VCPU_FEATURE_FLAGS(kvm, cpuid)	{			\
+	[0] = (!!(cpuid) << KVM_ARM_VCPU_POWER_OFF),			\
+}
+
+#endif /* KVM__KVM_CPU_ARCH_H */
diff --git a/tools/kvm/arm/aarch32/kvm-cpu.c b/tools/kvm/arm/aarch32/kvm-cpu.c
index f00a2f1..a528789 100644
--- a/tools/kvm/arm/aarch32/kvm-cpu.c
+++ b/tools/kvm/arm/aarch32/kvm-cpu.c
@@ -21,38 +21,33 @@ void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
 	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
 		die_perror("KVM_SET_ONE_REG failed (cpsr)");
 
-	if (vcpu->cpu_id == 0) {
-		/* r0 = 0 */
-		data	= 0;
-		reg.id	= ARM_CORE_REG(usr_regs.ARM_r0);
-		if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
-			die_perror("KVM_SET_ONE_REG failed (r0)");
-
-		/* r1 = machine type (-1) */
-		data	= -1;
-		reg.id	= ARM_CORE_REG(usr_regs.ARM_r1);
-		if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
-			die_perror("KVM_SET_ONE_REG failed (r1)");
-
-		/* r2 = physical address of the device tree blob */
-		data	= kvm->arch.dtb_guest_start;
-		reg.id	= ARM_CORE_REG(usr_regs.ARM_r2);
-		if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
-			die_perror("KVM_SET_ONE_REG failed (r2)");
-
-		/* pc = start of kernel image */
-		data	= kvm->arch.kern_guest_start;
-		reg.id	= ARM_CORE_REG(usr_regs.ARM_pc);
-		if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
-			die_perror("KVM_SET_ONE_REG failed (pc)");
-
-	} else {
-		/* Simply enter the pen */
-		data	= kvm->arch.smp_pen_guest_start;
-		reg.id	= ARM_CORE_REG(usr_regs.ARM_pc);
-		if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
-			die_perror("KVM_SET_ONE_REG failed (SMP pc)");
-	}
+	/* Secondary cores are stopped awaiting PSCI wakeup */
+	if (vcpu->cpu_id != 0)
+		return;
+
+	/* r0 = 0 */
+	data	= 0;
+	reg.id	= ARM_CORE_REG(usr_regs.ARM_r0);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (r0)");
+
+	/* r1 = machine type (-1) */
+	data	= -1;
+	reg.id	= ARM_CORE_REG(usr_regs.ARM_r1);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (r1)");
+
+	/* r2 = physical address of the device tree blob */
+	data	= kvm->arch.dtb_guest_start;
+	reg.id	= ARM_CORE_REG(usr_regs.ARM_r2);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (r2)");
+
+	/* pc = start of kernel image */
+	data	= kvm->arch.kern_guest_start;
+	reg.id	= ARM_CORE_REG(usr_regs.ARM_pc);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (pc)");
 }
 
 void kvm_cpu__show_code(struct kvm_cpu *vcpu)
diff --git a/tools/kvm/arm/aarch32/smp-pen.S b/tools/kvm/arm/aarch32/smp-pen.S
deleted file mode 100644
index 1e63c95..0000000
--- a/tools/kvm/arm/aarch32/smp-pen.S
+++ /dev/null
@@ -1,39 +0,0 @@
-#include "kvm/kvm-arch.h"
-
-#include "arm-common/gic.h"
-
-#define AARCH32_SMP_BAD_MAGIC	0xdeadc0de
-
-	.arm
-
-	.globl	smp_pen_start
-	.globl	smp_jump_addr
-	.globl	smp_pen_end
-
-	.align
-smp_pen_start:
-	@ Ensure that the CPU interface is enabled for the wfi wakeup
-	ldr	r0, =ARM_GIC_CPUI_BASE
-	mov	r1, #GIC_CPUI_CTLR_EN
-	str	r1, [r0]
-
-	@ Set the priority mask to accept any interrupt
-	mov	r1, #GIC_CPUI_PMR_MIN_PRIO
-	str	r1, [r0, #GIC_CPUI_OFF_PMR]
-
-	@ Now wait for the primary to poke us
-	adr	r0, smp_jump_addr
-	ldr	r1, =AARCH32_SMP_BAD_MAGIC
-	dsb
-1:	wfi
-	ldr	r2, [r0]
-	cmp	r1, r2
-	beq	1b
-	mov	pc, r2
-
-	.ltorg
-
-	.align
-smp_jump_addr:
-	.long	AARCH32_SMP_BAD_MAGIC
-smp_pen_end:
diff --git a/tools/kvm/arm/fdt.c b/tools/kvm/arm/fdt.c
index e52c10c..6c12e79 100644
--- a/tools/kvm/arm/fdt.c
+++ b/tools/kvm/arm/fdt.c
@@ -82,10 +82,6 @@ static int setup_fdt(struct kvm *kvm)
 
 	/* Create new tree without a reserve map */
 	_FDT(fdt_create(fdt, FDT_MAX_SIZE));
-	if (kvm->nrcpus > 1)
-		_FDT(fdt_add_reservemap_entry(fdt,
-					      kvm->arch.smp_pen_guest_start,
-					      ARM_SMP_PEN_SIZE));
 	_FDT(fdt_finish_reservemap(fdt));
 
 	/* Header */
@@ -129,6 +125,16 @@ static int setup_fdt(struct kvm *kvm)
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
+	/* PSCI firmware */
+	_FDT(fdt_begin_node(fdt, "psci"));
+	_FDT(fdt_property_string(fdt, "compatible", "arm,psci"));
+	_FDT(fdt_property_string(fdt, "method", "hvc"));
+	_FDT(fdt_property_cell(fdt, "cpu_suspend", KVM_PSCI_FN_CPU_SUSPEND));
+	_FDT(fdt_property_cell(fdt, "cpu_off", KVM_PSCI_FN_CPU_OFF));
+	_FDT(fdt_property_cell(fdt, "cpu_on", KVM_PSCI_FN_CPU_ON));
+	_FDT(fdt_property_cell(fdt, "migrate", KVM_PSCI_FN_MIGRATE));
+	_FDT(fdt_end_node(fdt));
+
 	/* Finalise. */
 	_FDT(fdt_end_node(fdt));
 	_FDT(fdt_finish(fdt));
@@ -157,7 +163,6 @@ static int read_image(int fd, void **pos, void *limit)
 
 #define FDT_ALIGN	SZ_2M
 #define INITRD_ALIGN	4
-#define SMP_PEN_ALIGN	PAGE_SIZE
 int load_flat_binary(struct kvm *kvm, int fd_kernel, int fd_initrd,
 		     const char *kernel_cmdline)
 {
@@ -168,8 +173,8 @@ int load_flat_binary(struct kvm *kvm, int fd_kernel, int fd_initrd,
 		die_perror("lseek");
 
 	/*
-	 * Linux requires the initrd, pen and dtb to be mapped inside
-	 * lowmem, so we can't just place them at the top of memory.
+	 * Linux requires the initrd and dtb to be mapped inside lowmem,
+	 * so we can't just place them at the top of memory.
 	 */
 	limit = kvm->ram_start + min(kvm->ram_size, (u64)SZ_256M) - 1;
 
@@ -186,24 +191,9 @@ int load_flat_binary(struct kvm *kvm, int fd_kernel, int fd_initrd,
 	/*
 	 * Now load backwards from the end of memory so the kernel
 	 * decompressor has plenty of space to work with. First up is
-	 * the SMP pen if we have more than one virtual CPU...
+	 * the device tree blob...
 	 */
 	pos = limit;
-	if (kvm->cfg.nrcpus > 1) {
-		pos -= (ARM_SMP_PEN_SIZE + SMP_PEN_ALIGN);
-		guest_addr = ALIGN(host_to_guest_flat(kvm, pos), SMP_PEN_ALIGN);
-		pos = guest_flat_to_host(kvm, guest_addr);
-		if (pos < kernel_end)
-			die("SMP pen overlaps with kernel image.");
-
-		kvm->arch.smp_pen_guest_start = guest_addr;
-		pr_info("Placing SMP pen at 0x%llx - 0x%llx",
-			kvm->arch.smp_pen_guest_start,
-			host_to_guest_flat(kvm, limit));
-		limit = pos;
-	}
-
-	/* ...now the device tree blob... */
 	pos -= (FDT_MAX_SIZE + FDT_ALIGN);
 	guest_addr = ALIGN(host_to_guest_flat(kvm, pos), FDT_ALIGN);
 	pos = guest_flat_to_host(kvm, guest_addr);
diff --git a/tools/kvm/arm/include/arm-common/gic.h b/tools/kvm/arm/include/arm-common/gic.h
index d6a18e1..850edc7 100644
--- a/tools/kvm/arm/include/arm-common/gic.h
+++ b/tools/kvm/arm/include/arm-common/gic.h
@@ -26,12 +26,10 @@
 #define GIC_MAX_CPUS			8
 #define GIC_MAX_IRQ			255
 
-#ifndef __ASSEMBLY__
 struct kvm;
 
 int gic__alloc_irqnum(void);
 int gic__init_irqchip(struct kvm *kvm);
 void gic__generate_fdt_nodes(void *fdt, u32 phandle);
 
-#endif /* __ASSEMBLY__ */
 #endif /* ARM_COMMON__GIC_H */
diff --git a/tools/kvm/arm/include/arm-common/kvm-arch.h b/tools/kvm/arm/include/arm-common/kvm-arch.h
index c910f07..798af59 100644
--- a/tools/kvm/arm/include/arm-common/kvm-arch.h
+++ b/tools/kvm/arm/include/arm-common/kvm-arch.h
@@ -3,8 +3,6 @@
 
 #define VIRTIO_DEFAULT_TRANS	VIRTIO_MMIO
 
-#ifndef __ASSEMBLY__
-
 #include <stdbool.h>
 #include <linux/types.h>
 
@@ -26,9 +24,6 @@ struct kvm_arch {
 	u64	initrd_guest_start;
 	u64	initrd_size;
 	u64	dtb_guest_start;
-	u64	smp_pen_guest_start;
-	u64	smp_jump_guest_start;
 };
 
-#endif /* __ASSEMBLY__ */
 #endif /* ARM_COMMON__KVM_ARCH_H */
diff --git a/tools/kvm/arm/include/kvm/kvm-cpu-arch.h b/tools/kvm/arm/include/arm-common/kvm-cpu-arch.h
similarity index 87%
rename from tools/kvm/arm/include/kvm/kvm-cpu-arch.h
rename to tools/kvm/arm/include/arm-common/kvm-cpu-arch.h
index f0aeca2..351fbe6 100644
--- a/tools/kvm/arm/include/kvm/kvm-cpu-arch.h
+++ b/tools/kvm/arm/include/arm-common/kvm-cpu-arch.h
@@ -1,5 +1,5 @@
-#ifndef KVM__KVM_CPU_ARCH_H
-#define KVM__KVM_CPU_ARCH_H
+#ifndef ARM_COMMON__KVM_CPU_ARCH_H
+#define ARM_COMMON__KVM_CPU_ARCH_H
 
 #include <linux/kvm.h>
 #include <pthread.h>
@@ -43,4 +43,4 @@ static inline bool kvm_cpu__emulate_io(struct kvm *kvm, u16 port, void *data,
 bool kvm_cpu__emulate_mmio(struct kvm *kvm, u64 phys_addr, u8 *data, u32 len,
 			   u8 is_write);
 
-#endif /* KVM__KVM_CPU_ARCH_H */
+#endif /* ARM_COMMON__KVM_CPU_ARCH_H */
diff --git a/tools/kvm/arm/kvm-cpu.c b/tools/kvm/arm/kvm-cpu.c
index 3b08e55..7a0eff45 100644
--- a/tools/kvm/arm/kvm-cpu.c
+++ b/tools/kvm/arm/kvm-cpu.c
@@ -33,7 +33,9 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
 	struct kvm_cpu *vcpu;
 	int coalesced_offset, mmap_size, err = -1;
 	unsigned int i;
-	struct kvm_vcpu_init vcpu_init = { };
+	struct kvm_vcpu_init vcpu_init = {
+		.features = ARM_VCPU_FEATURE_FLAGS(kvm, cpu_id)
+	};
 
 	vcpu = calloc(1, sizeof(struct kvm_cpu));
 	if (!vcpu)
diff --git a/tools/kvm/arm/kvm.c b/tools/kvm/arm/kvm.c
index bfce685..b10c857 100644
--- a/tools/kvm/arm/kvm.c
+++ b/tools/kvm/arm/kvm.c
@@ -11,6 +11,7 @@
 struct kvm_ext kvm_req_ext[] = {
 	{ DEFINE_KVM_EXT(KVM_CAP_IRQCHIP) },
 	{ DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
+	{ DEFINE_KVM_EXT(KVM_CAP_ARM_PSCI) },
 	{ 0, 0 },
 };
 
diff --git a/tools/kvm/arm/smp.c b/tools/kvm/arm/smp.c
deleted file mode 100644
index c1e59d2..0000000
--- a/tools/kvm/arm/smp.c
+++ /dev/null
@@ -1,21 +0,0 @@
-#include "kvm/kvm.h"
-
-extern u8 smp_pen_start, smp_pen_end, smp_jump_addr;
-
-static int smp_pen_init(struct kvm *kvm)
-{
-	unsigned long pen_size, pen_start, jump_offset;
-
-	if (!(kvm->nrcpus > 1))
-		return 0;
-
-	pen_size = &smp_pen_end - &smp_pen_start;
-	pen_start = kvm->arch.smp_pen_guest_start;
-	jump_offset = &smp_jump_addr - &smp_pen_start;
-
-	kvm->arch.smp_jump_guest_start = pen_start + jump_offset;
-	memcpy(guest_flat_to_host(kvm, pen_start), &smp_pen_start, pen_size);
-
-	return 0;
-}
-firmware_init(smp_pen_init);
-- 
1.8.0