[Qemu-devel] [PATCH v1 RFC 0/6] KVM: s390: cpu model implementation

[Qemu-devel] [PATCH v1 RFC 0/6] KVM: s390: cpu model implementation

[Michael Mueller]

The proposed patch set implements S390 cpu model support in kvm. A cpu
model is defined by a triple comprizing the cpu type , the cpu facility
set and instruction blocking control. A consumer requests a feasible
combination of that that triple before vcpu creation. All subsequently
created vcpu's will use this cpu configuration. 

Michael Mueller (6):
  s390/sclp: add sclp_get_ibc function
  KVM: s390: split SIE state guest prefix field
  KVM: s390: use facilities and cpu_id per KVM
  KVM: s390: add ibc api
  KVM: s390: initial implementation of soft facilities
  KVM: s390: add cpu model support

 arch/s390/include/asm/kvm_host.h |  13 +-
 arch/s390/include/asm/sclp.h     |   1 +
 arch/s390/include/uapi/asm/kvm.h |  23 ++++
 arch/s390/kvm/Makefile           |   2 +-
 arch/s390/kvm/diag.c             |   2 +-
 arch/s390/kvm/gaccess.c          |   4 +-
 arch/s390/kvm/gaccess.h          |   8 +-
 arch/s390/kvm/ibc.c              |  29 +++++
 arch/s390/kvm/kvm-s390.c         | 263 +++++++++++++++++++++++++++++++++------
 arch/s390/kvm/kvm-s390.h         |  43 ++++++-
 arch/s390/kvm/priv.c             |  16 ++-
 arch/s390/kvm/softfac.c          |  38 ++++++
 drivers/s390/char/sclp_early.c   |  12 +-
 13 files changed, 397 insertions(+), 57 deletions(-)
 create mode 100644 arch/s390/kvm/ibc.c
 create mode 100644 arch/s390/kvm/softfac.c

-- 
1.8.3.1

[Qemu-devel] [PATCH v1 RFC 1/6] s390/sclp: add sclp_get_ibc function

[Michael Mueller]

The patch adds functionality to retrieve the IBC configuration
by means of function sclp_get_ibc().

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Acked-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/s390/include/asm/sclp.h   |  1 +
 drivers/s390/char/sclp_early.c | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index 943d434..1aba89b 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -66,5 +66,6 @@ int memcpy_hsa(void *dest, unsigned long src, size_t count, int mode);
 unsigned long sclp_get_hsa_size(void);
 void sclp_early_detect(void);
 int sclp_has_siif(void);
+unsigned int sclp_get_ibc(void);
 
 #endif /* _ASM_S390_SCLP_H */
diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c
index b57fe0e..1918d9d 100644
--- a/drivers/s390/char/sclp_early.c
+++ b/drivers/s390/char/sclp_early.c
@@ -27,7 +27,9 @@ struct read_info_sccb {
 	u8	loadparm[8];		/* 24-31 */
 	u8	_reserved1[48 - 32];	/* 32-47 */
 	u64	facilities;		/* 48-55 */
-	u8	_reserved2[84 - 56];	/* 56-83 */
+	u8	_reserved2a[76 - 56];	/* 56-75 */
+	u32	ibc;			/* 76-79 */
+	u8	_reserved2b[84 - 80];	/* 80-83 */
 	u8	fac84;			/* 84 */
 	u8	fac85;			/* 85 */
 	u8	_reserved3[91 - 86];	/* 86-90 */
@@ -47,6 +49,7 @@ static unsigned long sclp_hsa_size;
 static unsigned int sclp_max_cpu;
 static struct sclp_ipl_info sclp_ipl_info;
 static unsigned char sclp_siif;
+static u32 sclp_ibc;
 
 u64 sclp_facilities;
 u8 sclp_fac84;
@@ -111,6 +114,7 @@ static void __init sclp_facilities_detect(struct read_info_sccb *sccb)
 	sclp_rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2;
 	sclp_rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2;
 	sclp_rzm <<= 20;
+	sclp_ibc = sccb->ibc;
 
 	if (!sccb->hcpua) {
 		if (MACHINE_IS_VM)
@@ -168,6 +172,12 @@ int sclp_has_siif(void)
 }
 EXPORT_SYMBOL(sclp_has_siif);
 
+unsigned int sclp_get_ibc(void)
+{
+	return sclp_ibc;
+}
+EXPORT_SYMBOL(sclp_get_ibc);
+
 /*
  * This function will be called after sclp_facilities_detect(), which gets
  * called from early.c code. The sclp_facilities_detect() function retrieves
-- 
1.8.3.1

[Qemu-devel] [PATCH v1 RFC 2/6] KVM: s390: split SIE state guest prefix field

[Michael Mueller]

This patch splits the SIE state guest prefix at offset 4
into a prefix bit field. Additionally it provides the
access functions:

 - kvm_s390_get_prefix()
 - kvm_s390_set_prefix()

to access the prefix per vcpu.

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
Reviewed-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Acked-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/s390/include/asm/kvm_host.h |  4 +++-
 arch/s390/kvm/diag.c             |  2 +-
 arch/s390/kvm/gaccess.h          |  8 ++++----
 arch/s390/kvm/kvm-s390.c         | 10 ++++++----
 arch/s390/kvm/kvm-s390.h         |  8 +++++++-
 arch/s390/kvm/priv.c             |  3 +--
 6 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 95f3dd4..38d487a 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -82,7 +82,9 @@ struct sca_block {
 
 struct kvm_s390_sie_block {
 	atomic_t cpuflags;		/* 0x0000 */
-	__u32	prefix;			/* 0x0004 */
+	__u32 : 1;			/* 0x0004 */
+	__u32 prefix : 18;
+	__u32 : 13;
 	__u8	reserved08[4];		/* 0x0008 */
 #define PROG_IN_SIE (1<<0)
 	__u32	prog0c;			/* 0x000c */
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 004d385..0161675 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -23,7 +23,7 @@
 static int diag_release_pages(struct kvm_vcpu *vcpu)
 {
 	unsigned long start, end;
-	unsigned long prefix  = vcpu->arch.sie_block->prefix;
+	unsigned long prefix  = kvm_s390_get_prefix(vcpu);
 
 	start = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4];
 	end = vcpu->run->s.regs.gprs[vcpu->arch.sie_block->ipa & 0xf] + 4096;
diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h
index 68db43e..a07ee08 100644
--- a/arch/s390/kvm/gaccess.h
+++ b/arch/s390/kvm/gaccess.h
@@ -30,7 +30,7 @@
 static inline unsigned long kvm_s390_real_to_abs(struct kvm_vcpu *vcpu,
 						 unsigned long gra)
 {
-	unsigned long prefix = vcpu->arch.sie_block->prefix;
+	unsigned long prefix  = kvm_s390_get_prefix(vcpu);
 
 	if (gra < 2 * PAGE_SIZE)
 		gra += prefix;
@@ -99,7 +99,7 @@ static inline unsigned long kvm_s390_logical_to_effective(struct kvm_vcpu *vcpu,
 	unsigned long __gpa;					\
 								\
 	__gpa = (unsigned long)(gra);				\
-	__gpa += __vcpu->arch.sie_block->prefix;		\
+	__gpa += kvm_s390_get_prefix(__vcpu);			\
 	kvm_write_guest(__vcpu->kvm, __gpa, &__x, sizeof(__x));	\
 })
 
@@ -124,7 +124,7 @@ static inline __must_check
 int write_guest_lc(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
 		   unsigned long len)
 {
-	unsigned long gpa = gra + vcpu->arch.sie_block->prefix;
+	unsigned long gpa = gra + kvm_s390_get_prefix(vcpu);
 
 	return kvm_write_guest(vcpu->kvm, gpa, data, len);
 }
@@ -150,7 +150,7 @@ static inline __must_check
 int read_guest_lc(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
 		  unsigned long len)
 {
-	unsigned long gpa = gra + vcpu->arch.sie_block->prefix;
+	unsigned long gpa = gra + kvm_s390_get_prefix(vcpu);
 
 	return kvm_read_guest(vcpu->kvm, gpa, data, len);
 }
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 5dff316..01a5212 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -846,7 +846,7 @@ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		/* match against both prefix pages */
-		if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
+		if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
 			VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 			kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 			exit_sie_sync(vcpu);
@@ -1110,7 +1110,7 @@ retry:
 	if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
 		int rc;
 		rc = gmap_ipte_notify(vcpu->arch.gmap,
-				      vcpu->arch.sie_block->prefix,
+				      kvm_s390_get_prefix(vcpu),
 				      PAGE_SIZE * 2);
 		if (rc)
 			return rc;
@@ -1413,7 +1413,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 	kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
 	kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
-	kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
+	kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
 	memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
 
 	if (vcpu->sigset_active)
@@ -1432,6 +1432,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
 {
 	unsigned char archmode = 1;
+	unsigned int px;
 	u64 clkcomp;
 	int rc;
 
@@ -1450,8 +1451,9 @@ int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
 			      vcpu->run->s.regs.gprs, 128);
 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
 			      &vcpu->arch.sie_block->gpsw, 16);
+	px = kvm_s390_get_prefix(vcpu);
 	rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
-			      &vcpu->arch.sie_block->prefix, 4);
+			      &px, 4);
 	rc |= write_guest_abs(vcpu,
 			      gpa + offsetof(struct save_area, fp_ctrl_reg),
 			      &vcpu->arch.guest_fpregs.fpc, 4);
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 62a5cb4..066fdbc 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -61,9 +61,15 @@ static inline int kvm_is_ucontrol(struct kvm *kvm)
 #endif
 }
 
+#define GUEST_PREFIX_SHIFT 13
+static inline u32 kvm_s390_get_prefix(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.sie_block->prefix << GUEST_PREFIX_SHIFT;
+}
+
 static inline void kvm_s390_set_prefix(struct kvm_vcpu *vcpu, u32 prefix)
 {
-	vcpu->arch.sie_block->prefix = prefix & 0x7fffe000u;
+	vcpu->arch.sie_block->prefix = prefix >> GUEST_PREFIX_SHIFT;
 	vcpu->arch.sie_block->ihcpu  = 0xffff;
 	kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 }
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index a4e5d86..19f3475 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -120,8 +120,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
 	if (operand2 & 3)
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 
-	address = vcpu->arch.sie_block->prefix;
-	address = address & 0x7fffe000u;
+	address = kvm_s390_get_prefix(vcpu);
 
 	/* get the value */
 	rc = write_guest(vcpu, operand2, &address, sizeof(address));
-- 
1.8.3.1

[Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Michael Mueller]

The patch introduces facilities and cpu_ids per virtual machine.
Different virtual machines may want to expose different facilities and
cpu ids to the guest, so let's make them per-vm instead of global.

In addition this patch renames all ocurrences of *facilities to *fac_list
smilar to the already exiting symbol stfl_fac_list in lowcore.

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/s390/include/asm/kvm_host.h |   7 +++
 arch/s390/kvm/gaccess.c          |   4 +-
 arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
 arch/s390/kvm/kvm-s390.h         |  23 +++++++--
 arch/s390/kvm/priv.c             |  13 +++--
 5 files changed, 113 insertions(+), 41 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 38d487a..b4751ba 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -414,6 +414,12 @@ struct kvm_s390_config {
 	struct kvm_s390_attr_name name;
 };
 
+struct kvm_s390_cpu_model {
+	unsigned long *sie_fac;
+	struct cpuid cpu_id;
+	unsigned long *fac_list;
+};
+
 struct kvm_arch{
 	struct sca_block *sca;
 	debug_info_t *dbf;
@@ -427,6 +433,7 @@ struct kvm_arch{
 	wait_queue_head_t ipte_wq;
 	struct kvm_s390_config *cfg;
 	spinlock_t start_stop_lock;
+	struct kvm_s390_cpu_model model;
 };
 
 #define KVM_HVA_ERR_BAD		(-1UL)
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index db608c3..4c7ca40 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
 	union asce asce;
 
 	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
-	edat1 = ctlreg0.edat && test_vfacility(8);
-	edat2 = edat1 && test_vfacility(78);
+	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
+	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
 	asce.val = get_vcpu_asce(vcpu);
 	if (asce.r)
 		goto real_address;
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 01a5212..a53652f 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1,5 +1,5 @@
 /*
- * hosting zSeries kernel virtual machines
+ * Hosting zSeries kernel virtual machines
  *
  * Copyright IBM Corp. 2008, 2009
  *
@@ -30,7 +30,6 @@
 #include <asm/pgtable.h>
 #include <asm/nmi.h>
 #include <asm/switch_to.h>
-#include <asm/facility.h>
 #include <asm/sclp.h>
 #include<asm/timex.h>
 #include "kvm-s390.h"
@@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ NULL }
 };
 
-unsigned long *vfacilities;
-static struct gmap_notifier gmap_notifier;
+/* upper facilities limit for kvm */
+unsigned long kvm_s390_fac_list_mask[] = {
+	0xff82fff3f47c2000UL,
+	0x005c000000000000UL,
+};
+
+unsigned long kvm_s390_fac_list_mask_size(void)
+{
+	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
+		     S390_ARCH_FAC_MASK_SIZE_U64);
+	return ARRAY_SIZE(kvm_s390_fac_list_mask);
+}
 
-/* test availability of vfacility */
-int test_vfacility(unsigned long nr)
+void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
 {
-	return __test_facility(nr, (void *) vfacilities);
+	unsigned int i;
+
+	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
+		if (i < kvm_s390_fac_list_mask_size())
+			fac_list[i] &= kvm_s390_fac_list_mask[i];
+		else
+			fac_list[i] &= 0UL;
+	}
 }
 
+static struct gmap_notifier gmap_notifier;
+
 /* Section: not file related */
 int kvm_arch_hardware_enable(void *garbage)
 {
@@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
 	return r;
 }
 
+/* make sure the memory used for fac_list is zeroed */
+void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
+{
+	int i;
+
+	if (!test_facility(7))
+		memcpy(fac_list, &S390_lowcore.stfl_fac_list, 4);
+	else {
+		register unsigned long reg0 asm("0") = size - 1;
+		asm volatile(".insn s,0xb2b00000,0(%1)" /* stfle */
+			     : "+d" (reg0)
+			     : "a" (fac_list)
+			     : "memory", "cc");
+	}
+	for (i = 0; i < size && i < kvm_s390_fac_list_mask_size(); i++)
+		fac_list[i] &= kvm_s390_fac_list_mask[i];
+}
+
+static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
+{
+	get_cpu_id(cpu_id);
+	cpu_id->version = 0xff;
+}
+
 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 {
 	int rc;
@@ -522,6 +563,23 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	if (!kvm->arch.dbf)
 		goto out_nodbf;
 
+	/*
+	 * The architected maximum amount of facilities is 16 kbit. To store
+	 * this amount, 2 kbyte of memory is required. Thus we need a full
+	 * page to hold the active copy (arch.model.sie_fac) and the current
+	 * facilities set (arch.model.facilities).
+	 */
+	kvm->arch.model.sie_fac = (unsigned long *)
+		get_zeroed_page(GFP_KERNEL|GFP_DMA);
+	if (!kvm->arch.model.sie_fac)
+		goto out_nofac;
+
+	kvm->arch.model.fac_list = kvm->arch.model.sie_fac;
+	kvm->arch.model.fac_list += S390_ARCH_FAC_LIST_SIZE_U64;
+	kvm_s390_get_hard_fac_list(kvm->arch.model.fac_list,
+				kvm_s390_fac_list_mask_size());
+	kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
+
 	spin_lock_init(&kvm->arch.float_int.lock);
 	INIT_LIST_HEAD(&kvm->arch.float_int.list);
 	init_waitqueue_head(&kvm->arch.ipte_wq);
@@ -554,6 +612,8 @@ out_gmap:
 out_nogmap:
 	debug_unregister(kvm->arch.dbf);
 out_nodbf:
+	free_page((unsigned long)kvm->arch.model.sie_fac);
+out_nofac:
 	free_page((unsigned long)(kvm->arch.sca));
 out_err:
 	return rc;
@@ -608,6 +668,7 @@ void kvm_arch_sync_events(struct kvm *kvm)
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvm_free_vcpus(kvm);
+	free_page((unsigned long)kvm->arch.model.sie_fac);
 	free_page((unsigned long)(kvm->arch.sca));
 	debug_unregister(kvm->arch.dbf);
 	if (!kvm_is_ucontrol(kvm))
@@ -720,14 +781,14 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 						    CPUSTAT_STOPPED |
 						    CPUSTAT_GED);
 	vcpu->arch.sie_block->ecb   = 6;
-	if (test_vfacility(50) && test_vfacility(73))
+	if (test_kvm_facility(vcpu->kvm, 50) &&
+	    test_kvm_facility(vcpu->kvm, 73))
 		vcpu->arch.sie_block->ecb |= 0x10;
 
 	vcpu->arch.sie_block->ecb2  = 8;
 	vcpu->arch.sie_block->eca   = 0xD1002000U;
 	if (sclp_has_siif())
 		vcpu->arch.sie_block->eca |= 1;
-	vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 	vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
 				      ICTL_TPROT;
 
@@ -740,8 +801,10 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 	tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 		     (unsigned long) vcpu);
 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
-	get_cpu_id(&vcpu->arch.cpu_id);
-	vcpu->arch.cpu_id.version = 0xff;
+	vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
+	memcpy(vcpu->kvm->arch.model.sie_fac, vcpu->kvm->arch.model.fac_list,
+	       S390_ARCH_FAC_LIST_SIZE_BYTE);
+
 	return rc;
 }
 
@@ -782,6 +845,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 		vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 		set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 	}
+	vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.sie_fac;
 
 	spin_lock_init(&vcpu->arch.local_int.lock);
 	INIT_LIST_HEAD(&vcpu->arch.local_int.list);
@@ -1841,30 +1905,11 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 
 static int __init kvm_s390_init(void)
 {
-	int ret;
-	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
-	if (ret)
-		return ret;
-
-	/*
-	 * guests can ask for up to 255+1 double words, we need a full page
-	 * to hold the maximum amount of facilities. On the other hand, we
-	 * only set facilities that are known to work in KVM.
-	 */
-	vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
-	if (!vfacilities) {
-		kvm_exit();
-		return -ENOMEM;
-	}
-	memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
-	vfacilities[0] &= 0xff82fff3f4fc2000UL;
-	vfacilities[1] &= 0x005c000000000000UL;
-	return 0;
+	return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
 }
 
 static void __exit kvm_s390_exit(void)
 {
-	free_page((unsigned long) vfacilities);
 	kvm_exit();
 }
 
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 066fdbc..ecadc8a 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -18,14 +18,20 @@
 #include <linux/hrtimer.h>
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
+#include <asm/facility.h>
 
 typedef int (*intercept_handler_t)(struct kvm_vcpu *vcpu);
 
-/* declare vfacilities extern */
-extern unsigned long *vfacilities;
-
 int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu);
 
+/* maximum size of facilities and facility mask is 2k bytes */
+#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11)
+#define S390_ARCH_FAC_LIST_SIZE_U64 \
+	(S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64))
+#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE
+#define S390_ARCH_FAC_MASK_SIZE_U64 \
+	(S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64))
+
 /* Transactional Memory Execution related macros */
 #define IS_TE_ENABLED(vcpu)	((vcpu->arch.sie_block->ecb & 0x10))
 #define TDB_FORMAT1		1
@@ -129,6 +135,12 @@ static inline void kvm_s390_set_psw_cc(struct kvm_vcpu *vcpu, unsigned long cc)
 	vcpu->arch.sie_block->gpsw.mask |= cc << 44;
 }
 
+/* test availability of kvm facility */
+static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr)
+{
+	return __test_facility(nr, kvm->arch.model.fac_list);
+}
+
 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu);
 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer);
 void kvm_s390_tasklet(unsigned long parm);
@@ -177,7 +189,10 @@ int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu);
 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu);
 /* is cmma enabled */
 bool kvm_s390_cmma_enabled(struct kvm *kvm);
-int test_vfacility(unsigned long nr);
+unsigned long kvm_s390_fac_list__mask_size(void);
+void kvm_s390_apply_fac_list_mask(unsigned long fac_list[]);
+extern unsigned long kvm_s390_fac_list_mask[];
+void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size);
 
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 19f3475..2de5baf 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -337,19 +337,24 @@ static int handle_io_inst(struct kvm_vcpu *vcpu)
 static int handle_stfl(struct kvm_vcpu *vcpu)
 {
 	int rc;
+	unsigned int fac;
 
 	vcpu->stat.instruction_stfl++;
 
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
+	/*
+	 * We need to shift the lower 32 facility bits (bit 0-31) from a u64
+	 * into a u32 memory representation. They will remain bits 0-31.
+	 */
+	fac = *vcpu->kvm->arch.model.sie_fac >> 32;
 	rc = write_guest_lc(vcpu, offsetof(struct _lowcore, stfl_fac_list),
-			    vfacilities, 4);
+			    &fac, sizeof(fac));
 	if (rc)
 		return rc;
-	VCPU_EVENT(vcpu, 5, "store facility list value %x",
-		   *(unsigned int *) vfacilities);
-	trace_kvm_s390_handle_stfl(vcpu, *(unsigned int *) vfacilities);
+	VCPU_EVENT(vcpu, 5, "store facility list value %x", fac);
+	trace_kvm_s390_handle_stfl(vcpu, fac);
 	return 0;
 }
 
-- 
1.8.3.1

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Alexander Graf]

On 13.05.14 16:58, Michael Mueller wrote:
> The patch introduces facilities and cpu_ids per virtual machine.
> Different virtual machines may want to expose different facilities and
> cpu ids to the guest, so let's make them per-vm instead of global.
>
> In addition this patch renames all ocurrences of *facilities to *fac_list
> smilar to the already exiting symbol stfl_fac_list in lowcore.
>
> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>   arch/s390/include/asm/kvm_host.h |   7 +++
>   arch/s390/kvm/gaccess.c          |   4 +-
>   arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
>   arch/s390/kvm/kvm-s390.h         |  23 +++++++--
>   arch/s390/kvm/priv.c             |  13 +++--
>   5 files changed, 113 insertions(+), 41 deletions(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 38d487a..b4751ba 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -414,6 +414,12 @@ struct kvm_s390_config {
>   	struct kvm_s390_attr_name name;
>   };
>   
> +struct kvm_s390_cpu_model {
> +	unsigned long *sie_fac;
> +	struct cpuid cpu_id;
> +	unsigned long *fac_list;
> +};
> +
>   struct kvm_arch{
>   	struct sca_block *sca;
>   	debug_info_t *dbf;
> @@ -427,6 +433,7 @@ struct kvm_arch{
>   	wait_queue_head_t ipte_wq;
>   	struct kvm_s390_config *cfg;
>   	spinlock_t start_stop_lock;
> +	struct kvm_s390_cpu_model model;
>   };
>   
>   #define KVM_HVA_ERR_BAD		(-1UL)
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index db608c3..4c7ca40 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
>   	union asce asce;
>   
>   	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
> -	edat1 = ctlreg0.edat && test_vfacility(8);
> -	edat2 = edat1 && test_vfacility(78);
> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
>   	asce.val = get_vcpu_asce(vcpu);
>   	if (asce.r)
>   		goto real_address;
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 01a5212..a53652f 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -1,5 +1,5 @@
>   /*
> - * hosting zSeries kernel virtual machines
> + * Hosting zSeries kernel virtual machines
>    *
>    * Copyright IBM Corp. 2008, 2009
>    *
> @@ -30,7 +30,6 @@
>   #include <asm/pgtable.h>
>   #include <asm/nmi.h>
>   #include <asm/switch_to.h>
> -#include <asm/facility.h>
>   #include <asm/sclp.h>
>   #include<asm/timex.h>
>   #include "kvm-s390.h"
> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>   	{ NULL }
>   };
>   
> -unsigned long *vfacilities;
> -static struct gmap_notifier gmap_notifier;
> +/* upper facilities limit for kvm */
> +unsigned long kvm_s390_fac_list_mask[] = {
> +	0xff82fff3f47c2000UL,
> +	0x005c000000000000UL,
> +};
> +
> +unsigned long kvm_s390_fac_list_mask_size(void)
> +{
> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
> +		     S390_ARCH_FAC_MASK_SIZE_U64);
> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
> +}
>   
> -/* test availability of vfacility */
> -int test_vfacility(unsigned long nr)
> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
>   {
> -	return __test_facility(nr, (void *) vfacilities);
> +	unsigned int i;
> +
> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
> +		if (i < kvm_s390_fac_list_mask_size())
> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
> +		else
> +			fac_list[i] &= 0UL;
> +	}
>   }
>   
> +static struct gmap_notifier gmap_notifier;
> +
>   /* Section: not file related */
>   int kvm_arch_hardware_enable(void *garbage)
>   {
> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
>   	return r;
>   }
>   
> +/* make sure the memory used for fac_list is zeroed */
> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)

Hard? Wouldn't "host" make more sense here?

I also think it makes sense to expose the native host facility list to 
user space via an ioctl somehow.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Michael Mueller]

On Fri, 16 May 2014 13:55:41 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 13.05.14 16:58, Michael Mueller wrote:
> > The patch introduces facilities and cpu_ids per virtual machine.
> > Different virtual machines may want to expose different facilities and
> > cpu ids to the guest, so let's make them per-vm instead of global.
> >
> > In addition this patch renames all ocurrences of *facilities to *fac_list
> > smilar to the already exiting symbol stfl_fac_list in lowcore.
> >
> > Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> > Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> > Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> > ---
> >   arch/s390/include/asm/kvm_host.h |   7 +++
> >   arch/s390/kvm/gaccess.c          |   4 +-
> >   arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
> >   arch/s390/kvm/kvm-s390.h         |  23 +++++++--
> >   arch/s390/kvm/priv.c             |  13 +++--
> >   5 files changed, 113 insertions(+), 41 deletions(-)
> >
> > diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> > index 38d487a..b4751ba 100644
> > --- a/arch/s390/include/asm/kvm_host.h
> > +++ b/arch/s390/include/asm/kvm_host.h
> > @@ -414,6 +414,12 @@ struct kvm_s390_config {
> >   	struct kvm_s390_attr_name name;
> >   };
> >   
> > +struct kvm_s390_cpu_model {
> > +	unsigned long *sie_fac;
> > +	struct cpuid cpu_id;
> > +	unsigned long *fac_list;
> > +};
> > +
> >   struct kvm_arch{
> >   	struct sca_block *sca;
> >   	debug_info_t *dbf;
> > @@ -427,6 +433,7 @@ struct kvm_arch{
> >   	wait_queue_head_t ipte_wq;
> >   	struct kvm_s390_config *cfg;
> >   	spinlock_t start_stop_lock;
> > +	struct kvm_s390_cpu_model model;
> >   };
> >   
> >   #define KVM_HVA_ERR_BAD		(-1UL)
> > diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> > index db608c3..4c7ca40 100644
> > --- a/arch/s390/kvm/gaccess.c
> > +++ b/arch/s390/kvm/gaccess.c
> > @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long
> > gva, union asce asce;
> >   
> >   	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
> > -	edat1 = ctlreg0.edat && test_vfacility(8);
> > -	edat2 = edat1 && test_vfacility(78);
> > +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
> > +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
> >   	asce.val = get_vcpu_asce(vcpu);
> >   	if (asce.r)
> >   		goto real_address;
> > diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> > index 01a5212..a53652f 100644
> > --- a/arch/s390/kvm/kvm-s390.c
> > +++ b/arch/s390/kvm/kvm-s390.c
> > @@ -1,5 +1,5 @@
> >   /*
> > - * hosting zSeries kernel virtual machines
> > + * Hosting zSeries kernel virtual machines
> >    *
> >    * Copyright IBM Corp. 2008, 2009
> >    *
> > @@ -30,7 +30,6 @@
> >   #include <asm/pgtable.h>
> >   #include <asm/nmi.h>
> >   #include <asm/switch_to.h>
> > -#include <asm/facility.h>
> >   #include <asm/sclp.h>
> >   #include<asm/timex.h>
> >   #include "kvm-s390.h"
> > @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
> >   	{ NULL }
> >   };
> >   
> > -unsigned long *vfacilities;
> > -static struct gmap_notifier gmap_notifier;
> > +/* upper facilities limit for kvm */
> > +unsigned long kvm_s390_fac_list_mask[] = {
> > +	0xff82fff3f47c2000UL,
> > +	0x005c000000000000UL,
> > +};
> > +
> > +unsigned long kvm_s390_fac_list_mask_size(void)
> > +{
> > +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
> > +		     S390_ARCH_FAC_MASK_SIZE_U64);
> > +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
> > +}
> >   
> > -/* test availability of vfacility */
> > -int test_vfacility(unsigned long nr)
> > +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
> >   {
> > -	return __test_facility(nr, (void *) vfacilities);
> > +	unsigned int i;
> > +
> > +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
> > +		if (i < kvm_s390_fac_list_mask_size())
> > +			fac_list[i] &= kvm_s390_fac_list_mask[i];
> > +		else
> > +			fac_list[i] &= 0UL;
> > +	}
> >   }
> >   
> > +static struct gmap_notifier gmap_notifier;
> > +
> >   /* Section: not file related */
> >   int kvm_arch_hardware_enable(void *garbage)
> >   {
> > @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
> >   	return r;
> >   }
> >   
> > +/* make sure the memory used for fac_list is zeroed */
> > +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
> 
> Hard? Wouldn't "host" make more sense here?

Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me. 

> 
> I also think it makes sense to expose the native host facility list to 
> user space via an ioctl somehow.
> 

In which situation do you need the full facility list. Do you have an example?

> 
> Alex
> 
> 

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Alexander Graf]

On 16.05.14 16:46, Michael Mueller wrote:
> On Fri, 16 May 2014 13:55:41 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 13.05.14 16:58, Michael Mueller wrote:
>>> The patch introduces facilities and cpu_ids per virtual machine.
>>> Different virtual machines may want to expose different facilities and
>>> cpu ids to the guest, so let's make them per-vm instead of global.
>>>
>>> In addition this patch renames all ocurrences of *facilities to *fac_list
>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
>>>
>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>>> ---
>>>    arch/s390/include/asm/kvm_host.h |   7 +++
>>>    arch/s390/kvm/gaccess.c          |   4 +-
>>>    arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
>>>    arch/s390/kvm/kvm-s390.h         |  23 +++++++--
>>>    arch/s390/kvm/priv.c             |  13 +++--
>>>    5 files changed, 113 insertions(+), 41 deletions(-)
>>>
>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>> index 38d487a..b4751ba 100644
>>> --- a/arch/s390/include/asm/kvm_host.h
>>> +++ b/arch/s390/include/asm/kvm_host.h
>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
>>>    	struct kvm_s390_attr_name name;
>>>    };
>>>    
>>> +struct kvm_s390_cpu_model {
>>> +	unsigned long *sie_fac;
>>> +	struct cpuid cpu_id;
>>> +	unsigned long *fac_list;
>>> +};
>>> +
>>>    struct kvm_arch{
>>>    	struct sca_block *sca;
>>>    	debug_info_t *dbf;
>>> @@ -427,6 +433,7 @@ struct kvm_arch{
>>>    	wait_queue_head_t ipte_wq;
>>>    	struct kvm_s390_config *cfg;
>>>    	spinlock_t start_stop_lock;
>>> +	struct kvm_s390_cpu_model model;
>>>    };
>>>    
>>>    #define KVM_HVA_ERR_BAD		(-1UL)
>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
>>> index db608c3..4c7ca40 100644
>>> --- a/arch/s390/kvm/gaccess.c
>>> +++ b/arch/s390/kvm/gaccess.c
>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long
>>> gva, union asce asce;
>>>    
>>>    	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
>>> -	edat2 = edat1 && test_vfacility(78);
>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
>>>    	asce.val = get_vcpu_asce(vcpu);
>>>    	if (asce.r)
>>>    		goto real_address;
>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>> index 01a5212..a53652f 100644
>>> --- a/arch/s390/kvm/kvm-s390.c
>>> +++ b/arch/s390/kvm/kvm-s390.c
>>> @@ -1,5 +1,5 @@
>>>    /*
>>> - * hosting zSeries kernel virtual machines
>>> + * Hosting zSeries kernel virtual machines
>>>     *
>>>     * Copyright IBM Corp. 2008, 2009
>>>     *
>>> @@ -30,7 +30,6 @@
>>>    #include <asm/pgtable.h>
>>>    #include <asm/nmi.h>
>>>    #include <asm/switch_to.h>
>>> -#include <asm/facility.h>
>>>    #include <asm/sclp.h>
>>>    #include<asm/timex.h>
>>>    #include "kvm-s390.h"
>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>>>    	{ NULL }
>>>    };
>>>    
>>> -unsigned long *vfacilities;
>>> -static struct gmap_notifier gmap_notifier;
>>> +/* upper facilities limit for kvm */
>>> +unsigned long kvm_s390_fac_list_mask[] = {
>>> +	0xff82fff3f47c2000UL,
>>> +	0x005c000000000000UL,
>>> +};
>>> +
>>> +unsigned long kvm_s390_fac_list_mask_size(void)
>>> +{
>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
>>> +}
>>>    
>>> -/* test availability of vfacility */
>>> -int test_vfacility(unsigned long nr)
>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
>>>    {
>>> -	return __test_facility(nr, (void *) vfacilities);
>>> +	unsigned int i;
>>> +
>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
>>> +		if (i < kvm_s390_fac_list_mask_size())
>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
>>> +		else
>>> +			fac_list[i] &= 0UL;
>>> +	}
>>>    }
>>>    
>>> +static struct gmap_notifier gmap_notifier;
>>> +
>>>    /* Section: not file related */
>>>    int kvm_arch_hardware_enable(void *garbage)
>>>    {
>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
>>>    	return r;
>>>    }
>>>    
>>> +/* make sure the memory used for fac_list is zeroed */
>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
>> Hard? Wouldn't "host" make more sense here?
> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me.
>
>> I also think it makes sense to expose the native host facility list to
>> user space via an ioctl somehow.
>>
> In which situation do you need the full facility list. Do you have an example?

If you want to just implement -cpu host to get the exact feature set 
that the host gives you, how do you know which set that is?


Alex

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Michael Mueller]

On Fri, 16 May 2014 16:49:37 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 16.05.14 16:46, Michael Mueller wrote:
> > On Fri, 16 May 2014 13:55:41 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 13.05.14 16:58, Michael Mueller wrote:
> >>> The patch introduces facilities and cpu_ids per virtual machine.
> >>> Different virtual machines may want to expose different facilities and
> >>> cpu ids to the guest, so let's make them per-vm instead of global.
> >>>
> >>> In addition this patch renames all ocurrences of *facilities to *fac_list
> >>> smilar to the already exiting symbol stfl_fac_list in lowcore.
> >>>
> >>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> >>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> >>> ---
> >>>    arch/s390/include/asm/kvm_host.h |   7 +++
> >>>    arch/s390/kvm/gaccess.c          |   4 +-
> >>>    arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
> >>>    arch/s390/kvm/kvm-s390.h         |  23 +++++++--
> >>>    arch/s390/kvm/priv.c             |  13 +++--
> >>>    5 files changed, 113 insertions(+), 41 deletions(-)
> >>>
> >>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>> index 38d487a..b4751ba 100644
> >>> --- a/arch/s390/include/asm/kvm_host.h
> >>> +++ b/arch/s390/include/asm/kvm_host.h
> >>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
> >>>    	struct kvm_s390_attr_name name;
> >>>    };
> >>>    
> >>> +struct kvm_s390_cpu_model {
> >>> +	unsigned long *sie_fac;
> >>> +	struct cpuid cpu_id;
> >>> +	unsigned long *fac_list;
> >>> +};
> >>> +
> >>>    struct kvm_arch{
> >>>    	struct sca_block *sca;
> >>>    	debug_info_t *dbf;
> >>> @@ -427,6 +433,7 @@ struct kvm_arch{
> >>>    	wait_queue_head_t ipte_wq;
> >>>    	struct kvm_s390_config *cfg;
> >>>    	spinlock_t start_stop_lock;
> >>> +	struct kvm_s390_cpu_model model;
> >>>    };
> >>>    
> >>>    #define KVM_HVA_ERR_BAD		(-1UL)
> >>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> >>> index db608c3..4c7ca40 100644
> >>> --- a/arch/s390/kvm/gaccess.c
> >>> +++ b/arch/s390/kvm/gaccess.c
> >>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
> >>> long gva, union asce asce;
> >>>    
> >>>    	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
> >>> -	edat1 = ctlreg0.edat && test_vfacility(8);
> >>> -	edat2 = edat1 && test_vfacility(78);
> >>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
> >>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
> >>>    	asce.val = get_vcpu_asce(vcpu);
> >>>    	if (asce.r)
> >>>    		goto real_address;
> >>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>> index 01a5212..a53652f 100644
> >>> --- a/arch/s390/kvm/kvm-s390.c
> >>> +++ b/arch/s390/kvm/kvm-s390.c
> >>> @@ -1,5 +1,5 @@
> >>>    /*
> >>> - * hosting zSeries kernel virtual machines
> >>> + * Hosting zSeries kernel virtual machines
> >>>     *
> >>>     * Copyright IBM Corp. 2008, 2009
> >>>     *
> >>> @@ -30,7 +30,6 @@
> >>>    #include <asm/pgtable.h>
> >>>    #include <asm/nmi.h>
> >>>    #include <asm/switch_to.h>
> >>> -#include <asm/facility.h>
> >>>    #include <asm/sclp.h>
> >>>    #include<asm/timex.h>
> >>>    #include "kvm-s390.h"
> >>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
> >>>    	{ NULL }
> >>>    };
> >>>    
> >>> -unsigned long *vfacilities;
> >>> -static struct gmap_notifier gmap_notifier;
> >>> +/* upper facilities limit for kvm */
> >>> +unsigned long kvm_s390_fac_list_mask[] = {
> >>> +	0xff82fff3f47c2000UL,
> >>> +	0x005c000000000000UL,
> >>> +};
> >>> +
> >>> +unsigned long kvm_s390_fac_list_mask_size(void)
> >>> +{
> >>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
> >>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
> >>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
> >>> +}
> >>>    
> >>> -/* test availability of vfacility */
> >>> -int test_vfacility(unsigned long nr)
> >>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
> >>>    {
> >>> -	return __test_facility(nr, (void *) vfacilities);
> >>> +	unsigned int i;
> >>> +
> >>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
> >>> +		if (i < kvm_s390_fac_list_mask_size())
> >>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
> >>> +		else
> >>> +			fac_list[i] &= 0UL;
> >>> +	}
> >>>    }
> >>>    
> >>> +static struct gmap_notifier gmap_notifier;
> >>> +
> >>>    /* Section: not file related */
> >>>    int kvm_arch_hardware_enable(void *garbage)
> >>>    {
> >>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
> >>>    	return r;
> >>>    }
> >>>    
> >>> +/* make sure the memory used for fac_list is zeroed */
> >>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
> >> Hard? Wouldn't "host" make more sense here?
> > Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me.
> >
> >> I also think it makes sense to expose the native host facility list to
> >> user space via an ioctl somehow.
> >>
> > In which situation do you need the full facility list. Do you have an example?
> 
> If you want to just implement -cpu host to get the exact feature set 
> that the host gives you, how do you know which set that is?

During qemu machine initalization I call:

kvm_s390_get_machine_props(&mach);

which returns the following information:

typedef struct S390MachineProps {
    uint64_t cpuid;
    uint32_t ibc_range;
    uint8_t  pad[4];
    uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
    uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
    uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
} S390MachineProps;

In addition, user space has a model that defines which CPU facility became available in
regard to the CPU model (TYPE-GA#).

Let Fn be the CPU facility set of an arbitrary machine type, Mh and Fh the facility mask and list
of the current host. Then I just calculate Fn & Mh and match it with Fh (Fh is STFLE & Mh). All
machines where Fn is a full or partial subset are allowed as cpu models, the latest/newest of them
represents "host".

That is basically all done in use space routine:

s390_setup_cpu_classes(S390MachineProps *prop);

> 
> 
> Alex
> 
> 

Michael

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Alexander Graf]

On 16.05.14 18:09, Michael Mueller wrote:
> On Fri, 16 May 2014 16:49:37 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 16.05.14 16:46, Michael Mueller wrote:
>>> On Fri, 16 May 2014 13:55:41 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>> The patch introduces facilities and cpu_ids per virtual machine.
>>>>> Different virtual machines may want to expose different facilities and
>>>>> cpu ids to the guest, so let's make them per-vm instead of global.
>>>>>
>>>>> In addition this patch renames all ocurrences of *facilities to *fac_list
>>>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
>>>>>
>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
>>>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>>>>> ---
>>>>>     arch/s390/include/asm/kvm_host.h |   7 +++
>>>>>     arch/s390/kvm/gaccess.c          |   4 +-
>>>>>     arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
>>>>>     arch/s390/kvm/kvm-s390.h         |  23 +++++++--
>>>>>     arch/s390/kvm/priv.c             |  13 +++--
>>>>>     5 files changed, 113 insertions(+), 41 deletions(-)
>>>>>
>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>> index 38d487a..b4751ba 100644
>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
>>>>>     	struct kvm_s390_attr_name name;
>>>>>     };
>>>>>     
>>>>> +struct kvm_s390_cpu_model {
>>>>> +	unsigned long *sie_fac;
>>>>> +	struct cpuid cpu_id;
>>>>> +	unsigned long *fac_list;
>>>>> +};
>>>>> +
>>>>>     struct kvm_arch{
>>>>>     	struct sca_block *sca;
>>>>>     	debug_info_t *dbf;
>>>>> @@ -427,6 +433,7 @@ struct kvm_arch{
>>>>>     	wait_queue_head_t ipte_wq;
>>>>>     	struct kvm_s390_config *cfg;
>>>>>     	spinlock_t start_stop_lock;
>>>>> +	struct kvm_s390_cpu_model model;
>>>>>     };
>>>>>     
>>>>>     #define KVM_HVA_ERR_BAD		(-1UL)
>>>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
>>>>> index db608c3..4c7ca40 100644
>>>>> --- a/arch/s390/kvm/gaccess.c
>>>>> +++ b/arch/s390/kvm/gaccess.c
>>>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
>>>>> long gva, union asce asce;
>>>>>     
>>>>>     	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
>>>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
>>>>> -	edat2 = edat1 && test_vfacility(78);
>>>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
>>>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
>>>>>     	asce.val = get_vcpu_asce(vcpu);
>>>>>     	if (asce.r)
>>>>>     		goto real_address;
>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>> index 01a5212..a53652f 100644
>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>> @@ -1,5 +1,5 @@
>>>>>     /*
>>>>> - * hosting zSeries kernel virtual machines
>>>>> + * Hosting zSeries kernel virtual machines
>>>>>      *
>>>>>      * Copyright IBM Corp. 2008, 2009
>>>>>      *
>>>>> @@ -30,7 +30,6 @@
>>>>>     #include <asm/pgtable.h>
>>>>>     #include <asm/nmi.h>
>>>>>     #include <asm/switch_to.h>
>>>>> -#include <asm/facility.h>
>>>>>     #include <asm/sclp.h>
>>>>>     #include<asm/timex.h>
>>>>>     #include "kvm-s390.h"
>>>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>>>>>     	{ NULL }
>>>>>     };
>>>>>     
>>>>> -unsigned long *vfacilities;
>>>>> -static struct gmap_notifier gmap_notifier;
>>>>> +/* upper facilities limit for kvm */
>>>>> +unsigned long kvm_s390_fac_list_mask[] = {
>>>>> +	0xff82fff3f47c2000UL,
>>>>> +	0x005c000000000000UL,
>>>>> +};
>>>>> +
>>>>> +unsigned long kvm_s390_fac_list_mask_size(void)
>>>>> +{
>>>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
>>>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
>>>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
>>>>> +}
>>>>>     
>>>>> -/* test availability of vfacility */
>>>>> -int test_vfacility(unsigned long nr)
>>>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
>>>>>     {
>>>>> -	return __test_facility(nr, (void *) vfacilities);
>>>>> +	unsigned int i;
>>>>> +
>>>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
>>>>> +		if (i < kvm_s390_fac_list_mask_size())
>>>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
>>>>> +		else
>>>>> +			fac_list[i] &= 0UL;
>>>>> +	}
>>>>>     }
>>>>>     
>>>>> +static struct gmap_notifier gmap_notifier;
>>>>> +
>>>>>     /* Section: not file related */
>>>>>     int kvm_arch_hardware_enable(void *garbage)
>>>>>     {
>>>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
>>>>>     	return r;
>>>>>     }
>>>>>     
>>>>> +/* make sure the memory used for fac_list is zeroed */
>>>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
>>>> Hard? Wouldn't "host" make more sense here?
>>> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me.
>>>
>>>> I also think it makes sense to expose the native host facility list to
>>>> user space via an ioctl somehow.
>>>>
>>> In which situation do you need the full facility list. Do you have an example?
>> If you want to just implement -cpu host to get the exact feature set
>> that the host gives you, how do you know which set that is?
> During qemu machine initalization I call:
>
> kvm_s390_get_machine_props(&mach);
>
> which returns the following information:
>
> typedef struct S390MachineProps {
>      uint64_t cpuid;
>      uint32_t ibc_range;
>      uint8_t  pad[4];
>      uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
>      uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
>      uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
> } S390MachineProps;

Ah, ok, I missed that part. So "kvm_s390_get_machine_props()" basically 
gets us the full facility list the host supports. That makes sense.

I still don't know whether we should care about hard/soft, but I suppose 
the rest makes sense.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Michael Mueller]

On Fri, 16 May 2014 22:35:34 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 16.05.14 18:09, Michael Mueller wrote:
> > On Fri, 16 May 2014 16:49:37 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 16.05.14 16:46, Michael Mueller wrote:
> >>> On Fri, 16 May 2014 13:55:41 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 13.05.14 16:58, Michael Mueller wrote:
> >>>>> The patch introduces facilities and cpu_ids per virtual machine.
> >>>>> Different virtual machines may want to expose different facilities and
> >>>>> cpu ids to the guest, so let's make them per-vm instead of global.
> >>>>>
> >>>>> In addition this patch renames all ocurrences of *facilities to *fac_list
> >>>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
> >>>>>
> >>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> >>>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> >>>>> ---
> >>>>>     arch/s390/include/asm/kvm_host.h |   7 +++
> >>>>>     arch/s390/kvm/gaccess.c          |   4 +-
> >>>>>     arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
> >>>>>     arch/s390/kvm/kvm-s390.h         |  23 +++++++--
> >>>>>     arch/s390/kvm/priv.c             |  13 +++--
> >>>>>     5 files changed, 113 insertions(+), 41 deletions(-)
> >>>>>
> >>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>>>> index 38d487a..b4751ba 100644
> >>>>> --- a/arch/s390/include/asm/kvm_host.h
> >>>>> +++ b/arch/s390/include/asm/kvm_host.h
> >>>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
> >>>>>     	struct kvm_s390_attr_name name;
> >>>>>     };
> >>>>>     
> >>>>> +struct kvm_s390_cpu_model {
> >>>>> +	unsigned long *sie_fac;
> >>>>> +	struct cpuid cpu_id;
> >>>>> +	unsigned long *fac_list;
> >>>>> +};
> >>>>> +
> >>>>>     struct kvm_arch{
> >>>>>     	struct sca_block *sca;
> >>>>>     	debug_info_t *dbf;
> >>>>> @@ -427,6 +433,7 @@ struct kvm_arch{
> >>>>>     	wait_queue_head_t ipte_wq;
> >>>>>     	struct kvm_s390_config *cfg;
> >>>>>     	spinlock_t start_stop_lock;
> >>>>> +	struct kvm_s390_cpu_model model;
> >>>>>     };
> >>>>>     
> >>>>>     #define KVM_HVA_ERR_BAD		(-1UL)
> >>>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> >>>>> index db608c3..4c7ca40 100644
> >>>>> --- a/arch/s390/kvm/gaccess.c
> >>>>> +++ b/arch/s390/kvm/gaccess.c
> >>>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
> >>>>> long gva, union asce asce;
> >>>>>     
> >>>>>     	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
> >>>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
> >>>>> -	edat2 = edat1 && test_vfacility(78);
> >>>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
> >>>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
> >>>>>     	asce.val = get_vcpu_asce(vcpu);
> >>>>>     	if (asce.r)
> >>>>>     		goto real_address;
> >>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>>>> index 01a5212..a53652f 100644
> >>>>> --- a/arch/s390/kvm/kvm-s390.c
> >>>>> +++ b/arch/s390/kvm/kvm-s390.c
> >>>>> @@ -1,5 +1,5 @@
> >>>>>     /*
> >>>>> - * hosting zSeries kernel virtual machines
> >>>>> + * Hosting zSeries kernel virtual machines
> >>>>>      *
> >>>>>      * Copyright IBM Corp. 2008, 2009
> >>>>>      *
> >>>>> @@ -30,7 +30,6 @@
> >>>>>     #include <asm/pgtable.h>
> >>>>>     #include <asm/nmi.h>
> >>>>>     #include <asm/switch_to.h>
> >>>>> -#include <asm/facility.h>
> >>>>>     #include <asm/sclp.h>
> >>>>>     #include<asm/timex.h>
> >>>>>     #include "kvm-s390.h"
> >>>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
> >>>>>     	{ NULL }
> >>>>>     };
> >>>>>     
> >>>>> -unsigned long *vfacilities;
> >>>>> -static struct gmap_notifier gmap_notifier;
> >>>>> +/* upper facilities limit for kvm */
> >>>>> +unsigned long kvm_s390_fac_list_mask[] = {
> >>>>> +	0xff82fff3f47c2000UL,
> >>>>> +	0x005c000000000000UL,
> >>>>> +};
> >>>>> +
> >>>>> +unsigned long kvm_s390_fac_list_mask_size(void)
> >>>>> +{
> >>>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
> >>>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
> >>>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
> >>>>> +}
> >>>>>     
> >>>>> -/* test availability of vfacility */
> >>>>> -int test_vfacility(unsigned long nr)
> >>>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
> >>>>>     {
> >>>>> -	return __test_facility(nr, (void *) vfacilities);
> >>>>> +	unsigned int i;
> >>>>> +
> >>>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
> >>>>> +		if (i < kvm_s390_fac_list_mask_size())
> >>>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
> >>>>> +		else
> >>>>> +			fac_list[i] &= 0UL;
> >>>>> +	}
> >>>>>     }
> >>>>>     
> >>>>> +static struct gmap_notifier gmap_notifier;
> >>>>> +
> >>>>>     /* Section: not file related */
> >>>>>     int kvm_arch_hardware_enable(void *garbage)
> >>>>>     {
> >>>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
> >>>>>     	return r;
> >>>>>     }
> >>>>>     
> >>>>> +/* make sure the memory used for fac_list is zeroed */
> >>>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
> >>>> Hard? Wouldn't "host" make more sense here?
> >>> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me.
> >>>
> >>>> I also think it makes sense to expose the native host facility list to
> >>>> user space via an ioctl somehow.
> >>>>
> >>> In which situation do you need the full facility list. Do you have an example?
> >> If you want to just implement -cpu host to get the exact feature set
> >> that the host gives you, how do you know which set that is?
> > During qemu machine initalization I call:
> >
> > kvm_s390_get_machine_props(&mach);
> >
> > which returns the following information:
> >
> > typedef struct S390MachineProps {
> >      uint64_t cpuid;
> >      uint32_t ibc_range;
> >      uint8_t  pad[4];
> >      uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
> >      uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
> >      uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
> > } S390MachineProps;
> 
> Ah, ok, I missed that part. So "kvm_s390_get_machine_props()" basically 
> gets us the full facility list the host supports. That makes sense.
> 
> I still don't know whether we should care about hard/soft, but I suppose 
> the rest makes sense.

The reason why I differentiate between hardware and software implemented
facilities has several reasons.

First, one should understand that both lists represent the same set of
features, there is no software feature not being implemented as hardware
feature on an existing S390 system. An optional implementation of a software
feature is usable on back-level hardware to implement functionality, not
quality or performance! That means, on OS and it's application is capable to
run most efficiently on the given hardware without that software feature.
If a system has the same facility in soft- and hardware implemented, KVM
always privileges the hardware implementation.

Second, separating both features allow user space to differentiate between
hardware and software implemented features and to give up on SW features
if not explicitly requested by the CPU model option "+sofl".

Third, and very important, the main reason for CPU model implementation
is to guarantee an identical environment on source and target side comprised
of facilities, instruction set and CPU id. The capability for migration is
lost as soon software implemented facilities become activated!

Once again the method Qemu uses to define the set of requested facilities.

The machine properties ioctl provides FMh, HFh, and SFh, ie. the mask and
hardware/software implemented facility sets of the hosts.

Qemu has knowledge on all real existing CPU models (TYPE-GA), particularly
on the facility set of these model (Fm)
 
During startup, Qemu retrives FMh, HFh and SFh. Then for all CPU models Qemu
is aware off, it calculates (Fm & FMh). If that is a subset of HFh, the model
is a supportable model on this host. If the user has specified to allow
software implemented facilities (-cpu <model>,+sofl), The facilities to
request are calculated like (Fm & FMh | SFh).

> 
> Alex
> 

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Alexander Graf]

On 19.05.14 12:13, Michael Mueller wrote:
> On Fri, 16 May 2014 22:35:34 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 16.05.14 18:09, Michael Mueller wrote:
>>> On Fri, 16 May 2014 16:49:37 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 16.05.14 16:46, Michael Mueller wrote:
>>>>> On Fri, 16 May 2014 13:55:41 +0200
>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>
>>>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>>>> The patch introduces facilities and cpu_ids per virtual machine.
>>>>>>> Different virtual machines may want to expose different facilities and
>>>>>>> cpu ids to the guest, so let's make them per-vm instead of global.
>>>>>>>
>>>>>>> In addition this patch renames all ocurrences of *facilities to *fac_list
>>>>>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
>>>>>>>
>>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
>>>>>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>>>>>>> ---
>>>>>>>      arch/s390/include/asm/kvm_host.h |   7 +++
>>>>>>>      arch/s390/kvm/gaccess.c          |   4 +-
>>>>>>>      arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
>>>>>>>      arch/s390/kvm/kvm-s390.h         |  23 +++++++--
>>>>>>>      arch/s390/kvm/priv.c             |  13 +++--
>>>>>>>      5 files changed, 113 insertions(+), 41 deletions(-)
>>>>>>>
>>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>>>> index 38d487a..b4751ba 100644
>>>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
>>>>>>>      	struct kvm_s390_attr_name name;
>>>>>>>      };
>>>>>>>      
>>>>>>> +struct kvm_s390_cpu_model {
>>>>>>> +	unsigned long *sie_fac;
>>>>>>> +	struct cpuid cpu_id;
>>>>>>> +	unsigned long *fac_list;
>>>>>>> +};
>>>>>>> +
>>>>>>>      struct kvm_arch{
>>>>>>>      	struct sca_block *sca;
>>>>>>>      	debug_info_t *dbf;
>>>>>>> @@ -427,6 +433,7 @@ struct kvm_arch{
>>>>>>>      	wait_queue_head_t ipte_wq;
>>>>>>>      	struct kvm_s390_config *cfg;
>>>>>>>      	spinlock_t start_stop_lock;
>>>>>>> +	struct kvm_s390_cpu_model model;
>>>>>>>      };
>>>>>>>      
>>>>>>>      #define KVM_HVA_ERR_BAD		(-1UL)
>>>>>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
>>>>>>> index db608c3..4c7ca40 100644
>>>>>>> --- a/arch/s390/kvm/gaccess.c
>>>>>>> +++ b/arch/s390/kvm/gaccess.c
>>>>>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
>>>>>>> long gva, union asce asce;
>>>>>>>      
>>>>>>>      	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
>>>>>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
>>>>>>> -	edat2 = edat1 && test_vfacility(78);
>>>>>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
>>>>>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
>>>>>>>      	asce.val = get_vcpu_asce(vcpu);
>>>>>>>      	if (asce.r)
>>>>>>>      		goto real_address;
>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>> index 01a5212..a53652f 100644
>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>> @@ -1,5 +1,5 @@
>>>>>>>      /*
>>>>>>> - * hosting zSeries kernel virtual machines
>>>>>>> + * Hosting zSeries kernel virtual machines
>>>>>>>       *
>>>>>>>       * Copyright IBM Corp. 2008, 2009
>>>>>>>       *
>>>>>>> @@ -30,7 +30,6 @@
>>>>>>>      #include <asm/pgtable.h>
>>>>>>>      #include <asm/nmi.h>
>>>>>>>      #include <asm/switch_to.h>
>>>>>>> -#include <asm/facility.h>
>>>>>>>      #include <asm/sclp.h>
>>>>>>>      #include<asm/timex.h>
>>>>>>>      #include "kvm-s390.h"
>>>>>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>>>>>>>      	{ NULL }
>>>>>>>      };
>>>>>>>      
>>>>>>> -unsigned long *vfacilities;
>>>>>>> -static struct gmap_notifier gmap_notifier;
>>>>>>> +/* upper facilities limit for kvm */
>>>>>>> +unsigned long kvm_s390_fac_list_mask[] = {
>>>>>>> +	0xff82fff3f47c2000UL,
>>>>>>> +	0x005c000000000000UL,
>>>>>>> +};
>>>>>>> +
>>>>>>> +unsigned long kvm_s390_fac_list_mask_size(void)
>>>>>>> +{
>>>>>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
>>>>>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
>>>>>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
>>>>>>> +}
>>>>>>>      
>>>>>>> -/* test availability of vfacility */
>>>>>>> -int test_vfacility(unsigned long nr)
>>>>>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
>>>>>>>      {
>>>>>>> -	return __test_facility(nr, (void *) vfacilities);
>>>>>>> +	unsigned int i;
>>>>>>> +
>>>>>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
>>>>>>> +		if (i < kvm_s390_fac_list_mask_size())
>>>>>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
>>>>>>> +		else
>>>>>>> +			fac_list[i] &= 0UL;
>>>>>>> +	}
>>>>>>>      }
>>>>>>>      
>>>>>>> +static struct gmap_notifier gmap_notifier;
>>>>>>> +
>>>>>>>      /* Section: not file related */
>>>>>>>      int kvm_arch_hardware_enable(void *garbage)
>>>>>>>      {
>>>>>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
>>>>>>>      	return r;
>>>>>>>      }
>>>>>>>      
>>>>>>> +/* make sure the memory used for fac_list is zeroed */
>>>>>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
>>>>>> Hard? Wouldn't "host" make more sense here?
>>>>> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with me.
>>>>>
>>>>>> I also think it makes sense to expose the native host facility list to
>>>>>> user space via an ioctl somehow.
>>>>>>
>>>>> In which situation do you need the full facility list. Do you have an example?
>>>> If you want to just implement -cpu host to get the exact feature set
>>>> that the host gives you, how do you know which set that is?
>>> During qemu machine initalization I call:
>>>
>>> kvm_s390_get_machine_props(&mach);
>>>
>>> which returns the following information:
>>>
>>> typedef struct S390MachineProps {
>>>       uint64_t cpuid;
>>>       uint32_t ibc_range;
>>>       uint8_t  pad[4];
>>>       uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
>>>       uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
>>>       uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
>>> } S390MachineProps;
>> Ah, ok, I missed that part. So "kvm_s390_get_machine_props()" basically
>> gets us the full facility list the host supports. That makes sense.
>>
>> I still don't know whether we should care about hard/soft, but I suppose
>> the rest makes sense.
> The reason why I differentiate between hardware and software implemented
> facilities has several reasons.
>
> First, one should understand that both lists represent the same set of
> features, there is no software feature not being implemented as hardware
> feature on an existing S390 system. An optional implementation of a software
> feature is usable on back-level hardware to implement functionality, not
> quality or performance! That means, on OS and it's application is capable to
> run most efficiently on the given hardware without that software feature.
> If a system has the same facility in soft- and hardware implemented, KVM
> always privileges the hardware implementation.
>
> Second, separating both features allow user space to differentiate between
> hardware and software implemented features and to give up on SW features
> if not explicitly requested by the CPU model option "+sofl".

Why? User space has a set of features it wants to expose to the guest. 
If the kernel can handle all of them, fine. If it can't, it can either 
decide that the VM can't run on that host or disable features and 
*remember it did so* so that on migration it can warn the target host.

> Third, and very important, the main reason for CPU model implementation
> is to guarantee an identical environment on source and target side comprised
> of facilities, instruction set and CPU id. The capability for migration is

Yes, exactly :).

> lost as soon software implemented facilities become activated!

Why? When something is done in software, I can surely migrate the state?

> Once again the method Qemu uses to define the set of requested facilities.
>
> The machine properties ioctl provides FMh, HFh, and SFh, ie. the mask and
> hardware/software implemented facility sets of the hosts.
>
> Qemu has knowledge on all real existing CPU models (TYPE-GA), particularly
> on the facility set of these model (Fm)
>   
> During startup, Qemu retrives FMh, HFh and SFh. Then for all CPU models Qemu
> is aware off, it calculates (Fm & FMh). If that is a subset of HFh, the model
> is a supportable model on this host. If the user has specified to allow
> software implemented facilities (-cpu <model>,+sofl), The facilities to
> request are calculated like (Fm & FMh | SFh).

Why so complicated? What's the difference between -cpu 
z9,cpu-id=z10,+featureA,+featureB,+featureC where A, B and C are the new 
features between z9 and z10?

There really shouldn't be any difference between those 2 different 
invocation types IMHO.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Michael Mueller]

On Mon, 19 May 2014 12:41:45 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 19.05.14 12:13, Michael Mueller wrote:
> > On Fri, 16 May 2014 22:35:34 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 16.05.14 18:09, Michael Mueller wrote:
> >>> On Fri, 16 May 2014 16:49:37 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 16.05.14 16:46, Michael Mueller wrote:
> >>>>> On Fri, 16 May 2014 13:55:41 +0200
> >>>>> Alexander Graf <agraf@suse.de> wrote:
> >>>>>
> >>>>>> On 13.05.14 16:58, Michael Mueller wrote:
> >>>>>>> The patch introduces facilities and cpu_ids per virtual machine.
> >>>>>>> Different virtual machines may want to expose different facilities and
> >>>>>>> cpu ids to the guest, so let's make them per-vm instead of global.
> >>>>>>>
> >>>>>>> In addition this patch renames all ocurrences of *facilities to *fac_list
> >>>>>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
> >>>>>>>
> >>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>>>>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
> >>>>>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> >>>>>>> ---
> >>>>>>>      arch/s390/include/asm/kvm_host.h |   7 +++
> >>>>>>>      arch/s390/kvm/gaccess.c          |   4 +-
> >>>>>>>      arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
> >>>>>>>      arch/s390/kvm/kvm-s390.h         |  23 +++++++--
> >>>>>>>      arch/s390/kvm/priv.c             |  13 +++--
> >>>>>>>      5 files changed, 113 insertions(+), 41 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>>>>>> index 38d487a..b4751ba 100644
> >>>>>>> --- a/arch/s390/include/asm/kvm_host.h
> >>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
> >>>>>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
> >>>>>>>      	struct kvm_s390_attr_name name;
> >>>>>>>      };
> >>>>>>>      
> >>>>>>> +struct kvm_s390_cpu_model {
> >>>>>>> +	unsigned long *sie_fac;
> >>>>>>> +	struct cpuid cpu_id;
> >>>>>>> +	unsigned long *fac_list;
> >>>>>>> +};
> >>>>>>> +
> >>>>>>>      struct kvm_arch{
> >>>>>>>      	struct sca_block *sca;
> >>>>>>>      	debug_info_t *dbf;
> >>>>>>> @@ -427,6 +433,7 @@ struct kvm_arch{
> >>>>>>>      	wait_queue_head_t ipte_wq;
> >>>>>>>      	struct kvm_s390_config *cfg;
> >>>>>>>      	spinlock_t start_stop_lock;
> >>>>>>> +	struct kvm_s390_cpu_model model;
> >>>>>>>      };
> >>>>>>>      
> >>>>>>>      #define KVM_HVA_ERR_BAD		(-1UL)
> >>>>>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> >>>>>>> index db608c3..4c7ca40 100644
> >>>>>>> --- a/arch/s390/kvm/gaccess.c
> >>>>>>> +++ b/arch/s390/kvm/gaccess.c
> >>>>>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
> >>>>>>> long gva, union asce asce;
> >>>>>>>      
> >>>>>>>      	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
> >>>>>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
> >>>>>>> -	edat2 = edat1 && test_vfacility(78);
> >>>>>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
> >>>>>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
> >>>>>>>      	asce.val = get_vcpu_asce(vcpu);
> >>>>>>>      	if (asce.r)
> >>>>>>>      		goto real_address;
> >>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>>>>>> index 01a5212..a53652f 100644
> >>>>>>> --- a/arch/s390/kvm/kvm-s390.c
> >>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
> >>>>>>> @@ -1,5 +1,5 @@
> >>>>>>>      /*
> >>>>>>> - * hosting zSeries kernel virtual machines
> >>>>>>> + * Hosting zSeries kernel virtual machines
> >>>>>>>       *
> >>>>>>>       * Copyright IBM Corp. 2008, 2009
> >>>>>>>       *
> >>>>>>> @@ -30,7 +30,6 @@
> >>>>>>>      #include <asm/pgtable.h>
> >>>>>>>      #include <asm/nmi.h>
> >>>>>>>      #include <asm/switch_to.h>
> >>>>>>> -#include <asm/facility.h>
> >>>>>>>      #include <asm/sclp.h>
> >>>>>>>      #include<asm/timex.h>
> >>>>>>>      #include "kvm-s390.h"
> >>>>>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
> >>>>>>>      	{ NULL }
> >>>>>>>      };
> >>>>>>>      
> >>>>>>> -unsigned long *vfacilities;
> >>>>>>> -static struct gmap_notifier gmap_notifier;
> >>>>>>> +/* upper facilities limit for kvm */
> >>>>>>> +unsigned long kvm_s390_fac_list_mask[] = {
> >>>>>>> +	0xff82fff3f47c2000UL,
> >>>>>>> +	0x005c000000000000UL,
> >>>>>>> +};
> >>>>>>> +
> >>>>>>> +unsigned long kvm_s390_fac_list_mask_size(void)
> >>>>>>> +{
> >>>>>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
> >>>>>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
> >>>>>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
> >>>>>>> +}
> >>>>>>>      
> >>>>>>> -/* test availability of vfacility */
> >>>>>>> -int test_vfacility(unsigned long nr)
> >>>>>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
> >>>>>>>      {
> >>>>>>> -	return __test_facility(nr, (void *) vfacilities);
> >>>>>>> +	unsigned int i;
> >>>>>>> +
> >>>>>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
> >>>>>>> +		if (i < kvm_s390_fac_list_mask_size())
> >>>>>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
> >>>>>>> +		else
> >>>>>>> +			fac_list[i] &= 0UL;
> >>>>>>> +	}
> >>>>>>>      }
> >>>>>>>      
> >>>>>>> +static struct gmap_notifier gmap_notifier;
> >>>>>>> +
> >>>>>>>      /* Section: not file related */
> >>>>>>>      int kvm_arch_hardware_enable(void *garbage)
> >>>>>>>      {
> >>>>>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
> >>>>>>>      	return r;
> >>>>>>>      }
> >>>>>>>      
> >>>>>>> +/* make sure the memory used for fac_list is zeroed */
> >>>>>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
> >>>>>> Hard? Wouldn't "host" make more sense here?
> >>>>> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with
> >>>>> me.
> >>>>>
> >>>>>> I also think it makes sense to expose the native host facility list to
> >>>>>> user space via an ioctl somehow.
> >>>>>>
> >>>>> In which situation do you need the full facility list. Do you have an example?
> >>>> If you want to just implement -cpu host to get the exact feature set
> >>>> that the host gives you, how do you know which set that is?
> >>> During qemu machine initalization I call:
> >>>
> >>> kvm_s390_get_machine_props(&mach);
> >>>
> >>> which returns the following information:
> >>>
> >>> typedef struct S390MachineProps {
> >>>       uint64_t cpuid;
> >>>       uint32_t ibc_range;
> >>>       uint8_t  pad[4];
> >>>       uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
> >>>       uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
> >>>       uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
> >>> } S390MachineProps;
> >> Ah, ok, I missed that part. So "kvm_s390_get_machine_props()" basically
> >> gets us the full facility list the host supports. That makes sense.
> >>
> >> I still don't know whether we should care about hard/soft, but I suppose
> >> the rest makes sense.
> > The reason why I differentiate between hardware and software implemented
> > facilities has several reasons.
> >
> > First, one should understand that both lists represent the same set of
> > features, there is no software feature not being implemented as hardware
> > feature on an existing S390 system. An optional implementation of a software
> > feature is usable on back-level hardware to implement functionality, not
> > quality or performance! That means, on OS and it's application is capable to
> > run most efficiently on the given hardware without that software feature.
> > If a system has the same facility in soft- and hardware implemented, KVM
> > always privileges the hardware implementation.
> >
> > Second, separating both features allow user space to differentiate between
> > hardware and software implemented features and to give up on SW features
> > if not explicitly requested by the CPU model option "+sofl".
> 
> Why? User space has a set of features it wants to expose to the guest.

You are talking about a different interface here. That is not the intension
of software / hardware facilities of KVM. They are never implemented in user
space!

> If the kernel can handle all of them, fine. If it can't, it can either 
> decide that the VM can't run on that host or disable features and 
> *remember it did so* so that on migration it can warn the target host.
> 
> > Third, and very important, the main reason for CPU model implementation
> > is to guarantee an identical environment on source and target side comprised
> > of facilities, instruction set and CPU id. The capability for migration is
> 
> Yes, exactly :).
> 
> > lost as soon software implemented facilities become activated!
> 
> Why? When something is done in software, I can surely migrate the state?

Sure, but in the former case it could be implemented in the source in HW and
the target in SW.

> 
> > Once again the method Qemu uses to define the set of requested facilities.
> >
> > The machine properties ioctl provides FMh, HFh, and SFh, ie. the mask and
> > hardware/software implemented facility sets of the hosts.
> >
> > Qemu has knowledge on all real existing CPU models (TYPE-GA), particularly
> > on the facility set of these model (Fm)
> >   
> > During startup, Qemu retrives FMh, HFh and SFh. Then for all CPU models Qemu
> > is aware off, it calculates (Fm & FMh). If that is a subset of HFh, the model
> > is a supportable model on this host. If the user has specified to allow
> > software implemented facilities (-cpu <model>,+sofl), The facilities to
> > request are calculated like (Fm & FMh | SFh).
> 
> Why so complicated? What's the difference between -cpu 
> z9,cpu-id=z10,+featureA,+featureB,+featureC where A, B and C are the new 
> features between z9 and z10?

I absolutely want to avoid a facility room of 2**14 bit being specified on the
command line by an administrator who is even unaware of facilities at all!
Also libvirt must not be scared by this facility flood when making migration
decisions. This is what I think would make things complicate.

I consider an soft defined KVM facility as exception and not the rule.

Michael

> 
> There really shouldn't be any difference between those 2 different 
> invocation types IMHO.
> 
> 
> Alex
> 

Re: [Qemu-devel] [PATCH v1 RFC 3/6] KVM: s390: use facilities and cpu_id per KVM

[Alexander Graf]

On 19.05.14 13:29, Michael Mueller wrote:
> On Mon, 19 May 2014 12:41:45 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 19.05.14 12:13, Michael Mueller wrote:
>>> On Fri, 16 May 2014 22:35:34 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 16.05.14 18:09, Michael Mueller wrote:
>>>>> On Fri, 16 May 2014 16:49:37 +0200
>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>
>>>>>> On 16.05.14 16:46, Michael Mueller wrote:
>>>>>>> On Fri, 16 May 2014 13:55:41 +0200
>>>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>>>
>>>>>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>>>>>> The patch introduces facilities and cpu_ids per virtual machine.
>>>>>>>>> Different virtual machines may want to expose different facilities and
>>>>>>>>> cpu ids to the guest, so let's make them per-vm instead of global.
>>>>>>>>>
>>>>>>>>> In addition this patch renames all ocurrences of *facilities to *fac_list
>>>>>>>>> smilar to the already exiting symbol stfl_fac_list in lowcore.
>>>>>>>>>
>>>>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>>>>>> Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
>>>>>>>>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>>>>>>>>> ---
>>>>>>>>>       arch/s390/include/asm/kvm_host.h |   7 +++
>>>>>>>>>       arch/s390/kvm/gaccess.c          |   4 +-
>>>>>>>>>       arch/s390/kvm/kvm-s390.c         | 107 +++++++++++++++++++++++++++------------
>>>>>>>>>       arch/s390/kvm/kvm-s390.h         |  23 +++++++--
>>>>>>>>>       arch/s390/kvm/priv.c             |  13 +++--
>>>>>>>>>       5 files changed, 113 insertions(+), 41 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>>>>>> index 38d487a..b4751ba 100644
>>>>>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>>>>>> @@ -414,6 +414,12 @@ struct kvm_s390_config {
>>>>>>>>>       	struct kvm_s390_attr_name name;
>>>>>>>>>       };
>>>>>>>>>       
>>>>>>>>> +struct kvm_s390_cpu_model {
>>>>>>>>> +	unsigned long *sie_fac;
>>>>>>>>> +	struct cpuid cpu_id;
>>>>>>>>> +	unsigned long *fac_list;
>>>>>>>>> +};
>>>>>>>>> +
>>>>>>>>>       struct kvm_arch{
>>>>>>>>>       	struct sca_block *sca;
>>>>>>>>>       	debug_info_t *dbf;
>>>>>>>>> @@ -427,6 +433,7 @@ struct kvm_arch{
>>>>>>>>>       	wait_queue_head_t ipte_wq;
>>>>>>>>>       	struct kvm_s390_config *cfg;
>>>>>>>>>       	spinlock_t start_stop_lock;
>>>>>>>>> +	struct kvm_s390_cpu_model model;
>>>>>>>>>       };
>>>>>>>>>       
>>>>>>>>>       #define KVM_HVA_ERR_BAD		(-1UL)
>>>>>>>>> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
>>>>>>>>> index db608c3..4c7ca40 100644
>>>>>>>>> --- a/arch/s390/kvm/gaccess.c
>>>>>>>>> +++ b/arch/s390/kvm/gaccess.c
>>>>>>>>> @@ -358,8 +358,8 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned
>>>>>>>>> long gva, union asce asce;
>>>>>>>>>       
>>>>>>>>>       	ctlreg0.val = vcpu->arch.sie_block->gcr[0];
>>>>>>>>> -	edat1 = ctlreg0.edat && test_vfacility(8);
>>>>>>>>> -	edat2 = edat1 && test_vfacility(78);
>>>>>>>>> +	edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
>>>>>>>>> +	edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
>>>>>>>>>       	asce.val = get_vcpu_asce(vcpu);
>>>>>>>>>       	if (asce.r)
>>>>>>>>>       		goto real_address;
>>>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> index 01a5212..a53652f 100644
>>>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> @@ -1,5 +1,5 @@
>>>>>>>>>       /*
>>>>>>>>> - * hosting zSeries kernel virtual machines
>>>>>>>>> + * Hosting zSeries kernel virtual machines
>>>>>>>>>        *
>>>>>>>>>        * Copyright IBM Corp. 2008, 2009
>>>>>>>>>        *
>>>>>>>>> @@ -30,7 +30,6 @@
>>>>>>>>>       #include <asm/pgtable.h>
>>>>>>>>>       #include <asm/nmi.h>
>>>>>>>>>       #include <asm/switch_to.h>
>>>>>>>>> -#include <asm/facility.h>
>>>>>>>>>       #include <asm/sclp.h>
>>>>>>>>>       #include<asm/timex.h>
>>>>>>>>>       #include "kvm-s390.h"
>>>>>>>>> @@ -92,15 +91,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>>>>>>>>>       	{ NULL }
>>>>>>>>>       };
>>>>>>>>>       
>>>>>>>>> -unsigned long *vfacilities;
>>>>>>>>> -static struct gmap_notifier gmap_notifier;
>>>>>>>>> +/* upper facilities limit for kvm */
>>>>>>>>> +unsigned long kvm_s390_fac_list_mask[] = {
>>>>>>>>> +	0xff82fff3f47c2000UL,
>>>>>>>>> +	0x005c000000000000UL,
>>>>>>>>> +};
>>>>>>>>> +
>>>>>>>>> +unsigned long kvm_s390_fac_list_mask_size(void)
>>>>>>>>> +{
>>>>>>>>> +	BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) >
>>>>>>>>> +		     S390_ARCH_FAC_MASK_SIZE_U64);
>>>>>>>>> +	return ARRAY_SIZE(kvm_s390_fac_list_mask);
>>>>>>>>> +}
>>>>>>>>>       
>>>>>>>>> -/* test availability of vfacility */
>>>>>>>>> -int test_vfacility(unsigned long nr)
>>>>>>>>> +void kvm_s390_apply_fac_list_mask(unsigned long fac_list[])
>>>>>>>>>       {
>>>>>>>>> -	return __test_facility(nr, (void *) vfacilities);
>>>>>>>>> +	unsigned int i;
>>>>>>>>> +
>>>>>>>>> +	for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
>>>>>>>>> +		if (i < kvm_s390_fac_list_mask_size())
>>>>>>>>> +			fac_list[i] &= kvm_s390_fac_list_mask[i];
>>>>>>>>> +		else
>>>>>>>>> +			fac_list[i] &= 0UL;
>>>>>>>>> +	}
>>>>>>>>>       }
>>>>>>>>>       
>>>>>>>>> +static struct gmap_notifier gmap_notifier;
>>>>>>>>> +
>>>>>>>>>       /* Section: not file related */
>>>>>>>>>       int kvm_arch_hardware_enable(void *garbage)
>>>>>>>>>       {
>>>>>>>>> @@ -485,6 +502,30 @@ long kvm_arch_vm_ioctl(struct file *filp,
>>>>>>>>>       	return r;
>>>>>>>>>       }
>>>>>>>>>       
>>>>>>>>> +/* make sure the memory used for fac_list is zeroed */
>>>>>>>>> +void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
>>>>>>>> Hard? Wouldn't "host" make more sense here?
>>>>>>> Renaming "*hard_fac_list" with "*host_fac_list" here and wherever it appears is ok with
>>>>>>> me.
>>>>>>>
>>>>>>>> I also think it makes sense to expose the native host facility list to
>>>>>>>> user space via an ioctl somehow.
>>>>>>>>
>>>>>>> In which situation do you need the full facility list. Do you have an example?
>>>>>> If you want to just implement -cpu host to get the exact feature set
>>>>>> that the host gives you, how do you know which set that is?
>>>>> During qemu machine initalization I call:
>>>>>
>>>>> kvm_s390_get_machine_props(&mach);
>>>>>
>>>>> which returns the following information:
>>>>>
>>>>> typedef struct S390MachineProps {
>>>>>        uint64_t cpuid;
>>>>>        uint32_t ibc_range;
>>>>>        uint8_t  pad[4];
>>>>>        uint64_t fac_mask[S390_ARCH_FAC_MASK_SIZE_UINT64];
>>>>>        uint64_t hard_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
>>>>>        uint64_t soft_fac_list[S390_ARCH_FAC_LIST_SIZE_UINT64];
>>>>> } S390MachineProps;
>>>> Ah, ok, I missed that part. So "kvm_s390_get_machine_props()" basically
>>>> gets us the full facility list the host supports. That makes sense.
>>>>
>>>> I still don't know whether we should care about hard/soft, but I suppose
>>>> the rest makes sense.
>>> The reason why I differentiate between hardware and software implemented
>>> facilities has several reasons.
>>>
>>> First, one should understand that both lists represent the same set of
>>> features, there is no software feature not being implemented as hardware
>>> feature on an existing S390 system. An optional implementation of a software
>>> feature is usable on back-level hardware to implement functionality, not
>>> quality or performance! That means, on OS and it's application is capable to
>>> run most efficiently on the given hardware without that software feature.
>>> If a system has the same facility in soft- and hardware implemented, KVM
>>> always privileges the hardware implementation.
>>>
>>> Second, separating both features allow user space to differentiate between
>>> hardware and software implemented features and to give up on SW features
>>> if not explicitly requested by the CPU model option "+sofl".
>> Why? User space has a set of features it wants to expose to the guest.
> You are talking about a different interface here.

No, I'm not.

> That is not the intension
> of software / hardware facilities of KVM. They are never implemented in user
> space!
>
>> If the kernel can handle all of them, fine. If it can't, it can either
>> decide that the VM can't run on that host or disable features and
>> *remember it did so* so that on migration it can warn the target host.
>>
>>> Third, and very important, the main reason for CPU model implementation
>>> is to guarantee an identical environment on source and target side comprised
>>> of facilities, instruction set and CPU id. The capability for migration is
>> Yes, exactly :).
>>
>>> lost as soon software implemented facilities become activated!
>> Why? When something is done in software, I can surely migrate the state?
> Sure, but in the former case it could be implemented in the source in HW and
> the target in SW.

So? For what user space (QEMU) cares about, it could be implemented in 
software on both sides, or in hardware on both sides. It doesn't care. 
It cares that it's compatible.

>
>>> Once again the method Qemu uses to define the set of requested facilities.
>>>
>>> The machine properties ioctl provides FMh, HFh, and SFh, ie. the mask and
>>> hardware/software implemented facility sets of the hosts.
>>>
>>> Qemu has knowledge on all real existing CPU models (TYPE-GA), particularly
>>> on the facility set of these model (Fm)
>>>    
>>> During startup, Qemu retrives FMh, HFh and SFh. Then for all CPU models Qemu
>>> is aware off, it calculates (Fm & FMh). If that is a subset of HFh, the model
>>> is a supportable model on this host. If the user has specified to allow
>>> software implemented facilities (-cpu <model>,+sofl), The facilities to
>>> request are calculated like (Fm & FMh | SFh).
>> Why so complicated? What's the difference between -cpu
>> z9,cpu-id=z10,+featureA,+featureB,+featureC where A, B and C are the new
>> features between z9 and z10?
> I absolutely want to avoid a facility room of 2**14 bit being specified on the
> command line by an administrator who is even unaware of facilities at all!
> Also libvirt must not be scared by this facility flood when making migration
> decisions. This is what I think would make things complicate.

We already have this with x86 - and for a good reason. When you're able 
to specify a CPU purely on the command line you can also specify a CPU 
purely using configuration files.


Alex

[Qemu-devel] [PATCH v1 RFC 4/6] KVM: s390: add ibc api

[Michael Mueller]

The patch addds an internal KVM API to access instruction blocking
control. It offers the following functions:

- has_ibc() : tests if the system offers instruction blocking control
- lowest_ibc() : first level supported
- latest_ibc() : latest level supported

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
Acked-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 arch/s390/kvm/Makefile   |  2 +-
 arch/s390/kvm/ibc.c      | 29 +++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.h |  8 ++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 arch/s390/kvm/ibc.c

diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile
index f0072bc..88aa28b 100644
--- a/arch/s390/kvm/Makefile
+++ b/arch/s390/kvm/Makefile
@@ -12,6 +12,6 @@ common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o  $(KVM)/async_pf.o $(KVM)/irqch
 ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
 kvm-objs := $(common-objs) kvm-s390.o intercept.o interrupt.o priv.o sigp.o
-kvm-objs += diag.o config.o gaccess.o guestdbg.o
+kvm-objs += diag.o config.o gaccess.o guestdbg.o ibc.o
 
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/arch/s390/kvm/ibc.c b/arch/s390/kvm/ibc.c
new file mode 100644
index 0000000..8171b02
--- /dev/null
+++ b/arch/s390/kvm/ibc.c
@@ -0,0 +1,29 @@
+/*
+ * Instruction Blocking related functions
+ *
+ * Copyright IBM Corp. 2014
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2 only)
+ * as published by the Free Software Foundation.
+ *
+ *    Author(s): Michael Mueller <mimu@linux.vnet.ibm.com>
+ */
+
+#include <asm/sclp.h>
+#include "kvm-s390.h"
+
+unsigned short kvm_s390_lowest_ibc(void)
+{
+	return (sclp_get_ibc() & IBC_MODE_MASK) >> 16;
+}
+
+unsigned short kvm_s390_latest_ibc(void)
+{
+	return sclp_get_ibc() & IBC_MODE_MASK;
+}
+
+bool kvm_s390_has_ibc(void)
+{
+	return kvm_s390_lowest_ibc() != 0;
+}
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index ecadc8a..3715917 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -32,6 +32,9 @@ int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu);
 #define S390_ARCH_FAC_MASK_SIZE_U64 \
 	(S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64))
 
+/* mask non defined IBC bits from SCLP read data */
+#define IBC_MODE_MASK 0x0fff0fffu
+
 /* Transactional Memory Execution related macros */
 #define IS_TE_ENABLED(vcpu)	((vcpu->arch.sie_block->ecb & 0x10))
 #define TDB_FORMAT1		1
@@ -253,4 +256,9 @@ void kvm_s390_clear_bp_data(struct kvm_vcpu *vcpu);
 void kvm_s390_prepare_debug_exit(struct kvm_vcpu *vcpu);
 void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu);
 
+/* implemented in ibc.c */
+unsigned short kvm_s390_lowest_ibc(void);
+unsigned short kvm_s390_latest_ibc(void);
+bool kvm_s390_has_ibc(void);
+
 #endif
-- 
1.8.3.1

[Qemu-devel] [PATCH v1 RFC 5/6] KVM: s390: initial implementation of soft facilities

[Michael Mueller]

This patch implements the new state soft_fac_bits. This state
defines which facilities are emulated in kernel and not in processor
or millicode. Currently all bits are cleared.

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
---
 arch/s390/kvm/Makefile   |  2 +-
 arch/s390/kvm/kvm-s390.h |  3 +++
 arch/s390/kvm/softfac.c  | 38 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 arch/s390/kvm/softfac.c

diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile
index 88aa28b..586f686 100644
--- a/arch/s390/kvm/Makefile
+++ b/arch/s390/kvm/Makefile
@@ -12,6 +12,6 @@ common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o  $(KVM)/async_pf.o $(KVM)/irqch
 ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
 
 kvm-objs := $(common-objs) kvm-s390.o intercept.o interrupt.o priv.o sigp.o
-kvm-objs += diag.o config.o gaccess.o guestdbg.o ibc.o
+kvm-objs += diag.o config.o gaccess.o guestdbg.o ibc.o softfac.o
 
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 3715917..10de678 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -261,4 +261,7 @@ unsigned short kvm_s390_lowest_ibc(void);
 unsigned short kvm_s390_latest_ibc(void);
 bool kvm_s390_has_ibc(void);
 
+/* implemented in softfac.c */
+unsigned int stsfle(unsigned long *fac, unsigned long size);
+
 #endif
diff --git a/arch/s390/kvm/softfac.c b/arch/s390/kvm/softfac.c
new file mode 100644
index 0000000..1c489d8
--- /dev/null
+++ b/arch/s390/kvm/softfac.c
@@ -0,0 +1,38 @@
+/*
+ * Registration of KVM emulated facilities
+ *
+ * Copyright IBM Corp. 2014
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2 only)
+ * as published by the Free Software Foundation.
+ *
+ *    Author(s): Michael Mueller <mimu@linux.vnet.ibm.com>
+ */
+
+#include "kvm-s390.h"
+
+/*
+ * The soft facilities define which of the facilities are emulatable
+ * in KVM. The size must be identical to the size of the cpu facility
+ * list.
+ */
+static unsigned long soft_fac_bits[S390_ARCH_FAC_LIST_SIZE_U64];
+
+/**
+ * stsfle():
+ *          store soft facility list extended, copies a subset of the
+ *          soft facility list
+ * @fac     address of buffer where the facilities should be stored
+ * @size    number of double words + 1 to store in @fac
+ * @returns number of bytes being copied
+ */
+unsigned int stsfle(unsigned long *fac, unsigned long size)
+{
+	if (size >= S390_ARCH_FAC_LIST_SIZE_U64)
+		size = S390_ARCH_FAC_LIST_SIZE_U64 - 1;
+	size = (!test_facility(7)) ? 4 : (size + 1) * sizeof(*fac);
+	memcpy(fac, soft_fac_bits, size);
+
+	return size;
+}
-- 
1.8.3.1

[Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

This patch enables cpu model support in kvm/s390 via the vm attribute
interface.

During KVM initialization, the host properties cpuid, IBC value and the
facility list are stored in the architecture specific cpu model structure.

During vcpu setup, these properties are taken to initialize the related SIE
state. This mechanism allows to adjust the properties from user space and thus
to implement different selectable cpu models.

This patch uses the IBC functionality to block instructions that have not
been implemented at the requested CPU type and GA level compared to the
full host capability.

Userspace has to initialize the cpu model before vcpu creation. A cpu model
change of running vcpus is currently not possible.

Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
---
 arch/s390/include/asm/kvm_host.h |   4 +-
 arch/s390/include/uapi/asm/kvm.h |  23 ++++++
 arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
 arch/s390/kvm/kvm-s390.h         |   1 +
 4 files changed, 172 insertions(+), 2 deletions(-)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index b4751ba..6b826cb 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
 	atomic_t cpuflags;		/* 0x0000 */
 	__u32 : 1;			/* 0x0004 */
 	__u32 prefix : 18;
-	__u32 : 13;
+	__u32 : 1;
+	__u32 ibc : 12;
 	__u8	reserved08[4];		/* 0x0008 */
 #define PROG_IN_SIE (1<<0)
 	__u32	prog0c;			/* 0x000c */
@@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
 	unsigned long *sie_fac;
 	struct cpuid cpu_id;
 	unsigned long *fac_list;
+	unsigned short ibc;
 };
 
 struct kvm_arch{
diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
index 313100a..82ef1b5 100644
--- a/arch/s390/include/uapi/asm/kvm.h
+++ b/arch/s390/include/uapi/asm/kvm.h
@@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
 
 /* kvm attr_group  on vm fd */
 #define KVM_S390_VM_MEM_CTRL		0
+#define KVM_S390_VM_CPU_MODEL		1
 
 /* kvm attributes for mem_ctrl */
 #define KVM_S390_VM_MEM_ENABLE_CMMA	0
 #define KVM_S390_VM_MEM_CLR_CMMA	1
 #define KVM_S390_VM_MEM_CLR_PAGES	2
 
+/* kvm attributes for cpu_model */
+
+/* the s390 processor related attributes are r/w */
+#define KVM_S390_VM_CPU_PROCESSOR	0
+struct kvm_s390_vm_cpu_processor {
+	__u64 cpuid;
+	__u16 ibc;
+	__u8  pad[6];
+	__u64 fac_list[256];
+};
+
+/* the machine related attributes are read only */
+#define KVM_S390_VM_CPU_MACHINE		1
+struct kvm_s390_vm_cpu_machine {
+	__u64 cpuid;
+	__u32 ibc_range;
+	__u8  pad[4];
+	__u64 fac_mask[256];
+	__u64 hard_fac_list[256];
+	__u64 soft_fac_list[256];
+};
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* general purpose regs for s390 */
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index a53652f..9965d8b 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 	return ret;
 }
 
+static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	struct kvm_s390_vm_cpu_processor *proc;
+
+	if (atomic_read(&kvm->online_vcpus))
+		return -EBUSY;
+
+	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
+	if (!proc)
+		return -ENOMEM;
+
+	if (copy_from_user(proc, (void __user *)attr->addr,
+			   sizeof(*proc))) {
+		kfree(proc);
+		return -EFAULT;
+	}
+
+	mutex_lock(&kvm->lock);
+	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
+	       sizeof(struct cpuid));
+	kvm->arch.model.ibc = proc->ibc;
+	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
+	memcpy(kvm->arch.model.fac_list, proc->fac_list,
+	       S390_ARCH_FAC_LIST_SIZE_BYTE);
+	mutex_unlock(&kvm->lock);
+	kfree(proc);
+
+	return 0;
+}
+
+static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	int ret = -ENXIO;
+
+	switch (attr->attr) {
+	case KVM_S390_VM_CPU_PROCESSOR:
+		ret = kvm_s390_set_processor(kvm, attr);
+		break;
+	}
+	return ret;
+}
+
+static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	struct kvm_s390_vm_cpu_processor *proc;
+	int rc = 0;
+
+	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
+	if (!proc) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
+	proc->ibc = kvm->arch.model.ibc;
+	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
+	       S390_ARCH_FAC_LIST_SIZE_BYTE);
+	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
+		rc = -EFAULT;
+	kfree(proc);
+out:
+	return rc;
+}
+
+static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	struct kvm_s390_vm_cpu_machine *mach;
+	int rc = 0;
+
+	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
+	if (!mach) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	get_cpu_id((struct cpuid *) &mach->cpuid);
+	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
+	mach->ibc_range |= kvm_s390_latest_ibc();
+	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
+	       kvm_s390_fac_list_mask_size() * sizeof(u64));
+	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
+				   S390_ARCH_FAC_LIST_SIZE_U64);
+	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
+				   S390_ARCH_FAC_LIST_SIZE_U64);
+	if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
+		rc = -EFAULT;
+	kfree(mach);
+out:
+	return rc;
+}
+
+static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	int ret = -ENXIO;
+
+	switch (attr->attr) {
+	case KVM_S390_VM_CPU_PROCESSOR:
+		ret = kvm_s390_get_processor(kvm, attr);
+		break;
+	case KVM_S390_VM_CPU_MACHINE:
+		ret = kvm_s390_get_machine(kvm, attr);
+		break;
+	}
+	return ret;
+}
+
 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 {
 	int ret;
@@ -377,6 +481,9 @@ static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 	case KVM_S390_VM_MEM_CTRL:
 		ret = kvm_s390_mem_control(kvm, attr);
 		break;
+	case KVM_S390_VM_CPU_MODEL:
+		ret = kvm_s390_set_cpu_model(kvm, attr);
+		break;
 	default:
 		ret = -ENXIO;
 		break;
@@ -387,7 +494,18 @@ static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 
 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 {
-	return -ENXIO;
+	int ret;
+
+	switch (attr->group) {
+	case KVM_S390_VM_CPU_MODEL:
+		ret = kvm_s390_get_cpu_model(kvm, attr);
+		break;
+	default:
+		ret = -ENXIO;
+		break;
+	}
+
+	return ret;
 }
 
 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
@@ -407,6 +525,17 @@ static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 			break;
 		}
 		break;
+	case KVM_S390_VM_CPU_MODEL:
+		switch (attr->attr) {
+		case KVM_S390_VM_CPU_PROCESSOR:
+		case KVM_S390_VM_CPU_MACHINE:
+			ret = 0;
+			break;
+		default:
+			ret = -ENXIO;
+			break;
+		}
+		break;
 	default:
 		ret = -ENXIO;
 		break;
@@ -520,6 +649,16 @@ void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size)
 		fac_list[i] &= kvm_s390_fac_list_mask[i];
 }
 
+/* make sure the memory used for fac_list is zeroed */
+void kvm_s390_get_soft_fac_list(unsigned long *fac_list, int size)
+{
+	int i;
+
+	stsfle(fac_list, size - 1);
+	for (i = 0; i < size && i < kvm_s390_fac_list_mask_size(); i++)
+		fac_list[i] &= kvm_s390_fac_list_mask[i];
+}
+
 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
 {
 	get_cpu_id(cpu_id);
@@ -579,6 +718,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	kvm_s390_get_hard_fac_list(kvm->arch.model.fac_list,
 				kvm_s390_fac_list_mask_size());
 	kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
+	kvm->arch.model.ibc = kvm_s390_latest_ibc();
 
 	spin_lock_init(&kvm->arch.float_int.lock);
 	INIT_LIST_HEAD(&kvm->arch.float_int.list);
@@ -801,9 +941,13 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 	tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 		     (unsigned long) vcpu);
 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
+
+	mutex_lock(&vcpu->kvm->lock);
 	vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
 	memcpy(vcpu->kvm->arch.model.sie_fac, vcpu->kvm->arch.model.fac_list,
 	       S390_ARCH_FAC_LIST_SIZE_BYTE);
+	vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
+	mutex_unlock(&vcpu->kvm->lock);
 
 	return rc;
 }
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 10de678..f3c7f83 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -196,6 +196,7 @@ unsigned long kvm_s390_fac_list__mask_size(void);
 void kvm_s390_apply_fac_list_mask(unsigned long fac_list[]);
 extern unsigned long kvm_s390_fac_list_mask[];
 void kvm_s390_get_hard_fac_list(unsigned long *fac_list, int size);
+void kvm_s390_get_soft_fac_list(unsigned long *fac_list, int size);
 
 /* implemented in diag.c */
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu);
-- 
1.8.3.1

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 13.05.14 16:58, Michael Mueller wrote:
> This patch enables cpu model support in kvm/s390 via the vm attribute
> interface.
>
> During KVM initialization, the host properties cpuid, IBC value and the
> facility list are stored in the architecture specific cpu model structure.
>
> During vcpu setup, these properties are taken to initialize the related SIE
> state. This mechanism allows to adjust the properties from user space and thus
> to implement different selectable cpu models.
>
> This patch uses the IBC functionality to block instructions that have not
> been implemented at the requested CPU type and GA level compared to the
> full host capability.
>
> Userspace has to initialize the cpu model before vcpu creation. A cpu model
> change of running vcpus is currently not possible.

Why is this VM global? It usually fits a lot better modeling wise when 
CPU types are vcpu properties.

>
> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> ---
>   arch/s390/include/asm/kvm_host.h |   4 +-
>   arch/s390/include/uapi/asm/kvm.h |  23 ++++++
>   arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
>   arch/s390/kvm/kvm-s390.h         |   1 +
>   4 files changed, 172 insertions(+), 2 deletions(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index b4751ba..6b826cb 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
>   	atomic_t cpuflags;		/* 0x0000 */
>   	__u32 : 1;			/* 0x0004 */
>   	__u32 prefix : 18;
> -	__u32 : 13;
> +	__u32 : 1;
> +	__u32 ibc : 12;
>   	__u8	reserved08[4];		/* 0x0008 */
>   #define PROG_IN_SIE (1<<0)
>   	__u32	prog0c;			/* 0x000c */
> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
>   	unsigned long *sie_fac;
>   	struct cpuid cpu_id;
>   	unsigned long *fac_list;
> +	unsigned short ibc;
>   };
>   
>   struct kvm_arch{
> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> index 313100a..82ef1b5 100644
> --- a/arch/s390/include/uapi/asm/kvm.h
> +++ b/arch/s390/include/uapi/asm/kvm.h
> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
>   
>   /* kvm attr_group  on vm fd */
>   #define KVM_S390_VM_MEM_CTRL		0
> +#define KVM_S390_VM_CPU_MODEL		1
>   
>   /* kvm attributes for mem_ctrl */
>   #define KVM_S390_VM_MEM_ENABLE_CMMA	0
>   #define KVM_S390_VM_MEM_CLR_CMMA	1
>   #define KVM_S390_VM_MEM_CLR_PAGES	2
>   
> +/* kvm attributes for cpu_model */
> +
> +/* the s390 processor related attributes are r/w */
> +#define KVM_S390_VM_CPU_PROCESSOR	0
> +struct kvm_s390_vm_cpu_processor {
> +	__u64 cpuid;
> +	__u16 ibc;
> +	__u8  pad[6];
> +	__u64 fac_list[256];
> +};
> +
> +/* the machine related attributes are read only */
> +#define KVM_S390_VM_CPU_MACHINE		1
> +struct kvm_s390_vm_cpu_machine {
> +	__u64 cpuid;
> +	__u32 ibc_range;
> +	__u8  pad[4];
> +	__u64 fac_mask[256];
> +	__u64 hard_fac_list[256];
> +	__u64 soft_fac_list[256];
> +};
> +
>   /* for KVM_GET_REGS and KVM_SET_REGS */
>   struct kvm_regs {
>   	/* general purpose regs for s390 */
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index a53652f..9965d8b 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
>   	return ret;
>   }
>   
> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> +{
> +	struct kvm_s390_vm_cpu_processor *proc;
> +
> +	if (atomic_read(&kvm->online_vcpus))
> +		return -EBUSY;
> +
> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> +	if (!proc)
> +		return -ENOMEM;
> +
> +	if (copy_from_user(proc, (void __user *)attr->addr,
> +			   sizeof(*proc))) {
> +		kfree(proc);
> +		return -EFAULT;
> +	}
> +
> +	mutex_lock(&kvm->lock);
> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> +	       sizeof(struct cpuid));
> +	kvm->arch.model.ibc = proc->ibc;
> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> +	mutex_unlock(&kvm->lock);
> +	kfree(proc);
> +
> +	return 0;
> +}
> +
> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> +{
> +	int ret = -ENXIO;
> +
> +	switch (attr->attr) {
> +	case KVM_S390_VM_CPU_PROCESSOR:
> +		ret = kvm_s390_set_processor(kvm, attr);
> +		break;
> +	}
> +	return ret;
> +}
> +
> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> +{
> +	struct kvm_s390_vm_cpu_processor *proc;
> +	int rc = 0;
> +
> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> +	if (!proc) {
> +		rc = -ENOMEM;
> +		goto out;
> +	}
> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> +	proc->ibc = kvm->arch.model.ibc;
> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> +		rc = -EFAULT;
> +	kfree(proc);
> +out:
> +	return rc;
> +}
> +
> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> +{
> +	struct kvm_s390_vm_cpu_machine *mach;
> +	int rc = 0;
> +
> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> +	if (!mach) {
> +		rc = -ENOMEM;
> +		goto out;
> +	}
> +	get_cpu_id((struct cpuid *) &mach->cpuid);
> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> +	mach->ibc_range |= kvm_s390_latest_ibc();
> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> +				   S390_ARCH_FAC_LIST_SIZE_U64);

I really have a hard time grasping what hard and soft means.

Also, if user space wants to make sure that its feature list is actually 
workable on the host kernel, it needs to set and get the features again 
and then compare that with the ones it set? That's different from x86's 
cpuid implementation but probably workable.

I also don't quite grasp what the story behind IBC is. Do you actually 
block instructions? Where do you match instructions that have to get 
blocked with instructions that user space wants to see exposed?


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Fri, 16 May 2014 14:08:24 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 13.05.14 16:58, Michael Mueller wrote:
> > This patch enables cpu model support in kvm/s390 via the vm attribute
> > interface.
> >
> > During KVM initialization, the host properties cpuid, IBC value and the
> > facility list are stored in the architecture specific cpu model structure.
> >
> > During vcpu setup, these properties are taken to initialize the related SIE
> > state. This mechanism allows to adjust the properties from user space and thus
> > to implement different selectable cpu models.
> >
> > This patch uses the IBC functionality to block instructions that have not
> > been implemented at the requested CPU type and GA level compared to the
> > full host capability.
> >
> > Userspace has to initialize the cpu model before vcpu creation. A cpu model
> > change of running vcpus is currently not possible.
> 
> Why is this VM global? It usually fits a lot better modeling wise when 
> CPU types are vcpu properties.

It simplifies the code substantially because it inherently guarantees the vcpus being configured
identical. In addition, there is no S390 hardware implementation containing inhomogeneous
processor types. Thus I consider the properties as machine specific.

> 
> >
> > Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> > ---
> >   arch/s390/include/asm/kvm_host.h |   4 +-
> >   arch/s390/include/uapi/asm/kvm.h |  23 ++++++
> >   arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
> >   arch/s390/kvm/kvm-s390.h         |   1 +
> >   4 files changed, 172 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> > index b4751ba..6b826cb 100644
> > --- a/arch/s390/include/asm/kvm_host.h
> > +++ b/arch/s390/include/asm/kvm_host.h
> > @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
> >   	atomic_t cpuflags;		/* 0x0000 */
> >   	__u32 : 1;			/* 0x0004 */
> >   	__u32 prefix : 18;
> > -	__u32 : 13;
> > +	__u32 : 1;
> > +	__u32 ibc : 12;
> >   	__u8	reserved08[4];		/* 0x0008 */
> >   #define PROG_IN_SIE (1<<0)
> >   	__u32	prog0c;			/* 0x000c */
> > @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
> >   	unsigned long *sie_fac;
> >   	struct cpuid cpu_id;
> >   	unsigned long *fac_list;
> > +	unsigned short ibc;
> >   };
> >   
> >   struct kvm_arch{
> > diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> > index 313100a..82ef1b5 100644
> > --- a/arch/s390/include/uapi/asm/kvm.h
> > +++ b/arch/s390/include/uapi/asm/kvm.h
> > @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
> >   
> >   /* kvm attr_group  on vm fd */
> >   #define KVM_S390_VM_MEM_CTRL		0
> > +#define KVM_S390_VM_CPU_MODEL		1
> >   
> >   /* kvm attributes for mem_ctrl */
> >   #define KVM_S390_VM_MEM_ENABLE_CMMA	0
> >   #define KVM_S390_VM_MEM_CLR_CMMA	1
> >   #define KVM_S390_VM_MEM_CLR_PAGES	2
> >   
> > +/* kvm attributes for cpu_model */
> > +
> > +/* the s390 processor related attributes are r/w */
> > +#define KVM_S390_VM_CPU_PROCESSOR	0
> > +struct kvm_s390_vm_cpu_processor {
> > +	__u64 cpuid;
> > +	__u16 ibc;
> > +	__u8  pad[6];
> > +	__u64 fac_list[256];
> > +};
> > +
> > +/* the machine related attributes are read only */
> > +#define KVM_S390_VM_CPU_MACHINE		1
> > +struct kvm_s390_vm_cpu_machine {
> > +	__u64 cpuid;
> > +	__u32 ibc_range;
> > +	__u8  pad[4];
> > +	__u64 fac_mask[256];
> > +	__u64 hard_fac_list[256];
> > +	__u64 soft_fac_list[256];
> > +};
> > +
> >   /* for KVM_GET_REGS and KVM_SET_REGS */
> >   struct kvm_regs {
> >   	/* general purpose regs for s390 */
> > diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> > index a53652f..9965d8b 100644
> > --- a/arch/s390/kvm/kvm-s390.c
> > +++ b/arch/s390/kvm/kvm-s390.c
> > @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr
> > *attr) return ret;
> >   }
> >   
> > +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> > +{
> > +	struct kvm_s390_vm_cpu_processor *proc;
> > +
> > +	if (atomic_read(&kvm->online_vcpus))
> > +		return -EBUSY;
> > +
> > +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> > +	if (!proc)
> > +		return -ENOMEM;
> > +
> > +	if (copy_from_user(proc, (void __user *)attr->addr,
> > +			   sizeof(*proc))) {
> > +		kfree(proc);
> > +		return -EFAULT;
> > +	}
> > +
> > +	mutex_lock(&kvm->lock);
> > +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> > +	       sizeof(struct cpuid));
> > +	kvm->arch.model.ibc = proc->ibc;
> > +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> > +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> > +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> > +	mutex_unlock(&kvm->lock);
> > +	kfree(proc);
> > +
> > +	return 0;
> > +}
> > +
> > +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> > +{
> > +	int ret = -ENXIO;
> > +
> > +	switch (attr->attr) {
> > +	case KVM_S390_VM_CPU_PROCESSOR:
> > +		ret = kvm_s390_set_processor(kvm, attr);
> > +		break;
> > +	}
> > +	return ret;
> > +}
> > +
> > +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> > +{
> > +	struct kvm_s390_vm_cpu_processor *proc;
> > +	int rc = 0;
> > +
> > +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> > +	if (!proc) {
> > +		rc = -ENOMEM;
> > +		goto out;
> > +	}
> > +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> > +	proc->ibc = kvm->arch.model.ibc;
> > +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> > +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> > +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> > +		rc = -EFAULT;
> > +	kfree(proc);
> > +out:
> > +	return rc;
> > +}
> > +
> > +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> > +{
> > +	struct kvm_s390_vm_cpu_machine *mach;
> > +	int rc = 0;
> > +
> > +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> > +	if (!mach) {
> > +		rc = -ENOMEM;
> > +		goto out;
> > +	}
> > +	get_cpu_id((struct cpuid *) &mach->cpuid);
> > +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> > +	mach->ibc_range |= kvm_s390_latest_ibc();
> > +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> > +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> > +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> > +				   S390_ARCH_FAC_LIST_SIZE_U64);
> > +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> > +				   S390_ARCH_FAC_LIST_SIZE_U64);
> 
> I really have a hard time grasping what hard and soft means.

Hard facilities are those that are implemented by the CPU itself, either through processor logic
or be means of firmware micro code. That's the list returned by the STFL/STFLE instruction. In
addition to that, one can imagine that in future some of that features are emulated on KVM side.
These will be placed in the soft facility list and are optionally to request by user space.

> 
> Also, if user space wants to make sure that its feature list is actually 
> workable on the host kernel, it needs to set and get the features again 
> and then compare that with the ones it set? That's different from x86's 
> cpuid implementation but probably workable.

User space will probe what facilities are available and match them with the predefined cpu model
set. Only those models which use a partial or full subset of the hard/host facility list are
selectable. 

> 
> I also don't quite grasp what the story behind IBC is. Do you actually 
> block instructions? Where do you match instructions that have to get 
> blocked with instructions that user space wants to see exposed?
> 

Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM System
z10.) The IBC value is part of the SIE state. Just consider it as a kind of parameter, that
allows only instructions that have been implemented up to a certain cpu type and GA level to
become executed, all other op codes will end in an illegal opcode abort. E.g. take the
"Transactional Memory" instructions, they are implemented since type 2827, GA1
(IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the GA level.
0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12 is 0x02) and
GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be able to use
TE instructions in contrast to a guest running with IBC value 0x022 given the host supports it.

> 
> Alex
> 

Michael

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 16.05.14 17:39, Michael Mueller wrote:
> On Fri, 16 May 2014 14:08:24 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 13.05.14 16:58, Michael Mueller wrote:
>>> This patch enables cpu model support in kvm/s390 via the vm attribute
>>> interface.
>>>
>>> During KVM initialization, the host properties cpuid, IBC value and the
>>> facility list are stored in the architecture specific cpu model structure.
>>>
>>> During vcpu setup, these properties are taken to initialize the related SIE
>>> state. This mechanism allows to adjust the properties from user space and thus
>>> to implement different selectable cpu models.
>>>
>>> This patch uses the IBC functionality to block instructions that have not
>>> been implemented at the requested CPU type and GA level compared to the
>>> full host capability.
>>>
>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
>>> change of running vcpus is currently not possible.
>> Why is this VM global? It usually fits a lot better modeling wise when
>> CPU types are vcpu properties.
> It simplifies the code substantially because it inherently guarantees the vcpus being configured
> identical. In addition, there is no S390 hardware implementation containing inhomogeneous
> processor types. Thus I consider the properties as machine specific.
>
>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>> ---
>>>    arch/s390/include/asm/kvm_host.h |   4 +-
>>>    arch/s390/include/uapi/asm/kvm.h |  23 ++++++
>>>    arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
>>>    arch/s390/kvm/kvm-s390.h         |   1 +
>>>    4 files changed, 172 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>> index b4751ba..6b826cb 100644
>>> --- a/arch/s390/include/asm/kvm_host.h
>>> +++ b/arch/s390/include/asm/kvm_host.h
>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
>>>    	atomic_t cpuflags;		/* 0x0000 */
>>>    	__u32 : 1;			/* 0x0004 */
>>>    	__u32 prefix : 18;
>>> -	__u32 : 13;
>>> +	__u32 : 1;
>>> +	__u32 ibc : 12;
>>>    	__u8	reserved08[4];		/* 0x0008 */
>>>    #define PROG_IN_SIE (1<<0)
>>>    	__u32	prog0c;			/* 0x000c */
>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
>>>    	unsigned long *sie_fac;
>>>    	struct cpuid cpu_id;
>>>    	unsigned long *fac_list;
>>> +	unsigned short ibc;
>>>    };
>>>    
>>>    struct kvm_arch{
>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
>>> index 313100a..82ef1b5 100644
>>> --- a/arch/s390/include/uapi/asm/kvm.h
>>> +++ b/arch/s390/include/uapi/asm/kvm.h
>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
>>>    
>>>    /* kvm attr_group  on vm fd */
>>>    #define KVM_S390_VM_MEM_CTRL		0
>>> +#define KVM_S390_VM_CPU_MODEL		1
>>>    
>>>    /* kvm attributes for mem_ctrl */
>>>    #define KVM_S390_VM_MEM_ENABLE_CMMA	0
>>>    #define KVM_S390_VM_MEM_CLR_CMMA	1
>>>    #define KVM_S390_VM_MEM_CLR_PAGES	2
>>>    
>>> +/* kvm attributes for cpu_model */
>>> +
>>> +/* the s390 processor related attributes are r/w */
>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
>>> +struct kvm_s390_vm_cpu_processor {
>>> +	__u64 cpuid;
>>> +	__u16 ibc;
>>> +	__u8  pad[6];
>>> +	__u64 fac_list[256];
>>> +};
>>> +
>>> +/* the machine related attributes are read only */
>>> +#define KVM_S390_VM_CPU_MACHINE		1
>>> +struct kvm_s390_vm_cpu_machine {
>>> +	__u64 cpuid;
>>> +	__u32 ibc_range;
>>> +	__u8  pad[4];
>>> +	__u64 fac_mask[256];
>>> +	__u64 hard_fac_list[256];
>>> +	__u64 soft_fac_list[256];
>>> +};
>>> +
>>>    /* for KVM_GET_REGS and KVM_SET_REGS */
>>>    struct kvm_regs {
>>>    	/* general purpose regs for s390 */
>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>> index a53652f..9965d8b 100644
>>> --- a/arch/s390/kvm/kvm-s390.c
>>> +++ b/arch/s390/kvm/kvm-s390.c
>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr
>>> *attr) return ret;
>>>    }
>>>    
>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>> +{
>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>> +
>>> +	if (atomic_read(&kvm->online_vcpus))
>>> +		return -EBUSY;
>>> +
>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>> +	if (!proc)
>>> +		return -ENOMEM;
>>> +
>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
>>> +			   sizeof(*proc))) {
>>> +		kfree(proc);
>>> +		return -EFAULT;
>>> +	}
>>> +
>>> +	mutex_lock(&kvm->lock);
>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
>>> +	       sizeof(struct cpuid));
>>> +	kvm->arch.model.ibc = proc->ibc;
>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>> +	mutex_unlock(&kvm->lock);
>>> +	kfree(proc);
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
>>> +{
>>> +	int ret = -ENXIO;
>>> +
>>> +	switch (attr->attr) {
>>> +	case KVM_S390_VM_CPU_PROCESSOR:
>>> +		ret = kvm_s390_set_processor(kvm, attr);
>>> +		break;
>>> +	}
>>> +	return ret;
>>> +}
>>> +
>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>> +{
>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>> +	int rc = 0;
>>> +
>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>> +	if (!proc) {
>>> +		rc = -ENOMEM;
>>> +		goto out;
>>> +	}
>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
>>> +	proc->ibc = kvm->arch.model.ibc;
>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
>>> +		rc = -EFAULT;
>>> +	kfree(proc);
>>> +out:
>>> +	return rc;
>>> +}
>>> +
>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
>>> +{
>>> +	struct kvm_s390_vm_cpu_machine *mach;
>>> +	int rc = 0;
>>> +
>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
>>> +	if (!mach) {
>>> +		rc = -ENOMEM;
>>> +		goto out;
>>> +	}
>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>> I really have a hard time grasping what hard and soft means.
> Hard facilities are those that are implemented by the CPU itself, either through processor logic
> or be means of firmware micro code. That's the list returned by the STFL/STFLE instruction. In
> addition to that, one can imagine that in future some of that features are emulated on KVM side.
> These will be placed in the soft facility list and are optionally to request by user space.

I don't see why we would have to differentiate between the two. User 
space wants features enabled. Whether they are done in hardware or in 
software doesn't matter.

So all we need is a list of "features the guest sees available" which is 
the same as "features user space wants the guest to see" which then gets 
masked through "features the host can do in hardware".

For emulation we can just check on the global feature availability on 
whether we should emulate them or not.

>
>> Also, if user space wants to make sure that its feature list is actually
>> workable on the host kernel, it needs to set and get the features again
>> and then compare that with the ones it set? That's different from x86's
>> cpuid implementation but probably workable.
> User space will probe what facilities are available and match them with the predefined cpu model
> set. Only those models which use a partial or full subset of the hard/host facility list are
> selectable.

Why?

Please take a look at how x86 does cpuid masking :).

In fact, I'm not 100% convinced that it's a good idea to link cpuid / 
feature list exposure to the guest and actual feature implementation 
inside the guest together. On POWER there is a patch set pending that 
implements these two things separately - admittedly mostly because 
hardware sucks and we can't change the PVR.

>
>> I also don't quite grasp what the story behind IBC is. Do you actually
>> block instructions? Where do you match instructions that have to get
>> blocked with instructions that user space wants to see exposed?
>>
> Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM System
> z10.) The IBC value is part of the SIE state. Just consider it as a kind of parameter, that
> allows only instructions that have been implemented up to a certain cpu type and GA level to
> become executed, all other op codes will end in an illegal opcode abort. E.g. take the
> "Transactional Memory" instructions, they are implemented since type 2827, GA1
> (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the GA level.
> 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12 is 0x02) and
> GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be able to use
> TE instructions in contrast to a guest running with IBC value 0x022 given the host supports it.

That sounds very similar to the "compat" cpu property that Alexey is 
introducing for POWER. Maybe we can model it identically?


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Fri, 16 May 2014 22:31:12 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 16.05.14 17:39, Michael Mueller wrote:
> > On Fri, 16 May 2014 14:08:24 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 13.05.14 16:58, Michael Mueller wrote:
> >>> This patch enables cpu model support in kvm/s390 via the vm attribute
> >>> interface.
> >>>
> >>> During KVM initialization, the host properties cpuid, IBC value and the
> >>> facility list are stored in the architecture specific cpu model structure.
> >>>
> >>> During vcpu setup, these properties are taken to initialize the related SIE
> >>> state. This mechanism allows to adjust the properties from user space and thus
> >>> to implement different selectable cpu models.
> >>>
> >>> This patch uses the IBC functionality to block instructions that have not
> >>> been implemented at the requested CPU type and GA level compared to the
> >>> full host capability.
> >>>
> >>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
> >>> change of running vcpus is currently not possible.
> >> Why is this VM global? It usually fits a lot better modeling wise when
> >> CPU types are vcpu properties.
> > It simplifies the code substantially because it inherently guarantees the vcpus being
> > configured identical. In addition, there is no S390 hardware implementation containing
> > inhomogeneous processor types. Thus I consider the properties as machine specific.
> >
> >>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>> ---
> >>>    arch/s390/include/asm/kvm_host.h |   4 +-
> >>>    arch/s390/include/uapi/asm/kvm.h |  23 ++++++
> >>>    arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
> >>>    arch/s390/kvm/kvm-s390.h         |   1 +
> >>>    4 files changed, 172 insertions(+), 2 deletions(-)
> >>>
> >>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>> index b4751ba..6b826cb 100644
> >>> --- a/arch/s390/include/asm/kvm_host.h
> >>> +++ b/arch/s390/include/asm/kvm_host.h
> >>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
> >>>    	atomic_t cpuflags;		/* 0x0000 */
> >>>    	__u32 : 1;			/* 0x0004 */
> >>>    	__u32 prefix : 18;
> >>> -	__u32 : 13;
> >>> +	__u32 : 1;
> >>> +	__u32 ibc : 12;
> >>>    	__u8	reserved08[4];		/* 0x0008 */
> >>>    #define PROG_IN_SIE (1<<0)
> >>>    	__u32	prog0c;			/* 0x000c */
> >>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
> >>>    	unsigned long *sie_fac;
> >>>    	struct cpuid cpu_id;
> >>>    	unsigned long *fac_list;
> >>> +	unsigned short ibc;
> >>>    };
> >>>    
> >>>    struct kvm_arch{
> >>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> >>> index 313100a..82ef1b5 100644
> >>> --- a/arch/s390/include/uapi/asm/kvm.h
> >>> +++ b/arch/s390/include/uapi/asm/kvm.h
> >>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
> >>>    
> >>>    /* kvm attr_group  on vm fd */
> >>>    #define KVM_S390_VM_MEM_CTRL		0
> >>> +#define KVM_S390_VM_CPU_MODEL		1
> >>>    
> >>>    /* kvm attributes for mem_ctrl */
> >>>    #define KVM_S390_VM_MEM_ENABLE_CMMA	0
> >>>    #define KVM_S390_VM_MEM_CLR_CMMA	1
> >>>    #define KVM_S390_VM_MEM_CLR_PAGES	2
> >>>    
> >>> +/* kvm attributes for cpu_model */
> >>> +
> >>> +/* the s390 processor related attributes are r/w */
> >>> +#define KVM_S390_VM_CPU_PROCESSOR	0
> >>> +struct kvm_s390_vm_cpu_processor {
> >>> +	__u64 cpuid;
> >>> +	__u16 ibc;
> >>> +	__u8  pad[6];
> >>> +	__u64 fac_list[256];
> >>> +};
> >>> +
> >>> +/* the machine related attributes are read only */
> >>> +#define KVM_S390_VM_CPU_MACHINE		1
> >>> +struct kvm_s390_vm_cpu_machine {
> >>> +	__u64 cpuid;
> >>> +	__u32 ibc_range;
> >>> +	__u8  pad[4];
> >>> +	__u64 fac_mask[256];
> >>> +	__u64 hard_fac_list[256];
> >>> +	__u64 soft_fac_list[256];
> >>> +};
> >>> +
> >>>    /* for KVM_GET_REGS and KVM_SET_REGS */
> >>>    struct kvm_regs {
> >>>    	/* general purpose regs for s390 */
> >>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>> index a53652f..9965d8b 100644
> >>> --- a/arch/s390/kvm/kvm-s390.c
> >>> +++ b/arch/s390/kvm/kvm-s390.c
> >>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
> >>> kvm_device_attr *attr) return ret;
> >>>    }
> >>>    
> >>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>> +{
> >>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>> +
> >>> +	if (atomic_read(&kvm->online_vcpus))
> >>> +		return -EBUSY;
> >>> +
> >>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>> +	if (!proc)
> >>> +		return -ENOMEM;
> >>> +
> >>> +	if (copy_from_user(proc, (void __user *)attr->addr,
> >>> +			   sizeof(*proc))) {
> >>> +		kfree(proc);
> >>> +		return -EFAULT;
> >>> +	}
> >>> +
> >>> +	mutex_lock(&kvm->lock);
> >>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> >>> +	       sizeof(struct cpuid));
> >>> +	kvm->arch.model.ibc = proc->ibc;
> >>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> >>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> >>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>> +	mutex_unlock(&kvm->lock);
> >>> +	kfree(proc);
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> >>> +{
> >>> +	int ret = -ENXIO;
> >>> +
> >>> +	switch (attr->attr) {
> >>> +	case KVM_S390_VM_CPU_PROCESSOR:
> >>> +		ret = kvm_s390_set_processor(kvm, attr);
> >>> +		break;
> >>> +	}
> >>> +	return ret;
> >>> +}
> >>> +
> >>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>> +{
> >>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>> +	int rc = 0;
> >>> +
> >>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>> +	if (!proc) {
> >>> +		rc = -ENOMEM;
> >>> +		goto out;
> >>> +	}
> >>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> >>> +	proc->ibc = kvm->arch.model.ibc;
> >>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> >>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> >>> +		rc = -EFAULT;
> >>> +	kfree(proc);
> >>> +out:
> >>> +	return rc;
> >>> +}
> >>> +
> >>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> >>> +{
> >>> +	struct kvm_s390_vm_cpu_machine *mach;
> >>> +	int rc = 0;
> >>> +
> >>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> >>> +	if (!mach) {
> >>> +		rc = -ENOMEM;
> >>> +		goto out;
> >>> +	}
> >>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
> >>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> >>> +	mach->ibc_range |= kvm_s390_latest_ibc();
> >>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> >>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> >>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> >>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> >>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >> I really have a hard time grasping what hard and soft means.
> > Hard facilities are those that are implemented by the CPU itself, either through processor
> > logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
> > instruction. In addition to that, one can imagine that in future some of that features are
> > emulated on KVM side. These will be placed in the soft facility list and are optionally to
> > request by user space.
> 
> I don't see why we would have to differentiate between the two. User 
> space wants features enabled. Whether they are done in hardware or in 
> software doesn't matter.

I've tried to make my point on that in last answer of patch 3/6. It's a mistake
to think that user space just wants to have features, they come with different
qualities! 

> 
> So all we need is a list of "features the guest sees available" which is 
> the same as "features user space wants the guest to see" which then gets 
> masked through "features the host can do in hardware".
> 
> For emulation we can just check on the global feature availability on 
> whether we should emulate them or not.
> 
> >
> >> Also, if user space wants to make sure that its feature list is actually
> >> workable on the host kernel, it needs to set and get the features again
> >> and then compare that with the ones it set? That's different from x86's
> >> cpuid implementation but probably workable.
> > User space will probe what facilities are available and match them with the predefined cpu
> > model set. Only those models which use a partial or full subset of the hard/host facility
> > list are selectable.
> 
> Why?

If a host does not offer the features required for a model it is not able to
run efficiently.

> 
> Please take a look at how x86 does cpuid masking :).
> 
> In fact, I'm not 100% convinced that it's a good idea to link cpuid / 
> feature list exposure to the guest and actual feature implementation 
> inside the guest together. On POWER there is a patch set pending that 
> implements these two things separately - admittedly mostly because 
> hardware sucks and we can't change the PVR.

That is maybe the big difference with s390. The cpuid in the S390 case is not
directly comparable with the processor version register of POWER.

In the S390 world we have a well defined CPU model room spanned by the machine
type and its GA count. Thus we can define a bijective mapping between
(type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
name which BTW is meaningful also for a human user.

By means of this name, a management interface (libvirt) will draw decisions if
migration to a remote hypervisor is a good idea or not. For that it just needs
to compare if the current model of the guest on the source hypervisor
("query-cpu-model"), is contained in the supported model list of the target
hypervisor ("query-cpu-definitions").

> 
> >
> >> I also don't quite grasp what the story behind IBC is. Do you actually
> >> block instructions? Where do you match instructions that have to get
> >> blocked with instructions that user space wants to see exposed?
> >>
> > Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM System
> > z10.) The IBC value is part of the SIE state. Just consider it as a kind of parameter, that
> > allows only instructions that have been implemented up to a certain cpu type and GA level to
> > become executed, all other op codes will end in an illegal opcode abort. E.g. take the
> > "Transactional Memory" instructions, they are implemented since type 2827, GA1
> > (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the GA level.
> > 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12 is 0x02) and
> > GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be able to use
> > TE instructions in contrast to a guest running with IBC value 0x022 given the host supports
> > it.
> 
> That sounds very similar to the "compat" cpu property that Alexey is 
> introducing for POWER. Maybe we can model it identically?

I think it is something different. With "compat" one might be able the express some kind
of compatibility between two processors of the some different generations, upon which
the management interface can draw conclusions if migration makes sense or not.

The IBC works totally different. It enforces that the instruction set defined for TYPE-GA. 

> 
> 
> Alex
> 

Michael

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 19.05.14 12:53, Michael Mueller wrote:
> On Fri, 16 May 2014 22:31:12 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 16.05.14 17:39, Michael Mueller wrote:
>>> On Fri, 16 May 2014 14:08:24 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
>>>>> interface.
>>>>>
>>>>> During KVM initialization, the host properties cpuid, IBC value and the
>>>>> facility list are stored in the architecture specific cpu model structure.
>>>>>
>>>>> During vcpu setup, these properties are taken to initialize the related SIE
>>>>> state. This mechanism allows to adjust the properties from user space and thus
>>>>> to implement different selectable cpu models.
>>>>>
>>>>> This patch uses the IBC functionality to block instructions that have not
>>>>> been implemented at the requested CPU type and GA level compared to the
>>>>> full host capability.
>>>>>
>>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
>>>>> change of running vcpus is currently not possible.
>>>> Why is this VM global? It usually fits a lot better modeling wise when
>>>> CPU types are vcpu properties.
>>> It simplifies the code substantially because it inherently guarantees the vcpus being
>>> configured identical. In addition, there is no S390 hardware implementation containing
>>> inhomogeneous processor types. Thus I consider the properties as machine specific.
>>>
>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>> ---
>>>>>     arch/s390/include/asm/kvm_host.h |   4 +-
>>>>>     arch/s390/include/uapi/asm/kvm.h |  23 ++++++
>>>>>     arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
>>>>>     arch/s390/kvm/kvm-s390.h         |   1 +
>>>>>     4 files changed, 172 insertions(+), 2 deletions(-)
>>>>>
>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>> index b4751ba..6b826cb 100644
>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
>>>>>     	atomic_t cpuflags;		/* 0x0000 */
>>>>>     	__u32 : 1;			/* 0x0004 */
>>>>>     	__u32 prefix : 18;
>>>>> -	__u32 : 13;
>>>>> +	__u32 : 1;
>>>>> +	__u32 ibc : 12;
>>>>>     	__u8	reserved08[4];		/* 0x0008 */
>>>>>     #define PROG_IN_SIE (1<<0)
>>>>>     	__u32	prog0c;			/* 0x000c */
>>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
>>>>>     	unsigned long *sie_fac;
>>>>>     	struct cpuid cpu_id;
>>>>>     	unsigned long *fac_list;
>>>>> +	unsigned short ibc;
>>>>>     };
>>>>>     
>>>>>     struct kvm_arch{
>>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
>>>>> index 313100a..82ef1b5 100644
>>>>> --- a/arch/s390/include/uapi/asm/kvm.h
>>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
>>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
>>>>>     
>>>>>     /* kvm attr_group  on vm fd */
>>>>>     #define KVM_S390_VM_MEM_CTRL		0
>>>>> +#define KVM_S390_VM_CPU_MODEL		1
>>>>>     
>>>>>     /* kvm attributes for mem_ctrl */
>>>>>     #define KVM_S390_VM_MEM_ENABLE_CMMA	0
>>>>>     #define KVM_S390_VM_MEM_CLR_CMMA	1
>>>>>     #define KVM_S390_VM_MEM_CLR_PAGES	2
>>>>>     
>>>>> +/* kvm attributes for cpu_model */
>>>>> +
>>>>> +/* the s390 processor related attributes are r/w */
>>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
>>>>> +struct kvm_s390_vm_cpu_processor {
>>>>> +	__u64 cpuid;
>>>>> +	__u16 ibc;
>>>>> +	__u8  pad[6];
>>>>> +	__u64 fac_list[256];
>>>>> +};
>>>>> +
>>>>> +/* the machine related attributes are read only */
>>>>> +#define KVM_S390_VM_CPU_MACHINE		1
>>>>> +struct kvm_s390_vm_cpu_machine {
>>>>> +	__u64 cpuid;
>>>>> +	__u32 ibc_range;
>>>>> +	__u8  pad[4];
>>>>> +	__u64 fac_mask[256];
>>>>> +	__u64 hard_fac_list[256];
>>>>> +	__u64 soft_fac_list[256];
>>>>> +};
>>>>> +
>>>>>     /* for KVM_GET_REGS and KVM_SET_REGS */
>>>>>     struct kvm_regs {
>>>>>     	/* general purpose regs for s390 */
>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>> index a53652f..9965d8b 100644
>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
>>>>> kvm_device_attr *attr) return ret;
>>>>>     }
>>>>>     
>>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>> +{
>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>> +
>>>>> +	if (atomic_read(&kvm->online_vcpus))
>>>>> +		return -EBUSY;
>>>>> +
>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>> +	if (!proc)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
>>>>> +			   sizeof(*proc))) {
>>>>> +		kfree(proc);
>>>>> +		return -EFAULT;
>>>>> +	}
>>>>> +
>>>>> +	mutex_lock(&kvm->lock);
>>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
>>>>> +	       sizeof(struct cpuid));
>>>>> +	kvm->arch.model.ibc = proc->ibc;
>>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
>>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>> +	mutex_unlock(&kvm->lock);
>>>>> +	kfree(proc);
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>> +{
>>>>> +	int ret = -ENXIO;
>>>>> +
>>>>> +	switch (attr->attr) {
>>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
>>>>> +		ret = kvm_s390_set_processor(kvm, attr);
>>>>> +		break;
>>>>> +	}
>>>>> +	return ret;
>>>>> +}
>>>>> +
>>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>> +{
>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>> +	int rc = 0;
>>>>> +
>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>> +	if (!proc) {
>>>>> +		rc = -ENOMEM;
>>>>> +		goto out;
>>>>> +	}
>>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
>>>>> +	proc->ibc = kvm->arch.model.ibc;
>>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
>>>>> +		rc = -EFAULT;
>>>>> +	kfree(proc);
>>>>> +out:
>>>>> +	return rc;
>>>>> +}
>>>>> +
>>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>> +{
>>>>> +	struct kvm_s390_vm_cpu_machine *mach;
>>>>> +	int rc = 0;
>>>>> +
>>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
>>>>> +	if (!mach) {
>>>>> +		rc = -ENOMEM;
>>>>> +		goto out;
>>>>> +	}
>>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
>>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
>>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
>>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
>>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
>>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>> I really have a hard time grasping what hard and soft means.
>>> Hard facilities are those that are implemented by the CPU itself, either through processor
>>> logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
>>> instruction. In addition to that, one can imagine that in future some of that features are
>>> emulated on KVM side. These will be placed in the soft facility list and are optionally to
>>> request by user space.
>> I don't see why we would have to differentiate between the two. User
>> space wants features enabled. Whether they are done in hardware or in
>> software doesn't matter.
> I've tried to make my point on that in last answer of patch 3/6. It's a mistake
> to think that user space just wants to have features, they come with different
> qualities!

So? If I want to run a z9 compatible guest, I do -cpu z9. I can either

   a) run it with emulation of a facility or
   b) not run it

which one would the user choose?

>
>> So all we need is a list of "features the guest sees available" which is
>> the same as "features user space wants the guest to see" which then gets
>> masked through "features the host can do in hardware".
>>
>> For emulation we can just check on the global feature availability on
>> whether we should emulate them or not.
>>
>>>> Also, if user space wants to make sure that its feature list is actually
>>>> workable on the host kernel, it needs to set and get the features again
>>>> and then compare that with the ones it set? That's different from x86's
>>>> cpuid implementation but probably workable.
>>> User space will probe what facilities are available and match them with the predefined cpu
>>> model set. Only those models which use a partial or full subset of the hard/host facility
>>> list are selectable.
>> Why?
> If a host does not offer the features required for a model it is not able to
> run efficiently.
>
>> Please take a look at how x86 does cpuid masking :).
>>
>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
>> feature list exposure to the guest and actual feature implementation
>> inside the guest together. On POWER there is a patch set pending that
>> implements these two things separately - admittedly mostly because
>> hardware sucks and we can't change the PVR.
> That is maybe the big difference with s390. The cpuid in the S390 case is not
> directly comparable with the processor version register of POWER.
>
> In the S390 world we have a well defined CPU model room spanned by the machine
> type and its GA count. Thus we can define a bijective mapping between
> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> name which BTW is meaningful also for a human user.

Same thing as POWER.

>
> By means of this name, a management interface (libvirt) will draw decisions if
> migration to a remote hypervisor is a good idea or not. For that it just needs
> to compare if the current model of the guest on the source hypervisor
> ("query-cpu-model"), is contained in the supported model list of the target
> hypervisor ("query-cpu-definitions").

I don't think this works, since QEMU should always return all the cpu 
definitions it's aware of on query-cpu-definitions, not just the ones 
that it thinks may be compatible with the host at a random point in time.

Please check with the x86 people to find out how they do this.

>
>>>> I also don't quite grasp what the story behind IBC is. Do you actually
>>>> block instructions? Where do you match instructions that have to get
>>>> blocked with instructions that user space wants to see exposed?
>>>>
>>> Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM System
>>> z10.) The IBC value is part of the SIE state. Just consider it as a kind of parameter, that
>>> allows only instructions that have been implemented up to a certain cpu type and GA level to
>>> become executed, all other op codes will end in an illegal opcode abort. E.g. take the
>>> "Transactional Memory" instructions, they are implemented since type 2827, GA1
>>> (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the GA level.
>>> 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12 is 0x02) and
>>> GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be able to use
>>> TE instructions in contrast to a guest running with IBC value 0x022 given the host supports
>>> it.
>> That sounds very similar to the "compat" cpu property that Alexey is
>> introducing for POWER. Maybe we can model it identically?
> I think it is something different. With "compat" one might be able the express some kind
> of compatibility between two processors of the some different generations, upon which
> the management interface can draw conclusions if migration makes sense or not.
>
> The IBC works totally different. It enforces that the instruction set defined for TYPE-GA.

Yes, which is the same as the PCR register on POWER which the "compat" 
option controls. I think we can simplify s390x because it's not as 
broken as POWER in what we can fake to the guest, but I think you should 
at least be aware of the concepts that are around.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Mon, 19 May 2014 13:48:08 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 19.05.14 12:53, Michael Mueller wrote:
> > On Fri, 16 May 2014 22:31:12 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 16.05.14 17:39, Michael Mueller wrote:
> >>> On Fri, 16 May 2014 14:08:24 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 13.05.14 16:58, Michael Mueller wrote:
> >>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
> >>>>> interface.
> >>>>>
> >>>>> During KVM initialization, the host properties cpuid, IBC value and the
> >>>>> facility list are stored in the architecture specific cpu model structure.
> >>>>>
> >>>>> During vcpu setup, these properties are taken to initialize the related SIE
> >>>>> state. This mechanism allows to adjust the properties from user space and thus
> >>>>> to implement different selectable cpu models.
> >>>>>
> >>>>> This patch uses the IBC functionality to block instructions that have not
> >>>>> been implemented at the requested CPU type and GA level compared to the
> >>>>> full host capability.
> >>>>>
> >>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
> >>>>> change of running vcpus is currently not possible.
> >>>> Why is this VM global? It usually fits a lot better modeling wise when
> >>>> CPU types are vcpu properties.
> >>> It simplifies the code substantially because it inherently guarantees the vcpus being
> >>> configured identical. In addition, there is no S390 hardware implementation containing
> >>> inhomogeneous processor types. Thus I consider the properties as machine specific.
> >>>
> >>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>>>> ---
> >>>>>     arch/s390/include/asm/kvm_host.h |   4 +-
> >>>>>     arch/s390/include/uapi/asm/kvm.h |  23 ++++++
> >>>>>     arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
> >>>>>     arch/s390/kvm/kvm-s390.h         |   1 +
> >>>>>     4 files changed, 172 insertions(+), 2 deletions(-)
> >>>>>
> >>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>>>> index b4751ba..6b826cb 100644
> >>>>> --- a/arch/s390/include/asm/kvm_host.h
> >>>>> +++ b/arch/s390/include/asm/kvm_host.h
> >>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
> >>>>>     	atomic_t cpuflags;		/* 0x0000 */
> >>>>>     	__u32 : 1;			/* 0x0004 */
> >>>>>     	__u32 prefix : 18;
> >>>>> -	__u32 : 13;
> >>>>> +	__u32 : 1;
> >>>>> +	__u32 ibc : 12;
> >>>>>     	__u8	reserved08[4];		/* 0x0008 */
> >>>>>     #define PROG_IN_SIE (1<<0)
> >>>>>     	__u32	prog0c;			/* 0x000c */
> >>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
> >>>>>     	unsigned long *sie_fac;
> >>>>>     	struct cpuid cpu_id;
> >>>>>     	unsigned long *fac_list;
> >>>>> +	unsigned short ibc;
> >>>>>     };
> >>>>>     
> >>>>>     struct kvm_arch{
> >>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> >>>>> index 313100a..82ef1b5 100644
> >>>>> --- a/arch/s390/include/uapi/asm/kvm.h
> >>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
> >>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
> >>>>>     
> >>>>>     /* kvm attr_group  on vm fd */
> >>>>>     #define KVM_S390_VM_MEM_CTRL		0
> >>>>> +#define KVM_S390_VM_CPU_MODEL		1
> >>>>>     
> >>>>>     /* kvm attributes for mem_ctrl */
> >>>>>     #define KVM_S390_VM_MEM_ENABLE_CMMA	0
> >>>>>     #define KVM_S390_VM_MEM_CLR_CMMA	1
> >>>>>     #define KVM_S390_VM_MEM_CLR_PAGES	2
> >>>>>     
> >>>>> +/* kvm attributes for cpu_model */
> >>>>> +
> >>>>> +/* the s390 processor related attributes are r/w */
> >>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
> >>>>> +struct kvm_s390_vm_cpu_processor {
> >>>>> +	__u64 cpuid;
> >>>>> +	__u16 ibc;
> >>>>> +	__u8  pad[6];
> >>>>> +	__u64 fac_list[256];
> >>>>> +};
> >>>>> +
> >>>>> +/* the machine related attributes are read only */
> >>>>> +#define KVM_S390_VM_CPU_MACHINE		1
> >>>>> +struct kvm_s390_vm_cpu_machine {
> >>>>> +	__u64 cpuid;
> >>>>> +	__u32 ibc_range;
> >>>>> +	__u8  pad[4];
> >>>>> +	__u64 fac_mask[256];
> >>>>> +	__u64 hard_fac_list[256];
> >>>>> +	__u64 soft_fac_list[256];
> >>>>> +};
> >>>>> +
> >>>>>     /* for KVM_GET_REGS and KVM_SET_REGS */
> >>>>>     struct kvm_regs {
> >>>>>     	/* general purpose regs for s390 */
> >>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>>>> index a53652f..9965d8b 100644
> >>>>> --- a/arch/s390/kvm/kvm-s390.c
> >>>>> +++ b/arch/s390/kvm/kvm-s390.c
> >>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
> >>>>> kvm_device_attr *attr) return ret;
> >>>>>     }
> >>>>>     
> >>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>> +{
> >>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>> +
> >>>>> +	if (atomic_read(&kvm->online_vcpus))
> >>>>> +		return -EBUSY;
> >>>>> +
> >>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>> +	if (!proc)
> >>>>> +		return -ENOMEM;
> >>>>> +
> >>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
> >>>>> +			   sizeof(*proc))) {
> >>>>> +		kfree(proc);
> >>>>> +		return -EFAULT;
> >>>>> +	}
> >>>>> +
> >>>>> +	mutex_lock(&kvm->lock);
> >>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> >>>>> +	       sizeof(struct cpuid));
> >>>>> +	kvm->arch.model.ibc = proc->ibc;
> >>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> >>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> >>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>> +	mutex_unlock(&kvm->lock);
> >>>>> +	kfree(proc);
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>> +{
> >>>>> +	int ret = -ENXIO;
> >>>>> +
> >>>>> +	switch (attr->attr) {
> >>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
> >>>>> +		ret = kvm_s390_set_processor(kvm, attr);
> >>>>> +		break;
> >>>>> +	}
> >>>>> +	return ret;
> >>>>> +}
> >>>>> +
> >>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>> +{
> >>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>> +	int rc = 0;
> >>>>> +
> >>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>> +	if (!proc) {
> >>>>> +		rc = -ENOMEM;
> >>>>> +		goto out;
> >>>>> +	}
> >>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> >>>>> +	proc->ibc = kvm->arch.model.ibc;
> >>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> >>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> >>>>> +		rc = -EFAULT;
> >>>>> +	kfree(proc);
> >>>>> +out:
> >>>>> +	return rc;
> >>>>> +}
> >>>>> +
> >>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>> +{
> >>>>> +	struct kvm_s390_vm_cpu_machine *mach;
> >>>>> +	int rc = 0;
> >>>>> +
> >>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> >>>>> +	if (!mach) {
> >>>>> +		rc = -ENOMEM;
> >>>>> +		goto out;
> >>>>> +	}
> >>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
> >>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> >>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
> >>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> >>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> >>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> >>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> >>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>> I really have a hard time grasping what hard and soft means.
> >>> Hard facilities are those that are implemented by the CPU itself, either through processor
> >>> logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
> >>> instruction. In addition to that, one can imagine that in future some of that features are
> >>> emulated on KVM side. These will be placed in the soft facility list and are optionally to
> >>> request by user space.
> >> I don't see why we would have to differentiate between the two. User
> >> space wants features enabled. Whether they are done in hardware or in
> >> software doesn't matter.
> > I've tried to make my point on that in last answer of patch 3/6. It's a mistake
> > to think that user space just wants to have features, they come with different
> > qualities!
> 
> So? If I want to run a z9 compatible guest, I do -cpu z9. I can either
> 
>    a) run it with emulation of a facility or
>    b) not run it
> 
> which one would the user choose?

If you run on a z990 host, you better use -cpu z990 because emulating some
fancy delta feature just cost additional CPU time. If the host is newer, please
go with -cpu z9.

What user and thus also user space wants depends on other factors:

1. reliability
2. performance
3. availability

It's not features, that's what programmers want.

That's why I have designed the model and migration capability around the hardware
and not around the software features and don't allow them to be enabled currently
together.

A software feature is a nice add on that is helpful for evaluation or development
purpose. There is few space for it on productions systems.

One option that I currently see to make software implemented facility migration
capable is to calculate some kind of hash value derived from the full set of
active software facilities. That value can be compared with pre-calculated
values also stored in the supported model table of qemu. This value could be
seen like a virtual model extension that has to match like the model name.

But I have said it elsewhere already, a soft facility should be an exception and
not the rule.

> 
> >
> >> So all we need is a list of "features the guest sees available" which is
> >> the same as "features user space wants the guest to see" which then gets
> >> masked through "features the host can do in hardware".
> >>
> >> For emulation we can just check on the global feature availability on
> >> whether we should emulate them or not.
> >>
> >>>> Also, if user space wants to make sure that its feature list is actually
> >>>> workable on the host kernel, it needs to set and get the features again
> >>>> and then compare that with the ones it set? That's different from x86's
> >>>> cpuid implementation but probably workable.
> >>> User space will probe what facilities are available and match them with the predefined cpu
> >>> model set. Only those models which use a partial or full subset of the hard/host facility
> >>> list are selectable.
> >> Why?
> > If a host does not offer the features required for a model it is not able to
> > run efficiently.
> >
> >> Please take a look at how x86 does cpuid masking :).
> >>
> >> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
> >> feature list exposure to the guest and actual feature implementation
> >> inside the guest together. On POWER there is a patch set pending that
> >> implements these two things separately - admittedly mostly because
> >> hardware sucks and we can't change the PVR.
> > That is maybe the big difference with s390. The cpuid in the S390 case is not
> > directly comparable with the processor version register of POWER.
> >
> > In the S390 world we have a well defined CPU model room spanned by the machine
> > type and its GA count. Thus we can define a bijective mapping between
> > (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> > name which BTW is meaningful also for a human user.
> 
> Same thing as POWER.
> 
> >
> > By means of this name, a management interface (libvirt) will draw decisions if
> > migration to a remote hypervisor is a good idea or not. For that it just needs
> > to compare if the current model of the guest on the source hypervisor
> > ("query-cpu-model"), is contained in the supported model list of the target
> > hypervisor ("query-cpu-definitions").
> 
> I don't think this works, since QEMU should always return all the cpu 
> definitions it's aware of on query-cpu-definitions, not just the ones 
> that it thinks may be compatible with the host at a random point in time.

It does not return model names that it thinks they are compatible at some point
in time. In s390 mode, it returns all definitions (CPU models) that a given host
system is capable to run. Together with the CPU model run by the guest, some upper
management interface knows if the hypervisor supports the required CPU model and
uses a guest definition with the same CPU model on the target hypervisor.

The information for that is taken from the model table which QEMU builds up during
startup time. This list limits the command line selectable CPU models as well.

This even easily allows to find some GCM (Greatest Common CPU Model) among a set
of hypervisors to allow migration in a a cluster with n>2 parties.

> 
> Please check with the x86 people to find out how they do this.

They currently just list their base model names:
{"return":
  [{"name": "qemu64"}, 
   {"name": "phenom"}, 
   {"name": "core2duo"},
   {"name": "kvm64"},
   {"name": "qemu32"},
   {"name": "kvm32"},
   {"name": "coreduo"},
   {"name": "486"},
   {"name": "pentium"},
   {"name": "pentium2"},
   {"name": "pentium3"},
   {"name": "athlon"},
   {"name": "n270"},
   {"name": "Conroe"},
   {"name": "Penryn"},
   {"name": "Nehalem"},
   {"name": "Westmere"},
   {"name": "SandyBridge"},
   {"name": "Opteron_G1"},
   {"name": "Opteron_G2"},
   {"name": "Opteron_G3"},
   {"name": "Opteron_G4"}]
}

> 
> >
> >>>> I also don't quite grasp what the story behind IBC is. Do you actually
> >>>> block instructions? Where do you match instructions that have to get
> >>>> blocked with instructions that user space wants to see exposed?
> >>>>
> >>> Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM
> >>> System z10.) The IBC value is part of the SIE state. Just consider it as a kind of
> >>> parameter, that allows only instructions that have been implemented up to a certain cpu
> >>> type and GA level to become executed, all other op codes will end in an illegal opcode
> >>> abort. E.g. take the "Transactional Memory" instructions, they are implemented since type
> >>> 2827, GA1 (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the
> >>> GA level. 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12
> >>> is 0x02) and GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be
> >>> able to use TE instructions in contrast to a guest running with IBC value 0x022 given the
> >>> host supports it.
> >> That sounds very similar to the "compat" cpu property that Alexey is
> >> introducing for POWER. Maybe we can model it identically?
> > I think it is something different. With "compat" one might be able the express some kind
> > of compatibility between two processors of the some different generations, upon which
> > the management interface can draw conclusions if migration makes sense or not.
> >
> > The IBC works totally different. It enforces that the instruction set defined for TYPE-GA.
> 
> Yes, which is the same as the PCR register on POWER which the "compat" 
> option controls. I think we can simplify s390x because it's not as 
> broken as POWER in what we can fake to the guest, but I think you should 
> at least be aware of the concepts that are around.

Sorry, I first thought you were talking about the PVR but it looks you are
talking about the LPCR (Logical Partition Control Register). This is as well
different from IBC. The LPCR allows the host POWER LPAR to run in a compatible
mode with back level CPUs. That will allow e.g. POWER8 guests to run safely on
POWER7 if they were started in this mode. But then all the guests in this KVM
hypervisor run in this compatibility mode.

With IBC, all guests have their own value, some might use the full instruction
set which the PR/SM LPAR is offering and others are limited, eventually to
different degrees depending on the CPU model request.

Michael

> 
> 
> Alex
> 

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 19.05.14 16:18, Michael Mueller wrote:
> On Mon, 19 May 2014 13:48:08 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 19.05.14 12:53, Michael Mueller wrote:
>>> On Fri, 16 May 2014 22:31:12 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 16.05.14 17:39, Michael Mueller wrote:
>>>>> On Fri, 16 May 2014 14:08:24 +0200
>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>
>>>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
>>>>>>> interface.
>>>>>>>
>>>>>>> During KVM initialization, the host properties cpuid, IBC value and the
>>>>>>> facility list are stored in the architecture specific cpu model structure.
>>>>>>>
>>>>>>> During vcpu setup, these properties are taken to initialize the related SIE
>>>>>>> state. This mechanism allows to adjust the properties from user space and thus
>>>>>>> to implement different selectable cpu models.
>>>>>>>
>>>>>>> This patch uses the IBC functionality to block instructions that have not
>>>>>>> been implemented at the requested CPU type and GA level compared to the
>>>>>>> full host capability.
>>>>>>>
>>>>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
>>>>>>> change of running vcpus is currently not possible.
>>>>>> Why is this VM global? It usually fits a lot better modeling wise when
>>>>>> CPU types are vcpu properties.
>>>>> It simplifies the code substantially because it inherently guarantees the vcpus being
>>>>> configured identical. In addition, there is no S390 hardware implementation containing
>>>>> inhomogeneous processor types. Thus I consider the properties as machine specific.
>>>>>
>>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>>>> ---
>>>>>>>      arch/s390/include/asm/kvm_host.h |   4 +-
>>>>>>>      arch/s390/include/uapi/asm/kvm.h |  23 ++++++
>>>>>>>      arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
>>>>>>>      arch/s390/kvm/kvm-s390.h         |   1 +
>>>>>>>      4 files changed, 172 insertions(+), 2 deletions(-)
>>>>>>>
>>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>>>> index b4751ba..6b826cb 100644
>>>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
>>>>>>>      	atomic_t cpuflags;		/* 0x0000 */
>>>>>>>      	__u32 : 1;			/* 0x0004 */
>>>>>>>      	__u32 prefix : 18;
>>>>>>> -	__u32 : 13;
>>>>>>> +	__u32 : 1;
>>>>>>> +	__u32 ibc : 12;
>>>>>>>      	__u8	reserved08[4];		/* 0x0008 */
>>>>>>>      #define PROG_IN_SIE (1<<0)
>>>>>>>      	__u32	prog0c;			/* 0x000c */
>>>>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
>>>>>>>      	unsigned long *sie_fac;
>>>>>>>      	struct cpuid cpu_id;
>>>>>>>      	unsigned long *fac_list;
>>>>>>> +	unsigned short ibc;
>>>>>>>      };
>>>>>>>      
>>>>>>>      struct kvm_arch{
>>>>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
>>>>>>> index 313100a..82ef1b5 100644
>>>>>>> --- a/arch/s390/include/uapi/asm/kvm.h
>>>>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
>>>>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
>>>>>>>      
>>>>>>>      /* kvm attr_group  on vm fd */
>>>>>>>      #define KVM_S390_VM_MEM_CTRL		0
>>>>>>> +#define KVM_S390_VM_CPU_MODEL		1
>>>>>>>      
>>>>>>>      /* kvm attributes for mem_ctrl */
>>>>>>>      #define KVM_S390_VM_MEM_ENABLE_CMMA	0
>>>>>>>      #define KVM_S390_VM_MEM_CLR_CMMA	1
>>>>>>>      #define KVM_S390_VM_MEM_CLR_PAGES	2
>>>>>>>      
>>>>>>> +/* kvm attributes for cpu_model */
>>>>>>> +
>>>>>>> +/* the s390 processor related attributes are r/w */
>>>>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
>>>>>>> +struct kvm_s390_vm_cpu_processor {
>>>>>>> +	__u64 cpuid;
>>>>>>> +	__u16 ibc;
>>>>>>> +	__u8  pad[6];
>>>>>>> +	__u64 fac_list[256];
>>>>>>> +};
>>>>>>> +
>>>>>>> +/* the machine related attributes are read only */
>>>>>>> +#define KVM_S390_VM_CPU_MACHINE		1
>>>>>>> +struct kvm_s390_vm_cpu_machine {
>>>>>>> +	__u64 cpuid;
>>>>>>> +	__u32 ibc_range;
>>>>>>> +	__u8  pad[4];
>>>>>>> +	__u64 fac_mask[256];
>>>>>>> +	__u64 hard_fac_list[256];
>>>>>>> +	__u64 soft_fac_list[256];
>>>>>>> +};
>>>>>>> +
>>>>>>>      /* for KVM_GET_REGS and KVM_SET_REGS */
>>>>>>>      struct kvm_regs {
>>>>>>>      	/* general purpose regs for s390 */
>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>> index a53652f..9965d8b 100644
>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
>>>>>>> kvm_device_attr *attr) return ret;
>>>>>>>      }
>>>>>>>      
>>>>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>> +{
>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>>>> +
>>>>>>> +	if (atomic_read(&kvm->online_vcpus))
>>>>>>> +		return -EBUSY;
>>>>>>> +
>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>>>> +	if (!proc)
>>>>>>> +		return -ENOMEM;
>>>>>>> +
>>>>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
>>>>>>> +			   sizeof(*proc))) {
>>>>>>> +		kfree(proc);
>>>>>>> +		return -EFAULT;
>>>>>>> +	}
>>>>>>> +
>>>>>>> +	mutex_lock(&kvm->lock);
>>>>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
>>>>>>> +	       sizeof(struct cpuid));
>>>>>>> +	kvm->arch.model.ibc = proc->ibc;
>>>>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
>>>>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>>>> +	mutex_unlock(&kvm->lock);
>>>>>>> +	kfree(proc);
>>>>>>> +
>>>>>>> +	return 0;
>>>>>>> +}
>>>>>>> +
>>>>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>> +{
>>>>>>> +	int ret = -ENXIO;
>>>>>>> +
>>>>>>> +	switch (attr->attr) {
>>>>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
>>>>>>> +		ret = kvm_s390_set_processor(kvm, attr);
>>>>>>> +		break;
>>>>>>> +	}
>>>>>>> +	return ret;
>>>>>>> +}
>>>>>>> +
>>>>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>> +{
>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>>>> +	int rc = 0;
>>>>>>> +
>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>>>> +	if (!proc) {
>>>>>>> +		rc = -ENOMEM;
>>>>>>> +		goto out;
>>>>>>> +	}
>>>>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
>>>>>>> +	proc->ibc = kvm->arch.model.ibc;
>>>>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
>>>>>>> +		rc = -EFAULT;
>>>>>>> +	kfree(proc);
>>>>>>> +out:
>>>>>>> +	return rc;
>>>>>>> +}
>>>>>>> +
>>>>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>> +{
>>>>>>> +	struct kvm_s390_vm_cpu_machine *mach;
>>>>>>> +	int rc = 0;
>>>>>>> +
>>>>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
>>>>>>> +	if (!mach) {
>>>>>>> +		rc = -ENOMEM;
>>>>>>> +		goto out;
>>>>>>> +	}
>>>>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
>>>>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
>>>>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
>>>>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
>>>>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
>>>>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>>>> I really have a hard time grasping what hard and soft means.
>>>>> Hard facilities are those that are implemented by the CPU itself, either through processor
>>>>> logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
>>>>> instruction. In addition to that, one can imagine that in future some of that features are
>>>>> emulated on KVM side. These will be placed in the soft facility list and are optionally to
>>>>> request by user space.
>>>> I don't see why we would have to differentiate between the two. User
>>>> space wants features enabled. Whether they are done in hardware or in
>>>> software doesn't matter.
>>> I've tried to make my point on that in last answer of patch 3/6. It's a mistake
>>> to think that user space just wants to have features, they come with different
>>> qualities!
>> So? If I want to run a z9 compatible guest, I do -cpu z9. I can either
>>
>>     a) run it with emulation of a facility or
>>     b) not run it
>>
>> which one would the user choose?
> If you run on a z990 host, you better use -cpu z990 because emulating some
> fancy delta feature just cost additional CPU time. If the host is newer, please
> go with -cpu z9.

Yes, I agree on that statement. Imagine a feature gets *dropped* though. 
In that case -cpu z9 should enable emulation of that feature to maintain 
migratability with a real z9 machine on newer hardware.

>
> What user and thus also user space wants depends on other factors:
>
> 1. reliability
> 2. performance
> 3. availability
>
> It's not features, that's what programmers want.
>
> That's why I have designed the model and migration capability around the hardware
> and not around the software features and don't allow them to be enabled currently
> together.
>
> A software feature is a nice add on that is helpful for evaluation or development
> purpose. There is few space for it on productions systems.
>
> One option that I currently see to make software implemented facility migration
> capable is to calculate some kind of hash value derived from the full set of
> active software facilities. That value can be compared with pre-calculated
> values also stored in the supported model table of qemu. This value could be
> seen like a virtual model extension that has to match like the model name.
>
> But I have said it elsewhere already, a soft facility should be an exception and
> not the rule.
>
>>>> So all we need is a list of "features the guest sees available" which is
>>>> the same as "features user space wants the guest to see" which then gets
>>>> masked through "features the host can do in hardware".
>>>>
>>>> For emulation we can just check on the global feature availability on
>>>> whether we should emulate them or not.
>>>>
>>>>>> Also, if user space wants to make sure that its feature list is actually
>>>>>> workable on the host kernel, it needs to set and get the features again
>>>>>> and then compare that with the ones it set? That's different from x86's
>>>>>> cpuid implementation but probably workable.
>>>>> User space will probe what facilities are available and match them with the predefined cpu
>>>>> model set. Only those models which use a partial or full subset of the hard/host facility
>>>>> list are selectable.
>>>> Why?
>>> If a host does not offer the features required for a model it is not able to
>>> run efficiently.
>>>
>>>> Please take a look at how x86 does cpuid masking :).
>>>>
>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
>>>> feature list exposure to the guest and actual feature implementation
>>>> inside the guest together. On POWER there is a patch set pending that
>>>> implements these two things separately - admittedly mostly because
>>>> hardware sucks and we can't change the PVR.
>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
>>> directly comparable with the processor version register of POWER.
>>>
>>> In the S390 world we have a well defined CPU model room spanned by the machine
>>> type and its GA count. Thus we can define a bijective mapping between
>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
>>> name which BTW is meaningful also for a human user.
>> Same thing as POWER.
>>
>>> By means of this name, a management interface (libvirt) will draw decisions if
>>> migration to a remote hypervisor is a good idea or not. For that it just needs
>>> to compare if the current model of the guest on the source hypervisor
>>> ("query-cpu-model"), is contained in the supported model list of the target
>>> hypervisor ("query-cpu-definitions").
>> I don't think this works, since QEMU should always return all the cpu
>> definitions it's aware of on query-cpu-definitions, not just the ones
>> that it thinks may be compatible with the host at a random point in time.
> It does not return model names that it thinks they are compatible at some point
> in time. In s390 mode, it returns all definitions (CPU models) that a given host
> system is capable to run. Together with the CPU model run by the guest, some upper
> management interface knows if the hypervisor supports the required CPU model and
> uses a guest definition with the same CPU model on the target hypervisor.
>
> The information for that is taken from the model table which QEMU builds up during
> startup time. This list limits the command line selectable CPU models as well.

This makes s390 derive from the way x86 handles things. NAK.

>
> This even easily allows to find some GCM (Greatest Common CPU Model) among a set
> of hypervisors to allow migration in a a cluster with n>2 parties.
>
>> Please check with the x86 people to find out how they do this.
> They currently just list their base model names:
> {"return":
>    [{"name": "qemu64"},
>     {"name": "phenom"},
>     {"name": "core2duo"},
>     {"name": "kvm64"},
>     {"name": "qemu32"},
>     {"name": "kvm32"},
>     {"name": "coreduo"},
>     {"name": "486"},
>     {"name": "pentium"},
>     {"name": "pentium2"},
>     {"name": "pentium3"},
>     {"name": "athlon"},
>     {"name": "n270"},
>     {"name": "Conroe"},
>     {"name": "Penryn"},
>     {"name": "Nehalem"},
>     {"name": "Westmere"},
>     {"name": "SandyBridge"},
>     {"name": "Opteron_G1"},
>     {"name": "Opteron_G2"},
>     {"name": "Opteron_G3"},
>     {"name": "Opteron_G4"}]
> }
>
>>>>>> I also don't quite grasp what the story behind IBC is. Do you actually
>>>>>> block instructions? Where do you match instructions that have to get
>>>>>> blocked with instructions that user space wants to see exposed?
>>>>>>
>>>>> Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM
>>>>> System z10.) The IBC value is part of the SIE state. Just consider it as a kind of
>>>>> parameter, that allows only instructions that have been implemented up to a certain cpu
>>>>> type and GA level to become executed, all other op codes will end in an illegal opcode
>>>>> abort. E.g. take the "Transactional Memory" instructions, they are implemented since type
>>>>> 2827, GA1 (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the
>>>>> GA level. 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation 12
>>>>> is 0x02) and GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will not be
>>>>> able to use TE instructions in contrast to a guest running with IBC value 0x022 given the
>>>>> host supports it.
>>>> That sounds very similar to the "compat" cpu property that Alexey is
>>>> introducing for POWER. Maybe we can model it identically?
>>> I think it is something different. With "compat" one might be able the express some kind
>>> of compatibility between two processors of the some different generations, upon which
>>> the management interface can draw conclusions if migration makes sense or not.
>>>
>>> The IBC works totally different. It enforces that the instruction set defined for TYPE-GA.
>> Yes, which is the same as the PCR register on POWER which the "compat"
>> option controls. I think we can simplify s390x because it's not as
>> broken as POWER in what we can fake to the guest, but I think you should
>> at least be aware of the concepts that are around.
> Sorry, I first thought you were talking about the PVR but it looks you are
> talking about the LPCR (Logical Partition Control Register). This is as well
> different from IBC. The LPCR allows the host POWER LPAR to run in a compatible
> mode with back level CPUs. That will allow e.g. POWER8 guests to run safely on
> POWER7 if they were started in this mode. But then all the guests in this KVM
> hypervisor run in this compatibility mode.

It's really the same. We context switch LPCR on POWER.

The only difference is that you guys can fake PVR as well while PCR can 
only limit feature availability. But the underlying problem that I was 
trying to point out is that the bits in IBC are different bits from the 
FAC bits. Some code has to know the correlation.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Mon, 19 May 2014 16:49:28 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 19.05.14 16:18, Michael Mueller wrote:
> > On Mon, 19 May 2014 13:48:08 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 19.05.14 12:53, Michael Mueller wrote:
> >>> On Fri, 16 May 2014 22:31:12 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 16.05.14 17:39, Michael Mueller wrote:
> >>>>> On Fri, 16 May 2014 14:08:24 +0200
> >>>>> Alexander Graf <agraf@suse.de> wrote:
> >>>>>
> >>>>>> On 13.05.14 16:58, Michael Mueller wrote:
> >>>>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
> >>>>>>> interface.
> >>>>>>>
> >>>>>>> During KVM initialization, the host properties cpuid, IBC value and the
> >>>>>>> facility list are stored in the architecture specific cpu model structure.
> >>>>>>>
> >>>>>>> During vcpu setup, these properties are taken to initialize the related SIE
> >>>>>>> state. This mechanism allows to adjust the properties from user space and thus
> >>>>>>> to implement different selectable cpu models.
> >>>>>>>
> >>>>>>> This patch uses the IBC functionality to block instructions that have not
> >>>>>>> been implemented at the requested CPU type and GA level compared to the
> >>>>>>> full host capability.
> >>>>>>>
> >>>>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
> >>>>>>> change of running vcpus is currently not possible.
> >>>>>> Why is this VM global? It usually fits a lot better modeling wise when
> >>>>>> CPU types are vcpu properties.
> >>>>> It simplifies the code substantially because it inherently guarantees the vcpus being
> >>>>> configured identical. In addition, there is no S390 hardware implementation containing
> >>>>> inhomogeneous processor types. Thus I consider the properties as machine specific.
> >>>>>
> >>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>>>>>> ---
> >>>>>>>      arch/s390/include/asm/kvm_host.h |   4 +-
> >>>>>>>      arch/s390/include/uapi/asm/kvm.h |  23 ++++++
> >>>>>>>      arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
> >>>>>>>      arch/s390/kvm/kvm-s390.h         |   1 +
> >>>>>>>      4 files changed, 172 insertions(+), 2 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>>>>>> index b4751ba..6b826cb 100644
> >>>>>>> --- a/arch/s390/include/asm/kvm_host.h
> >>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
> >>>>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
> >>>>>>>      	atomic_t cpuflags;		/* 0x0000 */
> >>>>>>>      	__u32 : 1;			/* 0x0004 */
> >>>>>>>      	__u32 prefix : 18;
> >>>>>>> -	__u32 : 13;
> >>>>>>> +	__u32 : 1;
> >>>>>>> +	__u32 ibc : 12;
> >>>>>>>      	__u8	reserved08[4];		/* 0x0008 */
> >>>>>>>      #define PROG_IN_SIE (1<<0)
> >>>>>>>      	__u32	prog0c;			/* 0x000c */
> >>>>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
> >>>>>>>      	unsigned long *sie_fac;
> >>>>>>>      	struct cpuid cpu_id;
> >>>>>>>      	unsigned long *fac_list;
> >>>>>>> +	unsigned short ibc;
> >>>>>>>      };
> >>>>>>>      
> >>>>>>>      struct kvm_arch{
> >>>>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> >>>>>>> index 313100a..82ef1b5 100644
> >>>>>>> --- a/arch/s390/include/uapi/asm/kvm.h
> >>>>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
> >>>>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
> >>>>>>>      
> >>>>>>>      /* kvm attr_group  on vm fd */
> >>>>>>>      #define KVM_S390_VM_MEM_CTRL		0
> >>>>>>> +#define KVM_S390_VM_CPU_MODEL		1
> >>>>>>>      
> >>>>>>>      /* kvm attributes for mem_ctrl */
> >>>>>>>      #define KVM_S390_VM_MEM_ENABLE_CMMA	0
> >>>>>>>      #define KVM_S390_VM_MEM_CLR_CMMA	1
> >>>>>>>      #define KVM_S390_VM_MEM_CLR_PAGES	2
> >>>>>>>      
> >>>>>>> +/* kvm attributes for cpu_model */
> >>>>>>> +
> >>>>>>> +/* the s390 processor related attributes are r/w */
> >>>>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
> >>>>>>> +struct kvm_s390_vm_cpu_processor {
> >>>>>>> +	__u64 cpuid;
> >>>>>>> +	__u16 ibc;
> >>>>>>> +	__u8  pad[6];
> >>>>>>> +	__u64 fac_list[256];
> >>>>>>> +};
> >>>>>>> +
> >>>>>>> +/* the machine related attributes are read only */
> >>>>>>> +#define KVM_S390_VM_CPU_MACHINE		1
> >>>>>>> +struct kvm_s390_vm_cpu_machine {
> >>>>>>> +	__u64 cpuid;
> >>>>>>> +	__u32 ibc_range;
> >>>>>>> +	__u8  pad[4];
> >>>>>>> +	__u64 fac_mask[256];
> >>>>>>> +	__u64 hard_fac_list[256];
> >>>>>>> +	__u64 soft_fac_list[256];
> >>>>>>> +};
> >>>>>>> +
> >>>>>>>      /* for KVM_GET_REGS and KVM_SET_REGS */
> >>>>>>>      struct kvm_regs {
> >>>>>>>      	/* general purpose regs for s390 */
> >>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>>>>>> index a53652f..9965d8b 100644
> >>>>>>> --- a/arch/s390/kvm/kvm-s390.c
> >>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
> >>>>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
> >>>>>>> kvm_device_attr *attr) return ret;
> >>>>>>>      }
> >>>>>>>      
> >>>>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>> +{
> >>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>>>> +
> >>>>>>> +	if (atomic_read(&kvm->online_vcpus))
> >>>>>>> +		return -EBUSY;
> >>>>>>> +
> >>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>>>> +	if (!proc)
> >>>>>>> +		return -ENOMEM;
> >>>>>>> +
> >>>>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
> >>>>>>> +			   sizeof(*proc))) {
> >>>>>>> +		kfree(proc);
> >>>>>>> +		return -EFAULT;
> >>>>>>> +	}
> >>>>>>> +
> >>>>>>> +	mutex_lock(&kvm->lock);
> >>>>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> >>>>>>> +	       sizeof(struct cpuid));
> >>>>>>> +	kvm->arch.model.ibc = proc->ibc;
> >>>>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> >>>>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> >>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>>>> +	mutex_unlock(&kvm->lock);
> >>>>>>> +	kfree(proc);
> >>>>>>> +
> >>>>>>> +	return 0;
> >>>>>>> +}
> >>>>>>> +
> >>>>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>> +{
> >>>>>>> +	int ret = -ENXIO;
> >>>>>>> +
> >>>>>>> +	switch (attr->attr) {
> >>>>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
> >>>>>>> +		ret = kvm_s390_set_processor(kvm, attr);
> >>>>>>> +		break;
> >>>>>>> +	}
> >>>>>>> +	return ret;
> >>>>>>> +}
> >>>>>>> +
> >>>>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>> +{
> >>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>>>> +	int rc = 0;
> >>>>>>> +
> >>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>>>> +	if (!proc) {
> >>>>>>> +		rc = -ENOMEM;
> >>>>>>> +		goto out;
> >>>>>>> +	}
> >>>>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> >>>>>>> +	proc->ibc = kvm->arch.model.ibc;
> >>>>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> >>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> >>>>>>> +		rc = -EFAULT;
> >>>>>>> +	kfree(proc);
> >>>>>>> +out:
> >>>>>>> +	return rc;
> >>>>>>> +}
> >>>>>>> +
> >>>>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>> +{
> >>>>>>> +	struct kvm_s390_vm_cpu_machine *mach;
> >>>>>>> +	int rc = 0;
> >>>>>>> +
> >>>>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> >>>>>>> +	if (!mach) {
> >>>>>>> +		rc = -ENOMEM;
> >>>>>>> +		goto out;
> >>>>>>> +	}
> >>>>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
> >>>>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> >>>>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
> >>>>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> >>>>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> >>>>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> >>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> >>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>>>> I really have a hard time grasping what hard and soft means.
> >>>>> Hard facilities are those that are implemented by the CPU itself, either through processor
> >>>>> logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
> >>>>> instruction. In addition to that, one can imagine that in future some of that features are
> >>>>> emulated on KVM side. These will be placed in the soft facility list and are optionally to
> >>>>> request by user space.
> >>>> I don't see why we would have to differentiate between the two. User
> >>>> space wants features enabled. Whether they are done in hardware or in
> >>>> software doesn't matter.
> >>> I've tried to make my point on that in last answer of patch 3/6. It's a mistake
> >>> to think that user space just wants to have features, they come with different
> >>> qualities!
> >> So? If I want to run a z9 compatible guest, I do -cpu z9. I can either
> >>
> >>     a) run it with emulation of a facility or
> >>     b) not run it
> >>
> >> which one would the user choose?
> > If you run on a z990 host, you better use -cpu z990 because emulating some
> > fancy delta feature just cost additional CPU time. If the host is newer, please
> > go with -cpu z9.
> 
> Yes, I agree on that statement. Imagine a feature gets *dropped* though. 
> In that case -cpu z9 should enable emulation of that feature to maintain 
> migratability with a real z9 machine on newer hardware.

Nice try, but think what's happening in real world. Let's assume the feature is
TE again, available since zEC12 but would go away with zNext. In that case the
CPU model zNext-GA1 and all successors will not have zEC12 as supported model.
The application will just not run on that model if it insists on executing TE
instructions.

> 
> >
> > What user and thus also user space wants depends on other factors:
> >
> > 1. reliability
> > 2. performance
> > 3. availability
> >
> > It's not features, that's what programmers want.
> >
> > That's why I have designed the model and migration capability around the hardware
> > and not around the software features and don't allow them to be enabled currently
> > together.
> >
> > A software feature is a nice add on that is helpful for evaluation or development
> > purpose. There is few space for it on productions systems.
> >
> > One option that I currently see to make software implemented facility migration
> > capable is to calculate some kind of hash value derived from the full set of
> > active software facilities. That value can be compared with pre-calculated
> > values also stored in the supported model table of qemu. This value could be
> > seen like a virtual model extension that has to match like the model name.
> >
> > But I have said it elsewhere already, a soft facility should be an exception and
> > not the rule.
> >
> >>>> So all we need is a list of "features the guest sees available" which is
> >>>> the same as "features user space wants the guest to see" which then gets
> >>>> masked through "features the host can do in hardware".
> >>>>
> >>>> For emulation we can just check on the global feature availability on
> >>>> whether we should emulate them or not.
> >>>>
> >>>>>> Also, if user space wants to make sure that its feature list is actually
> >>>>>> workable on the host kernel, it needs to set and get the features again
> >>>>>> and then compare that with the ones it set? That's different from x86's
> >>>>>> cpuid implementation but probably workable.
> >>>>> User space will probe what facilities are available and match them with the predefined cpu
> >>>>> model set. Only those models which use a partial or full subset of the hard/host facility
> >>>>> list are selectable.
> >>>> Why?
> >>> If a host does not offer the features required for a model it is not able to
> >>> run efficiently.
> >>>
> >>>> Please take a look at how x86 does cpuid masking :).
> >>>>
> >>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
> >>>> feature list exposure to the guest and actual feature implementation
> >>>> inside the guest together. On POWER there is a patch set pending that
> >>>> implements these two things separately - admittedly mostly because
> >>>> hardware sucks and we can't change the PVR.
> >>> That is maybe the big difference with s390. The cpuid in the S390 case is not
> >>> directly comparable with the processor version register of POWER.
> >>>
> >>> In the S390 world we have a well defined CPU model room spanned by the machine
> >>> type and its GA count. Thus we can define a bijective mapping between
> >>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> >>> name which BTW is meaningful also for a human user.
> >> Same thing as POWER.
> >>
> >>> By means of this name, a management interface (libvirt) will draw decisions if
> >>> migration to a remote hypervisor is a good idea or not. For that it just needs
> >>> to compare if the current model of the guest on the source hypervisor
> >>> ("query-cpu-model"), is contained in the supported model list of the target
> >>> hypervisor ("query-cpu-definitions").
> >> I don't think this works, since QEMU should always return all the cpu
> >> definitions it's aware of on query-cpu-definitions, not just the ones
> >> that it thinks may be compatible with the host at a random point in time.
> > It does not return model names that it thinks they are compatible at some point
> > in time. In s390 mode, it returns all definitions (CPU models) that a given host
> > system is capable to run. Together with the CPU model run by the guest, some upper
> > management interface knows if the hypervisor supports the required CPU model and
> > uses a guest definition with the same CPU model on the target hypervisor.
> >
> > The information for that is taken from the model table which QEMU builds up during
> > startup time. This list limits the command line selectable CPU models as well.
> 
> This makes s390 derive from the way x86 handles things. NAK.

One second, that goes a little fast here :-). x86 returns a list they support which happens to
be the full list they define and s390 does logically the same because we know that certain
models are not supported due to probing. BTW that happens only if you run Qemu on back
level hardware and that is perfectly correct.

The migration compatibility test is pretty much ARCH dependent. I looked into the
libvirt implementation and as one can see every architecture has its own implementation
there (libvirt/src/cpu/cpu_<arch>.c).

> >
> > This even easily allows to find some GCM (Greatest Common CPU Model) among a set
> > of hypervisors to allow migration in a a cluster with n>2 parties.
> >
> >> Please check with the x86 people to find out how they do this.
> > They currently just list their base model names:
> > {"return":
> >    [{"name": "qemu64"},
> >     {"name": "phenom"},
> >     {"name": "core2duo"},
> >     {"name": "kvm64"},
> >     {"name": "qemu32"},
> >     {"name": "kvm32"},
> >     {"name": "coreduo"},
> >     {"name": "486"},
> >     {"name": "pentium"},
> >     {"name": "pentium2"},
> >     {"name": "pentium3"},
> >     {"name": "athlon"},
> >     {"name": "n270"},
> >     {"name": "Conroe"},
> >     {"name": "Penryn"},
> >     {"name": "Nehalem"},
> >     {"name": "Westmere"},
> >     {"name": "SandyBridge"},
> >     {"name": "Opteron_G1"},
> >     {"name": "Opteron_G2"},
> >     {"name": "Opteron_G3"},
> >     {"name": "Opteron_G4"}]
> > }
> >
> >>>>>> I also don't quite grasp what the story behind IBC is. Do you actually
> >>>>>> block instructions? Where do you match instructions that have to get
> >>>>>> blocked with instructions that user space wants to see exposed?
> >>>>>>
> >>>>> Instruction Blocking Control is a feature that was first introduced with the 2097 (IBM
> >>>>> System z10.) The IBC value is part of the SIE state. Just consider it as a kind of
> >>>>> parameter, that allows only instructions that have been implemented up to a certain cpu
> >>>>> type and GA level to become executed, all other op codes will end in an illegal opcode
> >>>>> abort. E.g. take the "Transactional Memory" instructions, they are implemented since type
> >>>>> 2827, GA1 (IBM zEnterprise EC12.). The IBC value has 12 bits 8 for the type and 4 for the
> >>>>> GA level. 0x001 means its a z10, GA1. The value 0x021 means it's a 2827 (CMOS generation
> >>>>> 12 is 0x02) and GA1 and so forth. A guest running with IBC value 0x012 (z196 GA2) will
> >>>>> not be able to use TE instructions in contrast to a guest running with IBC value 0x022
> >>>>> given the host supports it.
> >>>> That sounds very similar to the "compat" cpu property that Alexey is
> >>>> introducing for POWER. Maybe we can model it identically?
> >>> I think it is something different. With "compat" one might be able the express some kind
> >>> of compatibility between two processors of the some different generations, upon which
> >>> the management interface can draw conclusions if migration makes sense or not.
> >>>
> >>> The IBC works totally different. It enforces that the instruction set defined for TYPE-GA.
> >> Yes, which is the same as the PCR register on POWER which the "compat"
> >> option controls. I think we can simplify s390x because it's not as
> >> broken as POWER in what we can fake to the guest, but I think you should
> >> at least be aware of the concepts that are around.
> > Sorry, I first thought you were talking about the PVR but it looks you are
> > talking about the LPCR (Logical Partition Control Register). This is as well
> > different from IBC. The LPCR allows the host POWER LPAR to run in a compatible
> > mode with back level CPUs. That will allow e.g. POWER8 guests to run safely on
> > POWER7 if they were started in this mode. But then all the guests in this KVM
> > hypervisor run in this compatibility mode.
> 
> It's really the same. We context switch LPCR on POWER.

Ok, I buy that. I just was aware of LPCR but have no access to full documentation here.

> 
> The only difference is that you guys can fake PVR as well while PCR can 
> only limit feature availability. But the underlying problem that I was 
> trying to point out is that the bits in IBC are different bits from the 
> FAC bits. Some code has to know the correlation.

Yes, IBC is the third dimension of the CPU model for us and I can calculate the value
from the type and ga, actually it is modeled into the CPU class model as the facilities and
also the cpuid is.

Please campare with
[PATCH v1 RFC 02/10] QEMU: s390: cpu model cpu class definition:

/* machine related properties */
typedef struct S390CPUMachineProps {
    uint16_t class;      /* machine class */
    uint16_t ga;         /* availability number of machine */
    uint16_t order;      /* order of availability */
} S390CPUMachineProps;

/* processor related properties */
typedef struct S390CPUProcessorProps {
    uint16_t gen;        /* S390 CMOS generation */
    uint16_t ver;        /* version of processor */
    uint32_t id;         /* processor identification*/
    uint16_t type;       /* machine type */
    uint16_t ibc;        /* IBC value */
    uint64_t *fac_list;  /* list of facilities */
} S390CPUProcessorProps;

/**
 * S390CPUClass:
 * @parent_realize: The parent class' realize handler.
 * @parent_reset: The parent class' reset handler.
 * @load_normal: Performs a load normal.
 * @cpu_reset: Performs a CPU reset.
 * @initial_cpu_reset: Performs an initial CPU reset.
 *
 * An S/390 CPU model.
 */
typedef struct S390CPUClass {
    /*< private >*/
    CPUClass parent_class;
    /*< public >*/

    DeviceRealize parent_realize;
    void (*parent_reset)(CPUState *cpu);
    void (*load_normal)(CPUState *cpu);
    void (*cpu_reset)(CPUState *cpu);
    void (*initial_cpu_reset)(CPUState *cpu);
    bool is_active;               /* true if cpu class supported by host */
    bool is_host;                 /* true if cpu class respresents "host" */
    uint64_t *fac_list;           /* active facility_list */
    S390CPUMachineProps   *mach;  /* machine specific properties */
    S390CPUProcessorProps *proc;  /* processor specific properties */
} S390CPUClass; 

Michael

> 
> 
> Alex
> 

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 19.05.14 19:03, Michael Mueller wrote:
> On Mon, 19 May 2014 16:49:28 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 19.05.14 16:18, Michael Mueller wrote:
>>> On Mon, 19 May 2014 13:48:08 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 19.05.14 12:53, Michael Mueller wrote:
>>>>> On Fri, 16 May 2014 22:31:12 +0200
>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>
>>>>>> On 16.05.14 17:39, Michael Mueller wrote:
>>>>>>> On Fri, 16 May 2014 14:08:24 +0200
>>>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>>>
>>>>>>>> On 13.05.14 16:58, Michael Mueller wrote:
>>>>>>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
>>>>>>>>> interface.
>>>>>>>>>
>>>>>>>>> During KVM initialization, the host properties cpuid, IBC value and the
>>>>>>>>> facility list are stored in the architecture specific cpu model structure.
>>>>>>>>>
>>>>>>>>> During vcpu setup, these properties are taken to initialize the related SIE
>>>>>>>>> state. This mechanism allows to adjust the properties from user space and thus
>>>>>>>>> to implement different selectable cpu models.
>>>>>>>>>
>>>>>>>>> This patch uses the IBC functionality to block instructions that have not
>>>>>>>>> been implemented at the requested CPU type and GA level compared to the
>>>>>>>>> full host capability.
>>>>>>>>>
>>>>>>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
>>>>>>>>> change of running vcpus is currently not possible.
>>>>>>>> Why is this VM global? It usually fits a lot better modeling wise when
>>>>>>>> CPU types are vcpu properties.
>>>>>>> It simplifies the code substantially because it inherently guarantees the vcpus being
>>>>>>> configured identical. In addition, there is no S390 hardware implementation containing
>>>>>>> inhomogeneous processor types. Thus I consider the properties as machine specific.
>>>>>>>
>>>>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
>>>>>>>>> ---
>>>>>>>>>       arch/s390/include/asm/kvm_host.h |   4 +-
>>>>>>>>>       arch/s390/include/uapi/asm/kvm.h |  23 ++++++
>>>>>>>>>       arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
>>>>>>>>>       arch/s390/kvm/kvm-s390.h         |   1 +
>>>>>>>>>       4 files changed, 172 insertions(+), 2 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>>>>>>>> index b4751ba..6b826cb 100644
>>>>>>>>> --- a/arch/s390/include/asm/kvm_host.h
>>>>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
>>>>>>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
>>>>>>>>>       	atomic_t cpuflags;		/* 0x0000 */
>>>>>>>>>       	__u32 : 1;			/* 0x0004 */
>>>>>>>>>       	__u32 prefix : 18;
>>>>>>>>> -	__u32 : 13;
>>>>>>>>> +	__u32 : 1;
>>>>>>>>> +	__u32 ibc : 12;
>>>>>>>>>       	__u8	reserved08[4];		/* 0x0008 */
>>>>>>>>>       #define PROG_IN_SIE (1<<0)
>>>>>>>>>       	__u32	prog0c;			/* 0x000c */
>>>>>>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
>>>>>>>>>       	unsigned long *sie_fac;
>>>>>>>>>       	struct cpuid cpu_id;
>>>>>>>>>       	unsigned long *fac_list;
>>>>>>>>> +	unsigned short ibc;
>>>>>>>>>       };
>>>>>>>>>       
>>>>>>>>>       struct kvm_arch{
>>>>>>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
>>>>>>>>> index 313100a..82ef1b5 100644
>>>>>>>>> --- a/arch/s390/include/uapi/asm/kvm.h
>>>>>>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
>>>>>>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
>>>>>>>>>       
>>>>>>>>>       /* kvm attr_group  on vm fd */
>>>>>>>>>       #define KVM_S390_VM_MEM_CTRL		0
>>>>>>>>> +#define KVM_S390_VM_CPU_MODEL		1
>>>>>>>>>       
>>>>>>>>>       /* kvm attributes for mem_ctrl */
>>>>>>>>>       #define KVM_S390_VM_MEM_ENABLE_CMMA	0
>>>>>>>>>       #define KVM_S390_VM_MEM_CLR_CMMA	1
>>>>>>>>>       #define KVM_S390_VM_MEM_CLR_PAGES	2
>>>>>>>>>       
>>>>>>>>> +/* kvm attributes for cpu_model */
>>>>>>>>> +
>>>>>>>>> +/* the s390 processor related attributes are r/w */
>>>>>>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
>>>>>>>>> +struct kvm_s390_vm_cpu_processor {
>>>>>>>>> +	__u64 cpuid;
>>>>>>>>> +	__u16 ibc;
>>>>>>>>> +	__u8  pad[6];
>>>>>>>>> +	__u64 fac_list[256];
>>>>>>>>> +};
>>>>>>>>> +
>>>>>>>>> +/* the machine related attributes are read only */
>>>>>>>>> +#define KVM_S390_VM_CPU_MACHINE		1
>>>>>>>>> +struct kvm_s390_vm_cpu_machine {
>>>>>>>>> +	__u64 cpuid;
>>>>>>>>> +	__u32 ibc_range;
>>>>>>>>> +	__u8  pad[4];
>>>>>>>>> +	__u64 fac_mask[256];
>>>>>>>>> +	__u64 hard_fac_list[256];
>>>>>>>>> +	__u64 soft_fac_list[256];
>>>>>>>>> +};
>>>>>>>>> +
>>>>>>>>>       /* for KVM_GET_REGS and KVM_SET_REGS */
>>>>>>>>>       struct kvm_regs {
>>>>>>>>>       	/* general purpose regs for s390 */
>>>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> index a53652f..9965d8b 100644
>>>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
>>>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
>>>>>>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
>>>>>>>>> kvm_device_attr *attr) return ret;
>>>>>>>>>       }
>>>>>>>>>       
>>>>>>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>>>> +{
>>>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>>>>>> +
>>>>>>>>> +	if (atomic_read(&kvm->online_vcpus))
>>>>>>>>> +		return -EBUSY;
>>>>>>>>> +
>>>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>>>>>> +	if (!proc)
>>>>>>>>> +		return -ENOMEM;
>>>>>>>>> +
>>>>>>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
>>>>>>>>> +			   sizeof(*proc))) {
>>>>>>>>> +		kfree(proc);
>>>>>>>>> +		return -EFAULT;
>>>>>>>>> +	}
>>>>>>>>> +
>>>>>>>>> +	mutex_lock(&kvm->lock);
>>>>>>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
>>>>>>>>> +	       sizeof(struct cpuid));
>>>>>>>>> +	kvm->arch.model.ibc = proc->ibc;
>>>>>>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
>>>>>>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
>>>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>>>>>> +	mutex_unlock(&kvm->lock);
>>>>>>>>> +	kfree(proc);
>>>>>>>>> +
>>>>>>>>> +	return 0;
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>>>> +{
>>>>>>>>> +	int ret = -ENXIO;
>>>>>>>>> +
>>>>>>>>> +	switch (attr->attr) {
>>>>>>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
>>>>>>>>> +		ret = kvm_s390_set_processor(kvm, attr);
>>>>>>>>> +		break;
>>>>>>>>> +	}
>>>>>>>>> +	return ret;
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>>>> +{
>>>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
>>>>>>>>> +	int rc = 0;
>>>>>>>>> +
>>>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
>>>>>>>>> +	if (!proc) {
>>>>>>>>> +		rc = -ENOMEM;
>>>>>>>>> +		goto out;
>>>>>>>>> +	}
>>>>>>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
>>>>>>>>> +	proc->ibc = kvm->arch.model.ibc;
>>>>>>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
>>>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
>>>>>>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
>>>>>>>>> +		rc = -EFAULT;
>>>>>>>>> +	kfree(proc);
>>>>>>>>> +out:
>>>>>>>>> +	return rc;
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
>>>>>>>>> +{
>>>>>>>>> +	struct kvm_s390_vm_cpu_machine *mach;
>>>>>>>>> +	int rc = 0;
>>>>>>>>> +
>>>>>>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
>>>>>>>>> +	if (!mach) {
>>>>>>>>> +		rc = -ENOMEM;
>>>>>>>>> +		goto out;
>>>>>>>>> +	}
>>>>>>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
>>>>>>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
>>>>>>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
>>>>>>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
>>>>>>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
>>>>>>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
>>>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>>>>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
>>>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
>>>>>>>> I really have a hard time grasping what hard and soft means.
>>>>>>> Hard facilities are those that are implemented by the CPU itself, either through processor
>>>>>>> logic or be means of firmware micro code. That's the list returned by the STFL/STFLE
>>>>>>> instruction. In addition to that, one can imagine that in future some of that features are
>>>>>>> emulated on KVM side. These will be placed in the soft facility list and are optionally to
>>>>>>> request by user space.
>>>>>> I don't see why we would have to differentiate between the two. User
>>>>>> space wants features enabled. Whether they are done in hardware or in
>>>>>> software doesn't matter.
>>>>> I've tried to make my point on that in last answer of patch 3/6. It's a mistake
>>>>> to think that user space just wants to have features, they come with different
>>>>> qualities!
>>>> So? If I want to run a z9 compatible guest, I do -cpu z9. I can either
>>>>
>>>>      a) run it with emulation of a facility or
>>>>      b) not run it
>>>>
>>>> which one would the user choose?
>>> If you run on a z990 host, you better use -cpu z990 because emulating some
>>> fancy delta feature just cost additional CPU time. If the host is newer, please
>>> go with -cpu z9.
>> Yes, I agree on that statement. Imagine a feature gets *dropped* though.
>> In that case -cpu z9 should enable emulation of that feature to maintain
>> migratability with a real z9 machine on newer hardware.
> Nice try, but think what's happening in real world. Let's assume the feature is
> TE again, available since zEC12 but would go away with zNext. In that case the
> CPU model zNext-GA1 and all successors will not have zEC12 as supported model.
> The application will just not run on that model if it insists on executing TE
> instructions.

So what's the point in software emulated features then? Either we can 
emulate a feature or we can't. If we can, we can be compatible. If we 
can't, we're not compatible.

>
>>> What user and thus also user space wants depends on other factors:
>>>
>>> 1. reliability
>>> 2. performance
>>> 3. availability
>>>
>>> It's not features, that's what programmers want.
>>>
>>> That's why I have designed the model and migration capability around the hardware
>>> and not around the software features and don't allow them to be enabled currently
>>> together.
>>>
>>> A software feature is a nice add on that is helpful for evaluation or development
>>> purpose. There is few space for it on productions systems.
>>>
>>> One option that I currently see to make software implemented facility migration
>>> capable is to calculate some kind of hash value derived from the full set of
>>> active software facilities. That value can be compared with pre-calculated
>>> values also stored in the supported model table of qemu. This value could be
>>> seen like a virtual model extension that has to match like the model name.
>>>
>>> But I have said it elsewhere already, a soft facility should be an exception and
>>> not the rule.
>>>
>>>>>> So all we need is a list of "features the guest sees available" which is
>>>>>> the same as "features user space wants the guest to see" which then gets
>>>>>> masked through "features the host can do in hardware".
>>>>>>
>>>>>> For emulation we can just check on the global feature availability on
>>>>>> whether we should emulate them or not.
>>>>>>
>>>>>>>> Also, if user space wants to make sure that its feature list is actually
>>>>>>>> workable on the host kernel, it needs to set and get the features again
>>>>>>>> and then compare that with the ones it set? That's different from x86's
>>>>>>>> cpuid implementation but probably workable.
>>>>>>> User space will probe what facilities are available and match them with the predefined cpu
>>>>>>> model set. Only those models which use a partial or full subset of the hard/host facility
>>>>>>> list are selectable.
>>>>>> Why?
>>>>> If a host does not offer the features required for a model it is not able to
>>>>> run efficiently.
>>>>>
>>>>>> Please take a look at how x86 does cpuid masking :).
>>>>>>
>>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
>>>>>> feature list exposure to the guest and actual feature implementation
>>>>>> inside the guest together. On POWER there is a patch set pending that
>>>>>> implements these two things separately - admittedly mostly because
>>>>>> hardware sucks and we can't change the PVR.
>>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
>>>>> directly comparable with the processor version register of POWER.
>>>>>
>>>>> In the S390 world we have a well defined CPU model room spanned by the machine
>>>>> type and its GA count. Thus we can define a bijective mapping between
>>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
>>>>> name which BTW is meaningful also for a human user.
>>>> Same thing as POWER.
>>>>
>>>>> By means of this name, a management interface (libvirt) will draw decisions if
>>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
>>>>> to compare if the current model of the guest on the source hypervisor
>>>>> ("query-cpu-model"), is contained in the supported model list of the target
>>>>> hypervisor ("query-cpu-definitions").
>>>> I don't think this works, since QEMU should always return all the cpu
>>>> definitions it's aware of on query-cpu-definitions, not just the ones
>>>> that it thinks may be compatible with the host at a random point in time.
>>> It does not return model names that it thinks they are compatible at some point
>>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
>>> system is capable to run. Together with the CPU model run by the guest, some upper
>>> management interface knows if the hypervisor supports the required CPU model and
>>> uses a guest definition with the same CPU model on the target hypervisor.
>>>
>>> The information for that is taken from the model table which QEMU builds up during
>>> startup time. This list limits the command line selectable CPU models as well.
>> This makes s390 derive from the way x86 handles things. NAK.
> One second, that goes a little fast here :-). x86 returns a list they support which happens to
> be the full list they define and s390 does logically the same because we know that certain
> models are not supported due to probing. BTW that happens only if you run Qemu on back
> level hardware and that is perfectly correct.

It's not what other architectures do and I'd hate to see s390 deviate 
just because.

> The migration compatibility test is pretty much ARCH dependent. I looked into the
> libvirt implementation and as one can see every architecture has its own implementation
> there (libvirt/src/cpu/cpu_<arch>.c).

So here's my question again. How does x86 evaluate whether a target 
machine is compatible with a source machine?


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Mon, 19 May 2014 22:14:00 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 19.05.14 19:03, Michael Mueller wrote:
> > On Mon, 19 May 2014 16:49:28 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 19.05.14 16:18, Michael Mueller wrote:
> >>> On Mon, 19 May 2014 13:48:08 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 19.05.14 12:53, Michael Mueller wrote:
> >>>>> On Fri, 16 May 2014 22:31:12 +0200
> >>>>> Alexander Graf <agraf@suse.de> wrote:
> >>>>>
> >>>>>> On 16.05.14 17:39, Michael Mueller wrote:
> >>>>>>> On Fri, 16 May 2014 14:08:24 +0200
> >>>>>>> Alexander Graf <agraf@suse.de> wrote:
> >>>>>>>
> >>>>>>>> On 13.05.14 16:58, Michael Mueller wrote:
> >>>>>>>>> This patch enables cpu model support in kvm/s390 via the vm attribute
> >>>>>>>>> interface.
> >>>>>>>>>
> >>>>>>>>> During KVM initialization, the host properties cpuid, IBC value and the
> >>>>>>>>> facility list are stored in the architecture specific cpu model structure.
> >>>>>>>>>
> >>>>>>>>> During vcpu setup, these properties are taken to initialize the related SIE
> >>>>>>>>> state. This mechanism allows to adjust the properties from user space and thus
> >>>>>>>>> to implement different selectable cpu models.
> >>>>>>>>>
> >>>>>>>>> This patch uses the IBC functionality to block instructions that have not
> >>>>>>>>> been implemented at the requested CPU type and GA level compared to the
> >>>>>>>>> full host capability.
> >>>>>>>>>
> >>>>>>>>> Userspace has to initialize the cpu model before vcpu creation. A cpu model
> >>>>>>>>> change of running vcpus is currently not possible.
> >>>>>>>> Why is this VM global? It usually fits a lot better modeling wise when
> >>>>>>>> CPU types are vcpu properties.
> >>>>>>> It simplifies the code substantially because it inherently guarantees the vcpus being
> >>>>>>> configured identical. In addition, there is no S390 hardware implementation containing
> >>>>>>> inhomogeneous processor types. Thus I consider the properties as machine specific.
> >>>>>>>
> >>>>>>>>> Signed-off-by: Michael Mueller <mimu@linux.vnet.ibm.com>
> >>>>>>>>> ---
> >>>>>>>>>       arch/s390/include/asm/kvm_host.h |   4 +-
> >>>>>>>>>       arch/s390/include/uapi/asm/kvm.h |  23 ++++++
> >>>>>>>>>       arch/s390/kvm/kvm-s390.c         | 146 ++++++++++++++++++++++++++++++++++++++-
> >>>>>>>>>       arch/s390/kvm/kvm-s390.h         |   1 +
> >>>>>>>>>       4 files changed, 172 insertions(+), 2 deletions(-)
> >>>>>>>>>
> >>>>>>>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> >>>>>>>>> index b4751ba..6b826cb 100644
> >>>>>>>>> --- a/arch/s390/include/asm/kvm_host.h
> >>>>>>>>> +++ b/arch/s390/include/asm/kvm_host.h
> >>>>>>>>> @@ -84,7 +84,8 @@ struct kvm_s390_sie_block {
> >>>>>>>>>       	atomic_t cpuflags;		/* 0x0000 */
> >>>>>>>>>       	__u32 : 1;			/* 0x0004 */
> >>>>>>>>>       	__u32 prefix : 18;
> >>>>>>>>> -	__u32 : 13;
> >>>>>>>>> +	__u32 : 1;
> >>>>>>>>> +	__u32 ibc : 12;
> >>>>>>>>>       	__u8	reserved08[4];		/* 0x0008 */
> >>>>>>>>>       #define PROG_IN_SIE (1<<0)
> >>>>>>>>>       	__u32	prog0c;			/* 0x000c */
> >>>>>>>>> @@ -418,6 +419,7 @@ struct kvm_s390_cpu_model {
> >>>>>>>>>       	unsigned long *sie_fac;
> >>>>>>>>>       	struct cpuid cpu_id;
> >>>>>>>>>       	unsigned long *fac_list;
> >>>>>>>>> +	unsigned short ibc;
> >>>>>>>>>       };
> >>>>>>>>>       
> >>>>>>>>>       struct kvm_arch{
> >>>>>>>>> diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h
> >>>>>>>>> index 313100a..82ef1b5 100644
> >>>>>>>>> --- a/arch/s390/include/uapi/asm/kvm.h
> >>>>>>>>> +++ b/arch/s390/include/uapi/asm/kvm.h
> >>>>>>>>> @@ -58,12 +58,35 @@ struct kvm_s390_io_adapter_req {
> >>>>>>>>>       
> >>>>>>>>>       /* kvm attr_group  on vm fd */
> >>>>>>>>>       #define KVM_S390_VM_MEM_CTRL		0
> >>>>>>>>> +#define KVM_S390_VM_CPU_MODEL		1
> >>>>>>>>>       
> >>>>>>>>>       /* kvm attributes for mem_ctrl */
> >>>>>>>>>       #define KVM_S390_VM_MEM_ENABLE_CMMA	0
> >>>>>>>>>       #define KVM_S390_VM_MEM_CLR_CMMA	1
> >>>>>>>>>       #define KVM_S390_VM_MEM_CLR_PAGES	2
> >>>>>>>>>       
> >>>>>>>>> +/* kvm attributes for cpu_model */
> >>>>>>>>> +
> >>>>>>>>> +/* the s390 processor related attributes are r/w */
> >>>>>>>>> +#define KVM_S390_VM_CPU_PROCESSOR	0
> >>>>>>>>> +struct kvm_s390_vm_cpu_processor {
> >>>>>>>>> +	__u64 cpuid;
> >>>>>>>>> +	__u16 ibc;
> >>>>>>>>> +	__u8  pad[6];
> >>>>>>>>> +	__u64 fac_list[256];
> >>>>>>>>> +};
> >>>>>>>>> +
> >>>>>>>>> +/* the machine related attributes are read only */
> >>>>>>>>> +#define KVM_S390_VM_CPU_MACHINE		1
> >>>>>>>>> +struct kvm_s390_vm_cpu_machine {
> >>>>>>>>> +	__u64 cpuid;
> >>>>>>>>> +	__u32 ibc_range;
> >>>>>>>>> +	__u8  pad[4];
> >>>>>>>>> +	__u64 fac_mask[256];
> >>>>>>>>> +	__u64 hard_fac_list[256];
> >>>>>>>>> +	__u64 soft_fac_list[256];
> >>>>>>>>> +};
> >>>>>>>>> +
> >>>>>>>>>       /* for KVM_GET_REGS and KVM_SET_REGS */
> >>>>>>>>>       struct kvm_regs {
> >>>>>>>>>       	/* general purpose regs for s390 */
> >>>>>>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> >>>>>>>>> index a53652f..9965d8b 100644
> >>>>>>>>> --- a/arch/s390/kvm/kvm-s390.c
> >>>>>>>>> +++ b/arch/s390/kvm/kvm-s390.c
> >>>>>>>>> @@ -369,6 +369,110 @@ static int kvm_s390_mem_control(struct kvm *kvm, struct
> >>>>>>>>> kvm_device_attr *attr) return ret;
> >>>>>>>>>       }
> >>>>>>>>>       
> >>>>>>>>> +static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>>>> +{
> >>>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>>>>>> +
> >>>>>>>>> +	if (atomic_read(&kvm->online_vcpus))
> >>>>>>>>> +		return -EBUSY;
> >>>>>>>>> +
> >>>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>>>>>> +	if (!proc)
> >>>>>>>>> +		return -ENOMEM;
> >>>>>>>>> +
> >>>>>>>>> +	if (copy_from_user(proc, (void __user *)attr->addr,
> >>>>>>>>> +			   sizeof(*proc))) {
> >>>>>>>>> +		kfree(proc);
> >>>>>>>>> +		return -EFAULT;
> >>>>>>>>> +	}
> >>>>>>>>> +
> >>>>>>>>> +	mutex_lock(&kvm->lock);
> >>>>>>>>> +	memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
> >>>>>>>>> +	       sizeof(struct cpuid));
> >>>>>>>>> +	kvm->arch.model.ibc = proc->ibc;
> >>>>>>>>> +	kvm_s390_apply_fac_list_mask((long unsigned *)proc->fac_list);
> >>>>>>>>> +	memcpy(kvm->arch.model.fac_list, proc->fac_list,
> >>>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>>>>>> +	mutex_unlock(&kvm->lock);
> >>>>>>>>> +	kfree(proc);
> >>>>>>>>> +
> >>>>>>>>> +	return 0;
> >>>>>>>>> +}
> >>>>>>>>> +
> >>>>>>>>> +static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>>>> +{
> >>>>>>>>> +	int ret = -ENXIO;
> >>>>>>>>> +
> >>>>>>>>> +	switch (attr->attr) {
> >>>>>>>>> +	case KVM_S390_VM_CPU_PROCESSOR:
> >>>>>>>>> +		ret = kvm_s390_set_processor(kvm, attr);
> >>>>>>>>> +		break;
> >>>>>>>>> +	}
> >>>>>>>>> +	return ret;
> >>>>>>>>> +}
> >>>>>>>>> +
> >>>>>>>>> +static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>>>> +{
> >>>>>>>>> +	struct kvm_s390_vm_cpu_processor *proc;
> >>>>>>>>> +	int rc = 0;
> >>>>>>>>> +
> >>>>>>>>> +	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
> >>>>>>>>> +	if (!proc) {
> >>>>>>>>> +		rc = -ENOMEM;
> >>>>>>>>> +		goto out;
> >>>>>>>>> +	}
> >>>>>>>>> +	memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
> >>>>>>>>> +	proc->ibc = kvm->arch.model.ibc;
> >>>>>>>>> +	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
> >>>>>>>>> +	       S390_ARCH_FAC_LIST_SIZE_BYTE);
> >>>>>>>>> +	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
> >>>>>>>>> +		rc = -EFAULT;
> >>>>>>>>> +	kfree(proc);
> >>>>>>>>> +out:
> >>>>>>>>> +	return rc;
> >>>>>>>>> +}
> >>>>>>>>> +
> >>>>>>>>> +static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
> >>>>>>>>> +{
> >>>>>>>>> +	struct kvm_s390_vm_cpu_machine *mach;
> >>>>>>>>> +	int rc = 0;
> >>>>>>>>> +
> >>>>>>>>> +	mach = kzalloc(sizeof(*mach), GFP_KERNEL);
> >>>>>>>>> +	if (!mach) {
> >>>>>>>>> +		rc = -ENOMEM;
> >>>>>>>>> +		goto out;
> >>>>>>>>> +	}
> >>>>>>>>> +	get_cpu_id((struct cpuid *) &mach->cpuid);
> >>>>>>>>> +	mach->ibc_range = kvm_s390_lowest_ibc() << 16;
> >>>>>>>>> +	mach->ibc_range |= kvm_s390_latest_ibc();
> >>>>>>>>> +	memcpy(&mach->fac_mask, kvm_s390_fac_list_mask,
> >>>>>>>>> +	       kvm_s390_fac_list_mask_size() * sizeof(u64));
> >>>>>>>>> +	kvm_s390_get_hard_fac_list((long unsigned int *) &mach->hard_fac_list,
> >>>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>>>>>>> +	kvm_s390_get_soft_fac_list((long unsigned int *) &mach->soft_fac_list,
> >>>>>>>>> +				   S390_ARCH_FAC_LIST_SIZE_U64);
> >>>>>>>> I really have a hard time grasping what hard and soft means.
> >>>>>>> Hard facilities are those that are implemented by the CPU itself, either through
> >>>>>>> processor logic or be means of firmware micro code. That's the list returned by the
> >>>>>>> STFL/STFLE instruction. In addition to that, one can imagine that in future some of
> >>>>>>> that features are emulated on KVM side. These will be placed in the soft facility list
> >>>>>>> and are optionally to request by user space.
> >>>>>> I don't see why we would have to differentiate between the two. User
> >>>>>> space wants features enabled. Whether they are done in hardware or in
> >>>>>> software doesn't matter.
> >>>>> I've tried to make my point on that in last answer of patch 3/6. It's a mistake
> >>>>> to think that user space just wants to have features, they come with different
> >>>>> qualities!
> >>>> So? If I want to run a z9 compatible guest, I do -cpu z9. I can either
> >>>>
> >>>>      a) run it with emulation of a facility or
> >>>>      b) not run it
> >>>>
> >>>> which one would the user choose?
> >>> If you run on a z990 host, you better use -cpu z990 because emulating some
> >>> fancy delta feature just cost additional CPU time. If the host is newer, please
> >>> go with -cpu z9.
> >> Yes, I agree on that statement. Imagine a feature gets *dropped* though.
> >> In that case -cpu z9 should enable emulation of that feature to maintain
> >> migratability with a real z9 machine on newer hardware.
> > Nice try, but think what's happening in real world. Let's assume the feature is
> > TE again, available since zEC12 but would go away with zNext. In that case the
> > CPU model zNext-GA1 and all successors will not have zEC12 as supported model.
> > The application will just not run on that model if it insists on executing TE
> > instructions.
> 
> So what's the point in software emulated features then? Either we can 
> emulate a feature or we can't. If we can, we can be compatible. If we 
> can't, we're not compatible.
> 
> >
> >>> What user and thus also user space wants depends on other factors:
> >>>
> >>> 1. reliability
> >>> 2. performance
> >>> 3. availability
> >>>
> >>> It's not features, that's what programmers want.
> >>>
> >>> That's why I have designed the model and migration capability around the hardware
> >>> and not around the software features and don't allow them to be enabled currently
> >>> together.
> >>>
> >>> A software feature is a nice add on that is helpful for evaluation or development
> >>> purpose. There is few space for it on productions systems.
> >>>
> >>> One option that I currently see to make software implemented facility migration
> >>> capable is to calculate some kind of hash value derived from the full set of
> >>> active software facilities. That value can be compared with pre-calculated
> >>> values also stored in the supported model table of qemu. This value could be
> >>> seen like a virtual model extension that has to match like the model name.
> >>>
> >>> But I have said it elsewhere already, a soft facility should be an exception and
> >>> not the rule.
> >>>
> >>>>>> So all we need is a list of "features the guest sees available" which is
> >>>>>> the same as "features user space wants the guest to see" which then gets
> >>>>>> masked through "features the host can do in hardware".
> >>>>>>
> >>>>>> For emulation we can just check on the global feature availability on
> >>>>>> whether we should emulate them or not.
> >>>>>>
> >>>>>>>> Also, if user space wants to make sure that its feature list is actually
> >>>>>>>> workable on the host kernel, it needs to set and get the features again
> >>>>>>>> and then compare that with the ones it set? That's different from x86's
> >>>>>>>> cpuid implementation but probably workable.
> >>>>>>> User space will probe what facilities are available and match them with the predefined
> >>>>>>> cpu model set. Only those models which use a partial or full subset of the hard/host
> >>>>>>> facility list are selectable.
> >>>>>> Why?
> >>>>> If a host does not offer the features required for a model it is not able to
> >>>>> run efficiently.
> >>>>>
> >>>>>> Please take a look at how x86 does cpuid masking :).
> >>>>>>
> >>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
> >>>>>> feature list exposure to the guest and actual feature implementation
> >>>>>> inside the guest together. On POWER there is a patch set pending that
> >>>>>> implements these two things separately - admittedly mostly because
> >>>>>> hardware sucks and we can't change the PVR.
> >>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
> >>>>> directly comparable with the processor version register of POWER.
> >>>>>
> >>>>> In the S390 world we have a well defined CPU model room spanned by the machine
> >>>>> type and its GA count. Thus we can define a bijective mapping between
> >>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> >>>>> name which BTW is meaningful also for a human user.
> >>>> Same thing as POWER.
> >>>>
> >>>>> By means of this name, a management interface (libvirt) will draw decisions if
> >>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
> >>>>> to compare if the current model of the guest on the source hypervisor
> >>>>> ("query-cpu-model"), is contained in the supported model list of the target
> >>>>> hypervisor ("query-cpu-definitions").
> >>>> I don't think this works, since QEMU should always return all the cpu
> >>>> definitions it's aware of on query-cpu-definitions, not just the ones
> >>>> that it thinks may be compatible with the host at a random point in time.
> >>> It does not return model names that it thinks they are compatible at some point
> >>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
> >>> system is capable to run. Together with the CPU model run by the guest, some upper
> >>> management interface knows if the hypervisor supports the required CPU model and
> >>> uses a guest definition with the same CPU model on the target hypervisor.
> >>>
> >>> The information for that is taken from the model table which QEMU builds up during
> >>> startup time. This list limits the command line selectable CPU models as well.
> >> This makes s390 derive from the way x86 handles things. NAK.
> > One second, that goes a little fast here :-). x86 returns a list they support which happens to
> > be the full list they define and s390 does logically the same because we know that certain
> > models are not supported due to probing. BTW that happens only if you run Qemu on back
> > level hardware and that is perfectly correct.
> 
> It's not what other architectures do and I'd hate to see s390 deviate 
> just because.

Only these four architectures implement the query and they all differ a little...

target-arm/helper.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
target-i386/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
target-ppc/translate_init.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
target-s390x/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)

arm walks through a list of all ARM CPU types
list = object_class_get_list(TYPE_ARM_CPU, false);
and returns the CpuDefinitionInfoList derived from that one to one

i386 loops over the static builtin_x86_defs[] array to retrieve the model names,
they don't even use the CPU class model as source

ppc walks through a list of all POWER CPU types
list = object_class_get_list(TYPE_POWERPC_CPU, false);
and then extends the produced list by all defined aliases

and s390x finally also walks through the defined S390 CPU types
list = object_class_get_list(TYPE_S390_CPU, false); 
but drops those which are not usable (!is_active)
Just consider them as not defined. I actually would undefine
them if I knew how.

Also the commands comment says "list of supported virtual CPU definitions" and the s390
list contains all supported models, that's no contradiction.

##                                                                                                                                                          
# @query-cpu-definitions:
#
# Return a list of supported virtual CPU definitions
#
# Returns: a list of CpuDefInfo 

> 
> > The migration compatibility test is pretty much ARCH dependent. I looked into the
> > libvirt implementation and as one can see every architecture has its own implementation
> > there (libvirt/src/cpu/cpu_<arch>.c).
> 
> So here's my question again. How does x86 evaluate whether a target 
> machine is compatible with a source machine?

Will again look into that during the afternoon...

> 
> 
> Alex
> 
> 

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 20.05.14 12:02, Michael Mueller wrote:
> On Mon, 19 May 2014 22:14:00 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 19.05.14 19:03, Michael Mueller wrote:
>>> On Mon, 19 May 2014 16:49:28 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>

[...]

>>>>> What user and thus also user space wants depends on other factors:
>>>>>
>>>>> 1. reliability
>>>>> 2. performance
>>>>> 3. availability
>>>>>
>>>>> It's not features, that's what programmers want.
>>>>>
>>>>> That's why I have designed the model and migration capability around the hardware
>>>>> and not around the software features and don't allow them to be enabled currently
>>>>> together.
>>>>>
>>>>> A software feature is a nice add on that is helpful for evaluation or development
>>>>> purpose. There is few space for it on productions systems.
>>>>>
>>>>> One option that I currently see to make software implemented facility migration
>>>>> capable is to calculate some kind of hash value derived from the full set of
>>>>> active software facilities. That value can be compared with pre-calculated
>>>>> values also stored in the supported model table of qemu. This value could be
>>>>> seen like a virtual model extension that has to match like the model name.
>>>>>
>>>>> But I have said it elsewhere already, a soft facility should be an exception and
>>>>> not the rule.
>>>>>
>>>>>>>> So all we need is a list of "features the guest sees available" which is
>>>>>>>> the same as "features user space wants the guest to see" which then gets
>>>>>>>> masked through "features the host can do in hardware".
>>>>>>>>
>>>>>>>> For emulation we can just check on the global feature availability on
>>>>>>>> whether we should emulate them or not.
>>>>>>>>
>>>>>>>>>> Also, if user space wants to make sure that its feature list is actually
>>>>>>>>>> workable on the host kernel, it needs to set and get the features again
>>>>>>>>>> and then compare that with the ones it set? That's different from x86's
>>>>>>>>>> cpuid implementation but probably workable.
>>>>>>>>> User space will probe what facilities are available and match them with the predefined
>>>>>>>>> cpu model set. Only those models which use a partial or full subset of the hard/host
>>>>>>>>> facility list are selectable.
>>>>>>>> Why?
>>>>>>> If a host does not offer the features required for a model it is not able to
>>>>>>> run efficiently.
>>>>>>>
>>>>>>>> Please take a look at how x86 does cpuid masking :).
>>>>>>>>
>>>>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
>>>>>>>> feature list exposure to the guest and actual feature implementation
>>>>>>>> inside the guest together. On POWER there is a patch set pending that
>>>>>>>> implements these two things separately - admittedly mostly because
>>>>>>>> hardware sucks and we can't change the PVR.
>>>>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
>>>>>>> directly comparable with the processor version register of POWER.
>>>>>>>
>>>>>>> In the S390 world we have a well defined CPU model room spanned by the machine
>>>>>>> type and its GA count. Thus we can define a bijective mapping between
>>>>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
>>>>>>> name which BTW is meaningful also for a human user.
>>>>>> Same thing as POWER.
>>>>>>
>>>>>>> By means of this name, a management interface (libvirt) will draw decisions if
>>>>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
>>>>>>> to compare if the current model of the guest on the source hypervisor
>>>>>>> ("query-cpu-model"), is contained in the supported model list of the target
>>>>>>> hypervisor ("query-cpu-definitions").
>>>>>> I don't think this works, since QEMU should always return all the cpu
>>>>>> definitions it's aware of on query-cpu-definitions, not just the ones
>>>>>> that it thinks may be compatible with the host at a random point in time.
>>>>> It does not return model names that it thinks they are compatible at some point
>>>>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
>>>>> system is capable to run. Together with the CPU model run by the guest, some upper
>>>>> management interface knows if the hypervisor supports the required CPU model and
>>>>> uses a guest definition with the same CPU model on the target hypervisor.
>>>>>
>>>>> The information for that is taken from the model table which QEMU builds up during
>>>>> startup time. This list limits the command line selectable CPU models as well.
>>>> This makes s390 derive from the way x86 handles things. NAK.
>>> One second, that goes a little fast here :-). x86 returns a list they support which happens to
>>> be the full list they define and s390 does logically the same because we know that certain
>>> models are not supported due to probing. BTW that happens only if you run Qemu on back
>>> level hardware and that is perfectly correct.
>> It's not what other architectures do and I'd hate to see s390 deviate
>> just because.
> Only these four architectures implement the query and they all differ a little...
>
> target-arm/helper.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> target-i386/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> target-ppc/translate_init.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> target-s390x/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
>
> arm walks through a list of all ARM CPU types
> list = object_class_get_list(TYPE_ARM_CPU, false);
> and returns the CpuDefinitionInfoList derived from that one to one
>
> i386 loops over the static builtin_x86_defs[] array to retrieve the model names,
> they don't even use the CPU class model as source
>
> ppc walks through a list of all POWER CPU types
> list = object_class_get_list(TYPE_POWERPC_CPU, false);
> and then extends the produced list by all defined aliases
>
> and s390x finally also walks through the defined S390 CPU types
> list = object_class_get_list(TYPE_S390_CPU, false);
> but drops those which are not usable (!is_active)
> Just consider them as not defined. I actually would undefine
> them if I knew how.
>
> Also the commands comment says "list of supported virtual CPU definitions" and the s390
> list contains all supported models, that's no contradiction.

So IMHO we can either

   a) change the definition of query_cpu_definitions to only return CPUs 
that are executable with KVM on a given machine (probably a bad idea) or
   b) return not only the CPU type, but also a hint whether it's 
available with KVM or
   c) add a parameter to query_cpu_definitions to say "only return KVM 
runnable CPUs" or
   d) introduce a new query_kvm_cpu_definitions qmp command

>
> ##
> # @query-cpu-definitions:
> #
> # Return a list of supported virtual CPU definitions
> #
> # Returns: a list of CpuDefInfo
>
>>> The migration compatibility test is pretty much ARCH dependent. I looked into the
>>> libvirt implementation and as one can see every architecture has its own implementation
>>> there (libvirt/src/cpu/cpu_<arch>.c).
>> So here's my question again. How does x86 evaluate whether a target
>> machine is compatible with a source machine?
> Will again look into that during the afternoon...

Yes, please. Someone else must have solved this before :).


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Tue, 20 May 2014 12:10:23 +0200
Alexander Graf <agraf@suse.de> wrote:

> 
> On 20.05.14 12:02, Michael Mueller wrote:
> > On Mon, 19 May 2014 22:14:00 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 19.05.14 19:03, Michael Mueller wrote:
> >>> On Mon, 19 May 2014 16:49:28 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> 
> [...]
> 
> >>>>> What user and thus also user space wants depends on other factors:
> >>>>>
> >>>>> 1. reliability
> >>>>> 2. performance
> >>>>> 3. availability
> >>>>>
> >>>>> It's not features, that's what programmers want.
> >>>>>
> >>>>> That's why I have designed the model and migration capability around the hardware
> >>>>> and not around the software features and don't allow them to be enabled currently
> >>>>> together.
> >>>>>
> >>>>> A software feature is a nice add on that is helpful for evaluation or development
> >>>>> purpose. There is few space for it on productions systems.
> >>>>>
> >>>>> One option that I currently see to make software implemented facility migration
> >>>>> capable is to calculate some kind of hash value derived from the full set of
> >>>>> active software facilities. That value can be compared with pre-calculated
> >>>>> values also stored in the supported model table of qemu. This value could be
> >>>>> seen like a virtual model extension that has to match like the model name.
> >>>>>
> >>>>> But I have said it elsewhere already, a soft facility should be an exception and
> >>>>> not the rule.
> >>>>>
> >>>>>>>> So all we need is a list of "features the guest sees available" which is
> >>>>>>>> the same as "features user space wants the guest to see" which then gets
> >>>>>>>> masked through "features the host can do in hardware".
> >>>>>>>>
> >>>>>>>> For emulation we can just check on the global feature availability on
> >>>>>>>> whether we should emulate them or not.
> >>>>>>>>
> >>>>>>>>>> Also, if user space wants to make sure that its feature list is actually
> >>>>>>>>>> workable on the host kernel, it needs to set and get the features again
> >>>>>>>>>> and then compare that with the ones it set? That's different from x86's
> >>>>>>>>>> cpuid implementation but probably workable.
> >>>>>>>>> User space will probe what facilities are available and match them with the predefined
> >>>>>>>>> cpu model set. Only those models which use a partial or full subset of the hard/host
> >>>>>>>>> facility list are selectable.
> >>>>>>>> Why?
> >>>>>>> If a host does not offer the features required for a model it is not able to
> >>>>>>> run efficiently.
> >>>>>>>
> >>>>>>>> Please take a look at how x86 does cpuid masking :).
> >>>>>>>>
> >>>>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
> >>>>>>>> feature list exposure to the guest and actual feature implementation
> >>>>>>>> inside the guest together. On POWER there is a patch set pending that
> >>>>>>>> implements these two things separately - admittedly mostly because
> >>>>>>>> hardware sucks and we can't change the PVR.
> >>>>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
> >>>>>>> directly comparable with the processor version register of POWER.
> >>>>>>>
> >>>>>>> In the S390 world we have a well defined CPU model room spanned by the machine
> >>>>>>> type and its GA count. Thus we can define a bijective mapping between
> >>>>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> >>>>>>> name which BTW is meaningful also for a human user.
> >>>>>> Same thing as POWER.
> >>>>>>
> >>>>>>> By means of this name, a management interface (libvirt) will draw decisions if
> >>>>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
> >>>>>>> to compare if the current model of the guest on the source hypervisor
> >>>>>>> ("query-cpu-model"), is contained in the supported model list of the target
> >>>>>>> hypervisor ("query-cpu-definitions").
> >>>>>> I don't think this works, since QEMU should always return all the cpu
> >>>>>> definitions it's aware of on query-cpu-definitions, not just the ones
> >>>>>> that it thinks may be compatible with the host at a random point in time.
> >>>>> It does not return model names that it thinks they are compatible at some point
> >>>>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
> >>>>> system is capable to run. Together with the CPU model run by the guest, some upper
> >>>>> management interface knows if the hypervisor supports the required CPU model and
> >>>>> uses a guest definition with the same CPU model on the target hypervisor.
> >>>>>
> >>>>> The information for that is taken from the model table which QEMU builds up during
> >>>>> startup time. This list limits the command line selectable CPU models as well.
> >>>> This makes s390 derive from the way x86 handles things. NAK.
> >>> One second, that goes a little fast here :-). x86 returns a list they support which happens
> >>> to be the full list they define and s390 does logically the same because we know that
> >>> certain models are not supported due to probing. BTW that happens only if you run Qemu on
> >>> back level hardware and that is perfectly correct.
> >> It's not what other architectures do and I'd hate to see s390 deviate
> >> just because.
> > Only these four architectures implement the query and they all differ a little...
> >
> > target-arm/helper.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> > target-i386/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> > target-ppc/translate_init.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> > target-s390x/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> >
> > arm walks through a list of all ARM CPU types
> > list = object_class_get_list(TYPE_ARM_CPU, false);
> > and returns the CpuDefinitionInfoList derived from that one to one
> >
> > i386 loops over the static builtin_x86_defs[] array to retrieve the model names,
> > they don't even use the CPU class model as source
> >
> > ppc walks through a list of all POWER CPU types
> > list = object_class_get_list(TYPE_POWERPC_CPU, false);
> > and then extends the produced list by all defined aliases
> >
> > and s390x finally also walks through the defined S390 CPU types
> > list = object_class_get_list(TYPE_S390_CPU, false);
> > but drops those which are not usable (!is_active)
> > Just consider them as not defined. I actually would undefine
> > them if I knew how.
> >
> > Also the commands comment says "list of supported virtual CPU definitions" and the s390
> > list contains all supported models, that's no contradiction.
> 
> So IMHO we can either
> 
>    a) change the definition of query_cpu_definitions to only return CPUs 
> that are executable with KVM on a given machine (probably a bad idea) or

no that is context dependent, yes in KVM case, no in TCG case. Actually for the TCG case one
would also report only those which can be emulated and not all.

>    b) return not only the CPU type, but also a hint whether it's 
> available with KVM or

That is also changing the semantics

>    c) add a parameter to query_cpu_definitions to say "only return KVM 
> runnable CPUs" or

That optional parameter is not used by libvirt

>    d) introduce a new query_kvm_cpu_definitions qmp command

The criterion is not KVM here, it is "supported by the current Qemu in conjuction with the given
host and kernel".

If I compare the command with it's use on the Qemu Page
(http://wiki.qemu.org/Features/CPUModels) it says: "Requirement: libvirt needs to know which CPU
models are available to be used with the "-cpu" option. Current solution: libvirt uses QMP
query-cpu-definitions command."

> 
> >
> > ##
> > # @query-cpu-definitions:
> > #
> > # Return a list of supported virtual CPU definitions
> > #
> > # Returns: a list of CpuDefInfo
> >
> >>> The migration compatibility test is pretty much ARCH dependent. I looked into the
> >>> libvirt implementation and as one can see every architecture has its own implementation
> >>> there (libvirt/src/cpu/cpu_<arch>.c).
> >> So here's my question again. How does x86 evaluate whether a target
> >> machine is compatible with a source machine?
> > Will again look into that during the afternoon...
> 
> Yes, please. Someone else must have solved this before :).

Well, in my eyes it was newer solved for x86! The issue is that libvirt has its one x86 model and
feature code, independent from Qemu's model and feature implementation. You might know
libvirt's cpu_map.xml file where in the meantime also POWER started to add a hand-full of model
names. That's their source, they even use asm instructions to identify the local CPU and to
derive a model list they (libvirt) think to support. x86 requires alone more the 2500 lines of
code to implement its from qemu deviating model space. There is no test like the one that I
suggest that guarantees a priori a target hypervisor is capable to run a specific CPU model until
it fails.

x86 basically calculates a cpu model to be used from it's domain xml cpu statements, the cpuid
that it retrieves by itself and the feature definitions in the cpu_map.xml. During migartion
the domain xml is just copied without any change to the model. And eventually qemu startup might
fail.  

That cpu_map.xml configuration is obsolete for other architectures like s390 and also for arm.

The usable cpu model name space is retrieved by "virsh cpu-models s390x" which in our case is
taken from libvirt's qemu capabilities cache which gets filled by "query-cpu-definitions" during
libvirtd's startup time.

In addition we add a test at some point in libvirt's qemuDomainMigrateBegin() phase where we
verify the remote hypervisor is suitable to run the current domain's CPU model and to prepare
the destination domain XML where the CPU model might differ from the local.

Assume the local domain XML specifies host as current model, then the test will retrieve the
normalized model name by means of "query-cpu-model" and and use it to verify it with the
remote Qemu cpu model capabilities (sitting in the remote Qemu capabilities cache) and choose
the remote domain XML CPU model accordingly.

> 
> 
> Alex
> 
> 

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 21.05.14 14:56, Michael Mueller wrote:
> On Tue, 20 May 2014 12:10:23 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
>> On 20.05.14 12:02, Michael Mueller wrote:
>>> On Mon, 19 May 2014 22:14:00 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>>> On 19.05.14 19:03, Michael Mueller wrote:
>>>>> On Mon, 19 May 2014 16:49:28 +0200
>>>>> Alexander Graf <agraf@suse.de> wrote:
>>>>>
>> [...]
>>
>>>>>>> What user and thus also user space wants depends on other factors:
>>>>>>>
>>>>>>> 1. reliability
>>>>>>> 2. performance
>>>>>>> 3. availability
>>>>>>>
>>>>>>> It's not features, that's what programmers want.
>>>>>>>
>>>>>>> That's why I have designed the model and migration capability around the hardware
>>>>>>> and not around the software features and don't allow them to be enabled currently
>>>>>>> together.
>>>>>>>
>>>>>>> A software feature is a nice add on that is helpful for evaluation or development
>>>>>>> purpose. There is few space for it on productions systems.
>>>>>>>
>>>>>>> One option that I currently see to make software implemented facility migration
>>>>>>> capable is to calculate some kind of hash value derived from the full set of
>>>>>>> active software facilities. That value can be compared with pre-calculated
>>>>>>> values also stored in the supported model table of qemu. This value could be
>>>>>>> seen like a virtual model extension that has to match like the model name.
>>>>>>>
>>>>>>> But I have said it elsewhere already, a soft facility should be an exception and
>>>>>>> not the rule.
>>>>>>>
>>>>>>>>>> So all we need is a list of "features the guest sees available" which is
>>>>>>>>>> the same as "features user space wants the guest to see" which then gets
>>>>>>>>>> masked through "features the host can do in hardware".
>>>>>>>>>>
>>>>>>>>>> For emulation we can just check on the global feature availability on
>>>>>>>>>> whether we should emulate them or not.
>>>>>>>>>>
>>>>>>>>>>>> Also, if user space wants to make sure that its feature list is actually
>>>>>>>>>>>> workable on the host kernel, it needs to set and get the features again
>>>>>>>>>>>> and then compare that with the ones it set? That's different from x86's
>>>>>>>>>>>> cpuid implementation but probably workable.
>>>>>>>>>>> User space will probe what facilities are available and match them with the predefined
>>>>>>>>>>> cpu model set. Only those models which use a partial or full subset of the hard/host
>>>>>>>>>>> facility list are selectable.
>>>>>>>>>> Why?
>>>>>>>>> If a host does not offer the features required for a model it is not able to
>>>>>>>>> run efficiently.
>>>>>>>>>
>>>>>>>>>> Please take a look at how x86 does cpuid masking :).
>>>>>>>>>>
>>>>>>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
>>>>>>>>>> feature list exposure to the guest and actual feature implementation
>>>>>>>>>> inside the guest together. On POWER there is a patch set pending that
>>>>>>>>>> implements these two things separately - admittedly mostly because
>>>>>>>>>> hardware sucks and we can't change the PVR.
>>>>>>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
>>>>>>>>> directly comparable with the processor version register of POWER.
>>>>>>>>>
>>>>>>>>> In the S390 world we have a well defined CPU model room spanned by the machine
>>>>>>>>> type and its GA count. Thus we can define a bijective mapping between
>>>>>>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
>>>>>>>>> name which BTW is meaningful also for a human user.
>>>>>>>> Same thing as POWER.
>>>>>>>>
>>>>>>>>> By means of this name, a management interface (libvirt) will draw decisions if
>>>>>>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
>>>>>>>>> to compare if the current model of the guest on the source hypervisor
>>>>>>>>> ("query-cpu-model"), is contained in the supported model list of the target
>>>>>>>>> hypervisor ("query-cpu-definitions").
>>>>>>>> I don't think this works, since QEMU should always return all the cpu
>>>>>>>> definitions it's aware of on query-cpu-definitions, not just the ones
>>>>>>>> that it thinks may be compatible with the host at a random point in time.
>>>>>>> It does not return model names that it thinks they are compatible at some point
>>>>>>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
>>>>>>> system is capable to run. Together with the CPU model run by the guest, some upper
>>>>>>> management interface knows if the hypervisor supports the required CPU model and
>>>>>>> uses a guest definition with the same CPU model on the target hypervisor.
>>>>>>>
>>>>>>> The information for that is taken from the model table which QEMU builds up during
>>>>>>> startup time. This list limits the command line selectable CPU models as well.
>>>>>> This makes s390 derive from the way x86 handles things. NAK.
>>>>> One second, that goes a little fast here :-). x86 returns a list they support which happens
>>>>> to be the full list they define and s390 does logically the same because we know that
>>>>> certain models are not supported due to probing. BTW that happens only if you run Qemu on
>>>>> back level hardware and that is perfectly correct.
>>>> It's not what other architectures do and I'd hate to see s390 deviate
>>>> just because.
>>> Only these four architectures implement the query and they all differ a little...
>>>
>>> target-arm/helper.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
>>> target-i386/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
>>> target-ppc/translate_init.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
>>> target-s390x/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
>>>
>>> arm walks through a list of all ARM CPU types
>>> list = object_class_get_list(TYPE_ARM_CPU, false);
>>> and returns the CpuDefinitionInfoList derived from that one to one
>>>
>>> i386 loops over the static builtin_x86_defs[] array to retrieve the model names,
>>> they don't even use the CPU class model as source
>>>
>>> ppc walks through a list of all POWER CPU types
>>> list = object_class_get_list(TYPE_POWERPC_CPU, false);
>>> and then extends the produced list by all defined aliases
>>>
>>> and s390x finally also walks through the defined S390 CPU types
>>> list = object_class_get_list(TYPE_S390_CPU, false);
>>> but drops those which are not usable (!is_active)
>>> Just consider them as not defined. I actually would undefine
>>> them if I knew how.
>>>
>>> Also the commands comment says "list of supported virtual CPU definitions" and the s390
>>> list contains all supported models, that's no contradiction.
>> So IMHO we can either
>>
>>     a) change the definition of query_cpu_definitions to only return CPUs
>> that are executable with KVM on a given machine (probably a bad idea) or
> no that is context dependent, yes in KVM case, no in TCG case. Actually for the TCG case one
> would also report only those which can be emulated and not all.

Yup, we're already in a messy position there today with -cpu host which 
we only expose if KVM is available.

>
>>     b) return not only the CPU type, but also a hint whether it's
>> available with KVM or
> That is also changing the semantics
>
>>     c) add a parameter to query_cpu_definitions to say "only return KVM
>> runnable CPUs" or
> That optional parameter is not used by libvirt

Well, we're trying to fix a generic bug here, so maybe it should be used ;).

>
>>     d) introduce a new query_kvm_cpu_definitions qmp command
> The criterion is not KVM here, it is "supported by the current Qemu in conjuction with the given
> host and kernel".

Only for KVM. For TCG it would be "supported by the current QEMU TCG 
engine on a given host". Maybe we'll implement transactional memory 
support in TCG one day but require host transactional memory support for it?

> If I compare the command with it's use on the Qemu Page
> (http://wiki.qemu.org/Features/CPUModels) it says: "Requirement: libvirt needs to know which CPU
> models are available to be used with the "-cpu" option. Current solution: libvirt uses QMP
> query-cpu-definitions command."
>
>>> ##
>>> # @query-cpu-definitions:
>>> #
>>> # Return a list of supported virtual CPU definitions
>>> #
>>> # Returns: a list of CpuDefInfo
>>>
>>>>> The migration compatibility test is pretty much ARCH dependent. I looked into the
>>>>> libvirt implementation and as one can see every architecture has its own implementation
>>>>> there (libvirt/src/cpu/cpu_<arch>.c).
>>>> So here's my question again. How does x86 evaluate whether a target
>>>> machine is compatible with a source machine?
>>> Will again look into that during the afternoon...
>> Yes, please. Someone else must have solved this before :).
> Well, in my eyes it was newer solved for x86! The issue is that libvirt has its one x86 model and
> feature code, independent from Qemu's model and feature implementation. You might know
> libvirt's cpu_map.xml file where in the meantime also POWER started to add a hand-full of model
> names. That's their source, they even use asm instructions to identify the local CPU and to
> derive a model list they (libvirt) think to support. x86 requires alone more the 2500 lines of
> code to implement its from qemu deviating model space. There is no test like the one that I
> suggest that guarantees a priori a target hypervisor is capable to run a specific CPU model until
> it fails.

Ok, maybe it's about time to start moving that code over into QEMU then. 
Maybe not. But this is not something you and me can just decide for s390 
alone - I want this solved for everyone.

>
> x86 basically calculates a cpu model to be used from it's domain xml cpu statements, the cpuid
> that it retrieves by itself and the feature definitions in the cpu_map.xml. During migartion
> the domain xml is just copied without any change to the model. And eventually qemu startup might
> fail.
>
> That cpu_map.xml configuration is obsolete for other architectures like s390 and also for arm.
>
> The usable cpu model name space is retrieved by "virsh cpu-models s390x" which in our case is
> taken from libvirt's qemu capabilities cache which gets filled by "query-cpu-definitions" during
> libvirtd's startup time.
>
> In addition we add a test at some point in libvirt's qemuDomainMigrateBegin() phase where we
> verify the remote hypervisor is suitable to run the current domain's CPU model and to prepare
> the destination domain XML where the CPU model might differ from the local.
>
> Assume the local domain XML specifies host as current model, then the test will retrieve the
> normalized model name by means of "query-cpu-model" and and use it to verify it with the
> remote Qemu cpu model capabilities (sitting in the remote Qemu capabilities cache) and choose
> the remote domain XML CPU model accordingly.

All of that sounds very reasonable. So now we only have to work on 
making that fit into the whole picture, not into a tiny s390 corner case.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

[-- Attachment #1: Type: text/plain, Size: 12188 bytes --]

On Wed, 21 May 2014 15:22:35 +0200
Alexander Graf <agraf@suse.de> wrote:

I have seen the slides from Eduardo which he presented during this years
DevConf in Brno and made my comments according the s390x implementation
on that. Is you will see, this is mostly overlapping except for the model
definition authority that I clearly see on qemu's side.

See pdf attachment.

> 
> On 21.05.14 14:56, Michael Mueller wrote:
> > On Tue, 20 May 2014 12:10:23 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> >> On 20.05.14 12:02, Michael Mueller wrote:
> >>> On Mon, 19 May 2014 22:14:00 +0200
> >>> Alexander Graf <agraf@suse.de> wrote:
> >>>
> >>>> On 19.05.14 19:03, Michael Mueller wrote:
> >>>>> On Mon, 19 May 2014 16:49:28 +0200
> >>>>> Alexander Graf <agraf@suse.de> wrote:
> >>>>>
> >> [...]
> >>
> >>>>>>> What user and thus also user space wants depends on other factors:
> >>>>>>>
> >>>>>>> 1. reliability
> >>>>>>> 2. performance
> >>>>>>> 3. availability
> >>>>>>>
> >>>>>>> It's not features, that's what programmers want.
> >>>>>>>
> >>>>>>> That's why I have designed the model and migration capability around the hardware
> >>>>>>> and not around the software features and don't allow them to be enabled currently
> >>>>>>> together.
> >>>>>>>
> >>>>>>> A software feature is a nice add on that is helpful for evaluation or development
> >>>>>>> purpose. There is few space for it on productions systems.
> >>>>>>>
> >>>>>>> One option that I currently see to make software implemented facility migration
> >>>>>>> capable is to calculate some kind of hash value derived from the full set of
> >>>>>>> active software facilities. That value can be compared with pre-calculated
> >>>>>>> values also stored in the supported model table of qemu. This value could be
> >>>>>>> seen like a virtual model extension that has to match like the model name.
> >>>>>>>
> >>>>>>> But I have said it elsewhere already, a soft facility should be an exception and
> >>>>>>> not the rule.
> >>>>>>>
> >>>>>>>>>> So all we need is a list of "features the guest sees available" which is
> >>>>>>>>>> the same as "features user space wants the guest to see" which then gets
> >>>>>>>>>> masked through "features the host can do in hardware".
> >>>>>>>>>>
> >>>>>>>>>> For emulation we can just check on the global feature availability on
> >>>>>>>>>> whether we should emulate them or not.
> >>>>>>>>>>
> >>>>>>>>>>>> Also, if user space wants to make sure that its feature list is actually
> >>>>>>>>>>>> workable on the host kernel, it needs to set and get the features again
> >>>>>>>>>>>> and then compare that with the ones it set? That's different from x86's
> >>>>>>>>>>>> cpuid implementation but probably workable.
> >>>>>>>>>>> User space will probe what facilities are available and match them with the
> >>>>>>>>>>> predefined cpu model set. Only those models which use a partial or full subset of
> >>>>>>>>>>> the hard/host facility list are selectable.
> >>>>>>>>>> Why?
> >>>>>>>>> If a host does not offer the features required for a model it is not able to
> >>>>>>>>> run efficiently.
> >>>>>>>>>
> >>>>>>>>>> Please take a look at how x86 does cpuid masking :).
> >>>>>>>>>>
> >>>>>>>>>> In fact, I'm not 100% convinced that it's a good idea to link cpuid /
> >>>>>>>>>> feature list exposure to the guest and actual feature implementation
> >>>>>>>>>> inside the guest together. On POWER there is a patch set pending that
> >>>>>>>>>> implements these two things separately - admittedly mostly because
> >>>>>>>>>> hardware sucks and we can't change the PVR.
> >>>>>>>>> That is maybe the big difference with s390. The cpuid in the S390 case is not
> >>>>>>>>> directly comparable with the processor version register of POWER.
> >>>>>>>>>
> >>>>>>>>> In the S390 world we have a well defined CPU model room spanned by the machine
> >>>>>>>>> type and its GA count. Thus we can define a bijective mapping between
> >>>>>>>>> (type, ga) <-> (cpuid, ibc, facility set). From type and ga we form the model
> >>>>>>>>> name which BTW is meaningful also for a human user.
> >>>>>>>> Same thing as POWER.
> >>>>>>>>
> >>>>>>>>> By means of this name, a management interface (libvirt) will draw decisions if
> >>>>>>>>> migration to a remote hypervisor is a good idea or not. For that it just needs
> >>>>>>>>> to compare if the current model of the guest on the source hypervisor
> >>>>>>>>> ("query-cpu-model"), is contained in the supported model list of the target
> >>>>>>>>> hypervisor ("query-cpu-definitions").
> >>>>>>>> I don't think this works, since QEMU should always return all the cpu
> >>>>>>>> definitions it's aware of on query-cpu-definitions, not just the ones
> >>>>>>>> that it thinks may be compatible with the host at a random point in time.
> >>>>>>> It does not return model names that it thinks they are compatible at some point
> >>>>>>> in time. In s390 mode, it returns all definitions (CPU models) that a given host
> >>>>>>> system is capable to run. Together with the CPU model run by the guest, some upper
> >>>>>>> management interface knows if the hypervisor supports the required CPU model and
> >>>>>>> uses a guest definition with the same CPU model on the target hypervisor.
> >>>>>>>
> >>>>>>> The information for that is taken from the model table which QEMU builds up during
> >>>>>>> startup time. This list limits the command line selectable CPU models as well.
> >>>>>> This makes s390 derive from the way x86 handles things. NAK.
> >>>>> One second, that goes a little fast here :-). x86 returns a list they support which
> >>>>> happens to be the full list they define and s390 does logically the same because we know
> >>>>> that certain models are not supported due to probing. BTW that happens only if you run
> >>>>> Qemu on back level hardware and that is perfectly correct.
> >>>> It's not what other architectures do and I'd hate to see s390 deviate
> >>>> just because.
> >>> Only these four architectures implement the query and they all differ a little...
> >>>
> >>> target-arm/helper.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> >>> target-i386/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> >>> target-ppc/translate_init.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> >>> target-s390x/cpu.c:CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
> >>>
> >>> arm walks through a list of all ARM CPU types
> >>> list = object_class_get_list(TYPE_ARM_CPU, false);
> >>> and returns the CpuDefinitionInfoList derived from that one to one
> >>>
> >>> i386 loops over the static builtin_x86_defs[] array to retrieve the model names,
> >>> they don't even use the CPU class model as source
> >>>
> >>> ppc walks through a list of all POWER CPU types
> >>> list = object_class_get_list(TYPE_POWERPC_CPU, false);
> >>> and then extends the produced list by all defined aliases
> >>>
> >>> and s390x finally also walks through the defined S390 CPU types
> >>> list = object_class_get_list(TYPE_S390_CPU, false);
> >>> but drops those which are not usable (!is_active)
> >>> Just consider them as not defined. I actually would undefine
> >>> them if I knew how.
> >>>
> >>> Also the commands comment says "list of supported virtual CPU definitions" and the s390
> >>> list contains all supported models, that's no contradiction.
> >> So IMHO we can either
> >>
> >>     a) change the definition of query_cpu_definitions to only return CPUs
> >> that are executable with KVM on a given machine (probably a bad idea) or
> > no that is context dependent, yes in KVM case, no in TCG case. Actually for the TCG case one
> > would also report only those which can be emulated and not all.
> 
> Yup, we're already in a messy position there today with -cpu host which 
> we only expose if KVM is available.
> 
> >
> >>     b) return not only the CPU type, but also a hint whether it's
> >> available with KVM or
> > That is also changing the semantics
> >
> >>     c) add a parameter to query_cpu_definitions to say "only return KVM
> >> runnable CPUs" or
> > That optional parameter is not used by libvirt
> 
> Well, we're trying to fix a generic bug here, so maybe it should be used ;).

I think Eduardo has a sample implementation here for already, let's see how its done there.
> 
> >
> >>     d) introduce a new query_kvm_cpu_definitions qmp command
> > The criterion is not KVM here, it is "supported by the current Qemu in conjuction with the
> > given host and kernel".
> 
> Only for KVM. For TCG it would be "supported by the current QEMU TCG 
> engine on a given host". Maybe we'll implement transactional memory 
> support in TCG one day but require host transactional memory support for it?
> 
> > If I compare the command with it's use on the Qemu Page
> > (http://wiki.qemu.org/Features/CPUModels) it says: "Requirement: libvirt needs to know which
> > CPU models are available to be used with the "-cpu" option. Current solution: libvirt uses QMP
> > query-cpu-definitions command."
> >
> >>> ##
> >>> # @query-cpu-definitions:
> >>> #
> >>> # Return a list of supported virtual CPU definitions
> >>> #
> >>> # Returns: a list of CpuDefInfo
> >>>
> >>>>> The migration compatibility test is pretty much ARCH dependent. I looked into the
> >>>>> libvirt implementation and as one can see every architecture has its own implementation
> >>>>> there (libvirt/src/cpu/cpu_<arch>.c).
> >>>> So here's my question again. How does x86 evaluate whether a target
> >>>> machine is compatible with a source machine?
> >>> Will again look into that during the afternoon...
> >> Yes, please. Someone else must have solved this before :).
> > Well, in my eyes it was newer solved for x86! The issue is that libvirt has its one x86 model
> > and feature code, independent from Qemu's model and feature implementation. You might know
> > libvirt's cpu_map.xml file where in the meantime also POWER started to add a hand-full of
> > model names. That's their source, they even use asm instructions to identify the local CPU
> > and to derive a model list they (libvirt) think to support. x86 requires alone more the 2500
> > lines of code to implement its from qemu deviating model space. There is no test like the one
> > that I suggest that guarantees a priori a target hypervisor is capable to run a specific CPU
> > model until it fails.
> 
> Ok, maybe it's about time to start moving that code over into QEMU then. 
> Maybe not. But this is not something you and me can just decide for s390 
> alone - I want this solved for everyone.

I will talk with Christian and Connie and see what they suggest how to proceed.

> 
> >
> > x86 basically calculates a cpu model to be used from it's domain xml cpu statements, the cpuid
> > that it retrieves by itself and the feature definitions in the cpu_map.xml. During migartion
> > the domain xml is just copied without any change to the model. And eventually qemu startup
> > might fail.
> >
> > That cpu_map.xml configuration is obsolete for other architectures like s390 and also for arm.
> >
> > The usable cpu model name space is retrieved by "virsh cpu-models s390x" which in our case is
> > taken from libvirt's qemu capabilities cache which gets filled by "query-cpu-definitions"
> > during libvirtd's startup time.
> >
> > In addition we add a test at some point in libvirt's qemuDomainMigrateBegin() phase where we
> > verify the remote hypervisor is suitable to run the current domain's CPU model and to prepare
> > the destination domain XML where the CPU model might differ from the local.
> >
> > Assume the local domain XML specifies host as current model, then the test will retrieve the
> > normalized model name by means of "query-cpu-model" and and use it to verify it with the
> > remote Qemu cpu model capabilities (sitting in the remote Qemu capabilities cache) and choose
> > the remote domain XML CPU model accordingly.
> 
> All of that sounds very reasonable. So now we only have to work on 
> making that fit into the whole picture, not into a tiny s390 corner case.
> 

That is definitely my goal but looking at x86 solely may lead into a special case.

Thanks
Michael 

> 
> Alex
> 


[-- Attachment #2: cpu-models-commented.pdf --]
[-- Type: application/pdf, Size: 190990 bytes --]

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Paolo Bonzini]

Il 22/05/2014 10:23, Michael Mueller ha scritto:
> On Wed, 21 May 2014 15:22:35 +0200
> Alexander Graf <agraf@suse.de> wrote:
>
> I have seen the slides from Eduardo which he presented during this years
> DevConf in Brno and made my comments according the s390x implementation
> on that. Is you will see, this is mostly overlapping except for the model
> definition authority that I clearly see on qemu's side.
>
> See pdf attachment.

More comments:

- "Only one machine type in s390 case which is -machine s390-virtio-ccw"

This probably should change sooner or later, as soon as the 
implementation becomes stable enough.  Versioning is necessary for live 
migration across different QEMU version.  Perhaps start versioning in 
2.2, i.e. start making s390-virtio-ccw-2.1 an alias for s390-virtio-ccw now?

Note that new virtio device features can appear at any time outside the 
s390 code, and will take part in versioning as well.

- "No enforce option"

Strongly suggest making enforce the only possible behavior.

- "Not in the s390x case, because the KVM facility mask limits the cpu 
model specific facilities"

What if the KVM facility mask changes?  For x86, nowadays new CPUID bits 
are only introduced in KVM when a new processors comes out.  But if we 
introduced an older CPUID bit, it would be a huge complication for 
backwards compatibility.  Is it different for s390?

Paolo

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Michael Mueller]

On Thu, 22 May 2014 10:53:38 +0200
Paolo Bonzini <pbonzini@redhat.com> wrote:

> Il 22/05/2014 10:23, Michael Mueller ha scritto:
> > On Wed, 21 May 2014 15:22:35 +0200
> > Alexander Graf <agraf@suse.de> wrote:
> >
> > I have seen the slides from Eduardo which he presented during this years
> > DevConf in Brno and made my comments according the s390x implementation
> > on that. Is you will see, this is mostly overlapping except for the model
> > definition authority that I clearly see on qemu's side.
> >
> > See pdf attachment.
> 
> More comments:
> 
> - "Only one machine type in s390 case which is -machine s390-virtio-ccw"
> 
> This probably should change sooner or later, as soon as the 
> implementation becomes stable enough.  Versioning is necessary for live 
> migration across different QEMU version.  Perhaps start versioning in 
> 2.2, i.e. start making s390-virtio-ccw-2.1 an alias for s390-virtio-ccw now?

Absolutely right, we did not do it yet.

> 
> Note that new virtio device features can appear at any time outside the 
> s390 code, and will take part in versioning as well.

These changes are then as well included in the machine version I guess. 

> 
> - "No enforce option"
> 
> Strongly suggest making enforce the only possible behavior.

Right that's the plan.

> 
> - "Not in the s390x case, because the KVM facility mask limits the cpu 
> model specific facilities"
> 
> What if the KVM facility mask changes?  For x86, nowadays new CPUID bits 
> are only introduced in KVM when a new processors comes out.  But if we 
> introduced an older CPUID bit, it would be a huge complication for 
> backwards compatibility.  Is it different for s390?
> 

I'm thinking about this one currently. The ABI itself will not change
because all facilities of the architecture will be defined in QEMU for the
respective CPU model even though some of them are masked out upon request
by KVM, but the mask is part of the interface itself and can be applied by
QEMU already.  

Michael 

> Paolo
> 

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Christian Borntraeger]

On 22/05/14 10:53, Paolo Bonzini wrote:
> Il 22/05/2014 10:23, Michael Mueller ha scritto:
>> On Wed, 21 May 2014 15:22:35 +0200
>> Alexander Graf <agraf@suse.de> wrote:
>>
>> I have seen the slides from Eduardo which he presented during this years
>> DevConf in Brno and made my comments according the s390x implementation
>> on that. Is you will see, this is mostly overlapping except for the model
>> definition authority that I clearly see on qemu's side.
>>
>> See pdf attachment.
> 
> More comments:
> 
> - "Only one machine type in s390 case which is -machine s390-virtio-ccw"
> 
> This probably should change sooner or later, as soon as the implementation becomes stable enough.  Versioning is necessary for live migration across different QEMU version.  Perhaps start versioning in 2.2, i.e. start making s390-virtio-ccw-2.1 an alias for s390-virtio-ccw now?
> 
> Note that new virtio device features can appear at any time outside the s390 code, and will take part in versioning as well.
> 
> - "No enforce option"
> 
> Strongly suggest making enforce the only possible behavior.
> 
> - "Not in the s390x case, because the KVM facility mask limits the cpu model specific facilities"
> 
> What if the KVM facility mask changes?  For x86, nowadays new CPUID bits are only introduced in KVM when a new processors comes out.  But if we introduced an older CPUID bit, it would be a huge complication for backwards compatibility.  Is it different for s390?
> 
> Paolo
> 

I guess we need to have a full picture here. Would this topic be suitable for the KVM call?

Christian

Re: [Qemu-devel] [PATCH v1 RFC 6/6] KVM: s390: add cpu model support

[Alexander Graf]

On 22.05.14 22:36, Christian Borntraeger wrote:
> On 22/05/14 10:53, Paolo Bonzini wrote:
>> Il 22/05/2014 10:23, Michael Mueller ha scritto:
>>> On Wed, 21 May 2014 15:22:35 +0200
>>> Alexander Graf <agraf@suse.de> wrote:
>>>
>>> I have seen the slides from Eduardo which he presented during this years
>>> DevConf in Brno and made my comments according the s390x implementation
>>> on that. Is you will see, this is mostly overlapping except for the model
>>> definition authority that I clearly see on qemu's side.
>>>
>>> See pdf attachment.
>> More comments:
>>
>> - "Only one machine type in s390 case which is -machine s390-virtio-ccw"
>>
>> This probably should change sooner or later, as soon as the implementation becomes stable enough.  Versioning is necessary for live migration across different QEMU version.  Perhaps start versioning in 2.2, i.e. start making s390-virtio-ccw-2.1 an alias for s390-virtio-ccw now?
>>
>> Note that new virtio device features can appear at any time outside the s390 code, and will take part in versioning as well.
>>
>> - "No enforce option"
>>
>> Strongly suggest making enforce the only possible behavior.
>>
>> - "Not in the s390x case, because the KVM facility mask limits the cpu model specific facilities"
>>
>> What if the KVM facility mask changes?  For x86, nowadays new CPUID bits are only introduced in KVM when a new processors comes out.  But if we introduced an older CPUID bit, it would be a huge complication for backwards compatibility.  Is it different for s390?
>>
>> Paolo
>>
> I guess we need to have a full picture here. Would this topic be suitable for the KVM call?

Very much so, yes. Please put it on the agenda.


Alex

Re: [Qemu-devel] [PATCH v1 RFC 0/6] KVM: s390: cpu model implementation

[Christian Borntraeger]

On 13/05/14 16:58, Michael Mueller wrote:
> The proposed patch set implements S390 cpu model support in kvm. A cpu
> model is defined by a triple comprizing the cpu type , the cpu facility
> set and instruction blocking control. A consumer requests a feasible
> combination of that that triple before vcpu creation. All subsequently
> created vcpu's will use this cpu configuration. 
> 
> Michael Mueller (6):
>   s390/sclp: add sclp_get_ibc function
>   KVM: s390: split SIE state guest prefix field
>   KVM: s390: use facilities and cpu_id per KVM
>   KVM: s390: add ibc api
>   KVM: s390: initial implementation of soft facilities
>   KVM: s390: add cpu model support
> 
>  arch/s390/include/asm/kvm_host.h |  13 +-
>  arch/s390/include/asm/sclp.h     |   1 +
>  arch/s390/include/uapi/asm/kvm.h |  23 ++++
>  arch/s390/kvm/Makefile           |   2 +-
>  arch/s390/kvm/diag.c             |   2 +-
>  arch/s390/kvm/gaccess.c          |   4 +-
>  arch/s390/kvm/gaccess.h          |   8 +-
>  arch/s390/kvm/ibc.c              |  29 +++++
>  arch/s390/kvm/kvm-s390.c         | 263 +++++++++++++++++++++++++++++++++------
>  arch/s390/kvm/kvm-s390.h         |  43 ++++++-
>  arch/s390/kvm/priv.c             |  16 ++-
>  arch/s390/kvm/softfac.c          |  38 ++++++
>  drivers/s390/char/sclp_early.c   |  12 +-
>  13 files changed, 397 insertions(+), 57 deletions(-)
>  create mode 100644 arch/s390/kvm/ibc.c
>  create mode 100644 arch/s390/kvm/softfac.c
> 

I will add patch 1 and 2 to the next KVM: s390: pull request as there 
does not seem to be potential for discussion / interface changes. 
Everything else looks ok, but I want to wait if additional feedback 
about the overall design comes in. 

Christian

Re: [Qemu-devel] [PATCH v1 RFC 0/6] KVM: s390: cpu model implementation

[Michael Mueller]

On Fri, 16 May 2014 13:32:09 +0200
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 13/05/14 16:58, Michael Mueller wrote:
> > The proposed patch set implements S390 cpu model support in kvm. A cpu
> > model is defined by a triple comprizing the cpu type , the cpu facility
> > set and instruction blocking control. A consumer requests a feasible
> > combination of that that triple before vcpu creation. All subsequently
> > created vcpu's will use this cpu configuration. 
> > 
> > Michael Mueller (6):
> >   s390/sclp: add sclp_get_ibc function
> >   KVM: s390: split SIE state guest prefix field
> >   KVM: s390: use facilities and cpu_id per KVM
> >   KVM: s390: add ibc api
> >   KVM: s390: initial implementation of soft facilities
> >   KVM: s390: add cpu model support
> > 
> >  arch/s390/include/asm/kvm_host.h |  13 +-
> >  arch/s390/include/asm/sclp.h     |   1 +
> >  arch/s390/include/uapi/asm/kvm.h |  23 ++++
> >  arch/s390/kvm/Makefile           |   2 +-
> >  arch/s390/kvm/diag.c             |   2 +-
> >  arch/s390/kvm/gaccess.c          |   4 +-
> >  arch/s390/kvm/gaccess.h          |   8 +-
> >  arch/s390/kvm/ibc.c              |  29 +++++
> >  arch/s390/kvm/kvm-s390.c         | 263 +++++++++++++++++++++++++++++++++------
> >  arch/s390/kvm/kvm-s390.h         |  43 ++++++-
> >  arch/s390/kvm/priv.c             |  16 ++-
> >  arch/s390/kvm/softfac.c          |  38 ++++++
> >  drivers/s390/char/sclp_early.c   |  12 +-
> >  13 files changed, 397 insertions(+), 57 deletions(-)
> >  create mode 100644 arch/s390/kvm/ibc.c
> >  create mode 100644 arch/s390/kvm/softfac.c
> > 
> 
> I will add patch 1 and 2 to the next KVM: s390: pull request as there 
> does not seem to be potential for discussion / interface changes. 
> Everything else looks ok, but I want to wait if additional feedback 
> about the overall design comes in. 

Perfect...

> 
> Christian