Re: [PATCH] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

Re: [PATCH] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

[Alexander Graf]

On 13.03.2012, at 03:49, Benjamin Herrenschmidt wrote:

> There is nothing in the code for emulating TCE tables in the kernel
> that prevents it from working on "PR" KVM... other than ifdef's and
> location of the code.
> 
> This renames book3s_64_vio_hv.c to book3s_64_vio.c and moves the
> bulk of the code there.
> 
> This speeds things up a bit on my G5.
> 
> ---
> arch/powerpc/include/asm/kvm_host.h |    6 +-
> arch/powerpc/include/asm/kvm_ppc.h  |    2 +
> arch/powerpc/kvm/Makefile           |    3 +-
> arch/powerpc/kvm/book3s_64_vio.c    |  187 +++++++++++++++++++++++++++++++++++
> arch/powerpc/kvm/book3s_64_vio_hv.c |   73 --------------
> arch/powerpc/kvm/book3s_hv.c        |  109 --------------------
> arch/powerpc/kvm/book3s_pr.c        |    3 +
> arch/powerpc/kvm/book3s_pr_papr.c   |   18 ++++
> arch/powerpc/kvm/powerpc.c          |    8 +-
> 9 files changed, 223 insertions(+), 186 deletions(-)
> create mode 100644 arch/powerpc/kvm/book3s_64_vio.c
> delete mode 100644 arch/powerpc/kvm/book3s_64_vio_hv.c
> 
> diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
> index bf8af5d..ff7fd33 100644
> --- a/arch/powerpc/include/asm/kvm_host.h
> +++ b/arch/powerpc/include/asm/kvm_host.h
> @@ -183,10 +183,14 @@ struct kvm_arch {
> 	unsigned long lpcr;
> 	unsigned long rmor;
> 	struct kvmppc_rma_info *rma;
> -	struct list_head spapr_tce_tables;
> 	unsigned short last_vcpu[NR_CPUS];
> 	struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
> +	struct list_head spapr_tce_tables;
> #endif /* CONFIG_KVM_BOOK3S_64_HV */
> +
> +#ifdef CONFIG_KVM_BOOK3S_64_PR
> +	struct list_head spapr_tce_tables;
> +#endif /* CONFIG_KVM_BOOK3S_64_PR */

Please move the definition to an #ifdef on CONFIG_KVM_BOOK3S_64. That way we get rid of the duplication here.

> };
> 
> /*
> diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
> index 46efd1a..ad2ca4b 100644
> --- a/arch/powerpc/include/asm/kvm_ppc.h
> +++ b/arch/powerpc/include/asm/kvm_ppc.h
> @@ -125,6 +125,8 @@ extern void kvmppc_map_vrma(struct kvm *kvm,
> extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu);
> extern long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
> 				struct kvm_create_spapr_tce *args);
> +extern long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
> +			     unsigned long ioba, unsigned long tce);
> extern long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
> 				struct kvm_allocate_rma *rma);
> extern struct kvmppc_rma_info *kvm_alloc_rma(void);
> diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile
> index 3688aee..fda9728 100644
> --- a/arch/powerpc/kvm/Makefile
> +++ b/arch/powerpc/kvm/Makefile
> @@ -44,6 +44,7 @@ kvm-book3s_64-objs-$(CONFIG_KVM_BOOK3S_64_PR) := \
> 	book3s_paired_singles.o \
> 	book3s_pr.o \
> 	book3s_pr_papr.o \
> +	book3s_64_vio.o \
> 	book3s_emulate.o \
> 	book3s_interrupts.o \
> 	book3s_mmu_hpte.o \
> @@ -60,7 +61,7 @@ kvm-book3s_64-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
> kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
> 	book3s_hv_rmhandlers.o \
> 	book3s_hv_rm_mmu.o \
> -	book3s_64_vio_hv.o \
> +	book3s_64_vio.o \
> 	book3s_hv_builtin.o
> 
> kvm-book3s_64-module-objs := \
> diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
> new file mode 100644
> index 0000000..193ba68
> --- /dev/null
> +++ b/arch/powerpc/kvm/book3s_64_vio.c
> @@ -0,0 +1,187 @@
> +/*
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License, version 2, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> + *
> + * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
> + * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
> + */
> +
> +#include <linux/types.h>
> +#include <linux/string.h>
> +#include <linux/kvm.h>
> +#include <linux/kvm_host.h>
> +#include <linux/highmem.h>
> +#include <linux/gfp.h>
> +#include <linux/slab.h>
> +#include <linux/hugetlb.h>
> +#include <linux/list.h>
> +#include <linux/anon_inodes.h>
> +
> +#include <asm/tlbflush.h>
> +#include <asm/kvm_ppc.h>
> +#include <asm/kvm_book3s.h>
> +#include <asm/mmu-hash64.h>
> +#include <asm/hvcall.h>
> +#include <asm/synch.h>
> +#include <asm/ppc-opcode.h>
> +#include <asm/kvm_host.h>
> +#include <asm/udbg.h>
> +
> +#define TCES_PER_PAGE	(PAGE_SIZE / sizeof(u64))
> +
> +/* WARNING: This will be called in real-mode on HV KVM and virtual
> + *          mode on PR KVM
> + */
> +long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
> +		      unsigned long ioba, unsigned long tce)
> +{
> +	struct kvm *kvm = vcpu->kvm;
> +	struct kvmppc_spapr_tce_table *stt;
> +
> +	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
> +	/* 	    liobn, ioba, tce); */
> +
> +	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
> +		if (stt->liobn == liobn) {
> +			unsigned long idx = ioba >> SPAPR_TCE_SHIFT;
> +			struct page *page;
> +			u64 *tbl;
> +
> +			/* udbg_printf("H_PUT_TCE: liobn 0x%lx => stt=%p  window_size=0x%x\n", */
> +			/* 	    liobn, stt, stt->window_size); */
> +			if (ioba >= stt->window_size)
> +				return H_PARAMETER;
> +
> +			page = stt->pages[idx / TCES_PER_PAGE];
> +			tbl = (u64 *)page_address(page);
> +
> +			/* FIXME: Need to validate the TCE itself */
> +			/* udbg_printf("tce @ %p\n", &tbl[idx % TCES_PER_PAGE]); */
> +			tbl[idx % TCES_PER_PAGE] = tce;
> +			return H_SUCCESS;
> +		}
> +	}
> +
> +	/* Didn't find the liobn, punt it to userspace */
> +	return H_TOO_HARD;
> +}
> +
> +static long kvmppc_stt_npages(unsigned long window_size)
> +{
> +	return ALIGN((window_size >> SPAPR_TCE_SHIFT)
> +		     * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
> +}
> +
> +static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
> +{
> +	struct kvm *kvm = stt->kvm;
> +	int i;
> +
> +	mutex_lock(&kvm->lock);
> +	list_del(&stt->list);
> +	for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
> +		__free_page(stt->pages[i]);
> +	kfree(stt);
> +	mutex_unlock(&kvm->lock);
> +
> +	kvm_put_kvm(kvm);
> +}
> +
> +static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
> +{
> +	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
> +	struct page *page;
> +
> +	if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
> +		return VM_FAULT_SIGBUS;
> +
> +	page = stt->pages[vmf->pgoff];
> +	get_page(page);
> +	vmf->page = page;
> +	return 0;
> +}
> +
> +static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
> +	.fault = kvm_spapr_tce_fault,
> +};
> +
> +static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> +	vma->vm_ops = &kvm_spapr_tce_vm_ops;
> +	return 0;
> +}
> +
> +static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
> +{
> +	struct kvmppc_spapr_tce_table *stt = filp->private_data;
> +
> +	release_spapr_tce_table(stt);
> +	return 0;
> +}
> +
> +static struct file_operations kvm_spapr_tce_fops = {
> +	.mmap           = kvm_spapr_tce_mmap,
> +	.release	= kvm_spapr_tce_release,
> +};
> +
> +long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
> +				   struct kvm_create_spapr_tce *args)
> +{
> +	struct kvmppc_spapr_tce_table *stt = NULL;
> +	long npages;
> +	int ret = -ENOMEM;
> +	int i;
> +
> +	/* Check this LIOBN hasn't been previously allocated */
> +	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
> +		if (stt->liobn == args->liobn)
> +			return -EBUSY;
> +	}
> +
> +	npages = kvmppc_stt_npages(args->window_size);
> +
> +	stt = kzalloc(sizeof(*stt) + npages* sizeof(struct page *),
> +		      GFP_KERNEL);
> +	if (!stt)
> +		goto fail;
> +
> +	stt->liobn = args->liobn;
> +	stt->window_size = args->window_size;
> +	stt->kvm = kvm;
> +
> +	for (i = 0; i < npages; i++) {
> +		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
> +		if (!stt->pages[i])
> +			goto fail;
> +	}
> +
> +	kvm_get_kvm(kvm);
> +
> +	mutex_lock(&kvm->lock);
> +	list_add(&stt->list, &kvm->arch.spapr_tce_tables);
> +
> +	mutex_unlock(&kvm->lock);
> +
> +	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
> +				stt, O_RDWR);
> +
> +fail:
> +	if (stt) {
> +		for (i = 0; i < npages; i++)
> +			if (stt->pages[i])
> +				__free_page(stt->pages[i]);
> +
> +		kfree(stt);
> +	}
> +	return ret;
> +}
> diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
> deleted file mode 100644
> index ea0f8c5..0000000
> --- a/arch/powerpc/kvm/book3s_64_vio_hv.c
> +++ /dev/null
> @@ -1,73 +0,0 @@
> -/*
> - * This program is free software; you can redistribute it and/or modify
> - * it under the terms of the GNU General Public License, version 2, as
> - * published by the Free Software Foundation.
> - *
> - * This program is distributed in the hope that it will be useful,
> - * but WITHOUT ANY WARRANTY; without even the implied warranty of
> - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> - * GNU General Public License for more details.
> - *
> - * You should have received a copy of the GNU General Public License
> - * along with this program; if not, write to the Free Software
> - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> - *
> - * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
> - * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
> - */
> -
> -#include <linux/types.h>
> -#include <linux/string.h>
> -#include <linux/kvm.h>
> -#include <linux/kvm_host.h>
> -#include <linux/highmem.h>
> -#include <linux/gfp.h>
> -#include <linux/slab.h>
> -#include <linux/hugetlb.h>
> -#include <linux/list.h>
> -
> -#include <asm/tlbflush.h>
> -#include <asm/kvm_ppc.h>
> -#include <asm/kvm_book3s.h>
> -#include <asm/mmu-hash64.h>
> -#include <asm/hvcall.h>
> -#include <asm/synch.h>
> -#include <asm/ppc-opcode.h>
> -#include <asm/kvm_host.h>
> -#include <asm/udbg.h>
> -
> -#define TCES_PER_PAGE	(PAGE_SIZE / sizeof(u64))
> -
> -long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
> -		      unsigned long ioba, unsigned long tce)
> -{
> -	struct kvm *kvm = vcpu->kvm;
> -	struct kvmppc_spapr_tce_table *stt;
> -
> -	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
> -	/* 	    liobn, ioba, tce); */
> -
> -	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
> -		if (stt->liobn == liobn) {
> -			unsigned long idx = ioba >> SPAPR_TCE_SHIFT;
> -			struct page *page;
> -			u64 *tbl;
> -
> -			/* udbg_printf("H_PUT_TCE: liobn 0x%lx => stt=%p  window_size=0x%x\n", */
> -			/* 	    liobn, stt, stt->window_size); */
> -			if (ioba >= stt->window_size)
> -				return H_PARAMETER;
> -
> -			page = stt->pages[idx / TCES_PER_PAGE];
> -			tbl = (u64 *)page_address(page);
> -
> -			/* FIXME: Need to validate the TCE itself */
> -			/* udbg_printf("tce @ %p\n", &tbl[idx % TCES_PER_PAGE]); */
> -			tbl[idx % TCES_PER_PAGE] = tce;
> -			return H_SUCCESS;
> -		}
> -	}
> -
> -	/* Didn't find the liobn, punt it to userspace */
> -	return H_TOO_HARD;
> -}
> diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
> index a726716..d2e4afe 100644
> --- a/arch/powerpc/kvm/book3s_hv.c
> +++ b/arch/powerpc/kvm/book3s_hv.c
> @@ -862,115 +862,6 @@ int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
> 	return r;
> }
> 
> -static long kvmppc_stt_npages(unsigned long window_size)
> -{
> -	return ALIGN((window_size >> SPAPR_TCE_SHIFT)
> -		     * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
> -}
> -
> -static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
> -{
> -	struct kvm *kvm = stt->kvm;
> -	int i;
> -
> -	mutex_lock(&kvm->lock);
> -	list_del(&stt->list);
> -	for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
> -		__free_page(stt->pages[i]);
> -	kfree(stt);
> -	mutex_unlock(&kvm->lock);
> -
> -	kvm_put_kvm(kvm);
> -}
> -
> -static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
> -{
> -	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
> -	struct page *page;
> -
> -	if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
> -		return VM_FAULT_SIGBUS;
> -
> -	page = stt->pages[vmf->pgoff];
> -	get_page(page);
> -	vmf->page = page;
> -	return 0;
> -}
> -
> -static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
> -	.fault = kvm_spapr_tce_fault,
> -};
> -
> -static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
> -{
> -	vma->vm_ops = &kvm_spapr_tce_vm_ops;
> -	return 0;
> -}
> -
> -static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
> -{
> -	struct kvmppc_spapr_tce_table *stt = filp->private_data;
> -
> -	release_spapr_tce_table(stt);
> -	return 0;
> -}
> -
> -static struct file_operations kvm_spapr_tce_fops = {
> -	.mmap           = kvm_spapr_tce_mmap,
> -	.release	= kvm_spapr_tce_release,
> -};
> -
> -long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
> -				   struct kvm_create_spapr_tce *args)
> -{
> -	struct kvmppc_spapr_tce_table *stt = NULL;
> -	long npages;
> -	int ret = -ENOMEM;
> -	int i;
> -
> -	/* Check this LIOBN hasn't been previously allocated */
> -	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
> -		if (stt->liobn == args->liobn)
> -			return -EBUSY;
> -	}
> -
> -	npages = kvmppc_stt_npages(args->window_size);
> -
> -	stt = kzalloc(sizeof(*stt) + npages* sizeof(struct page *),
> -		      GFP_KERNEL);
> -	if (!stt)
> -		goto fail;
> -
> -	stt->liobn = args->liobn;
> -	stt->window_size = args->window_size;
> -	stt->kvm = kvm;
> -
> -	for (i = 0; i < npages; i++) {
> -		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
> -		if (!stt->pages[i])
> -			goto fail;
> -	}
> -
> -	kvm_get_kvm(kvm);
> -
> -	mutex_lock(&kvm->lock);
> -	list_add(&stt->list, &kvm->arch.spapr_tce_tables);
> -
> -	mutex_unlock(&kvm->lock);
> -
> -	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
> -				stt, O_RDWR);
> -
> -fail:
> -	if (stt) {
> -		for (i = 0; i < npages; i++)
> -			if (stt->pages[i])
> -				__free_page(stt->pages[i]);
> -
> -		kfree(stt);
> -	}
> -	return ret;
> -}

Could you please enable rename support in git format-patch? This way it's really hard to see what you changed between the 2 files - if anything.

> 
> /* Work out RMLS (real mode limit selector) field value for a given RMA size.
>    Assumes POWER7 or PPC970. */
> diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
> index e2cfb9e..d6851a1 100644
> --- a/arch/powerpc/kvm/book3s_pr.c
> +++ b/arch/powerpc/kvm/book3s_pr.c
> @@ -1018,11 +1018,14 @@ void kvmppc_core_commit_memory_region(struct kvm *kvm,
> 
> int kvmppc_core_init_vm(struct kvm *kvm)
> {
> +	INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
> +
> 	return 0;
> }
> 
> void kvmppc_core_destroy_vm(struct kvm *kvm)
> {
> +	WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
> }
> 
> static int kvmppc_book3s_init(void)
> diff --git a/arch/powerpc/kvm/book3s_pr_papr.c b/arch/powerpc/kvm/book3s_pr_papr.c
> index b958932..0ec257e 100644
> --- a/arch/powerpc/kvm/book3s_pr_papr.c
> +++ b/arch/powerpc/kvm/book3s_pr_papr.c
> @@ -15,6 +15,8 @@
>  * published by the Free Software Foundation.
>  */
> 
> +#include <linux/anon_inodes.h>
> +
> #include <asm/uaccess.h>
> #include <asm/kvm_ppc.h>
> #include <asm/kvm_book3s.h>
> @@ -134,6 +136,20 @@ static int kvmppc_h_pr_protect(struct kvm_vcpu *vcpu)
> 	return EMULATE_DONE;
> }
> 
> +static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu)
> +{
> +	unsigned long liobn = kvmppc_get_gpr(vcpu, 4);
> +	unsigned long ioba = kvmppc_get_gpr(vcpu, 5);
> +	unsigned long tce = kvmppc_get_gpr(vcpu, 6);
> +	long rc;
> +
> +	rc = kvmppc_h_put_tce(vcpu, liobn, ioba, tce);
> +	if (rc == H_TOO_HARD)
> +		return EMULATE_FAIL;
> +	kvmppc_set_gpr(vcpu, 3, rc);
> +	return EMULATE_DONE;
> +}
> +
> int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
> {
> 	switch (cmd) {
> @@ -148,6 +164,8 @@ int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
> 		   handle the HPT modifications */
> 		kvmppc_mmu_pte_flush(vcpu, 0, 0);
> 		break;
> +	case H_PUT_TCE:
> +		return kvmppc_h_pr_put_tce(vcpu);
> 	case H_CEDE:
> 		kvm_vcpu_block(vcpu);
> 		vcpu->stat.halt_wakeup++;
> diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
> index 607fbdf..01fe58d 100644
> --- a/arch/powerpc/kvm/powerpc.c
> +++ b/arch/powerpc/kvm/powerpc.c
> @@ -225,10 +225,12 @@ int kvm_dev_ioctl_check_extension(long ext)
> 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
> 		break;
> #endif
> -#ifdef CONFIG_KVM_BOOK3S_64_HV
> +#if defined(CONFIG_KVM_BOOK3S_64_PR) || defined(CONFIG_KVM_BOOK3S_64_HV)

CONFIG_KVM_BOOK3S_64

> 	case KVM_CAP_SPAPR_TCE:
> 		r = 1;
> 		break;
> +#endif /* defined(CONFIG_KVM_BOOK3S_64_PR) || defined(CONFIG_KVM_BOOK3S_64_HV) */
> +#ifdef CONFIG_KVM_BOOK3S_64_HV
> 	case KVM_CAP_PPC_SMT:
> 		r = threads_per_core;
> 		break;
> @@ -699,7 +701,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
> 
> 		break;
> 	}
> -#ifdef CONFIG_KVM_BOOK3S_64_HV
> +#if defined(CONFIG_KVM_BOOK3S_64_PR) || defined(CONFIG_KVM_BOOK3S_64_HV)

CONFIG_KVM_BOOK3S_64

Otherwise a nice patch - thanks a lot for tackling this :). Also, please always CC kvm@vger in addition to kvm-ppc@vger, so Avi can't complain that he didn't see the patch earlier ;).


Alex

Re: [PATCH] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

[Benjamin Herrenschmidt]

On Tue, 2012-03-13 at 14:47 +0100, Alexander Graf wrote:

> > +++ b/arch/powerpc/include/asm/kvm_host.h
> > @@ -183,10 +183,14 @@ struct kvm_arch {
> > 	unsigned long lpcr;
> > 	unsigned long rmor;
> > 	struct kvmppc_rma_info *rma;
> > -	struct list_head spapr_tce_tables;
> > 	unsigned short last_vcpu[NR_CPUS];
> > 	struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
> > +	struct list_head spapr_tce_tables;
> > #endif /* CONFIG_KVM_BOOK3S_64_HV */
> > +
> > +#ifdef CONFIG_KVM_BOOK3S_64_PR
> > +	struct list_head spapr_tce_tables;
> > +#endif /* CONFIG_KVM_BOOK3S_64_PR */
> 
> Please move the definition to an #ifdef on CONFIG_KVM_BOOK3S_64. That way we get rid of the duplication here.

I did that initially but that doesn't work when it's a module, as
CONFIG_KVM_BOOK3S_64 is "m" in Kconfig and thus not defined as such
but CONFIG_KVM_BOOK3S_64_MODULE is)

Maybe we should change the way our Kconfig is organized but I din't
feel like doing so yesterday :-)

> Could you please enable rename support in git format-patch? This way it's
> really hard to see what you changed between the 2 files - if anything.

I thought I had, I'll dbl check. There was no code change, just moves,
the original file only had the one small function in it and I moved
over the rest from book3s_hv.c

> > -#ifdef CONFIG_KVM_BOOK3S_64_HV
> > +#if defined(CONFIG_KVM_BOOK3S_64_PR) || defined(CONFIG_KVM_BOOK3S_64_HV)
> 
> CONFIG_KVM_BOOK3S_64

Breaks modules.

> Otherwise a nice patch - thanks a lot for tackling this :). Also, please always CC kvm@vger in
> addition to kvm-ppc@vger, so Avi can't complain that he didn't see the patch earlier ;).

Heh ok.

Cheers,
Ben.

Re: [PATCH] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

[Alexander Graf]

Am 13.03.2012 um 21:09 schrieb Benjamin Herrenschmidt <benh@kernel.crashing.org>:

> On Tue, 2012-03-13 at 14:47 +0100, Alexander Graf wrote:
> 
>>> +++ b/arch/powerpc/include/asm/kvm_host.h
>>> @@ -183,10 +183,14 @@ struct kvm_arch {
>>>    unsigned long lpcr;
>>>    unsigned long rmor;
>>>    struct kvmppc_rma_info *rma;
>>> -    struct list_head spapr_tce_tables;
>>>    unsigned short last_vcpu[NR_CPUS];
>>>    struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
>>> +    struct list_head spapr_tce_tables;
>>> #endif /* CONFIG_KVM_BOOK3S_64_HV */
>>> +
>>> +#ifdef CONFIG_KVM_BOOK3S_64_PR
>>> +    struct list_head spapr_tce_tables;
>>> +#endif /* CONFIG_KVM_BOOK3S_64_PR */
>> 
>> Please move the definition to an #ifdef on CONFIG_KVM_BOOK3S_64. That way we get rid of the duplication here.
> 
> I did that initially but that doesn't work when it's a module, as
> CONFIG_KVM_BOOK3S_64 is "m" in Kconfig and thus not defined as such
> but CONFIG_KVM_BOOK3S_64_MODULE is)
> 
> Maybe we should change the way our Kconfig is organized but I din't
> feel like doing so yesterday :-)

So how about making it unconditional? Or just depend on book3s_64?


Alex

Re: [PATCH v2] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

[Benjamin Herrenschmidt]

On Wed, 2012-05-02 at 23:21 +0200, Alexander Graf wrote:
> If you're too busy to fix it up, I can give it another wuick try
> though...

That would be great... I'll have no time til at least next week.

Cheers,
Ben.

[PATCH] kvm/book3s: Make kernel emulated H_PUT_TCE available for "PR" KVM

[Alexander Graf]

From: Benjamin Herrenschmidt <benh@kernel.crashing.org>

There is nothing in the code for emulating TCE tables in the kernel
that prevents it from working on "PR" KVM... other than ifdef's and
location of the code.

This and moves the bulk of the code there to a new file called
book3s_64_vio.c.

This speeds things up a bit on my G5.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
[agraf: fix for hv kvm, whitespace]
Signed-off-by: Alexander Graf <agraf@suse.de>
---
 arch/powerpc/include/asm/kvm_host.h |    4 +-
 arch/powerpc/include/asm/kvm_ppc.h  |    2 +
 arch/powerpc/kvm/Makefile           |    2 +
 arch/powerpc/kvm/book3s_64_vio.c    |  150 +++++++++++++++++++++++++++++++++++
 arch/powerpc/kvm/book3s_64_vio_hv.c |    3 +
 arch/powerpc/kvm/book3s_hv.c        |  109 -------------------------
 arch/powerpc/kvm/book3s_pr.c        |    3 +
 arch/powerpc/kvm/book3s_pr_papr.c   |   18 ++++
 arch/powerpc/kvm/powerpc.c          |    8 ++-
 9 files changed, 187 insertions(+), 112 deletions(-)
 create mode 100644 arch/powerpc/kvm/book3s_64_vio.c

diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 42a527e..d848cdc 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -237,7 +237,6 @@ struct kvm_arch {
 	unsigned long vrma_slb_v;
 	int rma_setup_done;
 	int using_mmu_notifiers;
-	struct list_head spapr_tce_tables;
 	spinlock_t slot_phys_lock;
 	unsigned long *slot_phys[KVM_MEM_SLOTS_NUM];
 	int slot_npages[KVM_MEM_SLOTS_NUM];
@@ -245,6 +244,9 @@ struct kvm_arch {
 	struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
 	struct kvmppc_linear_info *hpt_li;
 #endif /* CONFIG_KVM_BOOK3S_64_HV */
+#ifdef CONFIG_PPC_BOOK3S_64
+	struct list_head spapr_tce_tables;
+#endif
 };
 
 /*
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index 7f0a3da..c1069f6 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -126,6 +126,8 @@ extern void kvmppc_map_vrma(struct kvm_vcpu *vcpu,
 extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu);
 extern long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 				struct kvm_create_spapr_tce *args);
+extern long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
+			     unsigned long ioba, unsigned long tce);
 extern long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
 				struct kvm_allocate_rma *rma);
 extern struct kvmppc_linear_info *kvm_alloc_rma(void);
diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile
index 25225ae..c2a0863 100644
--- a/arch/powerpc/kvm/Makefile
+++ b/arch/powerpc/kvm/Makefile
@@ -54,6 +54,7 @@ kvm-book3s_64-objs-$(CONFIG_KVM_BOOK3S_64_PR) := \
 	book3s_paired_singles.o \
 	book3s_pr.o \
 	book3s_pr_papr.o \
+	book3s_64_vio_hv.o \
 	book3s_emulate.o \
 	book3s_interrupts.o \
 	book3s_mmu_hpte.o \
@@ -78,6 +79,7 @@ kvm-book3s_64-module-objs := \
 	powerpc.o \
 	emulate.o \
 	book3s.o \
+	book3s_64_vio.o \
 	$(kvm-book3s_64-objs-y)
 
 kvm-objs-$(CONFIG_KVM_BOOK3S_64) := $(kvm-book3s_64-module-objs)
diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
new file mode 100644
index 0000000..72ffc89
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_64_vio.c
@@ -0,0 +1,150 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
+ * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
+ */
+
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/highmem.h>
+#include <linux/gfp.h>
+#include <linux/slab.h>
+#include <linux/hugetlb.h>
+#include <linux/list.h>
+#include <linux/anon_inodes.h>
+
+#include <asm/tlbflush.h>
+#include <asm/kvm_ppc.h>
+#include <asm/kvm_book3s.h>
+#include <asm/mmu-hash64.h>
+#include <asm/hvcall.h>
+#include <asm/synch.h>
+#include <asm/ppc-opcode.h>
+#include <asm/kvm_host.h>
+#include <asm/udbg.h>
+
+#define TCES_PER_PAGE	(PAGE_SIZE / sizeof(u64))
+
+static long kvmppc_stt_npages(unsigned long window_size)
+{
+	return ALIGN((window_size >> SPAPR_TCE_SHIFT)
+		     * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
+}
+
+static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
+{
+	struct kvm *kvm = stt->kvm;
+	int i;
+
+	mutex_lock(&kvm->lock);
+	list_del(&stt->list);
+	for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
+		__free_page(stt->pages[i]);
+	kfree(stt);
+	mutex_unlock(&kvm->lock);
+
+	kvm_put_kvm(kvm);
+}
+
+static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
+	struct page *page;
+
+	if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
+		return VM_FAULT_SIGBUS;
+
+	page = stt->pages[vmf->pgoff];
+	get_page(page);
+	vmf->page = page;
+	return 0;
+}
+
+static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
+	.fault = kvm_spapr_tce_fault,
+};
+
+static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	vma->vm_ops = &kvm_spapr_tce_vm_ops;
+	return 0;
+}
+
+static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
+{
+	struct kvmppc_spapr_tce_table *stt = filp->private_data;
+
+	release_spapr_tce_table(stt);
+	return 0;
+}
+
+static struct file_operations kvm_spapr_tce_fops = {
+	.mmap           = kvm_spapr_tce_mmap,
+	.release	= kvm_spapr_tce_release,
+};
+
+long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
+				   struct kvm_create_spapr_tce *args)
+{
+	struct kvmppc_spapr_tce_table *stt = NULL;
+	long npages;
+	int ret = -ENOMEM;
+	int i;
+
+	/* Check this LIOBN hasn't been previously allocated */
+	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
+		if (stt->liobn == args->liobn)
+			return -EBUSY;
+	}
+
+	npages = kvmppc_stt_npages(args->window_size);
+
+	stt = kzalloc(sizeof(*stt) + npages * sizeof(struct page *),
+		      GFP_KERNEL);
+	if (!stt)
+		goto fail;
+
+	stt->liobn = args->liobn;
+	stt->window_size = args->window_size;
+	stt->kvm = kvm;
+
+	for (i = 0; i < npages; i++) {
+		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!stt->pages[i])
+			goto fail;
+	}
+
+	kvm_get_kvm(kvm);
+
+	mutex_lock(&kvm->lock);
+	list_add(&stt->list, &kvm->arch.spapr_tce_tables);
+
+	mutex_unlock(&kvm->lock);
+
+	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
+				stt, O_RDWR);
+
+fail:
+	if (stt) {
+		for (i = 0; i < npages; i++)
+			if (stt->pages[i])
+				__free_page(stt->pages[i]);
+
+		kfree(stt);
+	}
+	return ret;
+}
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index ea0f8c5..30c2f3b 100644
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -38,6 +38,9 @@
 
 #define TCES_PER_PAGE	(PAGE_SIZE / sizeof(u64))
 
+/* WARNING: This will be called in real-mode on HV KVM and virtual
+ *          mode on PR KVM
+ */
 long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 		      unsigned long ioba, unsigned long tce)
 {
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 9079357..59c2967 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -1093,115 +1093,6 @@ int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
 	return r;
 }
 
-static long kvmppc_stt_npages(unsigned long window_size)
-{
-	return ALIGN((window_size >> SPAPR_TCE_SHIFT)
-		     * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
-}
-
-static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
-{
-	struct kvm *kvm = stt->kvm;
-	int i;
-
-	mutex_lock(&kvm->lock);
-	list_del(&stt->list);
-	for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
-		__free_page(stt->pages[i]);
-	kfree(stt);
-	mutex_unlock(&kvm->lock);
-
-	kvm_put_kvm(kvm);
-}
-
-static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
-{
-	struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
-	struct page *page;
-
-	if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
-		return VM_FAULT_SIGBUS;
-
-	page = stt->pages[vmf->pgoff];
-	get_page(page);
-	vmf->page = page;
-	return 0;
-}
-
-static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
-	.fault = kvm_spapr_tce_fault,
-};
-
-static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	vma->vm_ops = &kvm_spapr_tce_vm_ops;
-	return 0;
-}
-
-static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
-{
-	struct kvmppc_spapr_tce_table *stt = filp->private_data;
-
-	release_spapr_tce_table(stt);
-	return 0;
-}
-
-static struct file_operations kvm_spapr_tce_fops = {
-	.mmap           = kvm_spapr_tce_mmap,
-	.release	= kvm_spapr_tce_release,
-};
-
-long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
-				   struct kvm_create_spapr_tce *args)
-{
-	struct kvmppc_spapr_tce_table *stt = NULL;
-	long npages;
-	int ret = -ENOMEM;
-	int i;
-
-	/* Check this LIOBN hasn't been previously allocated */
-	list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
-		if (stt->liobn == args->liobn)
-			return -EBUSY;
-	}
-
-	npages = kvmppc_stt_npages(args->window_size);
-
-	stt = kzalloc(sizeof(*stt) + npages* sizeof(struct page *),
-		      GFP_KERNEL);
-	if (!stt)
-		goto fail;
-
-	stt->liobn = args->liobn;
-	stt->window_size = args->window_size;
-	stt->kvm = kvm;
-
-	for (i = 0; i < npages; i++) {
-		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!stt->pages[i])
-			goto fail;
-	}
-
-	kvm_get_kvm(kvm);
-
-	mutex_lock(&kvm->lock);
-	list_add(&stt->list, &kvm->arch.spapr_tce_tables);
-
-	mutex_unlock(&kvm->lock);
-
-	return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
-				stt, O_RDWR);
-
-fail:
-	if (stt) {
-		for (i = 0; i < npages; i++)
-			if (stt->pages[i])
-				__free_page(stt->pages[i]);
-
-		kfree(stt);
-	}
-	return ret;
-}
 
 /* Work out RMLS (real mode limit selector) field value for a given RMA size.
    Assumes POWER7 or PPC970. */
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
index d169a0a..63850cf 100644
--- a/arch/powerpc/kvm/book3s_pr.c
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -1171,11 +1171,14 @@ void kvmppc_core_commit_memory_region(struct kvm *kvm,
 
 int kvmppc_core_init_vm(struct kvm *kvm)
 {
+	INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
+
 	return 0;
 }
 
 void kvmppc_core_destroy_vm(struct kvm *kvm)
 {
+	WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
 }
 
 static int kvmppc_book3s_init(void)
diff --git a/arch/powerpc/kvm/book3s_pr_papr.c b/arch/powerpc/kvm/book3s_pr_papr.c
index 60ac0e7..3ff9013 100644
--- a/arch/powerpc/kvm/book3s_pr_papr.c
+++ b/arch/powerpc/kvm/book3s_pr_papr.c
@@ -15,6 +15,8 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/anon_inodes.h>
+
 #include <asm/uaccess.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -211,6 +213,20 @@ static int kvmppc_h_pr_protect(struct kvm_vcpu *vcpu)
 	return EMULATE_DONE;
 }
 
+static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu)
+{
+	unsigned long liobn = kvmppc_get_gpr(vcpu, 4);
+	unsigned long ioba = kvmppc_get_gpr(vcpu, 5);
+	unsigned long tce = kvmppc_get_gpr(vcpu, 6);
+	long rc;
+
+	rc = kvmppc_h_put_tce(vcpu, liobn, ioba, tce);
+	if (rc == H_TOO_HARD)
+		return EMULATE_FAIL;
+	kvmppc_set_gpr(vcpu, 3, rc);
+	return EMULATE_DONE;
+}
+
 int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
 {
 	switch (cmd) {
@@ -222,6 +238,8 @@ int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
 		return kvmppc_h_pr_protect(vcpu);
 	case H_BULK_REMOVE:
 		return kvmppc_h_pr_bulk_remove(vcpu);
+	case H_PUT_TCE:
+		return kvmppc_h_pr_put_tce(vcpu);
 	case H_CEDE:
 		kvm_vcpu_block(vcpu);
 		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 58ad860..6ac3115 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -244,10 +244,12 @@ int kvm_dev_ioctl_check_extension(long ext)
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 #endif
-#ifdef CONFIG_KVM_BOOK3S_64_HV
+#ifdef CONFIG_PPC_BOOK3S_64
 	case KVM_CAP_SPAPR_TCE:
 		r = 1;
 		break;
+#endif /* CONFIG_PPC_BOOK3S_64 */
+#ifdef CONFIG_KVM_BOOK3S_64_HV
 	case KVM_CAP_PPC_SMT:
 		r = threads_per_core;
 		break;
@@ -773,7 +775,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 
 		break;
 	}
-#ifdef CONFIG_KVM_BOOK3S_64_HV
+#ifdef CONFIG_PPC_BOOK3S_64
 	case KVM_CREATE_SPAPR_TCE: {
 		struct kvm_create_spapr_tce create_tce;
 		struct kvm *kvm = filp->private_data;
@@ -784,7 +786,9 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce);
 		goto out;
 	}
+#endif /* CONFIG_PPC_BOOK3S_64 */
 
+#ifdef CONFIG_KVM_BOOK3S_64_HV
 	case KVM_ALLOCATE_RMA: {
 		struct kvm *kvm = filp->private_data;
 		struct kvm_allocate_rma rma;
-- 
1.6.0.2