* [PATCH 05/10] s390/cio: introduce DMA pools to cio
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
In-Reply-To: <20190426183245.37939-1-pasic@linux.ibm.com>
To support protected virtualization cio will need to make sure the
memory used for communication with the hypervisor is DMA memory.
Let us introduce one global cio, and some tools for pools seated
at individual devices.
Our DMA pools are implemented as a gen_pool backed with DMA pages. The
idea is to avoid each allocation effectively wasting a page, as we
typically allocate much less than PAGE_SIZE.
Signed-off-by: Halil Pasic <pasic@linux.ibm.com>
---
arch/s390/Kconfig | 1 +
arch/s390/include/asm/cio.h | 11 +++++
drivers/s390/cio/cio.h | 1 +
drivers/s390/cio/css.c | 101 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 114 insertions(+)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 5500d05d4d53..5861311d95d9 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -195,6 +195,7 @@ config S390
select VIRT_TO_BUS
select HAVE_NMI
select SWIOTLB
+ select GENERIC_ALLOCATOR
config SCHED_OMIT_FRAME_POINTER
diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h
index 1727180e8ca1..43c007d2775a 100644
--- a/arch/s390/include/asm/cio.h
+++ b/arch/s390/include/asm/cio.h
@@ -328,6 +328,17 @@ static inline u8 pathmask_to_pos(u8 mask)
void channel_subsystem_reinit(void);
extern void css_schedule_reprobe(void);
+extern void *cio_dma_zalloc(size_t size);
+extern void cio_dma_free(void *cpu_addr, size_t size);
+extern struct device *cio_get_dma_css_dev(void);
+
+struct gen_pool;
+void *cio_gp_dma_zalloc(struct gen_pool *gp_dma, struct device *dma_dev,
+ size_t size);
+void cio_gp_dma_free(struct gen_pool *gp_dma, void *cpu_addr, size_t size);
+void cio_gp_dma_destroy(struct gen_pool *gp_dma, struct device *dma_dev);
+struct gen_pool *cio_gp_dma_create(struct device *dma_dev, int nr_pages);
+
/* Function from drivers/s390/cio/chsc.c */
int chsc_sstpc(void *page, unsigned int op, u16 ctrl, u64 *clock_delta);
int chsc_sstpi(void *page, void *result, size_t size);
diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h
index 92eabbb5f18d..f23f7e2c33f7 100644
--- a/drivers/s390/cio/cio.h
+++ b/drivers/s390/cio/cio.h
@@ -113,6 +113,7 @@ struct subchannel {
enum sch_todo todo;
struct work_struct todo_work;
struct schib_config config;
+ u64 dma_mask;
} __attribute__ ((aligned(8)));
DECLARE_PER_CPU_ALIGNED(struct irb, cio_irb);
diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
index aea502922646..7087cc314fe9 100644
--- a/drivers/s390/cio/css.c
+++ b/drivers/s390/cio/css.c
@@ -20,6 +20,8 @@
#include <linux/reboot.h>
#include <linux/suspend.h>
#include <linux/proc_fs.h>
+#include <linux/genalloc.h>
+#include <linux/dma-mapping.h>
#include <asm/isc.h>
#include <asm/crw.h>
@@ -199,6 +201,8 @@ static int css_validate_subchannel(struct subchannel_id schid,
return err;
}
+static u64 css_dev_dma_mask = DMA_BIT_MASK(31);
+
struct subchannel *css_alloc_subchannel(struct subchannel_id schid,
struct schib *schib)
{
@@ -224,6 +228,9 @@ struct subchannel *css_alloc_subchannel(struct subchannel_id schid,
INIT_WORK(&sch->todo_work, css_sch_todo);
sch->dev.release = &css_subchannel_release;
device_initialize(&sch->dev);
+ sch->dma_mask = css_dev_dma_mask;
+ sch->dev.dma_mask = &sch->dma_mask;
+ sch->dev.coherent_dma_mask = sch->dma_mask;
return sch;
err:
@@ -899,6 +906,9 @@ static int __init setup_css(int nr)
dev_set_name(&css->device, "css%x", nr);
css->device.groups = cssdev_attr_groups;
css->device.release = channel_subsystem_release;
+ /* some cio DMA memory needs to be 31 bit addressable */
+ css->device.coherent_dma_mask = css_dev_dma_mask,
+ css->device.dma_mask = &css_dev_dma_mask;
mutex_init(&css->mutex);
css->cssid = chsc_get_cssid(nr);
@@ -1018,6 +1028,96 @@ static struct notifier_block css_power_notifier = {
.notifier_call = css_power_event,
};
+#define POOL_INIT_PAGES 1
+static struct gen_pool *cio_dma_pool;
+/* Currently cio supports only a single css */
+#define CIO_DMA_GFP (GFP_KERNEL | __GFP_ZERO)
+
+
+struct device *cio_get_dma_css_dev(void)
+{
+ return &channel_subsystems[0]->device;
+}
+
+struct gen_pool *cio_gp_dma_create(struct device *dma_dev, int nr_pages)
+{
+ struct gen_pool *gp_dma;
+ void *cpu_addr;
+ dma_addr_t dma_addr;
+ int i;
+
+ gp_dma = gen_pool_create(3, -1);
+ if (!gp_dma)
+ return NULL;
+ for (i = 0; i < nr_pages; ++i) {
+ cpu_addr = dma_alloc_coherent(dma_dev, PAGE_SIZE, &dma_addr,
+ CIO_DMA_GFP);
+ if (!cpu_addr)
+ return gp_dma;
+ gen_pool_add_virt(gp_dma, (unsigned long) cpu_addr,
+ dma_addr, PAGE_SIZE, -1);
+ }
+ return gp_dma;
+}
+
+static void __gp_dma_free_dma(struct gen_pool *pool,
+ struct gen_pool_chunk *chunk, void *data)
+{
+ dma_free_coherent((struct device *) data, PAGE_SIZE,
+ (void *) chunk->start_addr,
+ (dma_addr_t) chunk->phys_addr);
+}
+
+void cio_gp_dma_destroy(struct gen_pool *gp_dma, struct device *dma_dev)
+{
+ if (!gp_dma)
+ return;
+ /* this is qite ugly but no better idea */
+ gen_pool_for_each_chunk(gp_dma, __gp_dma_free_dma, dma_dev);
+ gen_pool_destroy(gp_dma);
+}
+
+static void __init cio_dma_pool_init(void)
+{
+ /* No need to free up the resources: compiled in */
+ cio_dma_pool = cio_gp_dma_create(cio_get_dma_css_dev(), 1);
+}
+
+void *cio_gp_dma_zalloc(struct gen_pool *gp_dma, struct device *dma_dev,
+ size_t size)
+{
+ dma_addr_t dma_addr;
+ unsigned long addr = gen_pool_alloc(gp_dma, size);
+
+ if (!addr) {
+ addr = (unsigned long) dma_alloc_coherent(dma_dev,
+ PAGE_SIZE, &dma_addr, CIO_DMA_GFP);
+ if (!addr)
+ return NULL;
+ gen_pool_add_virt(gp_dma, addr, dma_addr, PAGE_SIZE, -1);
+ addr = gen_pool_alloc(gp_dma, size);
+ }
+ return (void *) addr;
+}
+
+void cio_gp_dma_free(struct gen_pool *gp_dma, void *cpu_addr, size_t size)
+{
+ if (!cpu_addr)
+ return;
+ memset(cpu_addr, 0, size);
+ gen_pool_free(gp_dma, (unsigned long) cpu_addr, size);
+}
+
+void *cio_dma_zalloc(size_t size)
+{
+ return cio_gp_dma_zalloc(cio_dma_pool, cio_get_dma_css_dev(), size);
+}
+
+void cio_dma_free(void *cpu_addr, size_t size)
+{
+ cio_gp_dma_free(cio_dma_pool, cpu_addr, size);
+}
+
/*
* Now that the driver core is running, we can setup our channel subsystem.
* The struct subchannel's are created during probing.
@@ -1063,6 +1163,7 @@ static int __init css_bus_init(void)
unregister_reboot_notifier(&css_reboot_notifier);
goto out_unregister;
}
+ cio_dma_pool_init();
css_init_done = 1;
/* Enable default isc for I/O subchannels. */
--
2.16.4
^ permalink raw reply related
* [PATCH 04/10] s390/mm: force swiotlb for protected virtualization
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
In-Reply-To: <20190426183245.37939-1-pasic@linux.ibm.com>
On s390, protected virtualization guests have to use bounced I/O
buffers. That requires some plumbing.
Let us make sure, any device that uses DMA API with direct ops correctly
is spared from the problems, that a hypervisor attempting I/O to a
non-shared page would bring.
Signed-off-by: Halil Pasic <pasic@linux.ibm.com>
---
arch/s390/Kconfig | 4 +++
arch/s390/include/asm/mem_encrypt.h | 18 +++++++++++++
arch/s390/mm/init.c | 50 +++++++++++++++++++++++++++++++++++++
3 files changed, 72 insertions(+)
create mode 100644 arch/s390/include/asm/mem_encrypt.h
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 1c3fcf19c3af..5500d05d4d53 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -1,4 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
+config ARCH_HAS_MEM_ENCRYPT
+ def_bool y
+
config MMU
def_bool y
@@ -191,6 +194,7 @@ config S390
select ARCH_HAS_SCALED_CPUTIME
select VIRT_TO_BUS
select HAVE_NMI
+ select SWIOTLB
config SCHED_OMIT_FRAME_POINTER
diff --git a/arch/s390/include/asm/mem_encrypt.h b/arch/s390/include/asm/mem_encrypt.h
new file mode 100644
index 000000000000..0898c09a888c
--- /dev/null
+++ b/arch/s390/include/asm/mem_encrypt.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef S390_MEM_ENCRYPT_H__
+#define S390_MEM_ENCRYPT_H__
+
+#ifndef __ASSEMBLY__
+
+#define sme_me_mask 0ULL
+
+static inline bool sme_active(void) { return false; }
+extern bool sev_active(void);
+
+int set_memory_encrypted(unsigned long addr, int numpages);
+int set_memory_decrypted(unsigned long addr, int numpages);
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* S390_MEM_ENCRYPT_H__ */
+
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 3e82f66d5c61..7e3cbd15dcfa 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -18,6 +18,7 @@
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/swap.h>
+#include <linux/swiotlb.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/pagemap.h>
@@ -29,6 +30,7 @@
#include <linux/export.h>
#include <linux/cma.h>
#include <linux/gfp.h>
+#include <linux/dma-mapping.h>
#include <asm/processor.h>
#include <linux/uaccess.h>
#include <asm/pgtable.h>
@@ -42,6 +44,8 @@
#include <asm/sclp.h>
#include <asm/set_memory.h>
#include <asm/kasan.h>
+#include <asm/dma-mapping.h>
+#include <asm/uv.h>
pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir);
@@ -126,6 +130,50 @@ void mark_rodata_ro(void)
pr_info("Write protected read-only-after-init data: %luk\n", size >> 10);
}
+int set_memory_encrypted(unsigned long addr, int numpages)
+{
+ int i;
+
+ /* make all pages shared, (swiotlb, dma_free) */
+ for (i = 0; i < numpages; ++i) {
+ uv_remove_shared(addr);
+ addr += PAGE_SIZE;
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(set_memory_encrypted);
+
+int set_memory_decrypted(unsigned long addr, int numpages)
+{
+ int i;
+ /* make all pages shared (swiotlb, dma_alloca) */
+ for (i = 0; i < numpages; ++i) {
+ uv_set_shared(addr);
+ addr += PAGE_SIZE;
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+/* are we a protected virtualization guest? */
+bool sev_active(void)
+{
+ return is_prot_virt_guest();
+}
+EXPORT_SYMBOL_GPL(sev_active);
+
+/* protected virtualization */
+static void pv_init(void)
+{
+ if (!sev_active())
+ return;
+
+ /* make sure bounce buffers are shared */
+ swiotlb_init(1);
+ swiotlb_update_mem_attributes();
+ swiotlb_force = SWIOTLB_FORCE;
+}
+
void __init mem_init(void)
{
cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask);
@@ -134,6 +182,8 @@ void __init mem_init(void)
set_max_mapnr(max_low_pfn);
high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
+ pv_init();
+
/* Setup guest page hinting */
cmma_init();
--
2.16.4
^ permalink raw reply related
* [PATCH 03/10] virtio/s390: enable packed ring
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
In-Reply-To: <20190426183245.37939-1-pasic@linux.ibm.com>
Nothing precludes to accepting VIRTIO_F_RING_PACKED any more.
Signed-off-by: Halil Pasic <pasic@linux.ibm.com>
---
drivers/s390/virtio/virtio_ccw.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 42832a164546..6d989c360f38 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -773,10 +773,8 @@ static u64 virtio_ccw_get_features(struct virtio_device *vdev)
static void ccw_transport_features(struct virtio_device *vdev)
{
/*
- * There shouldn't be anything that precludes supporting packed.
- * TODO: Remove the limitation after having another look into this.
+ * Currently nothing to do here.
*/
- __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED);
}
static int virtio_ccw_finalize_features(struct virtio_device *vdev)
--
2.16.4
^ permalink raw reply related
* [PATCH 02/10] virtio/s390: DMA support for virtio-ccw
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
In-Reply-To: <20190426183245.37939-1-pasic@linux.ibm.com>
Currently virtio-ccw devices do not work if the device has
VIRTIO_F_IOMMU_PLATFORM. In future we do want to support DMA API with
virtio-ccw.
Let us do the plumbing, so the feature VIRTIO_F_IOMMU_PLATFORM works
with virtio-ccw.
Let us also switch from legacy avail/used accessors to the DMA aware
ones (even if it isn't strictly necessary), and remove the legacy
accessors (we were the last users).
Signed-off-by: Halil Pasic <pasic@linux.ibm.com>
---
drivers/s390/virtio/virtio_ccw.c | 22 +++++++++++++++-------
include/linux/virtio.h | 17 -----------------
2 files changed, 15 insertions(+), 24 deletions(-)
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 2c66941ef3d0..42832a164546 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -66,6 +66,7 @@ struct virtio_ccw_device {
bool device_lost;
unsigned int config_ready;
void *airq_info;
+ u64 dma_mask;
};
struct vq_info_block_legacy {
@@ -539,8 +540,8 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
info->info_block->s.desc = queue;
info->info_block->s.index = i;
info->info_block->s.num = info->num;
- info->info_block->s.avail = (__u64)virtqueue_get_avail(vq);
- info->info_block->s.used = (__u64)virtqueue_get_used(vq);
+ info->info_block->s.avail = (__u64)virtqueue_get_avail_addr(vq);
+ info->info_block->s.used = (__u64)virtqueue_get_used_addr(vq);
ccw->count = sizeof(info->info_block->s);
}
ccw->cmd_code = CCW_CMD_SET_VQ;
@@ -772,10 +773,8 @@ static u64 virtio_ccw_get_features(struct virtio_device *vdev)
static void ccw_transport_features(struct virtio_device *vdev)
{
/*
- * Packed ring isn't enabled on virtio_ccw for now,
- * because virtio_ccw uses some legacy accessors,
- * e.g. virtqueue_get_avail() and virtqueue_get_used()
- * which aren't available in packed ring currently.
+ * There shouldn't be anything that precludes supporting packed.
+ * TODO: Remove the limitation after having another look into this.
*/
__virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED);
}
@@ -1258,6 +1257,16 @@ static int virtio_ccw_online(struct ccw_device *cdev)
ret = -ENOMEM;
goto out_free;
}
+
+ vcdev->vdev.dev.parent = &cdev->dev;
+ cdev->dev.dma_mask = &vcdev->dma_mask;
+ /* we are fine with common virtio infrastructure using 64 bit DMA */
+ ret = dma_set_mask_and_coherent(&cdev->dev, DMA_BIT_MASK(64));
+ if (ret) {
+ dev_warn(&cdev->dev, "Failed to enable 64-bit DMA.\n");
+ goto out_free;
+ }
+
vcdev->config_block = kzalloc(sizeof(*vcdev->config_block),
GFP_DMA | GFP_KERNEL);
if (!vcdev->config_block) {
@@ -1272,7 +1281,6 @@ static int virtio_ccw_online(struct ccw_device *cdev)
vcdev->is_thinint = virtio_ccw_use_airq; /* at least try */
- vcdev->vdev.dev.parent = &cdev->dev;
vcdev->vdev.dev.release = virtio_ccw_release_dev;
vcdev->vdev.config = &virtio_ccw_config_ops;
vcdev->cdev = cdev;
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 673fe3ef3607..15f906e4a748 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -90,23 +90,6 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq);
dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq);
dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq);
-/*
- * Legacy accessors -- in almost all cases, these are the wrong functions
- * to use.
- */
-static inline void *virtqueue_get_desc(struct virtqueue *vq)
-{
- return virtqueue_get_vring(vq)->desc;
-}
-static inline void *virtqueue_get_avail(struct virtqueue *vq)
-{
- return virtqueue_get_vring(vq)->avail;
-}
-static inline void *virtqueue_get_used(struct virtqueue *vq)
-{
- return virtqueue_get_vring(vq)->used;
-}
-
/**
* virtio_device - representation of a device using virtio
* @index: unique position on the virtio bus
--
2.16.4
^ permalink raw reply related
* [PATCH 01/10] virtio/s390: use vring_create_virtqueue
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
In-Reply-To: <20190426183245.37939-1-pasic@linux.ibm.com>
The commit 2a2d1382fe9d ("virtio: Add improved queue allocation API")
establishes a new way of allocating virtqueues (as a part of the effort
that taught DMA to virtio rings).
In the future we will want virtio-ccw to use the DMA API as well.
Let us switch from the legacy method of allocating virtqueues to
vring_create_virtqueue() as the first step into that direction.
Signed-off-by: Halil Pasic <pasic@linux.ibm.com>
---
drivers/s390/virtio/virtio_ccw.c | 30 +++++++++++-------------------
1 file changed, 11 insertions(+), 19 deletions(-)
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 74c328321889..2c66941ef3d0 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -108,7 +108,6 @@ struct virtio_rev_info {
struct virtio_ccw_vq_info {
struct virtqueue *vq;
int num;
- void *queue;
union {
struct vq_info_block s;
struct vq_info_block_legacy l;
@@ -423,7 +422,6 @@ static void virtio_ccw_del_vq(struct virtqueue *vq, struct ccw1 *ccw)
struct virtio_ccw_device *vcdev = to_vc_device(vq->vdev);
struct virtio_ccw_vq_info *info = vq->priv;
unsigned long flags;
- unsigned long size;
int ret;
unsigned int index = vq->index;
@@ -461,8 +459,6 @@ static void virtio_ccw_del_vq(struct virtqueue *vq, struct ccw1 *ccw)
ret, index);
vring_del_virtqueue(vq);
- size = PAGE_ALIGN(vring_size(info->num, KVM_VIRTIO_CCW_RING_ALIGN));
- free_pages_exact(info->queue, size);
kfree(info->info_block);
kfree(info);
}
@@ -494,8 +490,9 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
int err;
struct virtqueue *vq = NULL;
struct virtio_ccw_vq_info *info;
- unsigned long size = 0; /* silence the compiler */
+ u64 queue;
unsigned long flags;
+ bool may_reduce;
/* Allocate queue. */
info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL);
@@ -516,33 +513,30 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
err = info->num;
goto out_err;
}
- size = PAGE_ALIGN(vring_size(info->num, KVM_VIRTIO_CCW_RING_ALIGN));
- info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
- if (info->queue == NULL) {
- dev_warn(&vcdev->cdev->dev, "no queue\n");
- err = -ENOMEM;
- goto out_err;
- }
+ may_reduce = vcdev->revision > 0;
+ vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN,
+ vdev, true, may_reduce, ctx,
+ virtio_ccw_kvm_notify, callback, name);
- vq = vring_new_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN, vdev,
- true, ctx, info->queue, virtio_ccw_kvm_notify,
- callback, name);
if (!vq) {
/* For now, we fail if we can't get the requested size. */
dev_warn(&vcdev->cdev->dev, "no vq\n");
err = -ENOMEM;
goto out_err;
}
+ /* it may have been reduced */
+ info->num = virtqueue_get_vring_size(vq);
/* Register it with the host. */
+ queue = virtqueue_get_desc_addr(vq);
if (vcdev->revision == 0) {
- info->info_block->l.queue = (__u64)info->queue;
+ info->info_block->l.queue = queue;
info->info_block->l.align = KVM_VIRTIO_CCW_RING_ALIGN;
info->info_block->l.index = i;
info->info_block->l.num = info->num;
ccw->count = sizeof(info->info_block->l);
} else {
- info->info_block->s.desc = (__u64)info->queue;
+ info->info_block->s.desc = queue;
info->info_block->s.index = i;
info->info_block->s.num = info->num;
info->info_block->s.avail = (__u64)virtqueue_get_avail(vq);
@@ -572,8 +566,6 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
if (vq)
vring_del_virtqueue(vq);
if (info) {
- if (info->queue)
- free_pages_exact(info->queue, size);
kfree(info->info_block);
}
kfree(info);
--
2.16.4
^ permalink raw reply related
* [PATCH 00/10] s390: virtio: support protected virtualization
From: Halil Pasic @ 2019-04-26 18:32 UTC (permalink / raw)
To: kvm, linux-s390, Cornelia Huck, Martin Schwidefsky, Sebastian Ott
Cc: Christoph Hellwig, Thomas Huth, Claudio Imbrenda, Janosch Frank,
Vasily Gorbik, Michael S. Tsirkin, Farhan Ali, Eric Farman,
virtualization, Halil Pasic, Viktor Mihajlovski
Enhanced virtualization protection technology may require the use of
bounce buffers for I/O. While support for this was built into the virtio
core, virtio-ccw wasn't changed accordingly.
Some background on technology (not part of this series) and the
terminology used.
* Protected Virtualization (PV):
Protected Virtualization guarantees, that non-shared memory of a guest
that operates in PV mode private to that guest. I.e. any attempts by the
hypervisor or other guests to access it will result in an exception. If
supported by the environment (machine, KVM, guest VM) a guest can decide
to change into PV mode by doing the appropriate ultravisor calls. Unlike
some other enhanced virtualization protection technology,
* Ultravisor:
A hardware/firmware entity that manages PV guests, and polices access to
their memory. A PV guest prospect needs to interact with the ultravisor,
to enter PV mode, and potentially to share pages (for I/O which should
be encrypted by the guest). A guest interacts with the ultravisor via so
called ultravisor calls. A hypervisor needs to interact with the
ultravisor to facilitate interpretation, emulation and swapping. A
hypervisor interacts with the ultravisor via ultravisor calls and via
the SIE state description. Generally the ultravisor sanitizes hypervisor
inputs so that the guest can not be corrupted (except for denial of
service.
What needs to be done
=====================
Thus what needs to be done to bring virtio-ccw up to speed with respect
to protected virtualization is:
* use some 'new' common virtio stuff
* make sure that virtio-ccw specific stuff uses shared memory when
talking to the hypervisor (except control/communication blocks like ORB,
these are handled by the ultravisor)
* make sure the DMA API does what is necessary to talk through shared
memory if we are a protected virtualization guest.
* make sure the common IO layer plays along as well (airqs, sense).
Important notes
================
* This patch set is based on Martins features branch
(git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git branch
'features').
* Documentation is still very sketchy. I'm committed to improving this,
but I'm currently hampered by some dependencies currently.
* The existing naming in the common infrastructure (kernel internal
interfaces) is pretty much based on the AMD SEV terminology. Thus the
names aren't always perfect. There might be merit to changing these
names to more abstract ones. I did not put much thought into that at
the current stage.
* Testing: Please use iommu_platform=on for any virtio devices you are
going to test this code with (so virtio actually uses the DMA API).
Change log
==========
RFC --> v1:
* Fixed bugs found by Connie (may_reduce and handling reduced, warning,
split move -- thanks Connie!).
* Fixed console bug found by Sebastian (thanks Sebastian!).
* Removed the completely useless duplicate of dma-mapping.h spotted by
Christoph (thanks Christoph!).
* Don't use the global DMA pool for subchannel and ccw device
owned memory as requested by Sebastian. Consequences:
* Both subchannel and ccw devices have their dma masks
now (both specifying 31 bit addressable)
* We require at least 2 DMA pages per ccw device now, most of
this memory is wasted though.
* DMA memory allocated by virtio is also 31 bit addressable now
as virtio uses the parent (which is the ccw device).
* Enabled packed ring.
* Rebased onto Martins feature branch; using the actual uv (ultravisor)
interface instead of TODO comments.
* Added some explanations to the cover letter (Connie, David).
* Squashed a couple of patches together and fixed some text stuff.
Looking forward to your review, or any other type of input.
Halil Pasic (10):
virtio/s390: use vring_create_virtqueue
virtio/s390: DMA support for virtio-ccw
virtio/s390: enable packed ring
s390/mm: force swiotlb for protected virtualization
s390/cio: introduce DMA pools to cio
s390/cio: add basic protected virtualization support
s390/airq: use DMA memory for adapter interrupts
virtio/s390: add indirection to indicators access
virtio/s390: use DMA memory for ccw I/O and classic notifiers
virtio/s390: make airq summary indicators DMA
arch/s390/Kconfig | 5 +
arch/s390/include/asm/airq.h | 2 +
arch/s390/include/asm/ccwdev.h | 4 +
arch/s390/include/asm/cio.h | 11 ++
arch/s390/include/asm/mem_encrypt.h | 18 +++
arch/s390/mm/init.c | 50 +++++++
drivers/s390/cio/airq.c | 18 ++-
drivers/s390/cio/ccwreq.c | 8 +-
drivers/s390/cio/cio.h | 1 +
drivers/s390/cio/css.c | 101 +++++++++++++
drivers/s390/cio/device.c | 65 +++++++--
drivers/s390/cio/device_fsm.c | 40 +++---
drivers/s390/cio/device_id.c | 18 +--
drivers/s390/cio/device_ops.c | 21 ++-
drivers/s390/cio/device_pgid.c | 20 +--
drivers/s390/cio/device_status.c | 24 ++--
drivers/s390/cio/io_sch.h | 21 ++-
drivers/s390/virtio/virtio_ccw.c | 275 +++++++++++++++++++-----------------
include/linux/virtio.h | 17 ---
19 files changed, 499 insertions(+), 220 deletions(-)
create mode 100644 arch/s390/include/asm/mem_encrypt.h
--
2.16.4
^ permalink raw reply
* Re: [PATCH] Revert "drm/qxl: drop prime import/export callbacks"
From: Thomas Hellstrom @ 2019-04-26 15:42 UTC (permalink / raw)
To: Daniel Vetter, Gerd Hoffmann
Cc: David Airlie, open list, dri-devel,
open list:DRM DRIVER FOR QXL VIRTUAL GPU, David Airlie,
open list:DRM DRIVER FOR QXL VIRTUAL GPU
In-Reply-To: <CAKMK7uG+vMU0hqqiKAswu=LqpkcXtLPqbYLRWgoAPpsQQV4qzA@mail.gmail.com>
On 4/26/19 4:21 PM, Daniel Vetter wrote:
> On Fri, Apr 26, 2019 at 7:33 AM Gerd Hoffmann <kraxel@redhat.com> wrote:
>> This reverts commit f4c34b1e2a37d5676180901fa6ff188bcb6371f8.
>>
>> Simliar to commit a0cecc23cfcb Revert "drm/virtio: drop prime
>> import/export callbacks". We have to do the same with qxl,
>> for the same reasons (it breaks DRI3).
>>
>> Drop the WARN_ON_ONCE().
>>
>> Fixes: f4c34b1e2a37d5676180901fa6ff188bcb6371f8
>> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> Maybe we need some helpers for virtual drivers which only allow
> self-reimport and nothing else at all? I think there's qxl, virgl,
> vmwgfx and maybe also vbox one who could use this ... Just a quick
> idea.
> -Daniel
I think vmwgfx could, in theory, support the full range of operations,
at least for reasonably recent device versions. However, it wouldn't be
terribly efficient since the exported dma-buf sglist would basically be
a bounce-buffer.
/Thomas
>> ---
>> drivers/gpu/drm/qxl/qxl_drv.c | 4 ++++
>> drivers/gpu/drm/qxl/qxl_prime.c | 12 ++++++++++++
>> 2 files changed, 16 insertions(+)
>>
>> diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
>> index 578d867a81d5..f33e349c4ec5 100644
>> --- a/drivers/gpu/drm/qxl/qxl_drv.c
>> +++ b/drivers/gpu/drm/qxl/qxl_drv.c
>> @@ -255,10 +255,14 @@ static struct drm_driver qxl_driver = {
>> #if defined(CONFIG_DEBUG_FS)
>> .debugfs_init = qxl_debugfs_init,
>> #endif
>> + .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
>> + .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
>> .gem_prime_export = drm_gem_prime_export,
>> .gem_prime_import = drm_gem_prime_import,
>> .gem_prime_pin = qxl_gem_prime_pin,
>> .gem_prime_unpin = qxl_gem_prime_unpin,
>> + .gem_prime_get_sg_table = qxl_gem_prime_get_sg_table,
>> + .gem_prime_import_sg_table = qxl_gem_prime_import_sg_table,
>> .gem_prime_vmap = qxl_gem_prime_vmap,
>> .gem_prime_vunmap = qxl_gem_prime_vunmap,
>> .gem_prime_mmap = qxl_gem_prime_mmap,
>> diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c
>> index 8b448eca1cd9..114653b471c6 100644
>> --- a/drivers/gpu/drm/qxl/qxl_prime.c
>> +++ b/drivers/gpu/drm/qxl/qxl_prime.c
>> @@ -42,6 +42,18 @@ void qxl_gem_prime_unpin(struct drm_gem_object *obj)
>> qxl_bo_unpin(bo);
>> }
>>
>> +struct sg_table *qxl_gem_prime_get_sg_table(struct drm_gem_object *obj)
>> +{
>> + return ERR_PTR(-ENOSYS);
>> +}
>> +
>> +struct drm_gem_object *qxl_gem_prime_import_sg_table(
>> + struct drm_device *dev, struct dma_buf_attachment *attach,
>> + struct sg_table *table)
>> +{
>> + return ERR_PTR(-ENOSYS);
>> +}
>> +
>> void *qxl_gem_prime_vmap(struct drm_gem_object *obj)
>> {
>> struct qxl_bo *bo = gem_to_qxl_bo(obj);
>> --
>> 2.18.1
>>
>
^ permalink raw reply
* Re: [PATCH] Revert "drm/qxl: drop prime import/export callbacks"
From: Daniel Vetter @ 2019-04-26 14:21 UTC (permalink / raw)
To: Gerd Hoffmann
Cc: David Airlie, open list, dri-devel,
open list:DRM DRIVER FOR QXL VIRTUAL GPU,
open list:DRM DRIVER FOR QXL VIRTUAL GPU, David Airlie
In-Reply-To: <20190426053324.26443-1-kraxel@redhat.com>
On Fri, Apr 26, 2019 at 7:33 AM Gerd Hoffmann <kraxel@redhat.com> wrote:
>
> This reverts commit f4c34b1e2a37d5676180901fa6ff188bcb6371f8.
>
> Simliar to commit a0cecc23cfcb Revert "drm/virtio: drop prime
> import/export callbacks". We have to do the same with qxl,
> for the same reasons (it breaks DRI3).
>
> Drop the WARN_ON_ONCE().
>
> Fixes: f4c34b1e2a37d5676180901fa6ff188bcb6371f8
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
Maybe we need some helpers for virtual drivers which only allow
self-reimport and nothing else at all? I think there's qxl, virgl,
vmwgfx and maybe also vbox one who could use this ... Just a quick
idea.
-Daniel
> ---
> drivers/gpu/drm/qxl/qxl_drv.c | 4 ++++
> drivers/gpu/drm/qxl/qxl_prime.c | 12 ++++++++++++
> 2 files changed, 16 insertions(+)
>
> diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
> index 578d867a81d5..f33e349c4ec5 100644
> --- a/drivers/gpu/drm/qxl/qxl_drv.c
> +++ b/drivers/gpu/drm/qxl/qxl_drv.c
> @@ -255,10 +255,14 @@ static struct drm_driver qxl_driver = {
> #if defined(CONFIG_DEBUG_FS)
> .debugfs_init = qxl_debugfs_init,
> #endif
> + .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
> + .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
> .gem_prime_export = drm_gem_prime_export,
> .gem_prime_import = drm_gem_prime_import,
> .gem_prime_pin = qxl_gem_prime_pin,
> .gem_prime_unpin = qxl_gem_prime_unpin,
> + .gem_prime_get_sg_table = qxl_gem_prime_get_sg_table,
> + .gem_prime_import_sg_table = qxl_gem_prime_import_sg_table,
> .gem_prime_vmap = qxl_gem_prime_vmap,
> .gem_prime_vunmap = qxl_gem_prime_vunmap,
> .gem_prime_mmap = qxl_gem_prime_mmap,
> diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c
> index 8b448eca1cd9..114653b471c6 100644
> --- a/drivers/gpu/drm/qxl/qxl_prime.c
> +++ b/drivers/gpu/drm/qxl/qxl_prime.c
> @@ -42,6 +42,18 @@ void qxl_gem_prime_unpin(struct drm_gem_object *obj)
> qxl_bo_unpin(bo);
> }
>
> +struct sg_table *qxl_gem_prime_get_sg_table(struct drm_gem_object *obj)
> +{
> + return ERR_PTR(-ENOSYS);
> +}
> +
> +struct drm_gem_object *qxl_gem_prime_import_sg_table(
> + struct drm_device *dev, struct dma_buf_attachment *attach,
> + struct sg_table *table)
> +{
> + return ERR_PTR(-ENOSYS);
> +}
> +
> void *qxl_gem_prime_vmap(struct drm_gem_object *obj)
> {
> struct qxl_bo *bo = gem_to_qxl_bo(obj);
> --
> 2.18.1
>
--
Daniel Vetter
Software Engineer, Intel Corporation
+41 (0) 79 365 57 48 - http://blog.ffwll.ch
^ permalink raw reply
* Re: [PATCH net] vhost_net: fix possible infinite loop
From: Jason Wang @ 2019-04-26 7:35 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: netdev, ppandit, linux-kernel, kvm, virtualization
In-Reply-To: <20190425131021-mutt-send-email-mst@kernel.org>
On 2019/4/26 上午1:52, Michael S. Tsirkin wrote:
> On Thu, Apr 25, 2019 at 03:33:19AM -0400, Jason Wang wrote:
>> When the rx buffer is too small for a packet, we will discard the vq
>> descriptor and retry it for the next packet:
>>
>> while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
>> &busyloop_intr))) {
>> ...
>> /* On overrun, truncate and discard */
>> if (unlikely(headcount > UIO_MAXIOV)) {
>> iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
>> err = sock->ops->recvmsg(sock, &msg,
>> 1, MSG_DONTWAIT | MSG_TRUNC);
>> pr_debug("Discarded rx packet: len %zd\n", sock_len);
>> continue;
>> }
>> ...
>> }
>>
>> This makes it possible to trigger a infinite while..continue loop
>> through the co-opreation of two VMs like:
>>
>> 1) Malicious VM1 allocate 1 byte rx buffer and try to slow down the
>> vhost process as much as possible e.g using indirect descriptors or
>> other.
>> 2) Malicious VM2 generate packets to VM1 as fast as possible
>>
>> Fixing this by checking against weight at the end of RX and TX
>> loop. This also eliminate other similar cases when:
>>
>> - userspace is consuming the packets in the meanwhile
>> - theoretical TOCTOU attack if guest moving avail index back and forth
>> to hit the continue after vhost find guest just add new buffers
>>
>> This addresses CVE-2019-3900.
>>
>> Fixes: d8316f3991d20 ("vhost: fix total length when packets are too short")
> I agree this is the real issue.
>
>> Fixes: 3a4d5c94e9593 ("vhost_net: a kernel-level virtio server")
> This is just a red herring imho. We can stick this on any vhost patch :)
>
>> Signed-off-by: Jason Wang <jasowang@redhat.com>
>> ---
>> drivers/vhost/net.c | 41 +++++++++++++++++++++--------------------
>> 1 file changed, 21 insertions(+), 20 deletions(-)
>>
>> diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
>> index df51a35..fb46e6b 100644
>> --- a/drivers/vhost/net.c
>> +++ b/drivers/vhost/net.c
>> @@ -778,8 +778,9 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
>> int err;
>> int sent_pkts = 0;
>> bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);
>> + bool next_round = false;
>>
>> - for (;;) {
>> + do {
>> bool busyloop_intr = false;
>>
>> if (nvq->done_idx == VHOST_NET_BATCH)
>> @@ -845,11 +846,10 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
>> vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
>> vq->heads[nvq->done_idx].len = 0;
>> ++nvq->done_idx;
>> - if (vhost_exceeds_weight(++sent_pkts, total_len)) {
>> - vhost_poll_queue(&vq->poll);
>> - break;
>> - }
>> - }
>> + } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
>> +
>> + if (next_round)
>> + vhost_poll_queue(&vq->poll);
>>
>> vhost_tx_batch(net, nvq, sock, &msg);
>> }
>> @@ -873,8 +873,9 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
>> struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
>> bool zcopy_used;
>> int sent_pkts = 0;
>> + bool next_round = false;
>>
>> - for (;;) {
>> + do {
>> bool busyloop_intr;
>>
>> /* Release DMAs done buffers first */
>> @@ -951,11 +952,10 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
>> else
>> vhost_zerocopy_signal_used(net, vq);
>> vhost_net_tx_packet(net);
>> - if (unlikely(vhost_exceeds_weight(++sent_pkts, total_len))) {
>> - vhost_poll_queue(&vq->poll);
>> - break;
>> - }
>> - }
>> + } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
>> +
>> + if (next_round)
>> + vhost_poll_queue(&vq->poll);
>> }
>>
>> /* Expects to be always run from workqueue - which acts as
>> @@ -1134,6 +1134,7 @@ static void handle_rx(struct vhost_net *net)
>> struct iov_iter fixup;
>> __virtio16 num_buffers;
>> int recv_pkts = 0;
>> + bool next_round = false;
>>
>> mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX);
>> sock = vq->private_data;
>> @@ -1153,8 +1154,11 @@ static void handle_rx(struct vhost_net *net)
>> vq->log : NULL;
>> mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
>>
>> - while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
>> - &busyloop_intr))) {
>> + do {
>> + sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
>> + &busyloop_intr);
>> + if (!sock_len)
>> + break;
>> sock_len += sock_hlen;
>> vhost_len = sock_len + vhost_hlen;
>> headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
>> @@ -1239,12 +1243,9 @@ static void handle_rx(struct vhost_net *net)
>> vhost_log_write(vq, vq_log, log, vhost_len,
>> vq->iov, in);
>> total_len += vhost_len;
>> - if (unlikely(vhost_exceeds_weight(++recv_pkts, total_len))) {
>> - vhost_poll_queue(&vq->poll);
>> - goto out;
>> - }
>> - }
>> - if (unlikely(busyloop_intr))
>> + } while (!(next_round = vhost_exceeds_weight(++recv_pkts, total_len)));
>> +
>> + if (unlikely(busyloop_intr || next_round))
>> vhost_poll_queue(&vq->poll);
>> else
>> vhost_net_enable_vq(net, vq);
>
> I'm afraid with this addition the code is too much like spagetty. What
> does next_round mean? Just that we are breaking out of loop?
Yes, we can have a better name of course.
> That is
> what goto is for... Either let's have for(;;) with goto/break to get
> outside or a while loop with a condition. Both is just unreadable.
>
> All these checks in 3 places are exactly the same on all paths and they
> are slow path. Why don't we put this in a function?
The point I think is, we want the weight to be checked in both fast path
and slow path.
> E.g. like the below.
> Warning: completely untested.
>
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
>
> ---
>
> diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
> index df51a35cf537..a0f89a504cd9 100644
> --- a/drivers/vhost/net.c
> +++ b/drivers/vhost/net.c
> @@ -761,6 +761,23 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
> return 0;
> }
>
> +/* Returns true if caller needs to go back and re-read the ring. */
> +static bool empty_ring(struct vhost_net *net, struct vhost_virtqueue *vq,
> + int pkts, size_t total_len, bool busyloop_intr)
> +{
> + if (unlikely(busyloop_intr)) {
> + vhost_poll_queue(&vq->poll);
> + } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
> + /* They have slipped one in meanwhile: check again. */
> + vhost_disable_notify(&net->dev, vq);
> + if (!vhost_exceeds_weight(pkts, total_len))
> + return true;
> + vhost_poll_queue(&vq->poll);
> + }
> + /* Nothing new? Wait for eventfd to tell us they refilled. */
> + return false;
> +}
Ring empy is not the only places that needs care. E.g for RX, we need
care about overrun and when userspace is consuming the packet in the
same time. So there's no need to toggle vq notification in those two.
> +
> static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
> {
> struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
> @@ -790,15 +807,10 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
> /* On error, stop handling until the next kick. */
> if (unlikely(head < 0))
> break;
> - /* Nothing new? Wait for eventfd to tell us they refilled. */
> if (head == vq->num) {
> - if (unlikely(busyloop_intr)) {
> - vhost_poll_queue(&vq->poll);
> - } else if (unlikely(vhost_enable_notify(&net->dev,
> - vq))) {
> - vhost_disable_notify(&net->dev, vq);
> + if (unlikely(empty_ring(net, vq, ++sent_pkts,
> + total_len, busyloop_intr)))
> continue;
> - }
> break;
> }
>
> @@ -886,14 +898,10 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
> /* On error, stop handling until the next kick. */
> if (unlikely(head < 0))
> break;
> - /* Nothing new? Wait for eventfd to tell us they refilled. */
> if (head == vq->num) {
> - if (unlikely(busyloop_intr)) {
> - vhost_poll_queue(&vq->poll);
> - } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
> - vhost_disable_notify(&net->dev, vq);
> + if (unlikely(empty_ring(net, vq, ++sent_pkts,
> + total_len, busyloop_intr)))
> continue;
> - }
> break;
> }
>
> @@ -1163,18 +1171,10 @@ static void handle_rx(struct vhost_net *net)
> /* On error, stop handling until the next kick. */
> if (unlikely(headcount < 0))
> goto out;
> - /* OK, now we need to know about added descriptors. */
> if (!headcount) {
> - if (unlikely(busyloop_intr)) {
> - vhost_poll_queue(&vq->poll);
> - } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
> - /* They have slipped one in as we were
> - * doing that: check again. */
> - vhost_disable_notify(&net->dev, vq);
> - continue;
> - }
> - /* Nothing new? Wait for eventfd to tell us
> - * they refilled. */
> + if (unlikely(empty_ring(net, vq, ++recv_pkts,
> + total_len, busyloop_intr)))
> + continue;
> goto out;
> }
> busyloop_intr = false;
The patch misses several other continue that need cares and there's
another call of vhost_exceeds_weight() at the end of the loop.
So instead of duplicating check everywhere like:
for (;;) {
if (condition_x) {
if (empty_ring())
continue;
break;
}
if (condition_y) {
if (empty_ring());
continue;
break;
}
if (condition_z) {
if (empty_ring())
continue;
break;
}
}
What this patch did:
do {
if (condition_x)
continue;
if (condition_y)
continue;
if (condition_z)
continue;
} while(!need_break())
is much more compact and easier to read?
Thanks
>
>> --
>> 1.8.3.1
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply
* [PATCH] Revert "drm/qxl: drop prime import/export callbacks"
From: Gerd Hoffmann @ 2019-04-26 5:33 UTC (permalink / raw)
To: dri-devel, David Airlie
Cc: David Airlie, open list, open list:DRM DRIVER FOR QXL VIRTUAL GPU,
Daniel Vetter, open list:DRM DRIVER FOR QXL VIRTUAL GPU
This reverts commit f4c34b1e2a37d5676180901fa6ff188bcb6371f8.
Simliar to commit a0cecc23cfcb Revert "drm/virtio: drop prime
import/export callbacks". We have to do the same with qxl,
for the same reasons (it breaks DRI3).
Drop the WARN_ON_ONCE().
Fixes: f4c34b1e2a37d5676180901fa6ff188bcb6371f8
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
drivers/gpu/drm/qxl/qxl_drv.c | 4 ++++
drivers/gpu/drm/qxl/qxl_prime.c | 12 ++++++++++++
2 files changed, 16 insertions(+)
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index 578d867a81d5..f33e349c4ec5 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -255,10 +255,14 @@ static struct drm_driver qxl_driver = {
#if defined(CONFIG_DEBUG_FS)
.debugfs_init = qxl_debugfs_init,
#endif
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+ .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
.gem_prime_export = drm_gem_prime_export,
.gem_prime_import = drm_gem_prime_import,
.gem_prime_pin = qxl_gem_prime_pin,
.gem_prime_unpin = qxl_gem_prime_unpin,
+ .gem_prime_get_sg_table = qxl_gem_prime_get_sg_table,
+ .gem_prime_import_sg_table = qxl_gem_prime_import_sg_table,
.gem_prime_vmap = qxl_gem_prime_vmap,
.gem_prime_vunmap = qxl_gem_prime_vunmap,
.gem_prime_mmap = qxl_gem_prime_mmap,
diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c
index 8b448eca1cd9..114653b471c6 100644
--- a/drivers/gpu/drm/qxl/qxl_prime.c
+++ b/drivers/gpu/drm/qxl/qxl_prime.c
@@ -42,6 +42,18 @@ void qxl_gem_prime_unpin(struct drm_gem_object *obj)
qxl_bo_unpin(bo);
}
+struct sg_table *qxl_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+ return ERR_PTR(-ENOSYS);
+}
+
+struct drm_gem_object *qxl_gem_prime_import_sg_table(
+ struct drm_device *dev, struct dma_buf_attachment *attach,
+ struct sg_table *table)
+{
+ return ERR_PTR(-ENOSYS);
+}
+
void *qxl_gem_prime_vmap(struct drm_gem_object *obj)
{
struct qxl_bo *bo = gem_to_qxl_bo(obj);
--
2.18.1
^ permalink raw reply related
* [PATCH v7 6/6] xfs: disable map_sync for async flush
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
Dont support 'MAP_SYNC' with non-DAX files and DAX files
with asynchronous dax_device. Virtio pmem provides
asynchronous host page cache flush mechanism. We don't
support 'MAP_SYNC' with virtio pmem and xfs.
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
---
fs/xfs/xfs_file.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index a7ceae90110e..f17652cca5ff 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -1203,11 +1203,14 @@ xfs_file_mmap(
struct file *filp,
struct vm_area_struct *vma)
{
+ struct dax_device *dax_dev;
+
+ dax_dev = xfs_find_daxdev_for_inode(file_inode(filp));
/*
- * We don't support synchronous mappings for non-DAX files. At least
- * until someone comes with a sensible use case.
+ * We don't support synchronous mappings for non-DAX files and
+ * for DAX files if underneath dax_device is not synchronous.
*/
- if (!IS_DAX(file_inode(filp)) && (vma->vm_flags & VM_SYNC))
+ if (!daxdev_mapping_supported(vma, dax_dev))
return -EOPNOTSUPP;
file_accessed(filp);
--
2.20.1
^ permalink raw reply related
* [PATCH v7 5/6] ext4: disable map_sync for async flush
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
Dont support 'MAP_SYNC' with non-DAX files and DAX files
with asynchronous dax_device. Virtio pmem provides
asynchronous host page cache flush mechanism. We don't
support 'MAP_SYNC' with virtio pmem and ext4.
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
Reviewed-by: Jan Kara <jack@suse.cz>
---
fs/ext4/file.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index 98ec11f69cd4..dee549339e13 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -360,15 +360,17 @@ static const struct vm_operations_struct ext4_file_vm_ops = {
static int ext4_file_mmap(struct file *file, struct vm_area_struct *vma)
{
struct inode *inode = file->f_mapping->host;
+ struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
+ struct dax_device *dax_dev = sbi->s_daxdev;
- if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
+ if (unlikely(ext4_forced_shutdown(sbi)))
return -EIO;
/*
- * We don't support synchronous mappings for non-DAX files. At least
- * until someone comes with a sensible use case.
+ * We don't support synchronous mappings for non-DAX files and
+ * for DAX files if underneath dax_device is not synchronous.
*/
- if (!IS_DAX(file_inode(file)) && (vma->vm_flags & VM_SYNC))
+ if (!daxdev_mapping_supported(vma, dax_dev))
return -EOPNOTSUPP;
file_accessed(file);
--
2.20.1
^ permalink raw reply related
* [PATCH v7 4/6] dax: check synchronous mapping is supported
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
This patch introduces 'daxdev_mapping_supported' helper
which checks if 'MAP_SYNC' is supported with filesystem
mapping. It also checks if corresponding dax_device is
synchronous. Virtio pmem device is asynchronous and
does not not support VM_SYNC.
Suggested-by: Jan Kara <jack@suse.cz>
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
Reviewed-by: Jan Kara <jack@suse.cz>
---
include/linux/dax.h | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/include/linux/dax.h b/include/linux/dax.h
index c97fc0cc7167..41b4a5db6305 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -41,6 +41,18 @@ void kill_dax(struct dax_device *dax_dev);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
bool dax_synchronous(struct dax_device *dax_dev);
+/*
+ * Check if given mapping is supported by the file / underlying device.
+ */
+static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
+ struct dax_device *dax_dev)
+{
+ if (!(vma->vm_flags & VM_SYNC))
+ return true;
+ if (!IS_DAX(file_inode(vma->vm_file)))
+ return false;
+ return dax_synchronous(dax_dev);
+}
#else
static inline struct dax_device *dax_get_by_host(const char *host)
{
@@ -68,6 +80,11 @@ static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
return false;
}
+static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
+ struct dax_device *dax_dev)
+{
+ return !(vma->flags & VM_SYNC);
+}
#endif
struct writeback_control;
--
2.20.1
^ permalink raw reply related
* [PATCH v7 3/6] libnvdimm: add dax_dev sync flag
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
This patch adds 'DAXDEV_SYNC' flag which is set
for nd_region doing synchronous flush. This later
is used to disable MAP_SYNC functionality for
ext4 & xfs filesystem for devices don't support
synchronous flush.
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
---
drivers/dax/bus.c | 2 +-
drivers/dax/super.c | 13 ++++++++++++-
drivers/md/dm.c | 3 ++-
drivers/nvdimm/pmem.c | 3 ++-
drivers/nvdimm/region_devs.c | 7 +++++++
include/linux/dax.h | 8 ++++++--
include/linux/libnvdimm.h | 1 +
7 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
index 2109cfe80219..5f184e751c82 100644
--- a/drivers/dax/bus.c
+++ b/drivers/dax/bus.c
@@ -388,7 +388,7 @@ struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id,
* No 'host' or dax_operations since there is no access to this
* device outside of mmap of the resulting character device.
*/
- dax_dev = alloc_dax(dev_dax, NULL, NULL);
+ dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
if (!dax_dev)
goto err;
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 0a339b85133e..bd6509308d05 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -186,6 +186,8 @@ enum dax_device_flags {
DAXDEV_ALIVE,
/* gate whether dax_flush() calls the low level flush routine */
DAXDEV_WRITE_CACHE,
+ /* flag to check if device supports synchronous flush */
+ DAXDEV_SYNC,
};
/**
@@ -354,6 +356,12 @@ bool dax_write_cache_enabled(struct dax_device *dax_dev)
}
EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
+bool dax_synchronous(struct dax_device *dax_dev)
+{
+ return test_bit(DAXDEV_SYNC, &dax_dev->flags);
+}
+EXPORT_SYMBOL_GPL(dax_synchronous);
+
bool dax_alive(struct dax_device *dax_dev)
{
lockdep_assert_held(&dax_srcu);
@@ -511,7 +519,7 @@ static void dax_add_host(struct dax_device *dax_dev, const char *host)
}
struct dax_device *alloc_dax(void *private, const char *__host,
- const struct dax_operations *ops)
+ const struct dax_operations *ops, bool sync)
{
struct dax_device *dax_dev;
const char *host;
@@ -534,6 +542,9 @@ struct dax_device *alloc_dax(void *private, const char *__host,
dax_add_host(dax_dev, host);
dax_dev->ops = ops;
dax_dev->private = private;
+ if (sync)
+ set_bit(DAXDEV_SYNC, &dax_dev->flags);
+
return dax_dev;
err_dev:
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 68d24056d0b1..dd5266fb5471 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1965,7 +1965,8 @@ static struct mapped_device *alloc_dev(int minor)
sprintf(md->disk->disk_name, "dm-%d", minor);
if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
- dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops);
+ dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops,
+ DAXDEV_F_SYNC);
if (!dax_dev)
goto bad;
}
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index f719245da170..34fa20381c05 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -466,7 +466,8 @@ static int pmem_attach_disk(struct device *dev,
nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_res);
disk->bb = &pmem->bb;
- dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops);
+ dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops,
+ is_nvdimm_sync(nd_region));
if (!dax_dev) {
put_disk(disk);
return -ENOMEM;
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index e5b59708865e..427cf28380c6 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -1219,6 +1219,13 @@ int nvdimm_has_cache(struct nd_region *nd_region)
}
EXPORT_SYMBOL_GPL(nvdimm_has_cache);
+bool is_nvdimm_sync(struct nd_region *nd_region)
+{
+ return is_nd_pmem(&nd_region->dev) &&
+ !test_bit(ND_REGION_ASYNC, &nd_region->flags);
+}
+EXPORT_SYMBOL_GPL(is_nvdimm_sync);
+
struct conflict_context {
struct nd_region *nd_region;
resource_size_t start, size;
diff --git a/include/linux/dax.h b/include/linux/dax.h
index 0dd316a74a29..c97fc0cc7167 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -7,6 +7,9 @@
#include <linux/radix-tree.h>
#include <asm/pgtable.h>
+/* Flag for synchronous flush */
+#define DAXDEV_F_SYNC true
+
typedef unsigned long dax_entry_t;
struct iomap_ops;
@@ -32,18 +35,19 @@ extern struct attribute_group dax_attribute_group;
#if IS_ENABLED(CONFIG_DAX)
struct dax_device *dax_get_by_host(const char *host);
struct dax_device *alloc_dax(void *private, const char *host,
- const struct dax_operations *ops);
+ const struct dax_operations *ops, bool sync);
void put_dax(struct dax_device *dax_dev);
void kill_dax(struct dax_device *dax_dev);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
+bool dax_synchronous(struct dax_device *dax_dev);
#else
static inline struct dax_device *dax_get_by_host(const char *host)
{
return NULL;
}
static inline struct dax_device *alloc_dax(void *private, const char *host,
- const struct dax_operations *ops)
+ const struct dax_operations *ops, bool sync)
{
/*
* Callers should check IS_ENABLED(CONFIG_DAX) to know if this
diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h
index a5f369ec3726..8ae6b65d67e2 100644
--- a/include/linux/libnvdimm.h
+++ b/include/linux/libnvdimm.h
@@ -270,6 +270,7 @@ int generic_nvdimm_flush(struct nd_region *nd_region);
int nvdimm_has_flush(struct nd_region *nd_region);
int nvdimm_has_cache(struct nd_region *nd_region);
int nvdimm_in_overwrite(struct nvdimm *nvdimm);
+bool is_nvdimm_sync(struct nd_region *nd_region);
static inline int nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, void *buf,
unsigned int buf_len, int *cmd_rc)
--
2.20.1
^ permalink raw reply related
* [PATCH v7 2/6] virtio-pmem: Add virtio pmem driver
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
This patch adds virtio-pmem driver for KVM guest.
Guest reads the persistent memory range information from
Qemu over VIRTIO and registers it on nvdimm_bus. It also
creates a nd_region object with the persistent memory
range information so that existing 'nvdimm/pmem' driver
can reserve this into system memory map. This way
'virtio-pmem' driver uses existing functionality of pmem
driver to register persistent memory compatible for DAX
capable filesystems.
This also provides function to perform guest flush over
VIRTIO from 'pmem' driver when userspace performs flush
on DAX memory range.
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
---
drivers/nvdimm/virtio_pmem.c | 114 +++++++++++++++++++++++++++++
drivers/virtio/Kconfig | 10 +++
drivers/virtio/Makefile | 1 +
drivers/virtio/pmem.c | 118 +++++++++++++++++++++++++++++++
include/linux/virtio_pmem.h | 60 ++++++++++++++++
include/uapi/linux/virtio_ids.h | 1 +
include/uapi/linux/virtio_pmem.h | 10 +++
7 files changed, 314 insertions(+)
create mode 100644 drivers/nvdimm/virtio_pmem.c
create mode 100644 drivers/virtio/pmem.c
create mode 100644 include/linux/virtio_pmem.h
create mode 100644 include/uapi/linux/virtio_pmem.h
diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c
new file mode 100644
index 000000000000..66b582f751a3
--- /dev/null
+++ b/drivers/nvdimm/virtio_pmem.c
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * virtio_pmem.c: Virtio pmem Driver
+ *
+ * Discovers persistent memory range information
+ * from host and provides a virtio based flushing
+ * interface.
+ */
+#include <linux/virtio_pmem.h>
+#include "nd.h"
+
+ /* The interrupt handler */
+void host_ack(struct virtqueue *vq)
+{
+ unsigned int len;
+ unsigned long flags;
+ struct virtio_pmem_request *req, *req_buf;
+ struct virtio_pmem *vpmem = vq->vdev->priv;
+
+ spin_lock_irqsave(&vpmem->pmem_lock, flags);
+ while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
+ req->done = true;
+ wake_up(&req->host_acked);
+
+ if (!list_empty(&vpmem->req_list)) {
+ req_buf = list_first_entry(&vpmem->req_list,
+ struct virtio_pmem_request, list);
+ list_del(&vpmem->req_list);
+ req_buf->wq_buf_avail = true;
+ wake_up(&req_buf->wq_buf);
+ }
+ }
+ spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
+}
+EXPORT_SYMBOL_GPL(host_ack);
+
+ /* The request submission function */
+int virtio_pmem_flush(struct nd_region *nd_region)
+{
+ int err;
+ unsigned long flags;
+ struct scatterlist *sgs[2], sg, ret;
+ struct virtio_device *vdev = nd_region->provider_data;
+ struct virtio_pmem *vpmem = vdev->priv;
+ struct virtio_pmem_request *req;
+
+ might_sleep();
+ req = kmalloc(sizeof(*req), GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+
+ req->done = req->wq_buf_avail = false;
+ strcpy(req->name, "FLUSH");
+ init_waitqueue_head(&req->host_acked);
+ init_waitqueue_head(&req->wq_buf);
+ sg_init_one(&sg, req->name, strlen(req->name));
+ sgs[0] = &sg;
+ sg_init_one(&ret, &req->ret, sizeof(req->ret));
+ sgs[1] = &ret;
+
+ spin_lock_irqsave(&vpmem->pmem_lock, flags);
+ err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req, GFP_ATOMIC);
+ if (err) {
+ dev_err(&vdev->dev, "failed to send command to virtio pmem device\n");
+
+ list_add_tail(&vpmem->req_list, &req->list);
+ spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
+
+ /* When host has read buffer, this completes via host_ack */
+ wait_event(req->wq_buf, req->wq_buf_avail);
+ spin_lock_irqsave(&vpmem->pmem_lock, flags);
+ }
+ err = virtqueue_kick(vpmem->req_vq);
+ spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
+
+ if (!err) {
+ err = -EIO;
+ goto ret;
+ }
+ /* When host has read buffer, this completes via host_ack */
+ wait_event(req->host_acked, req->done);
+ err = req->ret;
+ret:
+ kfree(req);
+ return err;
+};
+
+ /* The asynchronous flush callback function */
+int async_pmem_flush(struct nd_region *nd_region, struct bio *bio)
+{
+ int rc = 0;
+
+ /* Create child bio for asynchronous flush and chain with
+ * parent bio. Otherwise directly call nd_region flush.
+ */
+ if (bio && bio->bi_iter.bi_sector != -1) {
+ struct bio *child = bio_alloc(GFP_ATOMIC, 0);
+
+ if (!child)
+ return -ENOMEM;
+ bio_copy_dev(child, bio);
+ child->bi_opf = REQ_PREFLUSH;
+ child->bi_iter.bi_sector = -1;
+ bio_chain(child, bio);
+ submit_bio(child);
+ } else {
+ if (virtio_pmem_flush(nd_region))
+ rc = -EIO;
+ }
+
+ return rc;
+};
+EXPORT_SYMBOL_GPL(async_pmem_flush);
+MODULE_LICENSE("GPL");
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 35897649c24f..9f634a2ed638 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -42,6 +42,16 @@ config VIRTIO_PCI_LEGACY
If unsure, say Y.
+config VIRTIO_PMEM
+ tristate "Support for virtio pmem driver"
+ depends on VIRTIO
+ depends on LIBNVDIMM
+ help
+ This driver provides support for virtio based flushing interface
+ for persistent memory range.
+
+ If unsure, say M.
+
config VIRTIO_BALLOON
tristate "Virtio balloon driver"
depends on VIRTIO
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index 3a2b5c5dcf46..143ce91eabe9 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -6,3 +6,4 @@ virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o
virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
+obj-$(CONFIG_VIRTIO_PMEM) += pmem.o ../nvdimm/virtio_pmem.o
diff --git a/drivers/virtio/pmem.c b/drivers/virtio/pmem.c
new file mode 100644
index 000000000000..309788628e41
--- /dev/null
+++ b/drivers/virtio/pmem.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * virtio_pmem.c: Virtio pmem Driver
+ *
+ * Discovers persistent memory range information
+ * from host and registers the virtual pmem device
+ * with libnvdimm core.
+ */
+#include <linux/virtio_pmem.h>
+#include <../../drivers/nvdimm/nd.h>
+
+static struct virtio_device_id id_table[] = {
+ { VIRTIO_ID_PMEM, VIRTIO_DEV_ANY_ID },
+ { 0 },
+};
+
+ /* Initialize virt queue */
+static int init_vq(struct virtio_pmem *vpmem)
+{
+ /* single vq */
+ vpmem->req_vq = virtio_find_single_vq(vpmem->vdev,
+ host_ack, "flush_queue");
+ if (IS_ERR(vpmem->req_vq))
+ return PTR_ERR(vpmem->req_vq);
+
+ spin_lock_init(&vpmem->pmem_lock);
+ INIT_LIST_HEAD(&vpmem->req_list);
+
+ return 0;
+};
+
+static int virtio_pmem_probe(struct virtio_device *vdev)
+{
+ int err = 0;
+ struct resource res;
+ struct virtio_pmem *vpmem;
+ struct nd_region_desc ndr_desc = {};
+ int nid = dev_to_node(&vdev->dev);
+ struct nd_region *nd_region;
+
+ if (!vdev->config->get) {
+ dev_err(&vdev->dev, "%s failure: config access disabled\n",
+ __func__);
+ return -EINVAL;
+ }
+
+ vpmem = devm_kzalloc(&vdev->dev, sizeof(*vpmem), GFP_KERNEL);
+ if (!vpmem) {
+ err = -ENOMEM;
+ goto out_err;
+ }
+
+ vpmem->vdev = vdev;
+ vdev->priv = vpmem;
+ err = init_vq(vpmem);
+ if (err)
+ goto out_err;
+
+ virtio_cread(vpmem->vdev, struct virtio_pmem_config,
+ start, &vpmem->start);
+ virtio_cread(vpmem->vdev, struct virtio_pmem_config,
+ size, &vpmem->size);
+
+ res.start = vpmem->start;
+ res.end = vpmem->start + vpmem->size-1;
+ vpmem->nd_desc.provider_name = "virtio-pmem";
+ vpmem->nd_desc.module = THIS_MODULE;
+
+ vpmem->nvdimm_bus = nvdimm_bus_register(&vdev->dev,
+ &vpmem->nd_desc);
+ if (!vpmem->nvdimm_bus)
+ goto out_vq;
+
+ dev_set_drvdata(&vdev->dev, vpmem->nvdimm_bus);
+
+ ndr_desc.res = &res;
+ ndr_desc.numa_node = nid;
+ ndr_desc.flush = async_pmem_flush;
+ set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags);
+ set_bit(ND_REGION_ASYNC, &ndr_desc.flags);
+ nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc);
+
+ if (!nd_region)
+ goto out_nd;
+ nd_region->provider_data = dev_to_virtio
+ (nd_region->dev.parent->parent);
+ return 0;
+out_nd:
+ err = -ENXIO;
+ nvdimm_bus_unregister(vpmem->nvdimm_bus);
+out_vq:
+ vdev->config->del_vqs(vdev);
+out_err:
+ dev_err(&vdev->dev, "failed to register virtio pmem memory\n");
+ return err;
+}
+
+static void virtio_pmem_remove(struct virtio_device *vdev)
+{
+ struct nvdimm_bus *nvdimm_bus = dev_get_drvdata(&vdev->dev);
+
+ nvdimm_bus_unregister(nvdimm_bus);
+ vdev->config->del_vqs(vdev);
+ vdev->config->reset(vdev);
+}
+
+static struct virtio_driver virtio_pmem_driver = {
+ .driver.name = KBUILD_MODNAME,
+ .driver.owner = THIS_MODULE,
+ .id_table = id_table,
+ .probe = virtio_pmem_probe,
+ .remove = virtio_pmem_remove,
+};
+
+module_virtio_driver(virtio_pmem_driver);
+MODULE_DEVICE_TABLE(virtio, id_table);
+MODULE_DESCRIPTION("Virtio pmem driver");
+MODULE_LICENSE("GPL");
diff --git a/include/linux/virtio_pmem.h b/include/linux/virtio_pmem.h
new file mode 100644
index 000000000000..ab1da877575d
--- /dev/null
+++ b/include/linux/virtio_pmem.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * virtio_pmem.h: virtio pmem Driver
+ *
+ * Discovers persistent memory range information
+ * from host and provides a virtio based flushing
+ * interface.
+ **/
+
+#ifndef _LINUX_VIRTIO_PMEM_H
+#define _LINUX_VIRTIO_PMEM_H
+
+#include <linux/virtio_ids.h>
+#include <linux/module.h>
+#include <linux/virtio_config.h>
+#include <uapi/linux/virtio_pmem.h>
+#include <linux/libnvdimm.h>
+#include <linux/spinlock.h>
+
+struct virtio_pmem_request {
+ /* Host return status corresponding to flush request */
+ int ret;
+
+ /* command name*/
+ char name[16];
+
+ /* Wait queue to process deferred work after ack from host */
+ wait_queue_head_t host_acked;
+ bool done;
+
+ /* Wait queue to process deferred work after virt queue buffer avail */
+ wait_queue_head_t wq_buf;
+ bool wq_buf_avail;
+ struct list_head list;
+};
+
+struct virtio_pmem {
+ struct virtio_device *vdev;
+
+ /* Virtio pmem request queue */
+ struct virtqueue *req_vq;
+
+ /* nvdimm bus registers virtio pmem device */
+ struct nvdimm_bus *nvdimm_bus;
+ struct nvdimm_bus_descriptor nd_desc;
+
+ /* List to store deferred work if virtqueue is full */
+ struct list_head req_list;
+
+ /* Synchronize virtqueue data */
+ spinlock_t pmem_lock;
+
+ /* Memory region information */
+ uint64_t start;
+ uint64_t size;
+};
+
+void host_ack(struct virtqueue *vq);
+int async_pmem_flush(struct nd_region *nd_region, struct bio *bio);
+#endif
diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h
index 6d5c3b2d4f4d..32b2f94d1f58 100644
--- a/include/uapi/linux/virtio_ids.h
+++ b/include/uapi/linux/virtio_ids.h
@@ -43,5 +43,6 @@
#define VIRTIO_ID_INPUT 18 /* virtio input */
#define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */
#define VIRTIO_ID_CRYPTO 20 /* virtio crypto */
+#define VIRTIO_ID_PMEM 27 /* virtio pmem */
#endif /* _LINUX_VIRTIO_IDS_H */
diff --git a/include/uapi/linux/virtio_pmem.h b/include/uapi/linux/virtio_pmem.h
new file mode 100644
index 000000000000..fa3f7d52717a
--- /dev/null
+++ b/include/uapi/linux/virtio_pmem.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _UAPI_LINUX_VIRTIO_PMEM_H
+#define _UAPI_LINUX_VIRTIO_PMEM_H
+
+struct virtio_pmem_config {
+ __le64 start;
+ __le64 size;
+};
+#endif
--
2.20.1
^ permalink raw reply related
* [PATCH v7 1/6] libnvdimm: nd_region flush callback support
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
In-Reply-To: <20190426050039.17460-1-pagupta@redhat.com>
This patch adds functionality to perform flush from guest
to host over VIRTIO. We are registering a callback based
on 'nd_region' type. virtio_pmem driver requires this special
flush function. For rest of the region types we are registering
existing flush function. Report error returned by host fsync
failure to userspace.
Signed-off-by: Pankaj Gupta <pagupta@redhat.com>
---
drivers/acpi/nfit/core.c | 4 ++--
drivers/nvdimm/claim.c | 6 ++++--
drivers/nvdimm/nd.h | 1 +
drivers/nvdimm/pmem.c | 13 ++++++++-----
drivers/nvdimm/region_devs.c | 26 ++++++++++++++++++++++++--
include/linux/libnvdimm.h | 8 +++++++-
6 files changed, 46 insertions(+), 12 deletions(-)
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 5a389a4f4f65..08dde76cf459 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -2434,7 +2434,7 @@ static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
offset = to_interleave_offset(offset, mmio);
writeq(cmd, mmio->addr.base + offset);
- nvdimm_flush(nfit_blk->nd_region);
+ nvdimm_flush(nfit_blk->nd_region, NULL);
if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
readq(mmio->addr.base + offset);
@@ -2483,7 +2483,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
}
if (rw)
- nvdimm_flush(nfit_blk->nd_region);
+ nvdimm_flush(nfit_blk->nd_region, NULL);
rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
return rc;
diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c
index fb667bf469c7..13510bae1e6f 100644
--- a/drivers/nvdimm/claim.c
+++ b/drivers/nvdimm/claim.c
@@ -263,7 +263,7 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns,
struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
unsigned int sz_align = ALIGN(size + (offset & (512 - 1)), 512);
sector_t sector = offset >> 9;
- int rc = 0;
+ int rc = 0, ret = 0;
if (unlikely(!size))
return 0;
@@ -301,7 +301,9 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns,
}
memcpy_flushcache(nsio->addr + offset, buf, size);
- nvdimm_flush(to_nd_region(ndns->dev.parent));
+ ret = nvdimm_flush(to_nd_region(ndns->dev.parent), NULL);
+ if (ret)
+ rc = ret;
return rc;
}
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index a5ac3b240293..0c74d2428bd7 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -159,6 +159,7 @@ struct nd_region {
struct badblocks bb;
struct nd_interleave_set *nd_set;
struct nd_percpu_lane __percpu *lane;
+ int (*flush)(struct nd_region *nd_region, struct bio *bio);
struct nd_mapping mapping[0];
};
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index bc2f700feef8..f719245da170 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -192,6 +192,7 @@ static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page,
static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
{
+ int ret = 0;
blk_status_t rc = 0;
bool do_acct;
unsigned long start;
@@ -201,7 +202,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
struct nd_region *nd_region = to_region(pmem);
if (bio->bi_opf & REQ_PREFLUSH)
- nvdimm_flush(nd_region);
+ ret = nvdimm_flush(nd_region, bio);
do_acct = nd_iostat_start(bio, &start);
bio_for_each_segment(bvec, bio, iter) {
@@ -216,7 +217,10 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
nd_iostat_end(bio, start);
if (bio->bi_opf & REQ_FUA)
- nvdimm_flush(nd_region);
+ ret = nvdimm_flush(nd_region, bio);
+
+ if (ret)
+ bio->bi_status = errno_to_blk_status(ret);
bio_endio(bio);
return BLK_QC_T_NONE;
@@ -469,7 +473,6 @@ static int pmem_attach_disk(struct device *dev,
}
dax_write_cache(dax_dev, nvdimm_has_cache(nd_region));
pmem->dax_dev = dax_dev;
-
gendev = disk_to_dev(disk);
gendev->groups = pmem_attribute_groups;
@@ -527,14 +530,14 @@ static int nd_pmem_remove(struct device *dev)
sysfs_put(pmem->bb_state);
pmem->bb_state = NULL;
}
- nvdimm_flush(to_nd_region(dev->parent));
+ nvdimm_flush(to_nd_region(dev->parent), NULL);
return 0;
}
static void nd_pmem_shutdown(struct device *dev)
{
- nvdimm_flush(to_nd_region(dev->parent));
+ nvdimm_flush(to_nd_region(dev->parent), NULL);
}
static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index b4ef7d9ff22e..e5b59708865e 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -295,7 +295,9 @@ static ssize_t deep_flush_store(struct device *dev, struct device_attribute *att
return rc;
if (!flush)
return -EINVAL;
- nvdimm_flush(nd_region);
+ rc = nvdimm_flush(nd_region, NULL);
+ if (rc)
+ return rc;
return len;
}
@@ -1085,6 +1087,11 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
dev->of_node = ndr_desc->of_node;
nd_region->ndr_size = resource_size(ndr_desc->res);
nd_region->ndr_start = ndr_desc->res->start;
+ if (ndr_desc->flush)
+ nd_region->flush = ndr_desc->flush;
+ else
+ nd_region->flush = NULL;
+
nd_device_register(dev);
return nd_region;
@@ -1125,11 +1132,24 @@ struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus,
}
EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
+int nvdimm_flush(struct nd_region *nd_region, struct bio *bio)
+{
+ int rc = 0;
+
+ if (!nd_region->flush)
+ rc = generic_nvdimm_flush(nd_region);
+ else {
+ if (nd_region->flush(nd_region, bio))
+ rc = -EIO;
+ }
+
+ return rc;
+}
/**
* nvdimm_flush - flush any posted write queues between the cpu and pmem media
* @nd_region: blk or interleaved pmem region
*/
-void nvdimm_flush(struct nd_region *nd_region)
+int generic_nvdimm_flush(struct nd_region *nd_region)
{
struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
int i, idx;
@@ -1153,6 +1173,8 @@ void nvdimm_flush(struct nd_region *nd_region)
if (ndrd_get_flush_wpq(ndrd, i, 0))
writeq(1, ndrd_get_flush_wpq(ndrd, i, idx));
wmb();
+
+ return 0;
}
EXPORT_SYMBOL_GPL(nvdimm_flush);
diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h
index feb342d026f2..a5f369ec3726 100644
--- a/include/linux/libnvdimm.h
+++ b/include/linux/libnvdimm.h
@@ -65,6 +65,9 @@ enum {
*/
ND_REGION_PERSIST_MEMCTRL = 2,
+ /* Platform provides asynchronous flush mechanism */
+ ND_REGION_ASYNC = 3,
+
/* mark newly adjusted resources as requiring a label update */
DPA_RESOURCE_ADJUSTED = 1 << 0,
};
@@ -121,6 +124,7 @@ struct nd_mapping_desc {
int position;
};
+struct nd_region;
struct nd_region_desc {
struct resource *res;
struct nd_mapping_desc *mapping;
@@ -133,6 +137,7 @@ struct nd_region_desc {
int target_node;
unsigned long flags;
struct device_node *of_node;
+ int (*flush)(struct nd_region *nd_region, struct bio *bio);
};
struct device;
@@ -260,7 +265,8 @@ unsigned long nd_blk_memremap_flags(struct nd_blk_region *ndbr);
unsigned int nd_region_acquire_lane(struct nd_region *nd_region);
void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane);
u64 nd_fletcher64(void *addr, size_t len, bool le);
-void nvdimm_flush(struct nd_region *nd_region);
+int nvdimm_flush(struct nd_region *nd_region, struct bio *bio);
+int generic_nvdimm_flush(struct nd_region *nd_region);
int nvdimm_has_flush(struct nd_region *nd_region);
int nvdimm_has_cache(struct nd_region *nd_region);
int nvdimm_in_overwrite(struct nvdimm *nvdimm);
--
2.20.1
^ permalink raw reply related
* [PATCH v7 0/6] virtio pmem driver
From: Pankaj Gupta @ 2019-04-26 5:00 UTC (permalink / raw)
To: linux-nvdimm, linux-kernel, virtualization, kvm, linux-fsdevel,
linux-acpi, qemu-devel, linux-ext4, linux-xfs
Cc: pagupta, jack, mst, david, lcapitulino, adilger.kernel, zwisler,
aarcange, dave.jiang, darrick.wong, vishal.l.verma, willy, hch,
jmoyer, nilal, lenb, kilobyte, riel, yuval.shaia, stefanha,
pbonzini, dan.j.williams, tytso, xiaoguangrong.eric, cohuck, rjw,
imammedo
This patch series has implementation for "virtio pmem".
"virtio pmem" is fake persistent memory(nvdimm) in guest
which allows to bypass the guest page cache. This also
implements a VIRTIO based asynchronous flush mechanism.
Sharing guest kernel driver in this patchset with the
changes suggested in v4. Tested with Qemu side device
emulation [6] for virtio-pmem. Documented the impact of
possible page cache side channel attacks with suggested
countermeasures.
Incorporated all the review suggestions.
Details of project idea for 'virtio pmem' flushing interface
is shared [3] & [4].
Implementation is divided into two parts:
New virtio pmem guest driver and qemu code changes for new
virtio pmem paravirtualized device.
1. Guest virtio-pmem kernel driver
---------------------------------
- Reads persistent memory range from paravirt device and
registers with 'nvdimm_bus'.
- 'nvdimm/pmem' driver uses this information to allocate
persistent memory region and setup filesystem operations
to the allocated memory.
- virtio pmem driver implements asynchronous flushing
interface to flush from guest to host.
2. Qemu virtio-pmem device
---------------------------------
- Creates virtio pmem device and exposes a memory range to
KVM guest.
- At host side this is file backed memory which acts as
persistent memory.
- Qemu side flush uses aio thread pool API's and virtio
for asynchronous guest multi request handling.
David Hildenbrand CCed also posted a modified version[7] of
qemu virtio-pmem code based on updated Qemu memory device API.
Virtio-pmem security implications and countermeasures:
-----------------------------------------------------
In previous posting of kernel driver, there was discussion [9]
on possible implications of page cache side channel attacks with
virtio pmem. After thorough analysis of details of known side
channel attacks, below are the suggestions:
- Depends entirely on how host backing image file is mapped
into guest address space.
- virtio-pmem device emulation, by default shared mapping is used
to map host backing file. It is recommended to use separate
backing file at host side for every guest. This will prevent
any possibility of executing common code from multiple guests
and any chance of inferring guest local data based based on
execution time.
- If backing file is required to be shared among multiple guests
it is recommended to don't support host page cache eviction
commands from the guest driver. This will avoid any possibility
of inferring guest local data or host data from another guest.
- Proposed device specification [8] for virtio-pmem device with
details of possible security implications and suggested
countermeasures for device emulation.
Virtio-pmem errors handling:
----------------------------------------
Checked behaviour of virtio-pmem for below types of errors
Need suggestions on expected behaviour for handling these errors?
- Hardware Errors: Uncorrectable recoverable Errors:
a] virtio-pmem:
- As per current logic if error page belongs to Qemu process,
host MCE handler isolates(hwpoison) that page and send SIGBUS.
Qemu SIGBUS handler injects exception to KVM guest.
- KVM guest then isolates the page and send SIGBUS to guest
userspace process which has mapped the page.
b] Existing implementation for ACPI pmem driver:
- Handles such errors with MCE notifier and creates a list
of bad blocks. Read/direct access DAX operation return EIO
if accessed memory page fall in bad block list.
- It also starts backgound scrubbing.
- Similar functionality can be reused in virtio-pmem with MCE
notifier but without scrubbing(no ACPI/ARS)? Need inputs to
confirm if this behaviour is ok or needs any change?
Changes from PATCH v6: [1]
- Corrected comment format in patch 5 & patch 6. [Dave]
- Changed variable declaration indentation in patch 6 [Darrick]
- Add Reviewed-by tag by 'Jan Kara' in patch 4 & patch 5
Changes from PATCH v5: [2]
Changes suggested in by - [Cornelia, Yuval]
- Remove assignment chaining in virtio driver
- Better error message and remove not required free
- Check nd_region before use
Changes suggested by - [Jan Kara]
- dax_synchronous() for !CONFIG_DAX
- Correct 'daxdev_mapping_supported' comment and non-dax implementation
Changes suggested by - [Dan Williams]
- Pass meaningful flag 'DAXDEV_F_SYNC' to alloc_dax
- Gate nvdimm_flush instead of additional async parameter
- Move block chaining logic to flush callback than common nvdimm_flush
- Use NULL flush callback for generic flush for better readability [Dan, Jan]
- Use virtio device id 27 from 25(already used) - [MST]
Changes from PATCH v4:
- Factor out MAP_SYNC supported functionality to a common helper
[Dave, Darrick, Jan]
- Comment, indentation and virtqueue_kick failure handle - Yuval Shaia
Changes from PATCH v3:
- Use generic dax_synchronous() helper to check for DAXDEV_SYNC
flag - [Dan, Darrick, Jan]
- Add 'is_nvdimm_async' function
- Document page cache side channel attacks implications &
countermeasures - [Dave Chinner, Michael]
Changes from PATCH v2:
- Disable MAP_SYNC for ext4 & XFS filesystems - [Dan]
- Use name 'virtio pmem' in place of 'fake dax'
Changes from PATCH v1:
- 0-day build test for build dependency on libnvdimm
Changes suggested by - [Dan Williams]
- Split the driver into two parts virtio & pmem
- Move queuing of async block request to block layer
- Add "sync" parameter in nvdimm_flush function
- Use indirect call for nvdimm_flush
- Don’t move declarations to common global header e.g nd.h
- nvdimm_flush() return 0 or -EIO if it fails
- Teach nsio_rw_bytes() that the flush can fail
- Rename nvdimm_flush() to generic_nvdimm_flush()
- Use 'nd_region->provider_data' for long dereferencing
- Remove virtio_pmem_freeze/restore functions
- Remove BSD license text with SPDX license text
- Add might_sleep() in virtio_pmem_flush - [Luiz]
- Make spin_lock_irqsave() narrow
Changes from RFC v3
- Rebase to latest upstream - Luiz
- Call ndregion->flush in place of nvdimm_flush- Luiz
- kmalloc return check - Luiz
- virtqueue full handling - Stefan
- Don't map entire virtio_pmem_req to device - Stefan
- request leak, correct sizeof req - Stefan
- Move declaration to virtio_pmem.c
Changes from RFC v2:
- Add flush function in the nd_region in place of switching
on a flag - Dan & Stefan
- Add flush completion function with proper locking and wait
for host side flush completion - Stefan & Dan
- Keep userspace API in uapi header file - Stefan, MST
- Use LE fields & New device id - MST
- Indentation & spacing suggestions - MST & Eric
- Remove extra header files & add licensing - Stefan
Changes from RFC v1:
- Reuse existing 'pmem' code for registering persistent
memory and other operations instead of creating an entirely
new block driver.
- Use VIRTIO driver to register memory information with
nvdimm_bus and create region_type accordingly.
- Call VIRTIO flush from existing pmem driver.
Pankaj Gupta (6):
libnvdimm: nd_region flush callback support
virtio-pmem: Add virtio-pmem guest driver
libnvdimm: add nd_region buffered dax_dev flag
dax: check synchronous mapping is supported
ext4: disable map_sync for virtio pmem
xfs: disable map_sync for virtio pmem
[1] https://lkml.org/lkml/2019/4/23/1092
[2] https://lkml.org/lkml/2019/4/10/3
[3] https://www.spinics.net/lists/kvm/msg149761.html
[4] https://www.spinics.net/lists/kvm/msg153095.html
[5] https://lkml.org/lkml/2018/8/31/413
[6] https://marc.info/?l=linux-kernel&m=153572228719237&w=2
[7] https://marc.info/?l=qemu-devel&m=153555721901824&w=2
[8] https://lists.oasis-open.org/archives/virtio-dev/201903/msg00083.html
[9] https://lkml.org/lkml/2019/1/9/1191
drivers/acpi/nfit/core.c | 4 -
drivers/dax/bus.c | 2
drivers/dax/super.c | 13 +++-
drivers/md/dm.c | 3
drivers/nvdimm/claim.c | 6 +
drivers/nvdimm/nd.h | 1
drivers/nvdimm/pmem.c | 16 +++--
drivers/nvdimm/region_devs.c | 33 ++++++++++
drivers/nvdimm/virtio_pmem.c | 114 +++++++++++++++++++++++++++++++++++++
drivers/virtio/Kconfig | 10 +++
drivers/virtio/Makefile | 1
drivers/virtio/pmem.c | 118 +++++++++++++++++++++++++++++++++++++++
fs/ext4/file.c | 10 +--
fs/xfs/xfs_file.c | 9 +-
include/linux/dax.h | 25 +++++++-
include/linux/libnvdimm.h | 9 ++
include/linux/virtio_pmem.h | 60 +++++++++++++++++++
include/uapi/linux/virtio_ids.h | 1
include/uapi/linux/virtio_pmem.h | 10 +++
19 files changed, 420 insertions(+), 25 deletions(-)
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply
* Re: [PATCH v3 1/4] mm/balloon_compaction: list interfaces
From: Nadav Amit via Virtualization @ 2019-04-25 19:08 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: Arnd Bergmann, Pv-drivers, Greg Kroah-Hartman, LKML,
virtualization@lists.linux-foundation.org, linux-mm@kvack.org,
Julien Freche
In-Reply-To: <20190424092829-mutt-send-email-mst@kernel.org>
> On Apr 24, 2019, at 6:49 AM, Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Tue, Apr 23, 2019 at 04:45:28PM -0700, Nadav Amit wrote:
>> Introduce interfaces for ballooning enqueueing and dequeueing of a list
>> of pages. These interfaces reduce the overhead of storing and restoring
>> IRQs by batching the operations. In addition they do not panic if the
>> list of pages is empty.
>>
>> Cc: "Michael S. Tsirkin" <mst@redhat.com>
>> Cc: Jason Wang <jasowang@redhat.com>
>> Cc: linux-mm@kvack.org
>> Cc: virtualization@lists.linux-foundation.org
>> Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
>> Signed-off-by: Nadav Amit <namit@vmware.com>
>
>
> Looks good overall. Two minor comments below.
>
>
>> ---
>> include/linux/balloon_compaction.h | 4 +
>> mm/balloon_compaction.c | 144 +++++++++++++++++++++--------
>> 2 files changed, 110 insertions(+), 38 deletions(-)
>>
>> diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h
>> index f111c780ef1d..430b6047cef7 100644
>> --- a/include/linux/balloon_compaction.h
>> +++ b/include/linux/balloon_compaction.h
>> @@ -64,6 +64,10 @@ extern struct page *balloon_page_alloc(void);
>> extern void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
>> struct page *page);
>> extern struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info);
>> +extern size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
>> + struct list_head *pages);
>> +extern size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
>> + struct list_head *pages, size_t n_req_pages);
>>
>> static inline void balloon_devinfo_init(struct balloon_dev_info *balloon)
>> {
>> diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
>> index ef858d547e2d..a2995002edc2 100644
>> --- a/mm/balloon_compaction.c
>> +++ b/mm/balloon_compaction.c
>> @@ -10,6 +10,105 @@
>> #include <linux/export.h>
>> #include <linux/balloon_compaction.h>
>>
>> +static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
>> + struct page *page)
>> +{
>> + /*
>> + * Block others from accessing the 'page' when we get around to
>> + * establishing additional references. We should be the only one
>> + * holding a reference to the 'page' at this point. If we are not, then
>> + * memory corruption is possible and we should stop execution.
>> + */
>> + BUG_ON(!trylock_page(page));
>> + list_del(&page->lru);
>> + balloon_page_insert(b_dev_info, page);
>> + unlock_page(page);
>> + __count_vm_event(BALLOON_INFLATE);
>> +}
>> +
>> +/**
>> + * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
>> + * list.
>> + * @b_dev_info: balloon device descriptor where we will insert a new page to
>> + * @pages: pages to enqueue - allocated using balloon_page_alloc.
>> + *
>> + * Driver must call it to properly enqueue a balloon pages before definitively
>> + * removing it from the guest system.
>> + *
>> + * Return: number of pages that were enqueued.
>> + */
>> +size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
>> + struct list_head *pages)
>> +{
>> + struct page *page, *tmp;
>> + unsigned long flags;
>> + size_t n_pages = 0;
>> +
>> + spin_lock_irqsave(&b_dev_info->pages_lock, flags);
>> + list_for_each_entry_safe(page, tmp, pages, lru) {
>> + balloon_page_enqueue_one(b_dev_info, page);
>> + n_pages++;
>> + }
>> + spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
>> + return n_pages;
>> +}
>> +EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
>> +
>> +/**
>> + * balloon_page_list_dequeue() - removes pages from balloon's page list and
>> + * returns a list of the pages.
>> + * @b_dev_info: balloon device decriptor where we will grab a page from.
>> + * @pages: pointer to the list of pages that would be returned to the caller.
>> + * @n_req_pages: number of requested pages.
>> + *
>> + * Driver must call this function to properly de-allocate a previous enlisted
>> + * balloon pages before definetively releasing it back to the guest system.
>> + * This function tries to remove @n_req_pages from the ballooned pages and
>> + * return them to the caller in the @pages list.
>> + *
>> + * Note that this function may fail to dequeue some pages temporarily empty due
>> + * to compaction isolated pages.
>> + *
>> + * Return: number of pages that were added to the @pages list.
>> + */
>> +size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
>> + struct list_head *pages, size_t n_req_pages)
>> +{
>> + struct page *page, *tmp;
>> + unsigned long flags;
>> + size_t n_pages = 0;
>> +
>> + spin_lock_irqsave(&b_dev_info->pages_lock, flags);
>> + list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
>> + if (n_pages == n_req_pages)
>> + break;
>> +
>> + /*
>> + * Block others from accessing the 'page' while we get around
>
> should be "get around to" - same as in other places
>
>
>> + * establishing additional references and preparing the 'page'
>> + * to be released by the balloon driver.
>> + */
>> + if (!trylock_page(page))
>> + continue;
>> +
>> + if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
>> + PageIsolated(page)) {
>> + /* raced with isolation */
>> + unlock_page(page);
>> + continue;
>> + }
>> + balloon_page_delete(page);
>> + __count_vm_event(BALLOON_DEFLATE);
>> + unlock_page(page);
>> + list_add(&page->lru, pages);
>
> I'm not sure whether this list_add must be under the page lock,
> but enqueue does list_del under page lock, so I think it's
> a good idea to keep dequeue consistent, operating in the
> reverse order of enqueue.
>
>> + ++n_pages;
>> + }
>> + spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
>> +
>> + return n_pages;
>> +}
>> +EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
>> +
>> /*
>> * balloon_page_alloc - allocates a new page for insertion into the balloon
>> * page list.
>> @@ -43,17 +142,9 @@ void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
>> {
>> unsigned long flags;
>>
>> - /*
>> - * Block others from accessing the 'page' when we get around to
>> - * establishing additional references. We should be the only one
>> - * holding a reference to the 'page' at this point.
>> - */
>> - BUG_ON(!trylock_page(page));
>> spin_lock_irqsave(&b_dev_info->pages_lock, flags);
>> - balloon_page_insert(b_dev_info, page);
>> - __count_vm_event(BALLOON_INFLATE);
>> + balloon_page_enqueue_one(b_dev_info, page);
>> spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
>> - unlock_page(page);
>> }
>> EXPORT_SYMBOL_GPL(balloon_page_enqueue);
>>
>> @@ -70,36 +161,13 @@ EXPORT_SYMBOL_GPL(balloon_page_enqueue);
>> */
>> struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
>> {
>> - struct page *page, *tmp;
>> unsigned long flags;
>> - bool dequeued_page;
>> + LIST_HEAD(pages);
>> + int n_pages;
>>
>> - dequeued_page = false;
>> - spin_lock_irqsave(&b_dev_info->pages_lock, flags);
>> - list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
>> - /*
>> - * Block others from accessing the 'page' while we get around
>> - * establishing additional references and preparing the 'page'
>> - * to be released by the balloon driver.
>> - */
>> - if (trylock_page(page)) {
>> -#ifdef CONFIG_BALLOON_COMPACTION
>> - if (PageIsolated(page)) {
>> - /* raced with isolation */
>> - unlock_page(page);
>> - continue;
>> - }
>> -#endif
>> - balloon_page_delete(page);
>> - __count_vm_event(BALLOON_DEFLATE);
>> - unlock_page(page);
>> - dequeued_page = true;
>> - break;
>> - }
>> - }
>> - spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
>> + n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
>>
>> - if (!dequeued_page) {
>> + if (n_pages != 1) {
>> /*
>> * If we are unable to dequeue a balloon page because the page
>> * list is empty and there is no isolated pages, then something
>> @@ -112,9 +180,9 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
>> !b_dev_info->isolated_pages))
>> BUG();
>> spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
>> - page = NULL;
>> + return NULL;
>> }
>> - return page;
>> + return list_first_entry(&pages, struct page, lru);
>> }
>> EXPORT_SYMBOL_GPL(balloon_page_dequeue);
>>
>> --
>> 2.19.1
>
>
> With above addressed:
>
> Acked-by: Michael S. Tsirkin <mst@redhat.com>
Thank you, Michael.
I addressed your feedback and I will send another version shortly.
^ permalink raw reply
* Re: [PATCH net] vhost_net: fix possible infinite loop
From: Michael S. Tsirkin @ 2019-04-25 17:52 UTC (permalink / raw)
To: Jason Wang; +Cc: netdev, ppandit, linux-kernel, kvm, virtualization
In-Reply-To: <1556177599-56248-1-git-send-email-jasowang@redhat.com>
On Thu, Apr 25, 2019 at 03:33:19AM -0400, Jason Wang wrote:
> When the rx buffer is too small for a packet, we will discard the vq
> descriptor and retry it for the next packet:
>
> while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
> &busyloop_intr))) {
> ...
> /* On overrun, truncate and discard */
> if (unlikely(headcount > UIO_MAXIOV)) {
> iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
> err = sock->ops->recvmsg(sock, &msg,
> 1, MSG_DONTWAIT | MSG_TRUNC);
> pr_debug("Discarded rx packet: len %zd\n", sock_len);
> continue;
> }
> ...
> }
>
> This makes it possible to trigger a infinite while..continue loop
> through the co-opreation of two VMs like:
>
> 1) Malicious VM1 allocate 1 byte rx buffer and try to slow down the
> vhost process as much as possible e.g using indirect descriptors or
> other.
> 2) Malicious VM2 generate packets to VM1 as fast as possible
>
> Fixing this by checking against weight at the end of RX and TX
> loop. This also eliminate other similar cases when:
>
> - userspace is consuming the packets in the meanwhile
> - theoretical TOCTOU attack if guest moving avail index back and forth
> to hit the continue after vhost find guest just add new buffers
>
> This addresses CVE-2019-3900.
>
> Fixes: d8316f3991d20 ("vhost: fix total length when packets are too short")
I agree this is the real issue.
> Fixes: 3a4d5c94e9593 ("vhost_net: a kernel-level virtio server")
This is just a red herring imho. We can stick this on any vhost patch :)
> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
> drivers/vhost/net.c | 41 +++++++++++++++++++++--------------------
> 1 file changed, 21 insertions(+), 20 deletions(-)
>
> diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
> index df51a35..fb46e6b 100644
> --- a/drivers/vhost/net.c
> +++ b/drivers/vhost/net.c
> @@ -778,8 +778,9 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
> int err;
> int sent_pkts = 0;
> bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);
> + bool next_round = false;
>
> - for (;;) {
> + do {
> bool busyloop_intr = false;
>
> if (nvq->done_idx == VHOST_NET_BATCH)
> @@ -845,11 +846,10 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
> vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
> vq->heads[nvq->done_idx].len = 0;
> ++nvq->done_idx;
> - if (vhost_exceeds_weight(++sent_pkts, total_len)) {
> - vhost_poll_queue(&vq->poll);
> - break;
> - }
> - }
> + } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
> +
> + if (next_round)
> + vhost_poll_queue(&vq->poll);
>
> vhost_tx_batch(net, nvq, sock, &msg);
> }
> @@ -873,8 +873,9 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
> struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
> bool zcopy_used;
> int sent_pkts = 0;
> + bool next_round = false;
>
> - for (;;) {
> + do {
> bool busyloop_intr;
>
> /* Release DMAs done buffers first */
> @@ -951,11 +952,10 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
> else
> vhost_zerocopy_signal_used(net, vq);
> vhost_net_tx_packet(net);
> - if (unlikely(vhost_exceeds_weight(++sent_pkts, total_len))) {
> - vhost_poll_queue(&vq->poll);
> - break;
> - }
> - }
> + } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
> +
> + if (next_round)
> + vhost_poll_queue(&vq->poll);
> }
>
> /* Expects to be always run from workqueue - which acts as
> @@ -1134,6 +1134,7 @@ static void handle_rx(struct vhost_net *net)
> struct iov_iter fixup;
> __virtio16 num_buffers;
> int recv_pkts = 0;
> + bool next_round = false;
>
> mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX);
> sock = vq->private_data;
> @@ -1153,8 +1154,11 @@ static void handle_rx(struct vhost_net *net)
> vq->log : NULL;
> mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
>
> - while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
> - &busyloop_intr))) {
> + do {
> + sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
> + &busyloop_intr);
> + if (!sock_len)
> + break;
> sock_len += sock_hlen;
> vhost_len = sock_len + vhost_hlen;
> headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
> @@ -1239,12 +1243,9 @@ static void handle_rx(struct vhost_net *net)
> vhost_log_write(vq, vq_log, log, vhost_len,
> vq->iov, in);
> total_len += vhost_len;
> - if (unlikely(vhost_exceeds_weight(++recv_pkts, total_len))) {
> - vhost_poll_queue(&vq->poll);
> - goto out;
> - }
> - }
> - if (unlikely(busyloop_intr))
> + } while (!(next_round = vhost_exceeds_weight(++recv_pkts, total_len)));
> +
> + if (unlikely(busyloop_intr || next_round))
> vhost_poll_queue(&vq->poll);
> else
> vhost_net_enable_vq(net, vq);
I'm afraid with this addition the code is too much like spagetty. What
does next_round mean? Just that we are breaking out of loop? That is
what goto is for... Either let's have for(;;) with goto/break to get
outside or a while loop with a condition. Both is just unreadable.
All these checks in 3 places are exactly the same on all paths and they
are slow path. Why don't we put this in a function? E.g. like the below.
Warning: completely untested.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index df51a35cf537..a0f89a504cd9 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -761,6 +761,23 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
return 0;
}
+/* Returns true if caller needs to go back and re-read the ring. */
+static bool empty_ring(struct vhost_net *net, struct vhost_virtqueue *vq,
+ int pkts, size_t total_len, bool busyloop_intr)
+{
+ if (unlikely(busyloop_intr)) {
+ vhost_poll_queue(&vq->poll);
+ } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
+ /* They have slipped one in meanwhile: check again. */
+ vhost_disable_notify(&net->dev, vq);
+ if (!vhost_exceeds_weight(pkts, total_len))
+ return true;
+ vhost_poll_queue(&vq->poll);
+ }
+ /* Nothing new? Wait for eventfd to tell us they refilled. */
+ return false;
+}
+
static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
{
struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
@@ -790,15 +807,10 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
/* On error, stop handling until the next kick. */
if (unlikely(head < 0))
break;
- /* Nothing new? Wait for eventfd to tell us they refilled. */
if (head == vq->num) {
- if (unlikely(busyloop_intr)) {
- vhost_poll_queue(&vq->poll);
- } else if (unlikely(vhost_enable_notify(&net->dev,
- vq))) {
- vhost_disable_notify(&net->dev, vq);
+ if (unlikely(empty_ring(net, vq, ++sent_pkts,
+ total_len, busyloop_intr)))
continue;
- }
break;
}
@@ -886,14 +898,10 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
/* On error, stop handling until the next kick. */
if (unlikely(head < 0))
break;
- /* Nothing new? Wait for eventfd to tell us they refilled. */
if (head == vq->num) {
- if (unlikely(busyloop_intr)) {
- vhost_poll_queue(&vq->poll);
- } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
- vhost_disable_notify(&net->dev, vq);
+ if (unlikely(empty_ring(net, vq, ++sent_pkts,
+ total_len, busyloop_intr)))
continue;
- }
break;
}
@@ -1163,18 +1171,10 @@ static void handle_rx(struct vhost_net *net)
/* On error, stop handling until the next kick. */
if (unlikely(headcount < 0))
goto out;
- /* OK, now we need to know about added descriptors. */
if (!headcount) {
- if (unlikely(busyloop_intr)) {
- vhost_poll_queue(&vq->poll);
- } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
- /* They have slipped one in as we were
- * doing that: check again. */
- vhost_disable_notify(&net->dev, vq);
- continue;
- }
- /* Nothing new? Wait for eventfd to tell us
- * they refilled. */
+ if (unlikely(empty_ring(net, vq, ++recv_pkts,
+ total_len, busyloop_intr)))
+ continue;
goto out;
}
busyloop_intr = false;
> --
> 1.8.3.1
^ permalink raw reply related
* [PATCH v4 4/4] vmw_balloon: Split refused pages
From: Nadav Amit via Virtualization @ 2019-04-25 11:54 UTC (permalink / raw)
To: Greg Kroah-Hartman, Michael S. Tsirkin
Cc: Arnd Bergmann, VMware, Inc., linux-kernel, virtualization,
linux-mm, Julien Freche, Nadav Amit
In-Reply-To: <20190425115445.20815-1-namit@vmware.com>
The hypervisor might refuse to inflate pages. While the balloon driver
handles this scenario correctly, a refusal to inflate a 2MB pages might
cause the same page to be allocated again later just for its inflation
to be refused again. This wastes energy and time.
To avoid this situation, split the 2MB page to 4KB pages, and then try
to inflate each one individually. Most of the 4KB pages out of the 2MB
should be inflated successfully, and the balloon is likely to prevent
the scenario of repeated refused inflation.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 63 +++++++++++++++++++++++++++++++-------
1 file changed, 52 insertions(+), 11 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 4b5e939ff4c8..043eed845246 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -239,6 +239,7 @@ static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
struct vmballoon_ctl {
struct list_head pages;
struct list_head refused_pages;
+ struct list_head prealloc_pages;
unsigned int n_refused_pages;
unsigned int n_pages;
enum vmballoon_page_size_type page_size;
@@ -668,15 +669,25 @@ static int vmballoon_alloc_page_list(struct vmballoon *b,
unsigned int i;
for (i = 0; i < req_n_pages; i++) {
- if (ctl->page_size == VMW_BALLOON_2M_PAGE)
- page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
+ /*
+ * First check if we happen to have pages that were allocated
+ * before. This happens when 2MB page rejected during inflation
+ * by the hypervisor, and then split into 4KB pages.
+ */
+ if (!list_empty(&ctl->prealloc_pages)) {
+ page = list_first_entry(&ctl->prealloc_pages,
+ struct page, lru);
+ list_del(&page->lru);
+ } else {
+ if (ctl->page_size == VMW_BALLOON_2M_PAGE)
+ page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
__GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
- else
- page = balloon_page_alloc();
+ else
+ page = balloon_page_alloc();
- /* Update statistics */
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
- ctl->page_size);
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
+ ctl->page_size);
+ }
if (page) {
vmballoon_mark_page_offline(page, ctl->page_size);
@@ -922,7 +933,8 @@ static void vmballoon_release_page_list(struct list_head *page_list,
__free_pages(page, vmballoon_page_order(page_size));
}
- *n_pages = 0;
+ if (n_pages)
+ *n_pages = 0;
}
@@ -1054,6 +1066,32 @@ static void vmballoon_dequeue_page_list(struct vmballoon *b,
*n_pages = i;
}
+/**
+ * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.
+ *
+ * If inflation of 2MB pages was denied by the hypervisor, it is likely to be
+ * due to one or few 4KB pages. These 2MB pages may keep being allocated and
+ * then being refused. To prevent this case, this function splits the refused
+ * pages into 4KB pages and adds them into @prealloc_pages list.
+ *
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ */
+static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl)
+{
+ struct page *page, *tmp;
+ unsigned int i, order;
+
+ order = vmballoon_page_order(ctl->page_size);
+
+ list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {
+ list_del(&page->lru);
+ split_page(page, order);
+ for (i = 0; i < (1 << order); i++)
+ list_add(&page[i].lru, &ctl->prealloc_pages);
+ }
+ ctl->n_refused_pages = 0;
+}
+
/**
* vmballoon_inflate() - Inflate the balloon towards its target size.
*
@@ -1065,6 +1103,7 @@ static void vmballoon_inflate(struct vmballoon *b)
struct vmballoon_ctl ctl = {
.pages = LIST_HEAD_INIT(ctl.pages),
.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages),
.page_size = b->max_page_size,
.op = VMW_BALLOON_INFLATE
};
@@ -1112,10 +1151,10 @@ static void vmballoon_inflate(struct vmballoon *b)
break;
/*
- * Ignore errors from locking as we now switch to 4k
- * pages and we might get different errors.
+ * Split the refused pages to 4k. This will also empty
+ * the refused pages list.
*/
- vmballoon_release_refused_pages(b, &ctl);
+ vmballoon_split_refused_pages(&ctl);
ctl.page_size--;
}
@@ -1129,6 +1168,8 @@ static void vmballoon_inflate(struct vmballoon *b)
*/
if (ctl.n_refused_pages != 0)
vmballoon_release_refused_pages(b, &ctl);
+
+ vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size);
}
/**
--
2.19.1
^ permalink raw reply related
* [PATCH v4 3/4] vmw_balloon: Add memory shrinker
From: Nadav Amit via Virtualization @ 2019-04-25 11:54 UTC (permalink / raw)
To: Greg Kroah-Hartman, Michael S. Tsirkin
Cc: Arnd Bergmann, VMware, Inc., linux-kernel, virtualization,
linux-mm, Julien Freche, Nadav Amit
In-Reply-To: <20190425115445.20815-1-namit@vmware.com>
Add a shrinker to the VMware balloon to prevent out-of-memory events.
We reuse the deflate logic for this matter. Deadlocks should not happen,
as no memory allocation is performed while the locks of the
communication (batch/page) and page-list are taken. In the unlikely
event in which the configuration semaphore is taken for write we bail
out and fail gracefully (causing processes to be killed).
Once the shrinker is called, inflation is postponed for few seconds.
The timeout is updated without any lock, but this should not cause any
races, as it is written and read atomically.
This feature is disabled by default, since it might cause performance
degradation.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/vmw_balloon.c | 133 ++++++++++++++++++++++++++++++++++++-
1 file changed, 131 insertions(+), 2 deletions(-)
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index 2136f6ad97d3..4b5e939ff4c8 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -40,6 +40,15 @@ MODULE_ALIAS("dmi:*:svnVMware*:*");
MODULE_ALIAS("vmware_vmmemctl");
MODULE_LICENSE("GPL");
+static bool __read_mostly vmwballoon_shrinker_enable;
+module_param(vmwballoon_shrinker_enable, bool, 0444);
+MODULE_PARM_DESC(vmwballoon_shrinker_enable,
+ "Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance.");
+
+/* Delay in seconds after shrink before inflation. */
+#define VMBALLOON_SHRINK_DELAY (5)
+
+/* Maximum number of refused pages we accumulate during inflation cycle */
#define VMW_BALLOON_MAX_REFUSED 16
/* Magic number for the balloon mount-point */
@@ -217,12 +226,13 @@ enum vmballoon_stat_general {
VMW_BALLOON_STAT_TIMER,
VMW_BALLOON_STAT_DOORBELL,
VMW_BALLOON_STAT_RESET,
- VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET
+ VMW_BALLOON_STAT_SHRINK,
+ VMW_BALLOON_STAT_SHRINK_FREE,
+ VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE
};
#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
-
static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
@@ -321,6 +331,15 @@ struct vmballoon {
*/
struct page *page;
+ /**
+ * @shrink_timeout: timeout until the next inflation.
+ *
+ * After an shrink event, indicates the time in jiffies after which
+ * inflation is allowed again. Can be written concurrently with reads,
+ * so must use READ_ONCE/WRITE_ONCE when accessing.
+ */
+ unsigned long shrink_timeout;
+
/* statistics */
struct vmballoon_stats *stats;
@@ -361,6 +380,20 @@ struct vmballoon {
* Lock ordering: @conf_sem -> @comm_lock .
*/
spinlock_t comm_lock;
+
+ /**
+ * @shrinker: shrinker interface that is used to avoid over-inflation.
+ */
+ struct shrinker shrinker;
+
+ /**
+ * @shrinker_registered: whether the shrinker was registered.
+ *
+ * The shrinker interface does not handle gracefully the removal of
+ * shrinker that was not registered before. This indication allows to
+ * simplify the unregistration process.
+ */
+ bool shrinker_registered;
};
static struct vmballoon balloon;
@@ -935,6 +968,10 @@ static int64_t vmballoon_change(struct vmballoon *b)
size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
return 0;
+ /* If an out-of-memory recently occurred, inflation is disallowed. */
+ if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))
+ return 0;
+
return target - size;
}
@@ -1430,6 +1467,90 @@ static void vmballoon_work(struct work_struct *work)
}
+/**
+ * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.
+ * @shrinker: pointer to the balloon shrinker.
+ * @sc: page reclaim information.
+ *
+ * Returns: number of pages that were freed during deflation.
+ */
+static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker,
+ struct shrink_control *sc)
+{
+ struct vmballoon *b = &balloon;
+ unsigned long deflated_frames;
+
+ pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));
+
+ vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);
+
+ /*
+ * If the lock is also contended for read, we cannot easily reclaim and
+ * we bail out.
+ */
+ if (!down_read_trylock(&b->conf_sem))
+ return 0;
+
+ deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);
+
+ vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,
+ deflated_frames);
+
+ /*
+ * Delay future inflation for some time to mitigate the situations in
+ * which balloon continuously grows and shrinks. Use WRITE_ONCE() since
+ * the access is asynchronous.
+ */
+ WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);
+
+ up_read(&b->conf_sem);
+
+ return deflated_frames;
+}
+
+/**
+ * vmballoon_shrinker_count() - return the number of ballooned pages.
+ * @shrinker: pointer to the balloon shrinker.
+ * @sc: page reclaim information.
+ *
+ * Returns: number of 4k pages that are allocated for the balloon and can
+ * therefore be reclaimed under pressure.
+ */
+static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker,
+ struct shrink_control *sc)
+{
+ struct vmballoon *b = &balloon;
+
+ return atomic64_read(&b->size);
+}
+
+static void vmballoon_unregister_shrinker(struct vmballoon *b)
+{
+ if (b->shrinker_registered)
+ unregister_shrinker(&b->shrinker);
+ b->shrinker_registered = false;
+}
+
+static int vmballoon_register_shrinker(struct vmballoon *b)
+{
+ int r;
+
+ /* Do nothing if the shrinker is not enabled */
+ if (!vmwballoon_shrinker_enable)
+ return 0;
+
+ b->shrinker.scan_objects = vmballoon_shrinker_scan;
+ b->shrinker.count_objects = vmballoon_shrinker_count;
+ b->shrinker.seeks = DEFAULT_SEEKS;
+
+ r = register_shrinker(&b->shrinker);
+
+ if (r == 0)
+ b->shrinker_registered = true;
+
+ return r;
+}
+
/*
* DEBUGFS Interface
*/
@@ -1447,6 +1568,8 @@ static const char * const vmballoon_stat_names[] = {
[VMW_BALLOON_STAT_TIMER] = "timer",
[VMW_BALLOON_STAT_DOORBELL] = "doorbell",
[VMW_BALLOON_STAT_RESET] = "reset",
+ [VMW_BALLOON_STAT_SHRINK] = "shrink",
+ [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree"
};
static int vmballoon_enable_stats(struct vmballoon *b)
@@ -1780,6 +1903,10 @@ static int __init vmballoon_init(void)
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
+ error = vmballoon_register_shrinker(&balloon);
+ if (error)
+ goto fail;
+
error = vmballoon_debugfs_init(&balloon);
if (error)
goto fail;
@@ -1805,6 +1932,7 @@ static int __init vmballoon_init(void)
return 0;
fail:
+ vmballoon_unregister_shrinker(&balloon);
vmballoon_compaction_deinit(&balloon);
return error;
}
@@ -1819,6 +1947,7 @@ late_initcall(vmballoon_init);
static void __exit vmballoon_exit(void)
{
+ vmballoon_unregister_shrinker(&balloon);
vmballoon_vmci_cleanup(&balloon);
cancel_delayed_work_sync(&balloon.dwork);
--
2.19.1
^ permalink raw reply related
* [PATCH v4 2/4] vmw_balloon: Compaction support
From: Nadav Amit via Virtualization @ 2019-04-25 11:54 UTC (permalink / raw)
To: Greg Kroah-Hartman, Michael S. Tsirkin
Cc: Arnd Bergmann, VMware, Inc., linux-kernel, virtualization,
linux-mm, Julien Freche, Nadav Amit
In-Reply-To: <20190425115445.20815-1-namit@vmware.com>
Add support for compaction for VMware balloon. Since unlike the virtio
balloon, we also support huge-pages, which are not going through
compaction, we keep these pages in vmballoon and handle this list
separately. We use the same lock to protect both lists, as this lock is
not supposed to be contended.
Doing so also eliminates the need for the page_size lists. We update the
accounting as needed to reflect inflation, deflation and migration to be
reflected in vmstat.
Since VMware balloon now provides statistics for inflation, deflation
and migration in vmstat, select MEMORY_BALLOON in Kconfig.
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
drivers/misc/Kconfig | 1 +
drivers/misc/vmw_balloon.c | 301 ++++++++++++++++++++++++++++++++-----
2 files changed, 264 insertions(+), 38 deletions(-)
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 42ab8ec92a04..427cf10579b4 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -420,6 +420,7 @@ config SPEAR13XX_PCIE_GADGET
config VMWARE_BALLOON
tristate "VMware Balloon Driver"
depends on VMWARE_VMCI && X86 && HYPERVISOR_GUEST
+ select MEMORY_BALLOON
help
This is VMware physical memory management driver which acts
like a "balloon" that can be inflated to reclaim physical pages
diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c
index ad807d5a3141..2136f6ad97d3 100644
--- a/drivers/misc/vmw_balloon.c
+++ b/drivers/misc/vmw_balloon.c
@@ -28,6 +28,8 @@
#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/mount.h>
+#include <linux/balloon_compaction.h>
#include <linux/vmw_vmci_defs.h>
#include <linux/vmw_vmci_api.h>
#include <asm/hypervisor.h>
@@ -38,25 +40,11 @@ MODULE_ALIAS("dmi:*:svnVMware*:*");
MODULE_ALIAS("vmware_vmmemctl");
MODULE_LICENSE("GPL");
-/*
- * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait
- * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page
- * allocation failure warnings. Disallow access to emergency low-memory pools.
- */
-#define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
- __GFP_NOMEMALLOC)
-
-/*
- * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight
- * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation
- * failure warnings. Disallow access to emergency low-memory pools.
- */
-#define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
- __GFP_NOMEMALLOC|__GFP_NORETRY)
-
-/* Maximum number of refused pages we accumulate during inflation cycle */
#define VMW_BALLOON_MAX_REFUSED 16
+/* Magic number for the balloon mount-point */
+#define BALLOON_VMW_MAGIC 0x0ba11007
+
/*
* Hypervisor communication port definitions.
*/
@@ -247,11 +235,6 @@ struct vmballoon_ctl {
enum vmballoon_op op;
};
-struct vmballoon_page_size {
- /* list of reserved physical pages */
- struct list_head pages;
-};
-
/**
* struct vmballoon_batch_entry - a batch entry for lock or unlock.
*
@@ -266,8 +249,6 @@ struct vmballoon_batch_entry {
} __packed;
struct vmballoon {
- struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
-
/**
* @max_page_size: maximum supported page size for ballooning.
*
@@ -348,8 +329,20 @@ struct vmballoon {
struct dentry *dbg_entry;
#endif
+ /**
+ * @b_dev_info: balloon device information descriptor.
+ */
+ struct balloon_dev_info b_dev_info;
+
struct delayed_work dwork;
+ /**
+ * @huge_pages - list of the inflated 2MB pages.
+ *
+ * Protected by @b_dev_info.pages_lock .
+ */
+ struct list_head huge_pages;
+
/**
* @vmci_doorbell.
*
@@ -643,10 +636,10 @@ static int vmballoon_alloc_page_list(struct vmballoon *b,
for (i = 0; i < req_n_pages; i++) {
if (ctl->page_size == VMW_BALLOON_2M_PAGE)
- page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
- VMW_BALLOON_2M_ORDER);
+ page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
+ __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
else
- page = alloc_page(VMW_PAGE_ALLOC_FLAGS);
+ page = balloon_page_alloc();
/* Update statistics */
vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
@@ -961,9 +954,22 @@ static void vmballoon_enqueue_page_list(struct vmballoon *b,
unsigned int *n_pages,
enum vmballoon_page_size_type page_size)
{
- struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+ unsigned long flags;
+
+ if (page_size == VMW_BALLOON_4K_PAGE) {
+ balloon_page_list_enqueue(&b->b_dev_info, pages);
+ } else {
+ /*
+ * Keep the huge pages in a local list which is not available
+ * for the balloon compaction mechanism.
+ */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+ list_splice_init(pages, &b->huge_pages);
+ __count_vm_events(BALLOON_INFLATE, *n_pages *
+ vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
+ }
- list_splice_init(pages, &page_size_info->pages);
*n_pages = 0;
}
@@ -986,15 +992,28 @@ static void vmballoon_dequeue_page_list(struct vmballoon *b,
enum vmballoon_page_size_type page_size,
unsigned int n_req_pages)
{
- struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
struct page *page, *tmp;
unsigned int i = 0;
+ unsigned long flags;
- list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) {
+ /* In the case of 4k pages, use the compaction infrastructure */
+ if (page_size == VMW_BALLOON_4K_PAGE) {
+ *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,
+ n_req_pages);
+ return;
+ }
+
+ /* 2MB pages */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {
list_move(&page->lru, pages);
if (++i == n_req_pages)
break;
}
+
+ __count_vm_events(BALLOON_DEFLATE,
+ i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
*n_pages = i;
}
@@ -1552,9 +1571,204 @@ static inline void vmballoon_debugfs_exit(struct vmballoon *b)
#endif /* CONFIG_DEBUG_FS */
+
+#ifdef CONFIG_BALLOON_COMPACTION
+
+static struct dentry *vmballoon_mount(struct file_system_type *fs_type,
+ int flags, const char *dev_name,
+ void *data)
+{
+ static const struct dentry_operations ops = {
+ .d_dname = simple_dname,
+ };
+
+ return mount_pseudo(fs_type, "balloon-vmware:", NULL, &ops,
+ BALLOON_VMW_MAGIC);
+}
+
+static struct file_system_type vmballoon_fs = {
+ .name = "balloon-vmware",
+ .mount = vmballoon_mount,
+ .kill_sb = kill_anon_super,
+};
+
+static struct vfsmount *vmballoon_mnt;
+
+/**
+ * vmballoon_migratepage() - migrates a balloon page.
+ * @b_dev_info: balloon device information descriptor.
+ * @newpage: the page to which @page should be migrated.
+ * @page: a ballooned page that should be migrated.
+ * @mode: migration mode, ignored.
+ *
+ * This function is really open-coded, but that is according to the interface
+ * that balloon_compaction provides.
+ *
+ * Return: zero on success, -EAGAIN when migration cannot be performed
+ * momentarily, and -EBUSY if migration failed and should be retried
+ * with that specific page.
+ */
+static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info,
+ struct page *newpage, struct page *page,
+ enum migrate_mode mode)
+{
+ unsigned long status, flags;
+ struct vmballoon *b;
+ int ret;
+
+ b = container_of(b_dev_info, struct vmballoon, b_dev_info);
+
+ /*
+ * If the semaphore is taken, there is ongoing configuration change
+ * (i.e., balloon reset), so try again.
+ */
+ if (!down_read_trylock(&b->conf_sem))
+ return -EAGAIN;
+
+ spin_lock(&b->comm_lock);
+ /*
+ * We must start by deflating and not inflating, as otherwise the
+ * hypervisor may tell us that it has enough memory and the new page is
+ * not needed. Since the old page is isolated, we cannot use the list
+ * interface to unlock it, as the LRU field is used for isolation.
+ * Instead, we use the native interface directly.
+ */
+ vmballoon_add_page(b, 0, page);
+ status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
+ VMW_BALLOON_DEFLATE);
+
+ if (status == VMW_BALLOON_SUCCESS)
+ status = vmballoon_status_page(b, 0, &page);
+
+ /*
+ * If a failure happened, let the migration mechanism know that it
+ * should not retry.
+ */
+ if (status != VMW_BALLOON_SUCCESS) {
+ spin_unlock(&b->comm_lock);
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ /*
+ * The page is isolated, so it is safe to delete it without holding
+ * @pages_lock . We keep holding @comm_lock since we will need it in a
+ * second.
+ */
+ balloon_page_delete(page);
+
+ put_page(page);
+
+ /* Inflate */
+ vmballoon_add_page(b, 0, newpage);
+ status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
+ VMW_BALLOON_INFLATE);
+
+ if (status == VMW_BALLOON_SUCCESS)
+ status = vmballoon_status_page(b, 0, &newpage);
+
+ spin_unlock(&b->comm_lock);
+
+ if (status != VMW_BALLOON_SUCCESS) {
+ /*
+ * A failure happened. While we can deflate the page we just
+ * inflated, this deflation can also encounter an error. Instead
+ * we will decrease the size of the balloon to reflect the
+ * change and report failure.
+ */
+ atomic64_dec(&b->size);
+ ret = -EBUSY;
+ } else {
+ /*
+ * Success. Take a reference for the page, and we will add it to
+ * the list after acquiring the lock.
+ */
+ get_page(newpage);
+ ret = MIGRATEPAGE_SUCCESS;
+ }
+
+ /* Update the balloon list under the @pages_lock */
+ spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
+
+ /*
+ * On inflation success, we already took a reference for the @newpage.
+ * If we succeed just insert it to the list and update the statistics
+ * under the lock.
+ */
+ if (ret == MIGRATEPAGE_SUCCESS) {
+ balloon_page_insert(&b->b_dev_info, newpage);
+ __count_vm_event(BALLOON_MIGRATE);
+ }
+
+ /*
+ * We deflated successfully, so regardless to the inflation success, we
+ * need to reduce the number of isolated_pages.
+ */
+ b->b_dev_info.isolated_pages--;
+ spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
+
+out_unlock:
+ up_read(&b->conf_sem);
+ return ret;
+}
+
+/**
+ * vmballoon_compaction_deinit() - removes compaction related data.
+ *
+ * @b: pointer to the balloon.
+ */
+static void vmballoon_compaction_deinit(struct vmballoon *b)
+{
+ if (!IS_ERR(b->b_dev_info.inode))
+ iput(b->b_dev_info.inode);
+
+ b->b_dev_info.inode = NULL;
+ kern_unmount(vmballoon_mnt);
+ vmballoon_mnt = NULL;
+}
+
+/**
+ * vmballoon_compaction_init() - initialized compaction for the balloon.
+ *
+ * @b: pointer to the balloon.
+ *
+ * If during the initialization a failure occurred, this function does not
+ * perform cleanup. The caller must call vmballoon_compaction_deinit() in this
+ * case.
+ *
+ * Return: zero on success or error code on failure.
+ */
+static __init int vmballoon_compaction_init(struct vmballoon *b)
+{
+ vmballoon_mnt = kern_mount(&vmballoon_fs);
+ if (IS_ERR(vmballoon_mnt))
+ return PTR_ERR(vmballoon_mnt);
+
+ b->b_dev_info.migratepage = vmballoon_migratepage;
+ b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb);
+
+ if (IS_ERR(b->b_dev_info.inode))
+ return PTR_ERR(b->b_dev_info.inode);
+
+ b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops;
+ return 0;
+}
+
+#else /* CONFIG_BALLOON_COMPACTION */
+
+static void vmballoon_compaction_deinit(struct vmballoon *b)
+{
+}
+
+static int vmballoon_compaction_init(struct vmballoon *b)
+{
+ return 0;
+}
+
+#endif /* CONFIG_BALLOON_COMPACTION */
+
static int __init vmballoon_init(void)
{
- enum vmballoon_page_size_type page_size;
int error;
/*
@@ -1564,17 +1778,22 @@ static int __init vmballoon_init(void)
if (x86_hyper_type != X86_HYPER_VMWARE)
return -ENODEV;
- for (page_size = VMW_BALLOON_4K_PAGE;
- page_size <= VMW_BALLOON_LAST_SIZE; page_size++)
- INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages);
-
-
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
error = vmballoon_debugfs_init(&balloon);
if (error)
- return error;
+ goto fail;
+ /*
+ * Initialization of compaction must be done after the call to
+ * balloon_devinfo_init() .
+ */
+ balloon_devinfo_init(&balloon.b_dev_info);
+ error = vmballoon_compaction_init(&balloon);
+ if (error)
+ goto fail;
+
+ INIT_LIST_HEAD(&balloon.huge_pages);
spin_lock_init(&balloon.comm_lock);
init_rwsem(&balloon.conf_sem);
balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
@@ -1585,6 +1804,9 @@ static int __init vmballoon_init(void)
queue_delayed_work(system_freezable_wq, &balloon.dwork, 0);
return 0;
+fail:
+ vmballoon_compaction_deinit(&balloon);
+ return error;
}
/*
@@ -1609,5 +1831,8 @@ static void __exit vmballoon_exit(void)
*/
vmballoon_send_start(&balloon, 0);
vmballoon_pop(&balloon);
+
+ /* Only once we popped the balloon, compaction can be deinit */
+ vmballoon_compaction_deinit(&balloon);
}
module_exit(vmballoon_exit);
--
2.19.1
^ permalink raw reply related
* [PATCH v4 1/4] mm/balloon_compaction: List interfaces
From: Nadav Amit via Virtualization @ 2019-04-25 11:54 UTC (permalink / raw)
To: Greg Kroah-Hartman, Michael S. Tsirkin
Cc: Arnd Bergmann, VMware, Inc., linux-kernel, virtualization,
linux-mm, Julien Freche, Nadav Amit
In-Reply-To: <20190425115445.20815-1-namit@vmware.com>
Introduce interfaces for ballooning enqueueing and dequeueing of a list
of pages. These interfaces reduce the overhead of storing and restoring
IRQs by batching the operations. In addition they do not panic if the
list of pages is empty.
Cc: Jason Wang <jasowang@redhat.com>
Cc: linux-mm@kvack.org
Cc: virtualization@lists.linux-foundation.org
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Reviewed-by: Xavier Deguillard <xdeguillard@vmware.com>
Signed-off-by: Nadav Amit <namit@vmware.com>
---
include/linux/balloon_compaction.h | 4 +
mm/balloon_compaction.c | 144 +++++++++++++++++++++--------
2 files changed, 110 insertions(+), 38 deletions(-)
diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h
index f111c780ef1d..430b6047cef7 100644
--- a/include/linux/balloon_compaction.h
+++ b/include/linux/balloon_compaction.h
@@ -64,6 +64,10 @@ extern struct page *balloon_page_alloc(void);
extern void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
struct page *page);
extern struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info);
+extern size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages);
+extern size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages, size_t n_req_pages);
static inline void balloon_devinfo_init(struct balloon_dev_info *balloon)
{
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index ef858d547e2d..b7bd72612c5a 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -10,6 +10,105 @@
#include <linux/export.h>
#include <linux/balloon_compaction.h>
+static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
+ struct page *page)
+{
+ /*
+ * Block others from accessing the 'page' when we get around to
+ * establishing additional references. We should be the only one
+ * holding a reference to the 'page' at this point. If we are not, then
+ * memory corruption is possible and we should stop execution.
+ */
+ BUG_ON(!trylock_page(page));
+ list_del(&page->lru);
+ balloon_page_insert(b_dev_info, page);
+ unlock_page(page);
+ __count_vm_event(BALLOON_INFLATE);
+}
+
+/**
+ * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
+ * list.
+ * @b_dev_info: balloon device descriptor where we will insert a new page to
+ * @pages: pages to enqueue - allocated using balloon_page_alloc.
+ *
+ * Driver must call it to properly enqueue a balloon pages before definitively
+ * removing it from the guest system.
+ *
+ * Return: number of pages that were enqueued.
+ */
+size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages)
+{
+ struct page *page, *tmp;
+ unsigned long flags;
+ size_t n_pages = 0;
+
+ spin_lock_irqsave(&b_dev_info->pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, pages, lru) {
+ balloon_page_enqueue_one(b_dev_info, page);
+ n_pages++;
+ }
+ spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+ return n_pages;
+}
+EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
+
+/**
+ * balloon_page_list_dequeue() - removes pages from balloon's page list and
+ * returns a list of the pages.
+ * @b_dev_info: balloon device decriptor where we will grab a page from.
+ * @pages: pointer to the list of pages that would be returned to the caller.
+ * @n_req_pages: number of requested pages.
+ *
+ * Driver must call this function to properly de-allocate a previous enlisted
+ * balloon pages before definetively releasing it back to the guest system.
+ * This function tries to remove @n_req_pages from the ballooned pages and
+ * return them to the caller in the @pages list.
+ *
+ * Note that this function may fail to dequeue some pages temporarily empty due
+ * to compaction isolated pages.
+ *
+ * Return: number of pages that were added to the @pages list.
+ */
+size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
+ struct list_head *pages, size_t n_req_pages)
+{
+ struct page *page, *tmp;
+ unsigned long flags;
+ size_t n_pages = 0;
+
+ spin_lock_irqsave(&b_dev_info->pages_lock, flags);
+ list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
+ if (n_pages == n_req_pages)
+ break;
+
+ /*
+ * Block others from accessing the 'page' while we get around to
+ * establishing additional references and preparing the 'page'
+ * to be released by the balloon driver.
+ */
+ if (!trylock_page(page))
+ continue;
+
+ if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
+ PageIsolated(page)) {
+ /* raced with isolation */
+ unlock_page(page);
+ continue;
+ }
+ balloon_page_delete(page);
+ __count_vm_event(BALLOON_DEFLATE);
+ list_add(&page->lru, pages);
+ unlock_page(page);
+ n_pages++;
+ }
+ spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+
+ return n_pages;
+}
+EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
+
/*
* balloon_page_alloc - allocates a new page for insertion into the balloon
* page list.
@@ -43,17 +142,9 @@ void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
{
unsigned long flags;
- /*
- * Block others from accessing the 'page' when we get around to
- * establishing additional references. We should be the only one
- * holding a reference to the 'page' at this point.
- */
- BUG_ON(!trylock_page(page));
spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- balloon_page_insert(b_dev_info, page);
- __count_vm_event(BALLOON_INFLATE);
+ balloon_page_enqueue_one(b_dev_info, page);
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
- unlock_page(page);
}
EXPORT_SYMBOL_GPL(balloon_page_enqueue);
@@ -70,36 +161,13 @@ EXPORT_SYMBOL_GPL(balloon_page_enqueue);
*/
struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
{
- struct page *page, *tmp;
unsigned long flags;
- bool dequeued_page;
+ LIST_HEAD(pages);
+ int n_pages;
- dequeued_page = false;
- spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
- /*
- * Block others from accessing the 'page' while we get around
- * establishing additional references and preparing the 'page'
- * to be released by the balloon driver.
- */
- if (trylock_page(page)) {
-#ifdef CONFIG_BALLOON_COMPACTION
- if (PageIsolated(page)) {
- /* raced with isolation */
- unlock_page(page);
- continue;
- }
-#endif
- balloon_page_delete(page);
- __count_vm_event(BALLOON_DEFLATE);
- unlock_page(page);
- dequeued_page = true;
- break;
- }
- }
- spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+ n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
- if (!dequeued_page) {
+ if (n_pages != 1) {
/*
* If we are unable to dequeue a balloon page because the page
* list is empty and there is no isolated pages, then something
@@ -112,9 +180,9 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
!b_dev_info->isolated_pages))
BUG();
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
- page = NULL;
+ return NULL;
}
- return page;
+ return list_first_entry(&pages, struct page, lru);
}
EXPORT_SYMBOL_GPL(balloon_page_dequeue);
--
2.19.1
^ permalink raw reply related
* [PATCH v4 0/4] vmw_balloon: Compaction and shrinker support
From: Nadav Amit via Virtualization @ 2019-04-25 11:54 UTC (permalink / raw)
To: Greg Kroah-Hartman, Michael S. Tsirkin
Cc: Arnd Bergmann, VMware, Inc., linux-kernel, virtualization,
linux-mm, Julien Freche, Nadav Amit
VMware balloon enhancements: adding support for memory compaction,
memory shrinker (to prevent OOM) and splitting of refused pages to
prevent recurring inflations.
Patches 1-2: Support for compaction
Patch 3: Support for memory shrinker - disabled by default
Patch 4: Split refused pages to improve performance
v3->v4:
* "get around to" comment [Michael]
* Put list_add under page lock [Michael]
v2->v3:
* Fixing wrong argument type (int->size_t) [Michael]
* Fixing a comment (it) [Michael]
* Reinstating the BUG_ON() when page is locked [Michael]
v1->v2:
* Return number of pages in list enqueue/dequeue interfaces [Michael]
* Removed first two patches which were already merged
Nadav Amit (4):
mm/balloon_compaction: List interfaces
vmw_balloon: Compaction support
vmw_balloon: Add memory shrinker
vmw_balloon: Split refused pages
drivers/misc/Kconfig | 1 +
drivers/misc/vmw_balloon.c | 489 ++++++++++++++++++++++++++---
include/linux/balloon_compaction.h | 4 +
mm/balloon_compaction.c | 144 ++++++---
4 files changed, 553 insertions(+), 85 deletions(-)
--
2.19.1
^ permalink raw reply
* [PATCH net] vhost_net: fix possible infinite loop
From: Jason Wang @ 2019-04-25 7:33 UTC (permalink / raw)
To: mst, jasowang, kvm, virtualization, netdev, linux-kernel; +Cc: ppandit
When the rx buffer is too small for a packet, we will discard the vq
descriptor and retry it for the next packet:
while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
&busyloop_intr))) {
...
/* On overrun, truncate and discard */
if (unlikely(headcount > UIO_MAXIOV)) {
iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
err = sock->ops->recvmsg(sock, &msg,
1, MSG_DONTWAIT | MSG_TRUNC);
pr_debug("Discarded rx packet: len %zd\n", sock_len);
continue;
}
...
}
This makes it possible to trigger a infinite while..continue loop
through the co-opreation of two VMs like:
1) Malicious VM1 allocate 1 byte rx buffer and try to slow down the
vhost process as much as possible e.g using indirect descriptors or
other.
2) Malicious VM2 generate packets to VM1 as fast as possible
Fixing this by checking against weight at the end of RX and TX
loop. This also eliminate other similar cases when:
- userspace is consuming the packets in the meanwhile
- theoretical TOCTOU attack if guest moving avail index back and forth
to hit the continue after vhost find guest just add new buffers
This addresses CVE-2019-3900.
Fixes: d8316f3991d20 ("vhost: fix total length when packets are too short")
Fixes: 3a4d5c94e9593 ("vhost_net: a kernel-level virtio server")
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
drivers/vhost/net.c | 41 +++++++++++++++++++++--------------------
1 file changed, 21 insertions(+), 20 deletions(-)
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index df51a35..fb46e6b 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -778,8 +778,9 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
int err;
int sent_pkts = 0;
bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);
+ bool next_round = false;
- for (;;) {
+ do {
bool busyloop_intr = false;
if (nvq->done_idx == VHOST_NET_BATCH)
@@ -845,11 +846,10 @@ static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
vq->heads[nvq->done_idx].len = 0;
++nvq->done_idx;
- if (vhost_exceeds_weight(++sent_pkts, total_len)) {
- vhost_poll_queue(&vq->poll);
- break;
- }
- }
+ } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
+
+ if (next_round)
+ vhost_poll_queue(&vq->poll);
vhost_tx_batch(net, nvq, sock, &msg);
}
@@ -873,8 +873,9 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
bool zcopy_used;
int sent_pkts = 0;
+ bool next_round = false;
- for (;;) {
+ do {
bool busyloop_intr;
/* Release DMAs done buffers first */
@@ -951,11 +952,10 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
else
vhost_zerocopy_signal_used(net, vq);
vhost_net_tx_packet(net);
- if (unlikely(vhost_exceeds_weight(++sent_pkts, total_len))) {
- vhost_poll_queue(&vq->poll);
- break;
- }
- }
+ } while (!(next_round = vhost_exceeds_weight(++sent_pkts, total_len)));
+
+ if (next_round)
+ vhost_poll_queue(&vq->poll);
}
/* Expects to be always run from workqueue - which acts as
@@ -1134,6 +1134,7 @@ static void handle_rx(struct vhost_net *net)
struct iov_iter fixup;
__virtio16 num_buffers;
int recv_pkts = 0;
+ bool next_round = false;
mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX);
sock = vq->private_data;
@@ -1153,8 +1154,11 @@ static void handle_rx(struct vhost_net *net)
vq->log : NULL;
mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
- while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
- &busyloop_intr))) {
+ do {
+ sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
+ &busyloop_intr);
+ if (!sock_len)
+ break;
sock_len += sock_hlen;
vhost_len = sock_len + vhost_hlen;
headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
@@ -1239,12 +1243,9 @@ static void handle_rx(struct vhost_net *net)
vhost_log_write(vq, vq_log, log, vhost_len,
vq->iov, in);
total_len += vhost_len;
- if (unlikely(vhost_exceeds_weight(++recv_pkts, total_len))) {
- vhost_poll_queue(&vq->poll);
- goto out;
- }
- }
- if (unlikely(busyloop_intr))
+ } while (!(next_round = vhost_exceeds_weight(++recv_pkts, total_len)));
+
+ if (unlikely(busyloop_intr || next_round))
vhost_poll_queue(&vq->poll);
else
vhost_net_enable_vq(net, vq);
--
1.8.3.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox