* Re: [PATCH RFC] vhost: basic device IOTLB support
From: Michael S. Tsirkin @ 2015-12-31 11:17 UTC (permalink / raw)
To: Jason Wang; +Cc: netdev, linux-api, linux-kernel, kvm, virtualization
In-Reply-To: <1451546025-15955-1-git-send-email-jasowang@redhat.com>
On Thu, Dec 31, 2015 at 03:13:45PM +0800, Jason Wang wrote:
> This patch tries to implement an device IOTLB for vhost. This could be
> used with for co-operation with userspace(qemu) implementation of
> iommu for a secure DMA environment in guest.
>
> The idea is simple. When vhost meets an IOTLB miss, it will request
> the assistance of userspace to do the translation, this is done
> through:
>
> - Fill the translation request in a preset userspace address (This
> address is set through ioctl VHOST_SET_IOTLB_REQUEST_ENTRY).
> - Notify userspace through eventfd (This eventfd was set through ioctl
> VHOST_SET_IOTLB_FD).
>
> When userspace finishes the translation, it will update the vhost
> IOTLB through VHOST_UPDATE_IOTLB ioctl. Userspace is also in charge of
> snooping the IOTLB invalidation of IOMMU IOTLB and use
> VHOST_UPDATE_IOTLB to invalidate the possible entry in vhost.
>
> For simplicity, IOTLB was implemented with a simple hash array. The
> index were calculated from IOVA page frame number which can only works
> at PAGE_SIZE level.
>
> An qemu implementation (for reference) is available at:
> git@github.com:jasowang/qemu.git iommu
>
> TODO & Known issues:
>
> - read/write permission validation was not implemented.
> - no feature negotiation.
> - VHOST_SET_MEM_TABLE is not reused (maybe there's a chance).
> - working at PAGE_SIZE level, don't support large mappings.
> - better data structure for IOTLB instead of simple hash array.
> - better API, e.g using mmap() instead of preset userspace address.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>
Interesting. I'm working on a slightly different approach
which is direct vt-d support in vhost.
This one has the advantage of being more portable.
> ---
> drivers/vhost/net.c | 2 +-
> drivers/vhost/vhost.c | 190 ++++++++++++++++++++++++++++++++++++++++++++-
> drivers/vhost/vhost.h | 13 ++++
> include/uapi/linux/vhost.h | 26 +++++++
> 4 files changed, 229 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
> index 9eda69e..a172be9 100644
> --- a/drivers/vhost/net.c
> +++ b/drivers/vhost/net.c
> @@ -1083,7 +1083,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
> r = vhost_dev_ioctl(&n->dev, ioctl, argp);
> if (r == -ENOIOCTLCMD)
> r = vhost_vring_ioctl(&n->dev, ioctl, argp);
> - else
> + else if (ioctl != VHOST_UPDATE_IOTLB)
> vhost_net_flush(n);
> mutex_unlock(&n->dev.mutex);
> return r;
> diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
> index eec2f11..729fe05 100644
> --- a/drivers/vhost/vhost.c
> +++ b/drivers/vhost/vhost.c
> @@ -113,6 +113,11 @@ static void vhost_init_is_le(struct vhost_virtqueue *vq)
> }
> #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
>
> +static inline int vhost_iotlb_hash(u64 iova)
> +{
> + return (iova >> PAGE_SHIFT) & (VHOST_IOTLB_SIZE - 1);
> +}
> +
> static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
> poll_table *pt)
> {
> @@ -384,8 +389,14 @@ void vhost_dev_init(struct vhost_dev *dev,
> dev->memory = NULL;
> dev->mm = NULL;
> spin_lock_init(&dev->work_lock);
> + spin_lock_init(&dev->iotlb_lock);
> + mutex_init(&dev->iotlb_req_mutex);
> INIT_LIST_HEAD(&dev->work_list);
> dev->worker = NULL;
> + dev->iotlb_request = NULL;
> + dev->iotlb_ctx = NULL;
> + dev->iotlb_file = NULL;
> + dev->pending_request.flags.type = VHOST_IOTLB_INVALIDATE;
>
> for (i = 0; i < dev->nvqs; ++i) {
> vq = dev->vqs[i];
> @@ -393,12 +404,17 @@ void vhost_dev_init(struct vhost_dev *dev,
> vq->indirect = NULL;
> vq->heads = NULL;
> vq->dev = dev;
> + vq->iotlb_request = NULL;
> mutex_init(&vq->mutex);
> vhost_vq_reset(dev, vq);
> if (vq->handle_kick)
> vhost_poll_init(&vq->poll, vq->handle_kick,
> POLLIN, dev);
> }
> +
> + init_completion(&dev->iotlb_completion);
> + for (i = 0; i < VHOST_IOTLB_SIZE; i++)
> + dev->iotlb[i].flags.valid = VHOST_IOTLB_INVALID;
> }
> EXPORT_SYMBOL_GPL(vhost_dev_init);
>
> @@ -940,9 +956,10 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
> {
> struct file *eventfp, *filep = NULL;
> struct eventfd_ctx *ctx = NULL;
> + struct vhost_iotlb_entry entry;
> u64 p;
> long r;
> - int i, fd;
> + int index, i, fd;
>
> /* If you are not the owner, you can become one */
> if (ioctl == VHOST_SET_OWNER) {
> @@ -1008,6 +1025,80 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
> if (filep)
> fput(filep);
> break;
> + case VHOST_SET_IOTLB_FD:
> + r = get_user(fd, (int __user *)argp);
> + if (r < 0)
> + break;
> + eventfp = fd == -1 ? NULL : eventfd_fget(fd);
> + if (IS_ERR(eventfp)) {
> + r = PTR_ERR(eventfp);
> + break;
> + }
> + if (eventfp != d->iotlb_file) {
> + filep = d->iotlb_file;
> + d->iotlb_file = eventfp;
> + ctx = d->iotlb_ctx;
> + d->iotlb_ctx = eventfp ?
> + eventfd_ctx_fileget(eventfp) : NULL;
> + } else
> + filep = eventfp;
> + for (i = 0; i < d->nvqs; ++i) {
> + mutex_lock(&d->vqs[i]->mutex);
> + d->vqs[i]->iotlb_ctx = d->iotlb_ctx;
> + mutex_unlock(&d->vqs[i]->mutex);
> + }
> + if (ctx)
> + eventfd_ctx_put(ctx);
> + if (filep)
> + fput(filep);
> + break;
> + case VHOST_SET_IOTLB_REQUEST_ENTRY:
> + if (!access_ok(VERIFY_READ, argp, sizeof(*d->iotlb_request)))
> + return -EFAULT;
> + if (!access_ok(VERIFY_WRITE, argp, sizeof(*d->iotlb_request)))
> + return -EFAULT;
> + d->iotlb_request = argp;
> + for (i = 0; i < d->nvqs; ++i) {
> + mutex_lock(&d->vqs[i]->mutex);
> + d->vqs[i]->iotlb_request = argp;
> + mutex_unlock(&d->vqs[i]->mutex);
> + }
> + break;
> + case VHOST_UPDATE_IOTLB:
> + r = copy_from_user(&entry, argp, sizeof(entry));
> + if (r < 0) {
> + r = -EFAULT;
> + goto done;
> + }
> +
> + index = vhost_iotlb_hash(entry.iova);
> +
> + spin_lock(&d->iotlb_lock);
> + switch (entry.flags.type) {
> + case VHOST_IOTLB_UPDATE:
> + d->iotlb[index] = entry;
> + break;
> + case VHOST_IOTLB_INVALIDATE:
> + if (d->iotlb[index].iova == entry.iova)
> + d->iotlb[index] = entry;
> + break;
> + default:
> + r = -EINVAL;
> + }
> + spin_unlock(&d->iotlb_lock);
> +
> + if (!r && entry.flags.type != VHOST_IOTLB_INVALIDATE) {
> + mutex_lock(&d->iotlb_req_mutex);
> + if (entry.iova == d->pending_request.iova &&
> + d->pending_request.flags.type ==
> + VHOST_IOTLB_MISS) {
> + d->pending_request = entry;
> + complete(&d->iotlb_completion);
> + }
> + mutex_unlock(&d->iotlb_req_mutex);
> + }
> +
> + break;
> default:
> r = -ENOIOCTLCMD;
> break;
> @@ -1177,9 +1268,104 @@ int vhost_init_used(struct vhost_virtqueue *vq)
> }
> EXPORT_SYMBOL_GPL(vhost_init_used);
>
> +static struct vhost_iotlb_entry vhost_iotlb_miss(struct vhost_virtqueue *vq,
> + u64 iova)
> +{
> + struct completion *c = &vq->dev->iotlb_completion;
> + struct vhost_iotlb_entry *pending = &vq->dev->pending_request;
> + struct vhost_iotlb_entry entry = {
> + .flags.valid = VHOST_IOTLB_INVALID,
> + };
> +
> + mutex_lock(&vq->dev->iotlb_req_mutex);
> +
> + if (!vq->iotlb_ctx)
> + goto err;
> +
> + if (!vq->dev->iotlb_request)
> + goto err;
> +
> + if (pending->flags.type == VHOST_IOTLB_MISS)
> + goto err;
> +
> + pending->iova = iova & PAGE_MASK;
> + pending->flags.type = VHOST_IOTLB_MISS;
> +
> + if (copy_to_user(vq->dev->iotlb_request, pending,
> + sizeof(struct vhost_iotlb_entry))) {
> + goto err;
> + }
> +
> + mutex_unlock(&vq->dev->iotlb_req_mutex);
> +
> + eventfd_signal(vq->iotlb_ctx, 1);
> + wait_for_completion_interruptible(c);
This can still be under vq lock, can it not?
Looks like this can cause deadlocks.
> +
> + mutex_lock(&vq->dev->iotlb_req_mutex);
> + entry = vq->dev->pending_request;
> + mutex_unlock(&vq->dev->iotlb_req_mutex);
> +
> + return entry;
> +err:
> + mutex_unlock(&vq->dev->iotlb_req_mutex);
> + return entry;
> +}
> +
> +static int translate_iotlb(struct vhost_virtqueue *vq, u64 iova, u32 len,
> + struct iovec iov[], int iov_size)
> +{
> + struct vhost_iotlb_entry *entry;
> + struct vhost_iotlb_entry miss;
> + struct vhost_dev *dev = vq->dev;
> + int ret = 0;
> + u64 s = 0, size;
> +
> + spin_lock(&dev->iotlb_lock);
> +
> + while ((u64) len > s) {
> + if (unlikely(ret >= iov_size)) {
> + ret = -ENOBUFS;
> + break;
> + }
> + entry = &vq->dev->iotlb[vhost_iotlb_hash(iova)];
> + if ((entry->iova != (iova & PAGE_MASK)) ||
> + (entry->flags.valid != VHOST_IOTLB_VALID)) {
> +
> + spin_unlock(&dev->iotlb_lock);
> + miss = vhost_iotlb_miss(vq, iova);
> + spin_lock(&dev->iotlb_lock);
> +
> + if (miss.flags.valid != VHOST_IOTLB_VALID ||
> + miss.iova != (iova & PAGE_MASK)) {
> + ret = -EFAULT;
> + goto err;
> + }
> + entry = &miss;
> + }
> +
> + if (entry->iova == (iova & PAGE_MASK)) {
> + size = entry->userspace_addr + entry->size - iova;
> + iov[ret].iov_base =
> + (void __user *)(entry->userspace_addr +
> + (iova & (PAGE_SIZE - 1)));
> + iov[ret].iov_len = min((u64)len - s, size);
> + s += size;
> + iova += size;
> + ret++;
> + } else {
> + BUG();
> + }
> + }
> +
> +err:
> + spin_unlock(&dev->iotlb_lock);
> + return ret;
> +}
> +
> static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
> struct iovec iov[], int iov_size)
> {
> +#if 0
> const struct vhost_memory_region *reg;
> struct vhost_memory *mem;
> struct iovec *_iov;
> @@ -1209,6 +1395,8 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
> }
>
> return ret;
> +#endif
> + return translate_iotlb(vq, addr, len, iov, iov_size);
> }
>
> /* Each buffer in the virtqueues is actually a chain of descriptors. This
> diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
> index d3f7674..d254efc 100644
> --- a/drivers/vhost/vhost.h
> +++ b/drivers/vhost/vhost.h
> @@ -68,6 +68,8 @@ struct vhost_virtqueue {
> struct eventfd_ctx *call_ctx;
> struct eventfd_ctx *error_ctx;
> struct eventfd_ctx *log_ctx;
> + struct eventfd_ctx *iotlb_ctx;
> + struct vhost_iotlb __user *iotlb_request;
>
> struct vhost_poll poll;
>
> @@ -116,6 +118,8 @@ struct vhost_virtqueue {
> #endif
> };
>
> +#define VHOST_IOTLB_SIZE 1024
> +
> struct vhost_dev {
> struct vhost_memory *memory;
> struct mm_struct *mm;
> @@ -124,9 +128,18 @@ struct vhost_dev {
> int nvqs;
> struct file *log_file;
> struct eventfd_ctx *log_ctx;
> + struct file *iotlb_file;
> + struct eventfd_ctx *iotlb_ctx;
> + struct mutex iotlb_req_mutex;
> + struct vhost_iotlb_entry __user *iotlb_request;
> + struct vhost_iotlb_entry pending_request;
> + struct completion iotlb_completion;
> + struct vhost_iotlb_entry request;
> spinlock_t work_lock;
> struct list_head work_list;
> struct task_struct *worker;
> + spinlock_t iotlb_lock;
> + struct vhost_iotlb_entry iotlb[VHOST_IOTLB_SIZE];
> };
>
> void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs);
> diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> index ab373191..400e513 100644
> --- a/include/uapi/linux/vhost.h
> +++ b/include/uapi/linux/vhost.h
> @@ -63,6 +63,26 @@ struct vhost_memory {
> struct vhost_memory_region regions[0];
> };
>
> +struct vhost_iotlb_entry {
> + __u64 iova;
> + __u64 size;
> + __u64 userspace_addr;
> + struct {
> +#define VHOST_IOTLB_PERM_READ 0x1
> +#define VHOST_IOTLB_PERM_WRITE 0x10
> + __u8 perm;
> +#define VHOST_IOTLB_MISS 1
> +#define VHOST_IOTLB_UPDATE 2
> +#define VHOST_IOTLB_INVALIDATE 3
> + __u8 type;
> +#define VHOST_IOTLB_INVALID 0x1
> +#define VHOST_IOTLB_VALID 0x2
> + __u8 valid;
> + __u8 u8_padding;
> + __u32 padding;
> + } flags;
> +};
> +
> /* ioctls */
>
> #define VHOST_VIRTIO 0xAF
> @@ -127,6 +147,12 @@ struct vhost_memory {
> /* Set eventfd to signal an error */
> #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
>
> +/* IOTLB */
> +/* Specify an eventfd file descriptor to signle on IOTLB miss */
> +#define VHOST_SET_IOTLB_FD _IOW(VHOST_VIRTIO, 0x23, int)
> +#define VHOST_UPDATE_IOTLB _IOW(VHOST_VIRTIO, 0x24, struct vhost_iotlb_entry)
> +#define VHOST_SET_IOTLB_REQUEST_ENTRY _IOW(VHOST_VIRTIO, 0x25, struct vhost_iotlb_entry)
> +
> /* VHOST_NET specific defines */
>
> /* Attach virtio net ring to a raw socket, or tap device.
> --
> 2.5.0
^ permalink raw reply
* [PATCH RFC] vhost: basic device IOTLB support
From: Jason Wang @ 2015-12-31 7:13 UTC (permalink / raw)
To: mst, kvm, virtualization, netdev, linux-kernel, linux-api
This patch tries to implement an device IOTLB for vhost. This could be
used with for co-operation with userspace(qemu) implementation of
iommu for a secure DMA environment in guest.
The idea is simple. When vhost meets an IOTLB miss, it will request
the assistance of userspace to do the translation, this is done
through:
- Fill the translation request in a preset userspace address (This
address is set through ioctl VHOST_SET_IOTLB_REQUEST_ENTRY).
- Notify userspace through eventfd (This eventfd was set through ioctl
VHOST_SET_IOTLB_FD).
When userspace finishes the translation, it will update the vhost
IOTLB through VHOST_UPDATE_IOTLB ioctl. Userspace is also in charge of
snooping the IOTLB invalidation of IOMMU IOTLB and use
VHOST_UPDATE_IOTLB to invalidate the possible entry in vhost.
For simplicity, IOTLB was implemented with a simple hash array. The
index were calculated from IOVA page frame number which can only works
at PAGE_SIZE level.
An qemu implementation (for reference) is available at:
git@github.com:jasowang/qemu.git iommu
TODO & Known issues:
- read/write permission validation was not implemented.
- no feature negotiation.
- VHOST_SET_MEM_TABLE is not reused (maybe there's a chance).
- working at PAGE_SIZE level, don't support large mappings.
- better data structure for IOTLB instead of simple hash array.
- better API, e.g using mmap() instead of preset userspace address.
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
drivers/vhost/net.c | 2 +-
drivers/vhost/vhost.c | 190 ++++++++++++++++++++++++++++++++++++++++++++-
drivers/vhost/vhost.h | 13 ++++
include/uapi/linux/vhost.h | 26 +++++++
4 files changed, 229 insertions(+), 2 deletions(-)
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 9eda69e..a172be9 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -1083,7 +1083,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
r = vhost_dev_ioctl(&n->dev, ioctl, argp);
if (r == -ENOIOCTLCMD)
r = vhost_vring_ioctl(&n->dev, ioctl, argp);
- else
+ else if (ioctl != VHOST_UPDATE_IOTLB)
vhost_net_flush(n);
mutex_unlock(&n->dev.mutex);
return r;
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index eec2f11..729fe05 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -113,6 +113,11 @@ static void vhost_init_is_le(struct vhost_virtqueue *vq)
}
#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
+static inline int vhost_iotlb_hash(u64 iova)
+{
+ return (iova >> PAGE_SHIFT) & (VHOST_IOTLB_SIZE - 1);
+}
+
static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
poll_table *pt)
{
@@ -384,8 +389,14 @@ void vhost_dev_init(struct vhost_dev *dev,
dev->memory = NULL;
dev->mm = NULL;
spin_lock_init(&dev->work_lock);
+ spin_lock_init(&dev->iotlb_lock);
+ mutex_init(&dev->iotlb_req_mutex);
INIT_LIST_HEAD(&dev->work_list);
dev->worker = NULL;
+ dev->iotlb_request = NULL;
+ dev->iotlb_ctx = NULL;
+ dev->iotlb_file = NULL;
+ dev->pending_request.flags.type = VHOST_IOTLB_INVALIDATE;
for (i = 0; i < dev->nvqs; ++i) {
vq = dev->vqs[i];
@@ -393,12 +404,17 @@ void vhost_dev_init(struct vhost_dev *dev,
vq->indirect = NULL;
vq->heads = NULL;
vq->dev = dev;
+ vq->iotlb_request = NULL;
mutex_init(&vq->mutex);
vhost_vq_reset(dev, vq);
if (vq->handle_kick)
vhost_poll_init(&vq->poll, vq->handle_kick,
POLLIN, dev);
}
+
+ init_completion(&dev->iotlb_completion);
+ for (i = 0; i < VHOST_IOTLB_SIZE; i++)
+ dev->iotlb[i].flags.valid = VHOST_IOTLB_INVALID;
}
EXPORT_SYMBOL_GPL(vhost_dev_init);
@@ -940,9 +956,10 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
{
struct file *eventfp, *filep = NULL;
struct eventfd_ctx *ctx = NULL;
+ struct vhost_iotlb_entry entry;
u64 p;
long r;
- int i, fd;
+ int index, i, fd;
/* If you are not the owner, you can become one */
if (ioctl == VHOST_SET_OWNER) {
@@ -1008,6 +1025,80 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
if (filep)
fput(filep);
break;
+ case VHOST_SET_IOTLB_FD:
+ r = get_user(fd, (int __user *)argp);
+ if (r < 0)
+ break;
+ eventfp = fd == -1 ? NULL : eventfd_fget(fd);
+ if (IS_ERR(eventfp)) {
+ r = PTR_ERR(eventfp);
+ break;
+ }
+ if (eventfp != d->iotlb_file) {
+ filep = d->iotlb_file;
+ d->iotlb_file = eventfp;
+ ctx = d->iotlb_ctx;
+ d->iotlb_ctx = eventfp ?
+ eventfd_ctx_fileget(eventfp) : NULL;
+ } else
+ filep = eventfp;
+ for (i = 0; i < d->nvqs; ++i) {
+ mutex_lock(&d->vqs[i]->mutex);
+ d->vqs[i]->iotlb_ctx = d->iotlb_ctx;
+ mutex_unlock(&d->vqs[i]->mutex);
+ }
+ if (ctx)
+ eventfd_ctx_put(ctx);
+ if (filep)
+ fput(filep);
+ break;
+ case VHOST_SET_IOTLB_REQUEST_ENTRY:
+ if (!access_ok(VERIFY_READ, argp, sizeof(*d->iotlb_request)))
+ return -EFAULT;
+ if (!access_ok(VERIFY_WRITE, argp, sizeof(*d->iotlb_request)))
+ return -EFAULT;
+ d->iotlb_request = argp;
+ for (i = 0; i < d->nvqs; ++i) {
+ mutex_lock(&d->vqs[i]->mutex);
+ d->vqs[i]->iotlb_request = argp;
+ mutex_unlock(&d->vqs[i]->mutex);
+ }
+ break;
+ case VHOST_UPDATE_IOTLB:
+ r = copy_from_user(&entry, argp, sizeof(entry));
+ if (r < 0) {
+ r = -EFAULT;
+ goto done;
+ }
+
+ index = vhost_iotlb_hash(entry.iova);
+
+ spin_lock(&d->iotlb_lock);
+ switch (entry.flags.type) {
+ case VHOST_IOTLB_UPDATE:
+ d->iotlb[index] = entry;
+ break;
+ case VHOST_IOTLB_INVALIDATE:
+ if (d->iotlb[index].iova == entry.iova)
+ d->iotlb[index] = entry;
+ break;
+ default:
+ r = -EINVAL;
+ }
+ spin_unlock(&d->iotlb_lock);
+
+ if (!r && entry.flags.type != VHOST_IOTLB_INVALIDATE) {
+ mutex_lock(&d->iotlb_req_mutex);
+ if (entry.iova == d->pending_request.iova &&
+ d->pending_request.flags.type ==
+ VHOST_IOTLB_MISS) {
+ d->pending_request = entry;
+ complete(&d->iotlb_completion);
+ }
+ mutex_unlock(&d->iotlb_req_mutex);
+ }
+
+ break;
default:
r = -ENOIOCTLCMD;
break;
@@ -1177,9 +1268,104 @@ int vhost_init_used(struct vhost_virtqueue *vq)
}
EXPORT_SYMBOL_GPL(vhost_init_used);
+static struct vhost_iotlb_entry vhost_iotlb_miss(struct vhost_virtqueue *vq,
+ u64 iova)
+{
+ struct completion *c = &vq->dev->iotlb_completion;
+ struct vhost_iotlb_entry *pending = &vq->dev->pending_request;
+ struct vhost_iotlb_entry entry = {
+ .flags.valid = VHOST_IOTLB_INVALID,
+ };
+
+ mutex_lock(&vq->dev->iotlb_req_mutex);
+
+ if (!vq->iotlb_ctx)
+ goto err;
+
+ if (!vq->dev->iotlb_request)
+ goto err;
+
+ if (pending->flags.type == VHOST_IOTLB_MISS)
+ goto err;
+
+ pending->iova = iova & PAGE_MASK;
+ pending->flags.type = VHOST_IOTLB_MISS;
+
+ if (copy_to_user(vq->dev->iotlb_request, pending,
+ sizeof(struct vhost_iotlb_entry))) {
+ goto err;
+ }
+
+ mutex_unlock(&vq->dev->iotlb_req_mutex);
+
+ eventfd_signal(vq->iotlb_ctx, 1);
+ wait_for_completion_interruptible(c);
+
+ mutex_lock(&vq->dev->iotlb_req_mutex);
+ entry = vq->dev->pending_request;
+ mutex_unlock(&vq->dev->iotlb_req_mutex);
+
+ return entry;
+err:
+ mutex_unlock(&vq->dev->iotlb_req_mutex);
+ return entry;
+}
+
+static int translate_iotlb(struct vhost_virtqueue *vq, u64 iova, u32 len,
+ struct iovec iov[], int iov_size)
+{
+ struct vhost_iotlb_entry *entry;
+ struct vhost_iotlb_entry miss;
+ struct vhost_dev *dev = vq->dev;
+ int ret = 0;
+ u64 s = 0, size;
+
+ spin_lock(&dev->iotlb_lock);
+
+ while ((u64) len > s) {
+ if (unlikely(ret >= iov_size)) {
+ ret = -ENOBUFS;
+ break;
+ }
+ entry = &vq->dev->iotlb[vhost_iotlb_hash(iova)];
+ if ((entry->iova != (iova & PAGE_MASK)) ||
+ (entry->flags.valid != VHOST_IOTLB_VALID)) {
+
+ spin_unlock(&dev->iotlb_lock);
+ miss = vhost_iotlb_miss(vq, iova);
+ spin_lock(&dev->iotlb_lock);
+
+ if (miss.flags.valid != VHOST_IOTLB_VALID ||
+ miss.iova != (iova & PAGE_MASK)) {
+ ret = -EFAULT;
+ goto err;
+ }
+ entry = &miss;
+ }
+
+ if (entry->iova == (iova & PAGE_MASK)) {
+ size = entry->userspace_addr + entry->size - iova;
+ iov[ret].iov_base =
+ (void __user *)(entry->userspace_addr +
+ (iova & (PAGE_SIZE - 1)));
+ iov[ret].iov_len = min((u64)len - s, size);
+ s += size;
+ iova += size;
+ ret++;
+ } else {
+ BUG();
+ }
+ }
+
+err:
+ spin_unlock(&dev->iotlb_lock);
+ return ret;
+}
+
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
struct iovec iov[], int iov_size)
{
+#if 0
const struct vhost_memory_region *reg;
struct vhost_memory *mem;
struct iovec *_iov;
@@ -1209,6 +1395,8 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
}
return ret;
+#endif
+ return translate_iotlb(vq, addr, len, iov, iov_size);
}
/* Each buffer in the virtqueues is actually a chain of descriptors. This
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index d3f7674..d254efc 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -68,6 +68,8 @@ struct vhost_virtqueue {
struct eventfd_ctx *call_ctx;
struct eventfd_ctx *error_ctx;
struct eventfd_ctx *log_ctx;
+ struct eventfd_ctx *iotlb_ctx;
+ struct vhost_iotlb __user *iotlb_request;
struct vhost_poll poll;
@@ -116,6 +118,8 @@ struct vhost_virtqueue {
#endif
};
+#define VHOST_IOTLB_SIZE 1024
+
struct vhost_dev {
struct vhost_memory *memory;
struct mm_struct *mm;
@@ -124,9 +128,18 @@ struct vhost_dev {
int nvqs;
struct file *log_file;
struct eventfd_ctx *log_ctx;
+ struct file *iotlb_file;
+ struct eventfd_ctx *iotlb_ctx;
+ struct mutex iotlb_req_mutex;
+ struct vhost_iotlb_entry __user *iotlb_request;
+ struct vhost_iotlb_entry pending_request;
+ struct completion iotlb_completion;
+ struct vhost_iotlb_entry request;
spinlock_t work_lock;
struct list_head work_list;
struct task_struct *worker;
+ spinlock_t iotlb_lock;
+ struct vhost_iotlb_entry iotlb[VHOST_IOTLB_SIZE];
};
void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs);
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index ab373191..400e513 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -63,6 +63,26 @@ struct vhost_memory {
struct vhost_memory_region regions[0];
};
+struct vhost_iotlb_entry {
+ __u64 iova;
+ __u64 size;
+ __u64 userspace_addr;
+ struct {
+#define VHOST_IOTLB_PERM_READ 0x1
+#define VHOST_IOTLB_PERM_WRITE 0x10
+ __u8 perm;
+#define VHOST_IOTLB_MISS 1
+#define VHOST_IOTLB_UPDATE 2
+#define VHOST_IOTLB_INVALIDATE 3
+ __u8 type;
+#define VHOST_IOTLB_INVALID 0x1
+#define VHOST_IOTLB_VALID 0x2
+ __u8 valid;
+ __u8 u8_padding;
+ __u32 padding;
+ } flags;
+};
+
/* ioctls */
#define VHOST_VIRTIO 0xAF
@@ -127,6 +147,12 @@ struct vhost_memory {
/* Set eventfd to signal an error */
#define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
+/* IOTLB */
+/* Specify an eventfd file descriptor to signle on IOTLB miss */
+#define VHOST_SET_IOTLB_FD _IOW(VHOST_VIRTIO, 0x23, int)
+#define VHOST_UPDATE_IOTLB _IOW(VHOST_VIRTIO, 0x24, struct vhost_iotlb_entry)
+#define VHOST_SET_IOTLB_REQUEST_ENTRY _IOW(VHOST_VIRTIO, 0x25, struct vhost_iotlb_entry)
+
/* VHOST_NET specific defines */
/* Attach virtio net ring to a raw socket, or tap device.
--
2.5.0
^ permalink raw reply related
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_xxx barriers for virt
From: Arnd Bergmann @ 2015-12-30 22:45 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: linux-arch, Stefano Stabellini, Peter Zijlstra, Andrew Cooper,
linux-kernel, virtualization
In-Reply-To: <20151230234521-mutt-send-email-mst@redhat.com>
On Wednesday 30 December 2015 23:45:41 Michael S. Tsirkin wrote:
> On Wed, Dec 30, 2015 at 02:49:53PM +0100, Arnd Bergmann wrote:
> > On Wednesday 30 December 2015 15:24:12 Michael S. Tsirkin wrote:
> > > This is really trying to cleanup some virt code, as suggested by Peter, who
> > > said
> > > > You could of course go fix that instead of mutilating things into
> > > > sort-of functional state.
> > >
> > > This work is needed for virtio, so it's probably easiest to
> > > merge it through my tree - is this fine by everyone?
> > > Arnd, if you agree, could you ack this please?
> >
> > I've looked over all patches now, and everything seems fine (I had one
> > small comment about a duplicate patch). It's a very nice cleanup, both
> > for using the asm-generic file, and for the virtualization users.
> >
> > Please add my "Acked-by: Arnd Bergmann <arnd@arndb.de>" when merging the
> > series through your tree.
> >
> > Arnd
>
> To all patches in the series?
Your choice, either all of them, or just the asm-generic ones.
Arnd
^ permalink raw reply
* Re: [PATCH 08/34] asm-generic: smp_store_mb should use smp_mb
From: Arnd Bergmann @ 2015-12-30 22:43 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: linux-arch, Davidlohr Bueso, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, linux-kernel, virtualization, Ingo Molnar
In-Reply-To: <20151230222703-mutt-send-email-mst@redhat.com>
On Wednesday 30 December 2015 22:30:38 Michael S. Tsirkin wrote:
> On Wed, Dec 30, 2015 at 02:44:21PM +0100, Arnd Bergmann wrote:
> > On Wednesday 30 December 2015 15:24:47 Michael S. Tsirkin wrote:
> > > #ifndef smp_store_mb
> > > -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); mb(); } while (0)
> > > +#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
> > > #endif
> > >
> > > #ifndef smp_mb__before_atomic
> > >
> >
> > The same patch is already in the tip tree scheduled for 4.5 as d5a73cadf3fd
> > ("lcoking/barriers, arch: Use smp barriers in smp_store_release()").
>
> Sorry which tree do you mean exactly?
$ git log --ancestry-path --oneline --merges d5a73cadf3fd..next/master | tail -n 17
cb17a685bed6 Merge remote-tracking branch 'tip/auto-latest'
f29c2e03f0b3 Merge branch 'x86/urgent'
8cd6990bf71d Merge branch 'x86/platform'
0541d92a5eb4 Merge branch 'x86/mm'
aa7c8013c8c0 Merge branch 'x86/fpu'
fcc9a1bd013c Merge branch 'x86/efi'
e74ef3f60886 Merge branch 'x86/cpu'
44a4f0063508 Merge branch 'x86/cleanups'
28c814578fcf Merge branch 'x86/cache'
d74ff99dada8 Merge branch 'x86/boot'
db3c55380b10 Merge branch 'x86/asm'
7cd91b91da20 Merge branch 'x86/apic'
7bfc343947e6 Merge branch 'timers/core'
1720bbcb66d1 Merge branch 'sched/core'
af9a59f26764 Merge branch 'ras/core'
984b85eca78d Merge branch 'perf/core'
d2b22d438aab Merge branch 'locking/core'
$ grep auto-latest Next/Trees
tip git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#auto-latest
So locking/core of tip.git has the patch and gets merged into linux-next
through auto-latest in tip.git.
> > I think you can drop your version.
> >
> > arnd
>
> Will drop mine, thanks.
> I kind of dislike that if I just drop it, some arches will temporarily
> regress to a slower implementation.
> I think I can just cherry-pick d5a73cadf3fd into my tree: git
> normally figures such duplicates out nicely.
> Does this sound good?
I don't think there is a perfect solution, you can either cherry-pick
it and get a duplicate commit in the git history, or you merge in the
whole locking/core branch from tip.
I'd say ask Ingo/PeterZ/Davidlohr which way they prefer.
Arnd
^ permalink raw reply
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_xxx barriers for virt
From: David Miller @ 2015-12-30 21:59 UTC (permalink / raw)
To: mst
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, virtualization
In-Reply-To: <20151230224855-mutt-send-email-mst@redhat.com>
From: "Michael S. Tsirkin" <mst@redhat.com>
Date: Wed, 30 Dec 2015 23:36:29 +0200
> On Wed, Dec 30, 2015 at 03:46:46PM -0500, David Miller wrote:
>> From: "Michael S. Tsirkin" <mst@redhat.com>
>> Date: Wed, 30 Dec 2015 14:58:19 +0200
>>
>> > -. Patch 1 documents the __smp APIs, and explains why they are
>> > useful for virt
>>
>> If virt is doing things like interacting with descriptors that are
>> shared with a (potentially SMP) host, why don't we just annotate those
>> specific cases?
>
> Using a bunch of per-arch ifdefs in virtio?
> That's fundamentally what we have now.
I was suggesting a generic interface to get what you want.
virt_mb() or something like that. You can name it something
else, that's not the important part.
> Or do you mean wrappers for __smp_XXX that explicitly
> say they are for talking to host?
> E.g. pv_mb() pv_rmb() etc.
> That sounds very reasonable to me.
Exactly!
^ permalink raw reply
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_xxx barriers for virt
From: Michael S. Tsirkin @ 2015-12-30 21:45 UTC (permalink / raw)
To: Arnd Bergmann
Cc: linux-arch, Stefano Stabellini, Peter Zijlstra, Andrew Cooper,
linux-kernel, virtualization
In-Reply-To: <11919001.9WHgFoPpmK@wuerfel>
On Wed, Dec 30, 2015 at 02:49:53PM +0100, Arnd Bergmann wrote:
> On Wednesday 30 December 2015 15:24:12 Michael S. Tsirkin wrote:
> > This is really trying to cleanup some virt code, as suggested by Peter, who
> > said
> > > You could of course go fix that instead of mutilating things into
> > > sort-of functional state.
> >
> > This work is needed for virtio, so it's probably easiest to
> > merge it through my tree - is this fine by everyone?
> > Arnd, if you agree, could you ack this please?
>
> I've looked over all patches now, and everything seems fine (I had one
> small comment about a duplicate patch). It's a very nice cleanup, both
> for using the asm-generic file, and for the virtualization users.
>
> Please add my "Acked-by: Arnd Bergmann <arnd@arndb.de>" when merging the
> series through your tree.
>
> Arnd
To all patches in the series?
^ permalink raw reply
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_xxx barriers for virt
From: Michael S. Tsirkin @ 2015-12-30 21:36 UTC (permalink / raw)
To: David Miller
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, virtualization
In-Reply-To: <20151230.154646.596597479469584255.davem@davemloft.net>
On Wed, Dec 30, 2015 at 03:46:46PM -0500, David Miller wrote:
> From: "Michael S. Tsirkin" <mst@redhat.com>
> Date: Wed, 30 Dec 2015 14:58:19 +0200
>
> > -. Patch 1 documents the __smp APIs, and explains why they are
> > useful for virt
>
> If virt is doing things like interacting with descriptors that are
> shared with a (potentially SMP) host, why don't we just annotate those
> specific cases?
Using a bunch of per-arch ifdefs in virtio?
That's fundamentally what we have now.
But basically the rework reduces the LOC count in kernel anyway
by moving all ifdef CONFIG_SMP hacks into asm-generic.
So why not let virt benefit?
Or do you mean wrappers for __smp_XXX that explicitly
say they are for talking to host?
E.g. pv_mb() pv_rmb() etc.
That sounds very reasonable to me.
__smp_XXX things then become an implementation detail.
> The other memory barriers in the kernel do not matter for SMP'ness
> when build UP.
^ permalink raw reply
* Re: [PATCH 07/34] sparc: reuse asm-generic/barrier.h
From: David Miller @ 2015-12-30 20:47 UTC (permalink / raw)
To: mst
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, andreyknvl, virtualization, mpe, sparclinux, mingo
In-Reply-To: <20151230215303-mutt-send-email-mst@redhat.com>
From: "Michael S. Tsirkin" <mst@redhat.com>
Date: Wed, 30 Dec 2015 21:55:33 +0200
> The problem I had in the past is that vger rejects email with
> too many CC addresses.
I can't think of a patch set where a > 1024 character CC: list is
legitimate anyways, even if all interested parties are included.
> I'll look for better solutions next time - for now, would it help if I
> bounced the whole patchset to you?
Not necessary, I provided a reply to your 00/NN repost.
^ permalink raw reply
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_xxx barriers for virt
From: David Miller @ 2015-12-30 20:46 UTC (permalink / raw)
To: mst
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
From: "Michael S. Tsirkin" <mst@redhat.com>
Date: Wed, 30 Dec 2015 14:58:19 +0200
> -. Patch 1 documents the __smp APIs, and explains why they are
> useful for virt
If virt is doing things like interacting with descriptors that are
shared with a (potentially SMP) host, why don't we just annotate those
specific cases?
The other memory barriers in the kernel do not matter for SMP'ness
when build UP.
^ permalink raw reply
* Re: [PATCH 08/34] asm-generic: smp_store_mb should use smp_mb
From: Michael S. Tsirkin @ 2015-12-30 20:30 UTC (permalink / raw)
To: Arnd Bergmann
Cc: linux-arch, Stefano Stabellini, Peter Zijlstra, Andrew Cooper,
linux-kernel, virtualization
In-Reply-To: <5946017.BCmPrpGMuh@wuerfel>
On Wed, Dec 30, 2015 at 02:44:21PM +0100, Arnd Bergmann wrote:
> On Wednesday 30 December 2015 15:24:47 Michael S. Tsirkin wrote:
> > asm-generic variant of smp_store_mb() calls mb() which is stronger
> > than implied by both the name and the documentation.
> >
> > smp_store_mb is only used by core kernel code at the moment, so
> > we know no one mis-uses it for an MMIO barrier.
> > Make it call smp_mb consistently before some arch-specific
> > code uses it as such by mistake.
> >
> > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > ---
> > include/asm-generic/barrier.h | 2 +-
> > 1 file changed, 1 insertion(+), 1 deletion(-)
> >
> > diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
> > index 538f8d1..987b2e0 100644
> > --- a/include/asm-generic/barrier.h
> > +++ b/include/asm-generic/barrier.h
> > @@ -93,7 +93,7 @@
> > #endif /* CONFIG_SMP */
> >
> > #ifndef smp_store_mb
> > -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); mb(); } while (0)
> > +#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
> > #endif
> >
> > #ifndef smp_mb__before_atomic
> >
>
> The same patch is already in the tip tree scheduled for 4.5 as d5a73cadf3fd
> ("lcoking/barriers, arch: Use smp barriers in smp_store_release()").
Sorry which tree do you mean exactly?
> I think you can drop your version.
>
> arnd
Will drop mine, thanks.
I kind of dislike that if I just drop it, some arches will temporarily
regress to a slower implementation.
I think I can just cherry-pick d5a73cadf3fd into my tree: git
normally figures such duplicates out nicely.
Does this sound good?
--
MST
^ permalink raw reply
* Re: [PATCH 07/34] sparc: reuse asm-generic/barrier.h
From: Michael S. Tsirkin @ 2015-12-30 19:55 UTC (permalink / raw)
To: David Miller
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, andreyknvl, virtualization, mpe, sparclinux, mingo
In-Reply-To: <20151230.125603.75638711551088484.davem@davemloft.net>
On Wed, Dec 30, 2015 at 12:56:03PM -0500, David Miller wrote:
>
> I find it a little bit irritating when I receive a patch directly
> addressed to me to review, but I don't see the "00/34" cover letter.
>
> I want to see what this series is about, and what the motiviation
> for this change is.
>
> I'm also highly frustrated, in general, with mix-and-match CC: lists
> for patch series. Just put everyone relevant in the CC: for the
> entire series. That way I can easily inspect other parts of the
> series to see infrastructure buildup, and examples of other
> conversions or versions of the change.
>
> Thanks.
Hi Dave.
Sorry about that.
The problem I had in the past is that vger rejects email with
too many CC addresses.
(It also rejected email with __smp_XXX in the subject
suspecting it's spam :( )
I'll look for better solutions next time - for now, would it help if I
bounced the whole patchset to you?
--
MST
^ permalink raw reply
* Re: [PATCH 07/34] sparc: reuse asm-generic/barrier.h
From: David Miller @ 2015-12-30 17:56 UTC (permalink / raw)
To: mst
Cc: linux-arch, arnd, stefano.stabellini, peterz, andrew.cooper3,
linux-kernel, andreyknvl, virtualization, mpe, sparclinux, mingo
In-Reply-To: <1451473761-30019-8-git-send-email-mst@redhat.com>
I find it a little bit irritating when I receive a patch directly
addressed to me to review, but I don't see the "00/34" cover letter.
I want to see what this series is about, and what the motiviation
for this change is.
I'm also highly frustrated, in general, with mix-and-match CC: lists
for patch series. Just put everyone relevant in the CC: for the
entire series. That way I can easily inspect other parts of the
series to see infrastructure buildup, and examples of other
conversions or versions of the change.
Thanks.
^ permalink raw reply
* RE: [PATCH 20/34] ia64: define __smp_XXX
From: Luck, Tony @ 2015-12-30 16:43 UTC (permalink / raw)
To: Michael S. Tsirkin, linux-kernel@vger.kernel.org
Cc: linux-arch@vger.kernel.org, Yu, Fenghua,
linux-ia64@vger.kernel.org, Arnd Bergmann, Stefano Stabellini,
Peter Zijlstra, Andrew Cooper, Davidlohr Bueso,
virtualization@lists.linux-foundation.org, Andrey Konovalov,
Ingo Molnar
In-Reply-To: <1451473761-30019-21-git-send-email-mst@redhat.com>
> This defines __smp_XXX barriers for ia64,
> for use by virtualization.
>
> smp_XXX barriers are removed as they are
> defined correctly by asm-generic/barriers.h
>
> This reduces the amount of arch-specific boiler-plate code.
Acked-by: Tony Luck <tony.luck@intel.com>
^ permalink raw reply
* RE: [PATCH 04/34] ia64: reuse asm-generic/barrier.h
From: Luck, Tony @ 2015-12-30 16:41 UTC (permalink / raw)
To: Michael S. Tsirkin, linux-kernel@vger.kernel.org
Cc: linux-arch@vger.kernel.org, Yu, Fenghua,
linux-ia64@vger.kernel.org, Arnd Bergmann, Stefano Stabellini,
Peter Zijlstra, Andrew Cooper, Andrey Konovalov,
virtualization@lists.linux-foundation.org, Michael Ellerman,
Ingo Molnar
In-Reply-To: <1451473761-30019-5-git-send-email-mst@redhat.com>
> On ia64 smp_rmb, smp_wmb, read_barrier_depends, smp_read_barrier_depends
> and smp_store_mb() match the asm-generic variants exactly. Drop the
> local definitions and pull in asm-generic/barrier.h instead.
>
> This is in preparation to refactoring this code area.
Acked-by: Tony Luck <tony.luck@intel.com>
^ permalink raw reply
* RE: [PATCH 03/34] ia64: rename nop->iosapic_nop
From: Luck, Tony @ 2015-12-30 16:39 UTC (permalink / raw)
To: Michael S. Tsirkin, linux-kernel@vger.kernel.org
Cc: linux-arch@vger.kernel.org, Yu, Fenghua,
linux-ia64@vger.kernel.org, Arnd Bergmann, Stefano Stabellini,
Peter Zijlstra, Andrew Cooper,
virtualization@lists.linux-foundation.org, Thomas Gleixner,
Jiang Liu
In-Reply-To: <1451473761-30019-4-git-send-email-mst@redhat.com>
> asm-generic/barrier.h defines a nop() macro.
> To be able to use this header on ia64, we shouldn't
> call local functions/variables nop().
>
> There's one instance where this breaks on ia64:
> rename the function to iosapic_nop to avoid the conflict.
Acked-by: Tony Luck <tony.luck@intel.com>
^ permalink raw reply
* [PATCH] virtio/s390: use dev_to_virtio
From: Geliang Tang @ 2015-12-30 14:05 UTC (permalink / raw)
To: Christian Borntraeger, Cornelia Huck, Martin Schwidefsky,
Heiko Carstens
Cc: Geliang Tang, virtualization, linux-kernel, kvm, linux-s390
Use dev_to_virtio() instead of open-coding it.
Signed-off-by: Geliang Tang <geliangtang@163.com>
---
drivers/s390/virtio/virtio_ccw.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 1b83159..97231e1 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -945,8 +945,7 @@ static struct virtio_config_ops virtio_ccw_config_ops = {
static void virtio_ccw_release_dev(struct device *_d)
{
- struct virtio_device *dev = container_of(_d, struct virtio_device,
- dev);
+ struct virtio_device *dev = dev_to_virtio(_d);
struct virtio_ccw_device *vcdev = to_vc_device(dev);
kfree(vcdev->status);
--
2.5.0
^ permalink raw reply related
* Re: [PATCH 00/34] arch: barrier cleanup + __smp_XXX barriers for virt
From: Arnd Bergmann @ 2015-12-30 13:49 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: linux-arch, Stefano Stabellini, Peter Zijlstra, Andrew Cooper,
linux-kernel, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
On Wednesday 30 December 2015 15:24:12 Michael S. Tsirkin wrote:
> This is really trying to cleanup some virt code, as suggested by Peter, who
> said
> > You could of course go fix that instead of mutilating things into
> > sort-of functional state.
>
> This work is needed for virtio, so it's probably easiest to
> merge it through my tree - is this fine by everyone?
> Arnd, if you agree, could you ack this please?
I've looked over all patches now, and everything seems fine (I had one
small comment about a duplicate patch). It's a very nice cleanup, both
for using the asm-generic file, and for the virtualization users.
Please add my "Acked-by: Arnd Bergmann <arnd@arndb.de>" when merging the
series through your tree.
Arnd
^ permalink raw reply
* Re: [PATCH 08/34] asm-generic: smp_store_mb should use smp_mb
From: Arnd Bergmann @ 2015-12-30 13:44 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: linux-arch, Stefano Stabellini, Peter Zijlstra, Andrew Cooper,
linux-kernel, virtualization
In-Reply-To: <1451473761-30019-9-git-send-email-mst@redhat.com>
On Wednesday 30 December 2015 15:24:47 Michael S. Tsirkin wrote:
> asm-generic variant of smp_store_mb() calls mb() which is stronger
> than implied by both the name and the documentation.
>
> smp_store_mb is only used by core kernel code at the moment, so
> we know no one mis-uses it for an MMIO barrier.
> Make it call smp_mb consistently before some arch-specific
> code uses it as such by mistake.
>
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> ---
> include/asm-generic/barrier.h | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
> index 538f8d1..987b2e0 100644
> --- a/include/asm-generic/barrier.h
> +++ b/include/asm-generic/barrier.h
> @@ -93,7 +93,7 @@
> #endif /* CONFIG_SMP */
>
> #ifndef smp_store_mb
> -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); mb(); } while (0)
> +#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
> #endif
>
> #ifndef smp_mb__before_atomic
>
The same patch is already in the tip tree scheduled for 4.5 as d5a73cadf3fd
("lcoking/barriers, arch: Use smp barriers in smp_store_release()").
I think you can drop your version.
arnd
^ permalink raw reply
* [PATCH 34/34] virtio_ring: use __smp_store_mb
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
We need a full barrier after writing out event index, using
__smp_store_mb there seems better than open-coding. As usual, we need a
wrapper to account for strong barriers.
It's tempting to use this in vhost as well, for that, we'll
need a variant of smp_store_mb that works on __user pointers.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
include/linux/virtio_ring.h | 12 ++++++++++++
drivers/virtio/virtio_ring.c | 15 +++++++++------
2 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 528cf81..8ff0c25 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -45,6 +45,18 @@ static inline void virtio_wmb(bool weak_barriers)
wmb();
}
+static inline void virtio_store_mb(bool weak_barriers,
+ __virtio16 *p, __virtio16 v)
+{
+ if (weak_barriers)
+ __smp_store_mb(*p, v);
+ else
+ {
+ WRITE_ONCE(*p, v);
+ mb();
+ }
+}
+
/* a load + acquire barrier, but only guaranteed to order reads */
static inline __virtio16 virtio_load_acquire_rmb(bool weak_barriers,
__virtio16 *p)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index edc01d0..9befa32 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -517,10 +517,10 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
/* If we expect an interrupt for the next entry, tell host
* by writing event index and flush out the write before
* the read in the next get_buf call. */
- if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
- vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);
- virtio_mb(vq->weak_barriers);
- }
+ if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
+ virtio_store_mb(vq->weak_barriers,
+ &vring_used_event(&vq->vring),
+ cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
#ifdef DEBUG
vq->last_add_time_valid = false;
@@ -653,8 +653,11 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
}
/* TODO: tune this threshold */
bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
- vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);
- virtio_mb(vq->weak_barriers);
+
+ virtio_store_mb(vq->weak_barriers,
+ &vring_used_event(&vq->vring),
+ cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
+
if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {
END_USE(vq);
return false;
--
MST
^ permalink raw reply related
* [PATCH 33/34] virtio: use __smp_load_acquire/__smp_store_release
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
virtio ring entries have exactly the acquire/release
semantics:
- reading used index acquires a ring entry from host
- updating the available index releases it to host
Thus when using weak barriers (as most people do), __smp_load_acquire
and __smp_store_release will do exactly the right thing to synchronize
with the host.
In fact, QEMU already uses __atomic_thread_fence(__ATOMIC_ACQUIRE) and
__atomic_thread_fence(__ATOMIC_RELEASE);
Documentation/circular-buffers.txt suggests smp_load_acquire and
smp_store_release for head and tail updates.
Since we have to worry about strong barrier users,
let's add APIs to wrap these, and use in virtio_ring.c:
in case of the string barriers, we fall back on
rmb/wmb as previously.
It is tempting to use this in vhost as well,
this needs a bit more work to make acquire/release macros
work on __user pointers.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
include/linux/virtio_ring.h | 26 ++++++++++++++++++++++++++
drivers/virtio/virtio_ring.c | 20 ++++++++++----------
2 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 71176be..528cf81 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -45,6 +45,32 @@ static inline void virtio_wmb(bool weak_barriers)
wmb();
}
+/* a load + acquire barrier, but only guaranteed to order reads */
+static inline __virtio16 virtio_load_acquire_rmb(bool weak_barriers,
+ __virtio16 *p)
+{
+ if (weak_barriers)
+ return __smp_load_acquire(p);
+ else {
+ __virtio16 v = READ_ONCE(*p);
+ rmb();
+ return v;
+ }
+}
+
+/* a release barrier + store, but only guaranteed to order writes */
+static inline void virtio_store_release_wmb(bool weak_barriers,
+ __virtio16 *p, __virtio16 v)
+{
+ if (weak_barriers)
+ __smp_store_release(p, v);
+ else {
+ wmb();
+ WRITE_ONCE(*p, v);
+ return;
+ }
+}
+
struct virtio_device;
struct virtqueue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index ee663c4..edc01d0 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -244,11 +244,11 @@ static inline int virtqueue_add(struct virtqueue *_vq,
avail = vq->avail_idx_shadow & (vq->vring.num - 1);
vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
+ vq->avail_idx_shadow++;
/* Descriptors and available array need to be set before we expose the
* new available array entries. */
- virtio_wmb(vq->weak_barriers);
- vq->avail_idx_shadow++;
- vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);
+ virtio_store_release_wmb(vq->weak_barriers, &vq->vring.avail->idx,
+ cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow));
vq->num_added++;
pr_debug("Added buffer head %i to %p\n", head, vq);
@@ -453,9 +453,10 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
vq->vq.num_free++;
}
-static inline bool more_used(const struct vring_virtqueue *vq)
+static inline
+bool more_used(const struct vring_virtqueue *vq, __virtio16 used_idx)
{
- return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);
+ return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, used_idx);
}
/**
@@ -488,15 +489,14 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
return NULL;
}
- if (!more_used(vq)) {
+ /* Only get used array entries after they have been exposed by host. */
+ if (!more_used(vq, virtio_load_acquire_rmb(vq->weak_barriers,
+ &vq->vring.used->idx))) {
pr_debug("No more buffers in queue\n");
END_USE(vq);
return NULL;
}
- /* Only get used array entries after they have been exposed by host. */
- virtio_rmb(vq->weak_barriers);
-
last_used = (vq->last_used_idx & (vq->vring.num - 1));
i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);
*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);
@@ -704,7 +704,7 @@ irqreturn_t vring_interrupt(int irq, void *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
- if (!more_used(vq)) {
+ if (!more_used(vq, vq->vring.used->idx)) {
pr_debug("virtqueue interrupt with no work for %p\n", vq);
return IRQ_NONE;
}
--
MST
^ permalink raw reply related
* [PATCH 32/34] xen/io: use __smp_XXX barriers
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, Konrad Rzeszutek Wilk, virtualization,
David Vrabel, xen-devel, Boris Ostrovsky
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
include/xen/interface/io/ring.h uses
full memory barriers to communicate with the other side.
For guests compiled with CONFIG_SMP, smp_wmb and smp_mb
would be sufficient, so mb() and wmb() here are only needed if
a non-SMP guest runs on an SMP host.
Switch to __smp_XXX barriers which serve this exact purpose.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
include/xen/interface/io/ring.h | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/include/xen/interface/io/ring.h b/include/xen/interface/io/ring.h
index 7dc685b..46dfc65 100644
--- a/include/xen/interface/io/ring.h
+++ b/include/xen/interface/io/ring.h
@@ -208,12 +208,12 @@ struct __name##_back_ring { \
#define RING_PUSH_REQUESTS(_r) do { \
- wmb(); /* back sees requests /before/ updated producer index */ \
+ __smp_wmb(); /* back sees requests /before/ updated producer index */ \
(_r)->sring->req_prod = (_r)->req_prod_pvt; \
} while (0)
#define RING_PUSH_RESPONSES(_r) do { \
- wmb(); /* front sees responses /before/ updated producer index */ \
+ __smp_wmb(); /* front sees responses /before/ updated producer index */ \
(_r)->sring->rsp_prod = (_r)->rsp_prod_pvt; \
} while (0)
@@ -250,9 +250,9 @@ struct __name##_back_ring { \
#define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do { \
RING_IDX __old = (_r)->sring->req_prod; \
RING_IDX __new = (_r)->req_prod_pvt; \
- wmb(); /* back sees requests /before/ updated producer index */ \
+ __smp_wmb(); /* back sees requests /before/ updated producer index */ \
(_r)->sring->req_prod = __new; \
- mb(); /* back sees new requests /before/ we check req_event */ \
+ __smp_mb(); /* back sees new requests /before/ we check req_event */ \
(_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) < \
(RING_IDX)(__new - __old)); \
} while (0)
@@ -260,9 +260,9 @@ struct __name##_back_ring { \
#define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do { \
RING_IDX __old = (_r)->sring->rsp_prod; \
RING_IDX __new = (_r)->rsp_prod_pvt; \
- wmb(); /* front sees responses /before/ updated producer index */ \
+ __smp_wmb(); /* front sees responses /before/ updated producer index */ \
(_r)->sring->rsp_prod = __new; \
- mb(); /* front sees new responses /before/ we check rsp_event */ \
+ __smp_mb(); /* front sees new responses /before/ we check rsp_event */ \
(_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) < \
(RING_IDX)(__new - __old)); \
} while (0)
@@ -271,7 +271,7 @@ struct __name##_back_ring { \
(_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
if (_work_to_do) break; \
(_r)->sring->req_event = (_r)->req_cons + 1; \
- mb(); \
+ __smp_mb(); \
(_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
} while (0)
@@ -279,7 +279,7 @@ struct __name##_back_ring { \
(_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
if (_work_to_do) break; \
(_r)->sring->rsp_event = (_r)->rsp_cons + 1; \
- mb(); \
+ __smp_mb(); \
(_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
} while (0)
--
MST
^ permalink raw reply related
* [PATCH 31/34] xenbus: use __smp_XXX barriers
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, Konrad Rzeszutek Wilk, virtualization,
David Vrabel, xen-devel, Boris Ostrovsky
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
drivers/xen/xenbus/xenbus_comms.c uses
full memory barriers to communicate with the other side.
For guests compiled with CONFIG_SMP, smp_wmb and smp_mb
would be sufficient, so mb() and wmb() here are only needed if
a non-SMP guest runs on an SMP host.
Switch to __smp_XXX barriers which serve this exact purpose.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
This is straight-forward, but untested.
I can either merge this patchset through my tree if this is
acked, or defer this and merge the patchset first,
and xen bits through xen tree afterwards.
Pls let me know.
drivers/xen/xenbus/xenbus_comms.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c
index fdb0f33..09b17c7 100644
--- a/drivers/xen/xenbus/xenbus_comms.c
+++ b/drivers/xen/xenbus/xenbus_comms.c
@@ -123,14 +123,14 @@ int xb_write(const void *data, unsigned len)
avail = len;
/* Must write data /after/ reading the consumer index. */
- mb();
+ __smp_mb();
memcpy(dst, data, avail);
data += avail;
len -= avail;
/* Other side must not see new producer until data is there. */
- wmb();
+ __smp_wmb();
intf->req_prod += avail;
/* Implies mb(): other side will see the updated producer. */
@@ -180,14 +180,14 @@ int xb_read(void *data, unsigned len)
avail = len;
/* Must read data /after/ reading the producer index. */
- rmb();
+ __smp_rmb();
memcpy(data, src, avail);
data += avail;
len -= avail;
/* Other side must not see free space until we've copied out */
- mb();
+ __smp_mb();
intf->rsp_cons += avail;
pr_debug("Finished read of %i bytes (%i to go)\n", avail, len);
--
MST
^ permalink raw reply related
* [PATCH 30/34] virtio_ring: update weak barriers to use __smp_XXX
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, Alexander Duyck, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
virtio ring uses smp_wmb on SMP and wmb on !SMP,
the reason for the later being that it might be
talking to another kernel on the same SMP machine.
This is exactly what __smp_XXX barriers do,
so switch to these instead of homegrown ifdef hacks.
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Alexander Duyck <alexander.duyck@gmail.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
include/linux/virtio_ring.h | 25 ++++---------------------
1 file changed, 4 insertions(+), 21 deletions(-)
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 67e06fe..71176be 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -12,7 +12,7 @@
* anyone care?
*
* For virtio_pci on SMP, we don't need to order with respect to MMIO
- * accesses through relaxed memory I/O windows, so smp_mb() et al are
+ * accesses through relaxed memory I/O windows, so __smp_mb() et al are
* sufficient.
*
* For using virtio to talk to real devices (eg. other heterogeneous
@@ -21,11 +21,10 @@
* actually quite cheap.
*/
-#ifdef CONFIG_SMP
static inline void virtio_mb(bool weak_barriers)
{
if (weak_barriers)
- smp_mb();
+ __smp_mb();
else
mb();
}
@@ -33,7 +32,7 @@ static inline void virtio_mb(bool weak_barriers)
static inline void virtio_rmb(bool weak_barriers)
{
if (weak_barriers)
- smp_rmb();
+ __smp_rmb();
else
rmb();
}
@@ -41,26 +40,10 @@ static inline void virtio_rmb(bool weak_barriers)
static inline void virtio_wmb(bool weak_barriers)
{
if (weak_barriers)
- smp_wmb();
+ __smp_wmb();
else
wmb();
}
-#else
-static inline void virtio_mb(bool weak_barriers)
-{
- mb();
-}
-
-static inline void virtio_rmb(bool weak_barriers)
-{
- rmb();
-}
-
-static inline void virtio_wmb(bool weak_barriers)
-{
- wmb();
-}
-#endif
struct virtio_device;
struct virtqueue;
--
MST
^ permalink raw reply related
* [PATCH 29/34] Revert "virtio_ring: Update weak barriers to use dma_wmb/rmb"
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, Alexander Duyck, virtualization
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
This reverts commit 9e1a27ea42691429e31f158cce6fc61bc79bb2e9.
While that commit optimizes !CONFIG_SMP, it mixes
up DMA and SMP concepts, making the code hard
to figure out.
A better way to optimize this is with the new __smp_XXX
barriers.
As a first step, go back to full rmb/wmb barriers
for !SMP.
We switch to __smp_XXX barriers in the next patch.
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Alexander Duyck <alexander.duyck@gmail.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
include/linux/virtio_ring.h | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 8e50888..67e06fe 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -21,20 +21,19 @@
* actually quite cheap.
*/
+#ifdef CONFIG_SMP
static inline void virtio_mb(bool weak_barriers)
{
-#ifdef CONFIG_SMP
if (weak_barriers)
smp_mb();
else
-#endif
mb();
}
static inline void virtio_rmb(bool weak_barriers)
{
if (weak_barriers)
- dma_rmb();
+ smp_rmb();
else
rmb();
}
@@ -42,10 +41,26 @@ static inline void virtio_rmb(bool weak_barriers)
static inline void virtio_wmb(bool weak_barriers)
{
if (weak_barriers)
- dma_wmb();
+ smp_wmb();
else
wmb();
}
+#else
+static inline void virtio_mb(bool weak_barriers)
+{
+ mb();
+}
+
+static inline void virtio_rmb(bool weak_barriers)
+{
+ rmb();
+}
+
+static inline void virtio_wmb(bool weak_barriers)
+{
+ wmb();
+}
+#endif
struct virtio_device;
struct virtqueue;
--
MST
^ permalink raw reply related
* [PATCH 28/34] x86: define __smp_XXX
From: Michael S. Tsirkin @ 2015-12-30 13:26 UTC (permalink / raw)
To: linux-kernel
Cc: linux-arch, Arnd Bergmann, Stefano Stabellini, Peter Zijlstra,
Andrew Cooper, x86, Andrey Konovalov, virtualization, Ingo Molnar,
Andy Lutomirski, H. Peter Anvin, Thomas Gleixner, Borislav Petkov
In-Reply-To: <1451473761-30019-1-git-send-email-mst@redhat.com>
This defines __smp_XXX barriers for x86,
for use by virtualization.
smp_XXX barriers are removed as they are
defined correctly by asm-generic/barriers.h
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
arch/x86/include/asm/barrier.h | 31 ++++++++++++-------------------
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index cc4c2a7..a584e1c 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -31,17 +31,10 @@
#endif
#define dma_wmb() barrier()
-#ifdef CONFIG_SMP
-#define smp_mb() mb()
-#define smp_rmb() dma_rmb()
-#define smp_wmb() barrier()
-#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
-#else /* !SMP */
-#define smp_mb() barrier()
-#define smp_rmb() barrier()
-#define smp_wmb() barrier()
-#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
-#endif /* SMP */
+#define __smp_mb() mb()
+#define __smp_rmb() dma_rmb()
+#define __smp_wmb() barrier()
+#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
#if defined(CONFIG_X86_PPRO_FENCE)
@@ -50,31 +43,31 @@
* model and we should fall back to full barriers.
*/
-#define smp_store_release(p, v) \
+#define __smp_store_release(p, v) \
do { \
compiletime_assert_atomic_type(*p); \
- smp_mb(); \
+ __smp_mb(); \
WRITE_ONCE(*p, v); \
} while (0)
-#define smp_load_acquire(p) \
+#define __smp_load_acquire(p) \
({ \
typeof(*p) ___p1 = READ_ONCE(*p); \
compiletime_assert_atomic_type(*p); \
- smp_mb(); \
+ __smp_mb(); \
___p1; \
})
#else /* regular x86 TSO memory ordering */
-#define smp_store_release(p, v) \
+#define __smp_store_release(p, v) \
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
WRITE_ONCE(*p, v); \
} while (0)
-#define smp_load_acquire(p) \
+#define __smp_load_acquire(p) \
({ \
typeof(*p) ___p1 = READ_ONCE(*p); \
compiletime_assert_atomic_type(*p); \
@@ -85,8 +78,8 @@ do { \
#endif
/* Atomic operations are already serializing on x86 */
-#define smp_mb__before_atomic() barrier()
-#define smp_mb__after_atomic() barrier()
+#define __smp_mb__before_atomic() barrier()
+#define __smp_mb__after_atomic() barrier()
#include <asm-generic/barrier.h>
--
MST
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox